diff --git a/VIEW_CKP_STATUS.md b/VIEW_CKP_STATUS.md new file mode 100644 index 0000000000000..78b3853af0e83 --- /dev/null +++ b/VIEW_CKP_STATUS.md @@ -0,0 +1,484 @@ +# view_ckp 分支状态总结 + +## 分支概述 + +`view_ckp` 分支主题是 **checkpoint 离线工具链**：checkpoint 浏览、表 schema 解析、CSV 数据 dump、LOAD DATA 回灌兼容，以及远程 S3/MinIO fileservice 读取。 + +| 提交 | 内容 | +|------|------| +| `913f27d09` | 添加远程 checkpoint 工具和逻辑表视图 | +| `eaa1bfc11` | 修复 checkpoint schema 回放和 CSV 列映射 | +| `953b6e76b` | 优化 checkpoint CSV dump 并添加 layout 兼容性 | +| `9a0e208a0` | 修复 checkpoint 系统表 dump | +| `721cab6e5` | 使 CSV dump 兼容 LOAD DATA | + +## 当前环境 + +- **mo-service**：本次验证结束后已停止；测试时 MySQL 端口为 `6001`，root 密码 `111` +- **存储后端**：当前 `etc/launch-minio-local/*.toml` 使用 MinIO 作为 `SHARED`/`ETL`，`LOCAL` 使用 DISK +- **checkpoint 目录**：`etc/launch-minio-local/mo-data` +- **当前 checkpoint**：`Total Entries: 44`，`Global: 11`，`Incremental: 33`，`Latest TS: 1781084792256166694-1` +- **构建命令**：`make mo-tool` + +注意：在当前 Codex/exec 环境里，`nohup ... &` 启动的后台进程可能会随命令会话结束被清理；本次验证使用前台 exec session 启动 `mo-service`。普通 shell 环境可继续按后台方式启动。 + +## 已修复问题 + +### 1. 远程 S3/MinIO checksum 不匹配 + +现象： + +```text +Error: open checkpoint dir: internal error: checksum not match +``` + +根因： + +- MO 写入 S3/MinIO 的 object 文件以 `pkg/objectio/object.go` 中的 `Magic = 0xFFFFFFFF` 开头。 +- `mo-tool` 的 `lazyCacheFS` 把远程文件下载到本地临时目录后，再通过 `LocalFS.Read` 读取。 +- `LocalFS.Read` 会按本地 fileservice checksum 格式解析，期望文件块以 4 字节 CRC32 开头。 +- 远程 object 文件的 Magic 前缀不是 LocalFS checksum，因此必然报 checksum mismatch。 + +修复： + +- `pkg/tools/toolfs/lazy_cache_fs.go` 中，缓存文件仍落本地，但读取缓存时直接按 OS file range 读取，不再通过 `LocalFS.Read` 的 checksum 包装路径。 +- `Write` 后清理本地缓存，避免读到旧缓存。 + +验证： + +```bash +go test ./pkg/tools/toolfs +``` + +结果：通过。 + +### 2. 用户表 CSV dump 无法解析 mo_columns + +原始报错： + +```text +cannot resolve visible columns for table 272535 from checkpoint metadata; +mo_columns data is unavailable or incomplete +``` + +根因不是单纯的固定列宽问题。实测当前 checkpoint raw logical view： + +- `mo_tables` 可见导出为 19 列，但 raw view 可能带尾部额外列。 +- `mo_columns` raw view 为 `data=28`，目标行字段从 `att_uniq_name` 开始，额外列不是稳定前缀。 +- 当前源码里的 `catalog.MoTablesSchema`/`catalog.MoColumnsSchema` 与 checkpoint 中的 catalog 布局不完全一致，尤其涉及 `CPKey`、`ExtraInfo`、生成列等字段。 +- 只按 `len(schema)+offset` 做一次 layout 推断，会在 `pre-cpk/current/legacy` 布局之间误判。 + +修复： + +- 新增 `preCPKLayout`，覆盖无 `CPKey`、无 `ExtraInfo` 的历史 catalog 布局。 +- `mo_columns` 和 `mo_tables` 解析改为候选布局试读：优先使用实际 header 名称；header 只有 `col_N` 时，按已知 catalog 布局和偏移逐个尝试，只有能读出目标 table_id 的列/DDL 时才采纳。 +- `mo_columns` 字段名改为当前 catalog 常量，例如 `attr_seqnum`、`attr_update`，避免旧的 `att_seqnum` 写法导致索引失配。 +- `show-create-table` 优先通过候选布局读取 `mo_tables.rel_createsql`，避免退化到 `mo_columns.atttyp` 的内部二进制类型编码。 + +相关文件： + +- `pkg/tools/checkpointtool/table_dump.go` +- `pkg/tools/checkpointtool/table_dump_test.go` + +验证： + +```bash +CGO_CFLAGS="-I$(pwd)/thirdparties/install/include " \ +CGO_LDFLAGS="-L$(pwd)/thirdparties/install/lib" \ +LD_LIBRARY_PATH="$(pwd)/thirdparties/install/lib:$LD_LIBRARY_PATH" \ +go test ./pkg/tools/checkpointtool + +make mo-tool +``` + +结果：通过。 + +## 实测结果 + +### 普通表导出与 LOAD DATA + +测试表： + +```sql +CREATE TABLE test_ckp.employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +table_id： + +```text +272535 +``` + +导出： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + -o /tmp/employees_dump.csv \ + etc/launch-minio-local/mo-data +``` + +结果： + +```text +104 /tmp/employees_dump.csv +id,name,age,salary,department,hire_date,is_active +1,"Alice",30,75000.00,"Engineering",2020-01-15,true +2,"Bob",\N,\N,\N,\N,\N +3,"Charlie",\N,65000.00,\N,2021-03-10,false +``` + +`show-create-table`： + +```bash +./mo-tool ckp show-create-table --table-id=272535 etc/launch-minio-local/mo-data +``` + +结果正确返回原始 DDL： + +```sql +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +) +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +103 103 + +diff_rows +0 +``` + +### MinIO 远程 checkpoint 导出与 LOAD DATA + +本次追加验证时间：`2026-06-11`。 + +测试环境： + +- 使用本地 `/usr/local/bin/minio` 启动 MinIO，API 为 `http://127.0.0.1:9000` +- bucket：`mo-test` +- `SHARED` key-prefix：`server/data` +- 访问密钥：`minio` / `minio123` +- Docker compose 路径未使用：`docker pull minio/minio:RELEASE.2023-11-01T18-37-25Z` 通过当前 Docker registry 代理返回 500，因此改用本地 MinIO 二进制 + +测试表： + +```sql +CREATE TABLE test_ckp_minio.employees_minio ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +table_id： + +```text +272535 +``` + +确认 MO 实际使用 MinIO： + +```text +new object storage {"sdk": "minio", "arguments": {"Name":"SHARED","Bucket":"mo-test","Endpoint":"127.0.0.1:9000","KeyPrefix":"server/data","IsMinio":true}} +new object storage {"sdk": "minio", "arguments": {"Name":"ETL","Bucket":"mo-test","Endpoint":"127.0.0.1:9000","KeyPrefix":"server/etl","IsMinio":true}} +``` + +远程 checkpoint ready： + +```bash +./mo-tool ckp dump --table-id=2 \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED \ + -o /tmp/minio_mo_tables_probe.csv +``` + +结果： + +```text +2008 /tmp/minio_mo_tables_probe.csv +``` + +`ckp info` 验证了两条远程打开路径： + +```bash +./mo-tool ckp info \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +./mo-tool ckp info \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +结果均为： + +```text +Total Entries: 2 + Global: 0 + Incremental: 2 +Latest TS: 1781142243226306710-0 +``` + +用户表 dump： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED \ + -o /tmp/minio_employees_dump.csv +``` + +结果： + +```text +104 /tmp/minio_employees_dump.csv +id,name,age,salary,department,hire_date,is_active +``` + +`--s3` 直连路径也通过，且与 `--fs-config` 输出一致： + +```bash +./mo-tool ckp dump --table-id=272535 --header \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 \ + -o /tmp/minio_employees_dump_s3.csv + +cmp -s /tmp/minio_employees_dump.csv /tmp/minio_employees_dump_s3.csv +``` + +结果： + +```text +104 /tmp/minio_employees_dump_s3.csv +cmp_exit=0 +``` + +`show-create-table`： + +```bash +./mo-tool ckp show-create-table --table-id=272535 \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +结果正确返回原始 DDL： + +```sql +CREATE TABLE employees_minio ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +) +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +103 103 + +diff_rows +0 +``` + +结论： + +- `mo-tool ckp info/dump/show-create-table` 通过 MinIO 远程 fileservice 可用。 +- `--fs-config ... --fs-name SHARED` 和 `--backend MINIO --s3 ...` 两条路径均可读取 checkpoint。 +- 未再出现 `checksum not match`。 +- 从 MinIO checkpoint 导出的 CSV 可直接 `LOAD DATA` 回灌，行数和差异校验通过。 + +### ALTER TABLE 后新旧数据兼容 + +测试过程： + +1. 创建旧布局表并插入 3 行： + +```sql +CREATE TABLE test_ckp.alter_compat ( + id INT PRIMARY KEY, + name VARCHAR(50) +); +INSERT INTO alter_compat VALUES (1, 'old_a'), (2, 'old_b'), (3, 'old_c'); +``` + +2. 等待旧 table_id `272537` 能从 checkpoint 导出： + +```text +id,name +1,"old_a" +2,"old_b" +3,"old_c" +``` + +3. 执行 ALTER 并插入新布局数据： + +```sql +ALTER TABLE alter_compat ADD COLUMN note VARCHAR(50) DEFAULT 'legacy'; +INSERT INTO alter_compat VALUES (4, 'new_d', 'fresh'), (5, 'new_e', NULL); +``` + +4. ALTER 后 MatrixOne 将当前 catalog table_id 切到 `272538`： + +```text +rel_id relname +272538 alter_compat +``` + +用旧 table_id `272537` 导出会失败，因为当前 catalog metadata 已经不再暴露旧 table_id 的 `mo_columns` 可见列；这是预期边界。应使用 ALTER 后的当前 table_id `272538` 导出。 + +导出 `272538` 结果： + +```text +id,name,note +1,"old_a","legacy" +2,"old_b","legacy" +3,"old_c","legacy" +4,"new_d","fresh" +5,"new_e",\N +``` + +LOAD DATA 校验： + +```text +original_rows loaded_rows +5 5 + +diff_rows +0 +``` + +兼容性结论： + +- ALTER 前旧对象和 ALTER 后新对象在当前 table_id `272538` 的 checkpoint 视图下可一起导出。 +- 新增列默认值已在导出结果中物化，旧行 `note='legacy'`。 +- 使用 ALTER 前旧 table_id 导出当前表不是支持目标；旧 ID 在 catalog 中已被新 ID 替代，schema 解析会失败。 + +## catalog/checkpoint 版本兼容性结论 + +当前实现支持以下 catalog layout： + +| Layout | 适用场景 | 处理方式 | +|--------|----------|----------| +| `current` | 当前源码中的 `catalog.MoTablesSchema` / `catalog.MoColumnsSchema` | 直接按当前 schema 或 header 名解析 | +| `pre-cpk` | checkpoint 中 catalog 尚未包含 `CPKey`/`ExtraInfo` 的布局 | 移除这些字段后按候选布局试读 | +| `3.0-dev` | 已知 3.0 开发期尾部字段较少的布局 | 保留原兼容路径 | + +重要规则： + +- 如果 checkpoint logical view 提供真实列名，优先按列名解析。 +- 如果只有 `col_N`，不再相信单一宽度推断；按候选 layout 和偏移逐个试读。 +- dataWidth 可能包含尾部额外物理列，不能只按 `len(schema)+offset` 精确匹配。 +- 未知未来 catalog layout 若无法通过候选试读解析出目标 table_id，会明确失败，不回退到物理列导出，避免把隐藏列或错列导出成用户数据。 + +## 推荐测试步骤 + +### 1. 构建 + +```bash +make mo-tool +``` + +### 2. 启动 mo-service + +普通 shell 可用： + +```bash +kill -9 $(pgrep mo-service) 2>/dev/null || true +rm -rf etc/launch-minio-local/mo-data +mkdir -p etc/launch-minio-local/mo-data +./mo-service -launch etc/launch-minio-local/launch.toml > /tmp/mo-service.log 2>&1 & +until mysqladmin -h 127.0.0.1 -P 6001 -u root -p111 ping >/dev/null 2>&1; do + sleep 1 +done +``` + +在 Codex exec 环境中建议前台运行： + +```bash +./mo-service -launch etc/launch-minio-local/launch.toml +``` + +### 3. 等 checkpoint 可用于 dump + +不要只等 `Incremental: [1-9]`。第一次 incremental 可能还没有目标表或系统表范围。更稳妥的方式是直接等系统表 dump 成功： + +```bash +until ./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables_probe.csv \ + etc/launch-minio-local/mo-data >/dev/null 2>&1; do + sleep 5 +done +``` + +### 4. 导出并回灌 + +```bash +TABLE_ID=272535 + +./mo-tool ckp dump --table-id=${TABLE_ID} --header \ + -o /tmp/employees_dump.csv \ + etc/launch-minio-local/mo-data + +wc -l /tmp/employees_dump.csv +head -5 /tmp/employees_dump.csv +``` + +```sql +CREATE TABLE test_ckp.employees_load ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); + +LOAD DATA INFILE '/tmp/employees_dump.csv' +INTO TABLE test_ckp.employees_load +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; + +SELECT (SELECT COUNT(*) FROM test_ckp.employees) AS original_rows, + (SELECT COUNT(*) FROM test_ckp.employees_load) AS loaded_rows; + +SELECT COUNT(*) FROM + (SELECT * FROM test_ckp.employees EXCEPT SELECT * FROM test_ckp.employees_load) AS diff; +``` + +## 常用命令 + +```bash +./mo-tool ckp info etc/launch-minio-local/mo-data +./mo-tool ckp show-create-table --table-id= etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id= --header -o /tmp/table.csv etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv etc/launch-minio-local/mo-data +./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv etc/launch-minio-local/mo-data +``` diff --git a/cmd/mo-object-tool/README.md b/cmd/mo-object-tool/README.md new file mode 100644 index 0000000000000..a5aa23cc2d912 --- /dev/null +++ b/cmd/mo-object-tool/README.md @@ -0,0 +1,322 @@ +# mo-tool Object And Checkpoint Usage + +`mo-tool` provides offline tools for inspecting MatrixOne checkpoint and object +files. It can read from a local data directory, or from a remote MatrixOne +fileservice such as S3 or MinIO. + +## Build + +```bash +go build -o mo-tool ./cmd/mo-tool +``` + +You can also run it directly during development: + +```bash +go run ./cmd/mo-tool --help +``` + +## Local Checkpoint Usage + +Read checkpoint files from a local MatrixOne data directory: + +```bash +./mo-tool ckp info ./mo-data +./mo-tool ckp view ./mo-data +``` + +If no directory is provided, the current working directory is used: + +```bash +./mo-tool ckp info +./mo-tool ckp view +``` + +The checkpoint viewer lists checkpoint entries and lets you drill down into +tables, object ranges, and object data. + +Inside table detail view: + +- Press `Enter` on a row to open the physical object view for that object. +- Press `L` to open a logical table view with tombstones applied to table rows. + +## Offline Table Schema And CSV Export + +`mo-tool ckp` can reconstruct table schema and export logical table rows directly +from checkpoint data. + +Show the `CREATE TABLE` statement for a table at the latest checkpoint: + +```bash +./mo-tool ckp show-create-table --table-id=272387 ./mo-data +``` + +Show the schema at a specific snapshot timestamp: + +```bash +./mo-tool ckp show-create-table \ + --table-id=272387 \ + --ts=1780574341899433000:0 \ + ./mo-data +``` + +Dump a logical table view to CSV: + +```bash +./mo-tool ckp dump --table-id=323820 ./mo-data +./mo-tool ckp dump --table-id=323820 -o workflow_cases.csv ./mo-data +./mo-tool ckp dump --table-id=323820 --header --meta-comments ./mo-data +./mo-tool ckp dump --table-id=323820 --row-order=lexical ./mo-data +``` + +The CSV dump: + +- applies tombstones at the selected checkpoint snapshot +- excludes hidden system columns +- uses catalog metadata from `mo_tables` and `mo_columns` +- uses MatrixOne SQL export style field formatting so the output can be loaded + back with `LOAD DATA` +- uses `\N` for nulls +- streams row output instead of materializing the full CSV in memory first when + `--row-order=storage` (the default) + +If catalog metadata cannot be resolved exactly, the command fails instead of +guessing from raw physical object columns. + +By default, `ckp dump` is geared toward reloadability: + +- no metadata comment lines +- no header row + +Use these flags when you want a more human-readable export: + +- `--meta-comments` to prepend `-- CREATE TABLE ...` and row count comment lines +- `--header` to include a header row with visible column names +- use `--row-order=lexical` if you want deterministic CSV ordering by visible + column values for diffing or reproducible snapshots + +Row order modes: + +- `storage`: object/block/row scan order after tombstone application; this is + the default and remains large-table friendly +- `lexical`: sorts the final visible CSV rows lexicographically by exported + column values; this buffers rows in memory and is intended for smaller exports + +## Local Object Usage + +Read a single local object file: + +```bash +./mo-tool object info ./mo-data/ +./mo-tool object view ./mo-data/ +``` + +Replace `` with the object path shown by the checkpoint viewer or +stored in checkpoint metadata. + +## Remote Fileservice From MatrixOne Config + +The recommended remote workflow is to reuse the MatrixOne service TOML file. +This avoids manually copying bucket, endpoint, key prefix, and credential +settings. + +For example, with a MinIO launch config: + +```bash +./mo-tool ckp info \ + --fs-config etc/launch-minio-local/tn.toml + +./mo-tool ckp view \ + --fs-config etc/launch-minio-local/tn.toml +``` + +By default, `mo-tool` uses the fileservice configured in +`[tn.Txn.Storage].fileservice`. If that field is absent, it uses `SHARED`. + +To select a specific fileservice: + +```bash +./mo-tool ckp view \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +The same fileservice options work for object inspection: + +```bash +./mo-tool object info \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +./mo-tool object view \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +When checkpoint data is opened from a remote fileservice, object drill-down from +the checkpoint viewer reuses the same fileservice. This means checkpoint meta, +checkpoint data objects, and table data objects are all read from the same S3 or +MinIO backend. + +The logical table view also uses the same fileservice and applies tombstones at +the selected checkpoint snapshot timestamp. + +The same remote options work for schema lookup and CSV export: + +```bash +./mo-tool ckp show-create-table \ + --table-id=272387 \ + --fs-config etc/launch-minio-local/tn.toml + +./mo-tool ckp dump \ + --table-id=323820 \ + --fs-config etc/launch-minio-local/tn.toml +``` + +## Remote S3 Or MinIO With Inline Arguments + +You can also provide object storage settings directly on the command line. + +MinIO example: + +```bash +./mo-tool ckp view \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +AWS S3 example: + +```bash +./mo-tool ckp info \ + --backend S3 \ + --s3 bucket=my-mo-bucket,region=us-east-1,key-prefix=prod/mo-data +``` + +Object viewer with inline S3 arguments: + +```bash +./mo-tool object view \ + --backend S3 \ + --s3 bucket=my-mo-bucket,region=us-east-1,key-prefix=prod/mo-data +``` + +Supported `--s3` keys include: + +- `bucket` +- `endpoint` +- `region` +- `key-prefix` +- `key-id` +- `key-secret` +- `session-token` +- `role-arn` +- `external-id` +- `shared-config-profile` +- `no-default-credentials` +- `no-bucket-validation` +- `is-minio` +- `cert-files` + +Credentials can also come from the default AWS credential chain when supported +by the selected backend and when `no-default-credentials` is not set. + +## Catalog Layout Compatibility + +Checkpoint catalog objects are not completely stable across MatrixOne branches. +In particular, older `3.0-dev` generated data may differ from newer builds in +the layout of `mo_tables` and `mo_columns`. + +Current tool behavior: + +- it first uses actual column names from checkpoint catalog rows when present +- if the checkpoint only exposes generic `col_N` headers, it infers the system + table layout from the observed column count +- it supports both: + - current layout, which includes `mo_tables.rel_logical_id` and + `mo_columns.attr_has_generated` / `mo_columns.attr_generated` + - older `3.0-dev` layout, which does not include those columns + +For `mo_columns`, CSV extraction uses: + +- `attnum` for SQL column order +- `att_seqnum` for physical object column position + +This matters for tables with hidden columns, index-related internal columns, or +wide schemas. Without `att_seqnum`, visible values can shift to the wrong CSV +column. + +In practice, you do not need a separate flag for `3.0-dev` generated data. The +tool auto-detects the known layout variants and falls back to the matching +built-in schema for system tables when needed. + +Example with older checkpoint data: + +```bash +./mo-tool ckp show-create-table --table-id=2 ./mo-data +./mo-tool ckp dump --table-id=323820 ./mo-data +``` + +If a checkpoint comes from an unknown future layout and neither named columns +nor known-width inference matches, schema reconstruction can still fail closed. +That is intentional. + +This means `3.0-dev` checkpoint data is supported for the known catalog layout +differences above, but compatibility is not an open-ended promise for arbitrary +older or future internal layouts. + +## Important Path Rules + +For remote checkpoint inspection, `key-prefix` should point to the MatrixOne +data root, not to the checkpoint directory itself. + +Correct: + +```text +s3://my-bucket/prod/mo-data/ + ckp/ + +``` + +Use: + +```bash +--s3 bucket=my-bucket,key-prefix=prod/mo-data,... +``` + +Do not use: + +```bash +--s3 bucket=my-bucket,key-prefix=prod/mo-data/ckp,... +``` + +Checkpoint metadata stores object names relative to the fileservice root. If the +fileservice root is pointed directly at `ckp/`, checkpoint metadata may be found +but table data object drill-down will fail. + +## Command Summary + +Checkpoint commands: + +```bash +./mo-tool ckp info [directory] [--fs-config FILE] [--fs-name NAME] +./mo-tool ckp view [directory] [--fs-config FILE] [--fs-name NAME] +./mo-tool ckp dump --table-id ID [directory] [--ts PHYSICAL:LOGICAL] [-o FILE] [--header] [--meta-comments] [--row-order storage|lexical] +./mo-tool ckp show-create-table --table-id ID [directory] [--ts PHYSICAL:LOGICAL] +./mo-tool ckp info --backend S3 --s3 key=value,... +./mo-tool ckp view --backend MINIO --s3 key=value,... +./mo-tool ckp dump --table-id ID --backend S3 --s3 key=value,... [--header] [--meta-comments] [--row-order storage|lexical] +./mo-tool ckp show-create-table --table-id ID --backend S3 --s3 key=value,... +``` + +Object commands: + +```bash +./mo-tool object info [--fs-config FILE] [--fs-name NAME] +./mo-tool object view [--fs-config FILE] [--fs-name NAME] +./mo-tool object info --backend S3 --s3 key=value,... +./mo-tool object view --backend MINIO --s3 key=value,... +``` + +Local mode treats paths as local filesystem paths. Remote mode treats object +names as paths relative to the selected fileservice root. diff --git a/cmd/mo-object-tool/ckp/checkpoint.go b/cmd/mo-object-tool/ckp/checkpoint.go index 59d4889416169..a262669bfccfd 100644 --- a/cmd/mo-object-tool/ckp/checkpoint.go +++ b/cmd/mo-object-tool/ckp/checkpoint.go @@ -15,17 +15,38 @@ package ckp import ( + "bytes" "context" + "encoding/json" + "errors" "fmt" + "io" "os" + "os/signal" + "path" + "path/filepath" + "runtime/pprof" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "text/tabwriter" + "time" + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool/interactive" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) func PrepareCommand() *cobra.Command { + var storage toolfs.StorageOptions cmd := &cobra.Command{ Use: "ckp [directory]", Short: "Checkpoint viewer tool", @@ -36,16 +57,34 @@ func PrepareCommand() *cobra.Command { if len(args) == 1 { dir = args[0] } - return runViewer(dir) + return runViewer(dir, storage) }, } + addStorageFlags(cmd, &storage) - cmd.AddCommand(infoCommand()) - cmd.AddCommand(viewCommand()) + cmd.AddCommand(infoCommand(&storage)) + cmd.AddCommand(viewCommand(&storage)) + cmd.AddCommand(listCommand(&storage)) + cmd.AddCommand(dumpCommand(&storage)) + cmd.AddCommand(showCreateTableCommand(&storage)) return cmd } +func addStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.PersistentFlags().StringVar(&storage.FSConfig, "fs-config", "", "MO config TOML containing fileservice settings") + cmd.PersistentFlags().StringVar(&storage.FSName, "fs-name", "SHARED", "fileservice name to use from --fs-config") + cmd.PersistentFlags().StringVar(&storage.S3, "s3", "", "S3 arguments, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.PersistentFlags().StringVar(&storage.Backend, "backend", "", "remote backend for --s3: S3 or MINIO") +} + +func addOutputStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.Flags().StringVar(&storage.FSConfig, "out-fs-config", "", "MO config TOML containing fileservice settings for dump output") + cmd.Flags().StringVar(&storage.FSName, "out-fs-name", "SHARED", "fileservice name to use from --out-fs-config") + cmd.Flags().StringVar(&storage.S3, "out-s3", "", "S3 arguments for dump output, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.Flags().StringVar(&storage.Backend, "out-backend", "", "remote backend for --out-s3: S3 or MINIO") +} + func setupLogFile() (*os.File, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -76,7 +115,7 @@ func setupLogFile() (*os.File, error) { return logFile, nil } -func runViewer(dir string) error { +func runViewer(dir string, storage toolfs.StorageOptions) error { logFile, err := setupLogFile() if err != nil { return fmt.Errorf("setup log file: %w", err) @@ -84,7 +123,7 @@ func runViewer(dir string) error { defer logFile.Close() ctx := context.Background() - reader, err := checkpointtool.Open(ctx, dir) + reader, err := openReader(ctx, dir, storage) if err != nil { return fmt.Errorf("open checkpoint dir: %w", err) } @@ -93,7 +132,7 @@ func runViewer(dir string) error { return interactive.Run(reader) } -func infoCommand() *cobra.Command { +func infoCommand(storage *toolfs.StorageOptions) *cobra.Command { return &cobra.Command{ Use: "info [directory]", Short: "Show checkpoint summary", @@ -111,7 +150,7 @@ func infoCommand() *cobra.Command { defer logFile.Close() ctx := context.Background() - reader, err := checkpointtool.Open(ctx, dir) + reader, err := openReader(ctx, dir, *storage) if err != nil { return fmt.Errorf("open checkpoint dir: %w", err) } @@ -134,7 +173,7 @@ func infoCommand() *cobra.Command { } } -func viewCommand() *cobra.Command { +func viewCommand(storage *toolfs.StorageOptions) *cobra.Command { return &cobra.Command{ Use: "view [directory]", Short: "Interactive checkpoint viewer", @@ -144,7 +183,2035 @@ func viewCommand() *cobra.Command { if len(args) == 1 { dir = args[0] } - return runViewer(dir) + return runViewer(dir, *storage) + }, + } +} + +func listCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + accountID uint32 + databaseID uint64 + tsStr string + includeViews bool + listType string + ) + + cmd := &cobra.Command{ + Use: "list [directory]", + Short: "List checkpoint catalog tables", + Long: `List table metadata from the checkpoint catalog. + +By default this lists all catalog relations. Use --database-id or --account-id +to narrow the result.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) + } + + var accountFilter *uint32 + if cmd.Flags().Changed("account-id") { + accountFilter = &accountID + } + var dbFilter *uint64 + if cmd.Flags().Changed("database-id") { + dbFilter = &databaseID + } + opts := checkpointtool.TableListOptions{ + AccountID: accountFilter, + DatabaseID: dbFilter, + IncludeViews: includeViews, + } + var tables []checkpointtool.TableCatalogEntry + if listType == "databases" || listType == "dbs" { + tables, err = reader.ListCatalogDatabases(ctx, snapshotTS, opts) + } else { + tables, err = reader.ListCatalogTables(ctx, snapshotTS, opts) + } + if err != nil { + return fmt.Errorf("list checkpoint catalog tables: %w", err) + } + if err := printCatalogList(cmd.OutOrStdout(), tables, listType); err != nil { + return err + } + return nil + }, + } + cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to list") + cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to list") + cmd.Flags().BoolVar(&includeViews, "include-views", false, "Deprecated no-op; views are listed by default") + cmd.Flags().StringVar(&listType, "type", "tables", "List type: tables, databases, or accounts") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") + return cmd +} + +func printCatalogList(w io.Writer, tables []checkpointtool.TableCatalogEntry, listType string) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + switch listType { + case "", "tables": + fmt.Fprintln(tw, "ACCOUNT_ID\tDATABASE\tTABLE\tTABLE_ID\tREL_KIND") + for _, table := range tables { + fmt.Fprintf(tw, "%d\t%s\t%s\t%d\t%s\n", table.AccountID, table.DatabaseName, table.TableName, table.TableID, table.RelKind) + } + case "databases", "dbs": + type dbKey struct { + accountID uint32 + name string + id uint64 + } + seen := make(map[dbKey]struct{}) + var dbs []dbKey + for _, table := range tables { + key := dbKey{accountID: table.AccountID, name: table.DatabaseName, id: table.DatabaseID} + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + dbs = append(dbs, key) + } + sort.Slice(dbs, func(i, j int) bool { + if dbs[i].accountID != dbs[j].accountID { + return dbs[i].accountID < dbs[j].accountID + } + return dbs[i].name < dbs[j].name + }) + fmt.Fprintln(tw, "ACCOUNT_ID\tDATABASE\tDATABASE_ID") + for _, db := range dbs { + fmt.Fprintf(tw, "%d\t%s\t%d\n", db.accountID, db.name, db.id) + } + case "accounts", "tenants": + seen := make(map[uint32]struct{}) + var accounts []uint32 + for _, table := range tables { + if _, ok := seen[table.AccountID]; ok { + continue + } + seen[table.AccountID] = struct{}{} + accounts = append(accounts, table.AccountID) + } + sort.Slice(accounts, func(i, j int) bool { return accounts[i] < accounts[j] }) + fmt.Fprintln(tw, "ACCOUNT_ID") + for _, accountID := range accounts { + fmt.Fprintf(tw, "%d\n", accountID) + } + default: + return fmt.Errorf("unknown --type %q; expected tables, databases, or accounts", listType) + } + return tw.Flush() +} + +func openReader(ctx context.Context, dir string, storage toolfs.StorageOptions) (*checkpointtool.CheckpointReader, error) { + if !storage.IsRemote() { + return checkpointtool.Open(ctx, dir) + } + fs, display, err := toolfs.Open(ctx, storage) + if err != nil { + return nil, err + } + if display == "" { + display = dir + } + return checkpointtool.OpenWithFS(ctx, fs, display, checkpointtool.WithCloseFS()) +} + +type dumpOutput struct { + fs fileservice.FileService + remote bool +} + +func openDumpOutput(ctx context.Context, storage toolfs.StorageOptions) (*dumpOutput, error) { + if !storage.IsRemote() { + return &dumpOutput{}, nil + } + fs, display, err := toolfs.Open(ctx, storage) + if err != nil { + return nil, err + } + logutil.Infof("using fileservice %s for dump output", display) + return &dumpOutput{fs: fs, remote: true}, nil +} + +func (o *dumpOutput) Close(ctx context.Context) { + if o != nil && o.fs != nil { + o.fs.Close(ctx) + } +} + +func (o *dumpOutput) MkdirAll(dir string) error { + if o == nil || !o.remote { + return os.MkdirAll(dir, 0o755) + } + return nil +} + +func (o *dumpOutput) Create(ctx context.Context, filePath string) (io.WriteCloser, error) { + if o == nil || !o.remote { + return os.Create(filePath) + } + return newFileServiceWriteCloser(ctx, o.fs, filePath) +} + +type fileServiceWriteCloser struct { + pw *io.PipeWriter + done chan error + closeOnce sync.Once + closeErr error +} + +func newFileServiceWriteCloser(ctx context.Context, fs fileservice.FileService, filePath string) (io.WriteCloser, error) { + pr, pw := io.Pipe() + w := &fileServiceWriteCloser{ + pw: pw, + done: make(chan error, 1), + } + go func() { + var err error + defer func() { + w.done <- err + }() + ctx = fileservice.WithParallelMode(ctx, fileservice.ParallelAuto) + logutil.Infof("streaming dump output to fileservice with parallel-mode=auto: %s", cleanObjectPath(filePath)) + err = fs.Write(ctx, fileservice.IOVector{ + FilePath: cleanObjectPath(filePath), + Entries: []fileservice.IOEntry{{ + ReaderForWrite: pr, + Size: -1, + }}, + }) + if err != nil { + _ = pr.CloseWithError(err) + } + }() + return w, nil +} + +func (w *fileServiceWriteCloser) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *fileServiceWriteCloser) Close() error { + w.closeOnce.Do(func() { + err := w.pw.Close() + writeErr := <-w.done + if err != nil { + w.closeErr = err + return + } + w.closeErr = writeErr + }) + return w.closeErr +} + +func cleanObjectPath(filePath string) string { + filePath = filepath.ToSlash(filePath) + return strings.TrimPrefix(path.Clean(filePath), "/") +} + +// dumpCommand implements the "ckp dump" subcommand for offline CSV export. +// +// Usage: +// +// mo-tool ckp dump --table-id=12345 [--ts=...] [--output=table.csv] [directory] +func dumpCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + tableID uint64 + accountID uint32 + databaseID uint64 + tsStr string + output string + outputDir string + jobs int + metaComments bool + header bool + loadScript bool + noLoad bool + rowOrder string + cpuProfile string + outputStorage toolfs.StorageOptions + ) + + cmd := &cobra.Command{ + Use: "dump [directory]", + Short: "Dump a table from checkpoint to CSV (offline)", + Long: `Dump a table from checkpoint to CSV with tombstone filtering applied. + +The schema (column names and visible columns) is resolved from mo_tables +and mo_columns system tables in the checkpoint. If that catalog metadata is +unavailable or incomplete, the command fails instead of exporting hidden +physical columns. + +Examples: + mo-tool ckp dump --table-id=12345 /path/to/ckp # dump to stdout + mo-tool ckp dump --table-id=12345 -o users.csv . # dump to file + mo-tool ckp dump --table-id=12345 --ts=1749001234567890:1 . # dump at specific TS + mo-tool ckp dump --table-id=12345 --load-script -o /tmp/ . + mo-tool ckp dump --database-id=9001 --output-dir=/tmp/test-dump --jobs=4 . + mo-tool ckp dump --database-id=9001 --load-script -o /tmp/test-dump . + mo-tool ckp dump --table-id=12345 -o dump/users.csv --out-s3='endpoint=...,bucket=...,key-prefix=...,key-id=...,key-secret=...' .`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + if cpuProfile != "" { + f, err := os.Create(cpuProfile) + if err != nil { + return fmt.Errorf("create cpuprofile: %w", err) + } + defer f.Close() + if err := pprof.StartCPUProfile(f); err != nil { + return fmt.Errorf("start cpuprofile: %w", err) + } + + // Ensure CPU profile is flushed on SIGINT (Ctrl+C) + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + pprof.StopCPUProfile() + f.Close() + os.Exit(1) + }() + defer pprof.StopCPUProfile() + } + + accountIDSet := cmd.Flags().Changed("account-id") + batchDump := tableID == 0 + databaseIDSet := cmd.Flags().Changed("database-id") + if tableID == 0 && !databaseIDSet && !accountIDSet { + return fmt.Errorf("--table-id, or at least one of --database-id/--account-id is required") + } + if noLoad && !loadScript { + return fmt.Errorf("--no-load requires --load-script") + } + if tableID != 0 && (databaseIDSet || accountIDSet || (outputDir != "" && !loadScript)) { + return fmt.Errorf("--table-id cannot be combined with --database-id, --account-id, or --output-dir") + } + if batchDump && output != "" && !loadScript { + return fmt.Errorf("--output cannot be used when dumping by --database-id or --account-id; use --output-dir") + } + if loadScript && output == "" { + return fmt.Errorf("--output/-o directory is required with --load-script") + } + if loadScript && outputDir == "" { + outputDir = output + } + if batchDump && outputDir == "" { + return fmt.Errorf("--output-dir is required when dumping by --database-id or --account-id") + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + dumpOut, err := openDumpOutput(ctx, outputStorage) + if err != nil { + return fmt.Errorf("open dump output fileservice: %w", err) + } + defer dumpOut.Close(ctx) + loadPathResolver, err := newLoadDataPathResolver(outputStorage) + if err != nil { + return err + } + + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) + } + + var w = cmd.OutOrStdout() + var outFile io.WriteCloser + if output != "" && !loadScript { + outFile, err = dumpOut.Create(ctx, output) + if err != nil { + return fmt.Errorf("create output file: %w", err) + } + defer func() { + if outFile != nil { + _ = outFile.Close() + } + }() + w = outFile + } + + parsedRowOrder, err := checkpointtool.ParseCSVRowOrder(rowOrder) + if err != nil { + return err + } + + effectiveHeader := header || (loadScript && !noLoad) + + if batchDump { + var accountFilter *uint32 + if accountIDSet { + accountFilter = &accountID + } + var dbFilter *uint64 + if databaseIDSet { + dbFilter = &databaseID + } + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + AccountID: accountFilter, + DatabaseID: dbFilter, + }) + if err != nil { + return fmt.Errorf("list checkpoint catalog tables: %w", err) + } + if len(tables) == 0 { + return fmt.Errorf("no checkpoint tables match database-id-set=%v database-id=%d account-id-set=%v account-id=%d", databaseIDSet, databaseID, accountIDSet, accountID) + } + if err := dumpOut.MkdirAll(outputDir); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + var dumpPlans []tableDumpPlan + if loadScript || !noLoad { + dumpPlans, err = prepareTableDumpPlans(ctx, reader, tables, snapshotTS) + if err != nil { + return err + } + } + if !noLoad { + if jobs < 1 { + jobs = 1 + } + if jobs > len(tables) { + jobs = len(tables) + } + if err := dumpTablesConcurrently(ctx, reader, dumpOut, dumpPlans, snapshotTS, outputDir, jobs, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout()); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Dumped %d tables to %s\n", len(tables), outputDir) + } + if loadScript { + scriptTables, skippedSystemTables := filterAccountRestoreScriptTables(tables, accountIDSet) + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, scriptTables, dumpDataByTableID(dumpPlans), snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) + if err != nil { + return err + } + if skippedSystemTables > 0 { + fmt.Fprintf(cmd.OutOrStdout(), "Restore script skipped %d system tables that are not restorable by tenant admin; CSV files were still dumped.\n", skippedSystemTables) + } + fmt.Fprintf(cmd.OutOrStdout(), "Restore script written to %s\n", scriptPath) + } + return nil + } + + var tableEntry checkpointtool.TableCatalogEntry + if loadScript { + tableEntry, err = resolveTableByID(ctx, reader, snapshotTS, tableID) + if err != nil { + return err + } + if err := dumpOut.MkdirAll(outputDir); err != nil { + return fmt.Errorf("create output dir: %w", err) + } + dumpData, err := reader.PrepareTableDumpData(ctx, tableEntry.TableID, snapshotTS) + if err != nil { + return fmt.Errorf("prepare table %d (%s.%s): %w", tableEntry.TableID, tableEntry.DatabaseName, tableEntry.TableName, err) + } + if !noLoad { + if err := dumpOneTable(ctx, reader, dumpOut, tableDumpPlan{table: tableEntry, data: dumpData}, snapshotTS, outputDir, parsedRowOrder, metaComments, effectiveHeader, cmd.OutOrStdout(), &sync.Mutex{}); err != nil { + return err + } + } + scriptPath, err := writeRestoreScript(ctx, reader, dumpOut, []checkpointtool.TableCatalogEntry{tableEntry}, map[uint64]*checkpointtool.TableDumpData{tableEntry.TableID: dumpData}, snapshotTS, output, outputDir, loadPathResolver, !noLoad, effectiveHeader) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "Restore script written to %s\n", scriptPath) + return nil + } + + dumpErr := reader.DumpTableCSVComposed( + ctx, + w, + tableID, + snapshotTS, + checkpointtool.WithCSVMetaComments(metaComments), + checkpointtool.WithCSVHeader(effectiveHeader), + checkpointtool.WithCSVRowOrder(parsedRowOrder), + ) + if outFile != nil { + dumpErr = errors.Join(dumpErr, outFile.Close()) + outFile = nil + } + if dumpErr != nil { + return fmt.Errorf("dump table %d: %w", tableID, dumpErr) + } + + if output != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Table %d dumped to %s\n", tableID, output) + } + return nil }, } + + cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to dump") + cmd.Flags().Uint32Var(&accountID, "account-id", 0, "Account ID to dump tables for; combine with --database-id to narrow the result") + cmd.Flags().Uint64Var(&databaseID, "database-id", 0, "Database ID to dump tables for") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or '2006-01-02 15:04:05' in local time (default: latest)") + cmd.Flags().StringVarP(&output, "output", "o", "", "Output CSV file path (default: stdout)") + cmd.Flags().StringVar(&outputDir, "output-dir", "", "Output directory for database/account dumps") + cmd.Flags().IntVar(&jobs, "jobs", 5, "Concurrent table dumps for --database-id/--account-id batch dumps") + cmd.Flags().BoolVar(&metaComments, "meta-comments", false, "Prepend DDL and row-count comment lines (disabled by default so output can be loaded directly)") + cmd.Flags().BoolVar(&header, "header", false, "Include a CSV header row with column names") + cmd.Flags().BoolVar(&loadScript, "load-script", false, "Generate restore.sql with CREATE DATABASE, CREATE TABLE, and LOAD DATA statements; --output/-o is treated as a directory") + cmd.Flags().BoolVar(&noLoad, "no-load", false, "With --load-script, generate only DDL and skip CSV dump and LOAD DATA statements") + cmd.Flags().StringVar(&rowOrder, "row-order", string(checkpointtool.CSVRowOrderStorage), "CSV row order: storage (streaming, large-table friendly) or lexical (sort by visible CSV values in memory)") + cmd.Flags().StringVar(&cpuProfile, "cpuprofile", "", "Write CPU profile to file") + addOutputStorageFlags(cmd, &outputStorage) + + return cmd +} + +func parseTS(s string) (types.TS, error) { + s = strings.TrimSpace(s) + for _, sep := range []string{":", "-"} { + parts := strings.SplitN(s, sep, 2) + if len(parts) != 2 { + continue + } + if !allDigits(parts[0]) || !allDigits(parts[1]) { + continue + } + physical, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return types.TS{}, fmt.Errorf("invalid physical in timestamp: %w", err) + } + logical, err := strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return types.TS{}, fmt.Errorf("invalid logical in timestamp: %w", err) + } + return types.BuildTS(physical, uint32(logical)), nil + } + + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02 15:04", + "2006-01-02", + } { + t, err := time.ParseInLocation(layout, s, time.Local) + if err == nil { + return types.BuildTS(t.UnixNano(), 0), nil + } + } + return types.TS{}, fmt.Errorf("timestamp must be physical:logical, physical-logical, RFC3339, or local time, got %q", s) +} + +func resolveSnapshotTS(ctx context.Context, reader *checkpointtool.CheckpointReader, tsStr string) (types.TS, error) { + info := reader.Info() + if strings.TrimSpace(tsStr) == "" { + if info.LatestTS.IsEmpty() { + return types.TS{}, fmt.Errorf("no checkpoint timestamp is available") + } + if err := reader.ValidateSnapshot(ctx, info.LatestTS); err != nil { + return types.TS{}, err + } + return info.LatestTS, nil + } + + ts, err := parseTS(tsStr) + if err != nil { + return types.TS{}, err + } + if ts.IsEmpty() { + return types.TS{}, fmt.Errorf("timestamp must not be empty") + } + if !info.EarliestTS.IsEmpty() && ts.LT(&info.EarliestTS) { + return types.TS{}, fmt.Errorf("timestamp %s is earlier than earliest checkpoint %s", ts.ToString(), info.EarliestTS.ToString()) + } + if !info.LatestTS.IsEmpty() && info.LatestTS.LT(&ts) { + return types.TS{}, fmt.Errorf("timestamp %s is newer than latest checkpoint %s", ts.ToString(), info.LatestTS.ToString()) + } + if err := reader.ValidateSnapshot(ctx, ts); err != nil { + return types.TS{}, err + } + return ts, nil +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func resolveTableByID( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + snapshotTS types.TS, + tableID uint64, +) (checkpointtool.TableCatalogEntry, error) { + tables, err := reader.ListCatalogTables(ctx, snapshotTS, checkpointtool.TableListOptions{ + IncludeViews: true, + }) + if err != nil { + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("list checkpoint catalog tables: %w", err) + } + for _, table := range tables { + if table.TableID == tableID { + return table, nil + } + } + return checkpointtool.TableCatalogEntry{}, fmt.Errorf("table %d not found in checkpoint catalog", tableID) +} + +func writeRestoreScript( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, + tables []checkpointtool.TableCatalogEntry, + dumpDataByTable map[uint64]*checkpointtool.TableDumpData, + snapshotTS types.TS, + scriptDir string, + csvRoot string, + loadPathResolver loadDataPathResolver, + includeLoad bool, + csvHasHeader bool, +) (string, error) { + if err := dumpOut.MkdirAll(scriptDir); err != nil { + return "", fmt.Errorf("create script dir: %w", err) + } + scriptPath := outputPathJoin(scriptDir, "restore.sql") + + var script bytes.Buffer + currentDB := "" + for _, table := range orderTablesForRestore(tables, dumpDataByTable) { + if table.DatabaseName == "" { + return "", fmt.Errorf("table %d has empty database name in checkpoint catalog", table.TableID) + } + if table.DatabaseName != currentDB { + if currentDB != "" { + if _, err := fmt.Fprintln(&script); err != nil { + return "", err + } + } + if _, err := fmt.Fprintf(&script, "CREATE DATABASE IF NOT EXISTS %s;\n", quoteSQLIdent(table.DatabaseName)); err != nil { + return "", err + } + if _, err := fmt.Fprintf(&script, "USE %s;\n\n", quoteSQLIdent(table.DatabaseName)); err != nil { + return "", err + } + currentDB = table.DatabaseName + } + + ddl, err := restoreCreateTableDDL(ctx, reader, table, dumpDataByTable[table.TableID], snapshotTS) + if err != nil { + return "", err + } + if isExternalRelation(table) { + ddl, err = packageExternalTableSource(ctx, dumpOut, csvRoot, table, ddl) + if err != nil { + return "", err + } + } + if !isViewRelation(table) { + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, table.TableID, table.TableName, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create indexes for table %d: %w", table.TableID, err) + } + ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) + if err != nil { + return "", fmt.Errorf("merge create table indexes for table %d: %w", table.TableID, err) + } + } + if _, err := fmt.Fprintln(&script, strings.TrimRight(ddl, " ;\n\t")); err != nil { + return "", err + } + if _, err := fmt.Fprintln(&script, ";"); err != nil { + return "", err + } + if shouldWriteLoadData(includeLoad, table) { + if _, err := fmt.Fprintf(&script, "\n%s\n", loadPathResolver.loadDataSource(csvRoot, table)); err != nil { + return "", err + } + if _, err := fmt.Fprintf(&script, "INTO TABLE %s\n", quoteSQLIdent(table.TableName)); err != nil { + return "", err + } + if _, err := fmt.Fprintln(&script, "FIELDS TERMINATED BY ','"); err != nil { + return "", err + } + if _, err := fmt.Fprintln(&script, "ENCLOSED BY '\"'"); err != nil { + return "", err + } + if _, err := fmt.Fprintln(&script, "LINES TERMINATED BY '\\n'"); err != nil { + return "", err + } + if csvHasHeader { + if _, err := fmt.Fprintln(&script, "IGNORE 1 LINES"); err != nil { + return "", err + } + } + if _, err := fmt.Fprintln(&script, "parallel 'true'"); err != nil { + return "", err + } + if _, err := fmt.Fprintln(&script, ";"); err != nil { + return "", err + } + } + if _, err := fmt.Fprintln(&script); err != nil { + return "", err + } + } + + f, err := dumpOut.Create(ctx, scriptPath) + if err != nil { + return "", fmt.Errorf("create restore script: %w", err) + } + _, writeErr := io.Copy(f, &script) + if err := f.Close(); err != nil { + return "", fmt.Errorf("close restore script: %w", err) + } + if writeErr != nil { + return "", fmt.Errorf("write restore script: %w", writeErr) + } + return scriptPath, nil +} + +func filterAccountRestoreScriptTables( + tables []checkpointtool.TableCatalogEntry, + accountDump bool, +) ([]checkpointtool.TableCatalogEntry, int) { + if !accountDump { + return tables, 0 + } + filtered := make([]checkpointtool.TableCatalogEntry, 0, len(tables)) + skipped := 0 + for _, table := range tables { + if isSystemDatabase(table.DatabaseName) { + skipped++ + continue + } + filtered = append(filtered, table) + } + return filtered, skipped +} + +func isSystemDatabase(databaseName string) bool { + for _, systemDatabase := range catalog.SystemDatabases { + if strings.EqualFold(databaseName, systemDatabase) { + return true + } + } + return false +} + +func orderTablesForRestore( + tables []checkpointtool.TableCatalogEntry, + dumpDataByTable map[uint64]*checkpointtool.TableDumpData, +) []checkpointtool.TableCatalogEntry { + if len(tables) <= 1 { + return tables + } + type tableKey struct { + database string + table string + } + keyFor := func(table checkpointtool.TableCatalogEntry) tableKey { + return tableKey{database: table.DatabaseName, table: table.TableName} + } + tableByKey := make(map[tableKey]checkpointtool.TableCatalogEntry, len(tables)) + orderByID := make(map[uint64]int, len(tables)) + for i, table := range tables { + tableByKey[keyFor(table)] = table + orderByID[table.TableID] = i + } + ordered := make([]checkpointtool.TableCatalogEntry, 0, len(tables)) + visiting := make(map[uint64]bool, len(tables)) + visited := make(map[uint64]bool, len(tables)) + var visit func(checkpointtool.TableCatalogEntry) + visit = func(table checkpointtool.TableCatalogEntry) { + if visited[table.TableID] { + return + } + if visiting[table.TableID] { + return + } + visiting[table.TableID] = true + if dumpData := dumpDataByTable[table.TableID]; dumpData != nil && dumpData.Schema != nil { + refs := append([]checkpointtool.TableForeignKey(nil), dumpData.Schema.ForeignKeys...) + sort.SliceStable(refs, func(i, j int) bool { + left, leftOK := tableByKey[tableKey{database: refs[i].ReferDatabase, table: refs[i].ReferTable}] + right, rightOK := tableByKey[tableKey{database: refs[j].ReferDatabase, table: refs[j].ReferTable}] + if leftOK != rightOK { + return leftOK + } + if !leftOK { + return false + } + return orderByID[left.TableID] < orderByID[right.TableID] + }) + for _, fk := range refs { + parent, ok := tableByKey[tableKey{database: fk.ReferDatabase, table: fk.ReferTable}] + if !ok { + continue + } + visit(parent) + } + } + visiting[table.TableID] = false + visited[table.TableID] = true + ordered = append(ordered, table) + } + for _, table := range tables { + visit(table) + } + return ordered +} + +func restoreCreateTableDDL( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + table checkpointtool.TableCatalogEntry, + dumpData *checkpointtool.TableDumpData, + snapshotTS types.TS, +) (string, error) { + ddl := "" + if dumpData != nil && dumpData.Schema != nil { + schema := dumpData.Schema + if isViewRelation(table) { + ddl = strings.TrimSpace(schema.CreateSQL) + if ddl == "" { + return "", fmt.Errorf("view %d (%s.%s): missing CREATE VIEW SQL in checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + if !isCreateViewSQL(ddl) { + return "", fmt.Errorf("view %d (%s.%s): rel_createsql is not CREATE VIEW: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(ddl)) + } + } else if isExternalRelation(table) { + if strings.TrimSpace(schema.CreateSQL) == "" { + return "", fmt.Errorf("external table %d (%s.%s): missing external table parameter JSON in checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + var err error + ddl, err = renderExternalCreateTableDDLFromParamJSON(table, schema) + if err != nil { + return "", err + } + } else { + schemaCopy := *schema + schemaCopy.TableName = table.TableName + schemaCopy.DatabaseName = table.DatabaseName + ddl = checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) + } + } + if ddl == "" { + var err error + ddl, err = reader.ShowCreateTable(ctx, table.TableID, snapshotTS) + if err != nil { + return "", fmt.Errorf("show create table %d: %w", table.TableID, err) + } + } + return normalizeCreateTableDDLName(ddl, table), nil +} + +func renderExternalCreateTableDDLFromParamJSON(table checkpointtool.TableCatalogEntry, schema *checkpointtool.TableSchema) (string, error) { + var param tree.ExternParam + if err := json.Unmarshal([]byte(schema.CreateSQL), ¶m); err != nil { + return "", fmt.Errorf("external table %d (%s.%s): decode external table parameter JSON: %w", table.TableID, table.DatabaseName, table.TableName, err) + } + applyExternalParamOptions(¶m) + + schemaCopy := *schema + schemaCopy.TableName = table.TableName + schemaCopy.DatabaseName = table.DatabaseName + schemaCopy.CreateSQL = "" + base := checkpointtool.RenderCreateTableDDLFromSchema(&schemaCopy) + if base == "" { + return "", fmt.Errorf("external table %d (%s.%s): cannot render external table columns from checkpoint metadata", table.TableID, table.DatabaseName, table.TableName) + } + base = strings.TrimRight(base, " ;\n\t") + if strings.HasPrefix(strings.ToUpper(base), "CREATE TABLE ") { + base = "CREATE EXTERNAL TABLE " + strings.TrimSpace(base[len("CREATE TABLE "):]) + } else { + return "", fmt.Errorf("external table %d (%s.%s): unexpected generated DDL: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(base)) + } + return base + renderExternalParamClause(¶m), nil +} + +func applyExternalParamOptions(param *tree.ExternParam) { + for i := 0; i+1 < len(param.Option); i += 2 { + key := strings.ToLower(param.Option[i]) + value := param.Option[i+1] + switch key { + case "filepath": + param.Filepath = value + case "compression": + param.CompressType = value + case "format": + param.Format = strings.ToLower(value) + case "jsondata": + param.JsonData = strings.ToLower(value) + case "hive_partitioning": + param.HivePartitioning = strings.EqualFold(value, "true") + case "hive_partition_columns": + if value != "" { + param.HivePartitionCols = strings.Split(value, ",") + } + case "endpoint": + ensureExternalS3Param(param).Endpoint = value + case "region": + ensureExternalS3Param(param).Region = value + case "access_key_id": + ensureExternalS3Param(param).APIKey = value + case "secret_access_key": + ensureExternalS3Param(param).APISecret = value + case "bucket": + ensureExternalS3Param(param).Bucket = value + case "provider": + ensureExternalS3Param(param).Provider = value + case "role_arn": + ensureExternalS3Param(param).RoleArn = value + case "external_id": + ensureExternalS3Param(param).ExternalId = value + } + } + if param.CompressType == "" { + param.CompressType = "auto" + } + if param.Format == "" { + param.Format = "csv" + } +} + +func ensureExternalS3Param(param *tree.ExternParam) *tree.S3Parameter { + if param.S3Param == nil { + param.S3Param = &tree.S3Parameter{} + } + return param.S3Param +} + +func renderExternalParamClause(param *tree.ExternParam) string { + if param.ScanType == tree.S3 { + return renderExternalS3Clause(param) + } + return renderExternalInfileClause(param) +} + +func renderExternalInfileClause(param *tree.ExternParam) string { + options := []string{ + "filepath", param.Filepath, + "compression", param.CompressType, + "format", param.Format, + } + if param.JsonData != "" { + options = append(options, "jsondata", param.JsonData) + } + if param.HivePartitioning { + options = append(options, "hive_partitioning", "true") + if len(param.HivePartitionCols) > 0 { + options = append(options, "hive_partition_columns", strings.Join(param.HivePartitionCols, ",")) + } + } + return " INFILE {" + formatExternalOptions(options) + "}" + renderExternalTailClause(param) +} + +func renderExternalS3Clause(param *tree.ExternParam) string { + options := make([]string, 0, 20) + if param.S3Param != nil { + options = appendExternalOptionIfSet(options, "endpoint", param.S3Param.Endpoint) + options = appendExternalOptionIfSet(options, "region", param.S3Param.Region) + options = appendExternalOptionIfSet(options, "access_key_id", param.S3Param.APIKey) + options = appendExternalOptionIfSet(options, "secret_access_key", param.S3Param.APISecret) + options = appendExternalOptionIfSet(options, "bucket", param.S3Param.Bucket) + } + options = appendExternalOptionIfSet(options, "filepath", param.Filepath) + if param.S3Param != nil { + options = appendExternalOptionIfSet(options, "provider", param.S3Param.Provider) + options = appendExternalOptionIfSet(options, "role_arn", param.S3Param.RoleArn) + options = appendExternalOptionIfSet(options, "external_id", param.S3Param.ExternalId) + } + options = appendExternalOptionIfSet(options, "compression", param.CompressType) + options = appendExternalOptionIfSet(options, "format", param.Format) + options = appendExternalOptionIfSet(options, "jsondata", param.JsonData) + return " URL s3option {" + formatExternalOptions(options) + "}" + renderExternalTailClause(param) +} + +func appendExternalOptionIfSet(options []string, key, value string) []string { + if value == "" { + return options + } + return append(options, key, value) +} + +func formatExternalOptions(options []string) string { + parts := make([]string, 0, len(options)/2) + for i := 0; i+1 < len(options); i += 2 { + parts = append(parts, quoteSQLString(options[i])+"="+quoteSQLString(options[i+1])) + } + return strings.Join(parts, ", ") +} + +func renderExternalTailClause(param *tree.ExternParam) string { + if param.Tail == nil { + return "" + } + var sb strings.Builder + if param.Tail.Fields != nil { + var fields []string + if param.Tail.Fields.Terminated != nil { + fields = append(fields, "TERMINATED BY "+quoteExternalLiteral(param.Tail.Fields.Terminated.Value)) + } + if param.Tail.Fields.EnclosedBy != nil { + fields = append(fields, "ENCLOSED BY "+quoteExternalLiteral(byteSQLString(param.Tail.Fields.EnclosedBy.Value))) + } + if param.Tail.Fields.EscapedBy != nil { + fields = append(fields, "ESCAPED BY "+quoteExternalLiteral(byteSQLString(param.Tail.Fields.EscapedBy.Value))) + } + if len(fields) > 0 { + sb.WriteString(" FIELDS ") + sb.WriteString(strings.Join(fields, " ")) + } + } + if param.Tail.Lines != nil { + var lines []string + if param.Tail.Lines.StartingBy != "" { + lines = append(lines, "STARTING BY "+quoteExternalLiteral(param.Tail.Lines.StartingBy)) + } + if param.Tail.Lines.TerminatedBy != nil { + lines = append(lines, "TERMINATED BY "+quoteExternalLiteral(param.Tail.Lines.TerminatedBy.Value)) + } + if len(lines) > 0 { + sb.WriteString(" LINES ") + sb.WriteString(strings.Join(lines, " ")) + } + } + if param.Tail.IgnoredLines > 0 { + fmt.Fprintf(&sb, " IGNORE %d LINES", param.Tail.IgnoredLines) + } + return sb.String() +} + +func quoteExternalLiteral(s string) string { + s = strings.ReplaceAll(s, "\n", `\n`) + s = strings.ReplaceAll(s, "\r", `\r`) + s = strings.ReplaceAll(s, "\t", `\t`) + return quoteSQLString(s) +} + +func byteSQLString(value byte) string { + if value == 0 { + return "" + } + return string([]byte{value}) +} + +func shouldWriteLoadData(includeLoad bool, table checkpointtool.TableCatalogEntry) bool { + return includeLoad && !isExternalRelation(table) && !isViewRelation(table) +} + +func isExternalRelation(table checkpointtool.TableCatalogEntry) bool { + switch strings.ToLower(strings.TrimSpace(table.RelKind)) { + case "e", "external": + return true + default: + return false + } +} + +func isViewRelation(table checkpointtool.TableCatalogEntry) bool { + switch strings.ToLower(strings.TrimSpace(table.RelKind)) { + case "v", "view": + return true + default: + return false + } +} + +func isCreateViewSQL(ddl string) bool { + upper := strings.ToUpper(strings.TrimSpace(ddl)) + return strings.HasPrefix(upper, "CREATE VIEW ") || strings.HasPrefix(upper, "CREATE OR REPLACE VIEW ") +} + +func packageExternalTableSource( + ctx context.Context, + dumpOut *dumpOutput, + outputDir string, + table checkpointtool.TableCatalogEntry, + ddl string, +) (string, error) { + sourcePath, valueStart, valueEnd, ok := externalTableFilepathValueRange(ddl) + if !ok { + return "", fmt.Errorf("external table %d (%s.%s): CREATE EXTERNAL TABLE SQL does not contain INFILE filepath: %s", table.TableID, table.DatabaseName, table.TableName, summarizeSQLForError(ddl)) + } + if dumpOut != nil && dumpOut.remote { + return "", fmt.Errorf("external table %d (%s.%s): packaging local external source is not supported for remote dump output", table.TableID, table.DatabaseName, table.TableName) + } + + destPath := externalSourcePath(outputDir, table, filepath.Base(sourcePath)) + if err := dumpOut.MkdirAll(path.Dir(destPath)); err != nil { + return "", fmt.Errorf("create external source output dir: %w", err) + } + if err := copyLocalFileToDumpOutput(ctx, dumpOut, sourcePath, destPath); err != nil { + return "", fmt.Errorf("copy external source for table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, err) + } + restorePath := destPath + if absPath, err := filepath.Abs(destPath); err == nil { + restorePath = absPath + } + return ddl[:valueStart] + quoteSQLString(restorePath) + ddl[valueEnd:], nil +} + +func copyLocalFileToDumpOutput(ctx context.Context, dumpOut *dumpOutput, sourcePath, destPath string) error { + in, err := os.Open(sourcePath) + if err != nil { + return err + } + defer in.Close() + + out, err := dumpOut.Create(ctx, destPath) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} + +func externalSourcePath(outputDir string, table checkpointtool.TableCatalogEntry, sourceName string) string { + sourceName = safePathPart(sourceName) + if sourceName == "_" || sourceName == "." || sourceName == string(filepath.Separator) { + sourceName = safePathPart(table.TableName) + ".data" + } + return outputPathJoin( + outputDir, + "external_sources", + fmt.Sprintf("account_%d", table.AccountID), + fmt.Sprintf("db_%d", table.DatabaseID), + fmt.Sprintf("%s_%d_%s", safePathPart(table.TableName), table.TableID, sourceName), + ) +} + +func externalTableFilepathValueRange(sql string) (string, int, int, bool) { + lower := strings.ToLower(sql) + searchFrom := 0 + for { + idx := strings.Index(lower[searchFrom:], "filepath") + if idx < 0 { + return "", 0, 0, false + } + idx += searchFrom + i := idx + len("filepath") + for i < len(sql) { + ch := sql[i] + if ch == '=' { + break + } + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' && ch != '\'' && ch != '"' { + searchFrom = i + 1 + break + } + i++ + } + if searchFrom > idx { + continue + } + if i >= len(sql) || sql[i] != '=' { + return "", 0, 0, false + } + i = skipSQLSpace(sql, i+1) + valueStart := i + value, valueEnd, ok := readSQLStringOrBareValue(sql, i) + if !ok { + return "", 0, 0, false + } + return value, valueStart, valueEnd, true + } +} + +func readSQLStringOrBareValue(sql string, start int) (string, int, bool) { + if start >= len(sql) { + return "", 0, false + } + if sql[start] == '\'' || sql[start] == '"' { + quote := sql[start] + var b strings.Builder + for i := start + 1; i < len(sql); i++ { + ch := sql[i] + if ch == quote { + if i+1 < len(sql) && sql[i+1] == quote { + b.WriteByte(quote) + i++ + continue + } + return b.String(), i + 1, true + } + if ch == '\\' && i+1 < len(sql) { + i++ + b.WriteByte(sql[i]) + continue + } + b.WriteByte(ch) + } + return "", 0, false + } + end := start + for end < len(sql) && sql[end] != ',' && sql[end] != '}' && sql[end] != ')' && sql[end] != ' ' && sql[end] != '\t' && sql[end] != '\n' && sql[end] != '\r' { + end++ + } + value := strings.TrimSpace(sql[start:end]) + return value, end, value != "" +} + +func summarizeSQLForError(sql string) string { + sql = strings.Join(strings.Fields(sql), " ") + if len(sql) > 240 { + return sql[:240] + "..." + } + return sql +} + +func quoteSQLIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +func quoteSQLString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `''`) + return "'" + s + "'" +} + +func normalizeCreateTableDDLName(ddl string, table checkpointtool.TableCatalogEntry) string { + nameStart, nameEnd, ok := createTableNameRange(ddl) + if !ok { + return ddl + } + target := quoteSQLIdent(table.TableName) + if table.DatabaseName != "" { + target = quoteSQLIdent(table.DatabaseName) + "." + target + } + return ddl[:nameStart] + target + ddl[nameEnd:] +} + +func mergeCreateTableIndexDDLs(createDDL string, indexDDLs []string) (string, error) { + indexDDLs = filterExistingIndexDDLs(createDDL, indexDDLs) + if len(indexDDLs) == 0 { + return createDDL, nil + } + clauses := make([]string, 0, len(indexDDLs)) + for _, indexDDL := range indexDDLs { + clause, ok := alterTableAddClause(indexDDL) + if !ok { + return "", fmt.Errorf("unsupported index DDL %q", indexDDL) + } + clauses = append(clauses, clause) + } + ddl, err := injectCreateTableClauses(createDDL, clauses) + if err == nil { + return ddl, nil + } + return appendIndexDDLsAfterCreateTable(createDDL, indexDDLs), nil +} + +func alterTableAddClause(sql string) (string, bool) { + i, ok := consumeSQLKeyword(sql, 0, "alter") + if !ok { + return "", false + } + i, ok = consumeSQLKeyword(sql, i, "table") + if !ok { + return "", false + } + i = skipSQLSpace(sql, i) + i, ok = consumeSQLIdentifier(sql, i) + if !ok { + return "", false + } + if j := skipSQLSpace(sql, i); j < len(sql) && sql[j] == '.' { + j = skipSQLSpace(sql, j+1) + if end, ok := consumeSQLIdentifier(sql, j); ok { + i = end + } + } + i, ok = consumeSQLKeyword(sql, i, "add") + if !ok { + return "", false + } + clause := trimSQLStatementTerminator(sql[i:]) + _, hasUnique := consumeSQLKeyword(clause, 0, "unique") + _, hasKey := consumeSQLKeyword(clause, 0, "key") + _, hasIndex := consumeSQLKeyword(clause, 0, "index") + if _, hasFullText := consumeSQLKeyword(clause, 0, "fulltext"); hasFullText { + hasKey = true + } + if !hasUnique && !hasKey && !hasIndex { + return "", false + } + return clause, clause != "" +} + +func trimSQLStatementTerminator(sql string) string { + sql = strings.TrimSpace(sql) + for strings.HasSuffix(sql, ";") { + sql = strings.TrimSpace(strings.TrimSuffix(sql, ";")) + } + return sql +} + +func injectCreateTableClauses(createDDL string, clauses []string) (string, error) { + open, close, ok := createTableDefinitionParens(createDDL) + if !ok { + return "", fmt.Errorf("cannot locate CREATE TABLE definition") + } + body := createDDL[open+1 : close] + multiline := strings.Contains(body, "\n") + if multiline { + indent := inferCreateTableClauseIndent(body) + insert := ",\n" + indent + strings.Join(clauses, ",\n"+indent) + insertAt := close + for insertAt > open+1 { + ch := createDDL[insertAt-1] + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' { + break + } + insertAt-- + } + return createDDL[:insertAt] + insert + createDDL[insertAt:], nil + } + separator := ", " + if strings.TrimSpace(body) == "" { + separator = "" + } + return createDDL[:close] + separator + strings.Join(clauses, ", ") + createDDL[close:], nil +} + +func appendIndexDDLsAfterCreateTable(createDDL string, indexDDLs []string) string { + var sb strings.Builder + sb.WriteString(strings.TrimRight(createDDL, " ;\n\t")) + for _, indexDDL := range indexDDLs { + sb.WriteString(";\n") + sb.WriteString(strings.TrimRight(strings.TrimSpace(indexDDL), " ;\n\t")) + } + return sb.String() +} + +func inferCreateTableClauseIndent(body string) string { + lines := strings.Split(body, "\n") + for _, line := range lines[1:] { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + return line[:len(line)-len(strings.TrimLeft(line, " \t"))] + } + return " " +} + +func createTableDefinitionParens(sql string) (int, int, bool) { + _, nameEnd, ok := createTableNameRange(sql) + if !ok { + return 0, 0, false + } + open := skipSQLSpace(sql, nameEnd) + if open >= len(sql) || sql[open] != '(' { + return 0, 0, false + } + depth := 0 + inBacktick := false + inString := byte(0) + for i := open; i < len(sql); i++ { + ch := sql[i] + if inBacktick { + if ch == '`' { + if i+1 < len(sql) && sql[i+1] == '`' { + i++ + continue + } + inBacktick = false + } + continue + } + if inString != 0 { + if ch == '\\' && i+1 < len(sql) { + i++ + continue + } + if ch == inString { + if i+1 < len(sql) && sql[i+1] == inString { + i++ + continue + } + inString = 0 + } + continue + } + switch ch { + case '`': + inBacktick = true + case '\'', '"': + inString = ch + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + return open, i, true + } + if depth < 0 { + return 0, 0, false + } + } + } + return 0, 0, false +} + +func ddlTableName(ddl string, tableID uint64) string { + nameStart, nameEnd, ok := createTableNameRange(ddl) + if !ok { + return strconv.FormatUint(tableID, 10) + } + name := strings.TrimSpace(ddl[nameStart:nameEnd]) + if dot := strings.LastIndex(name, "."); dot >= 0 { + name = name[dot+1:] + } + if unquoted, ok := unquoteSQLIdent(name); ok && unquoted != "" { + return unquoted + } + return strings.Trim(name, "`") +} + +func unquoteSQLIdent(s string) (string, bool) { + s = strings.TrimSpace(s) + if len(s) < 2 || s[0] != '`' || s[len(s)-1] != '`' { + return s, false + } + return strings.ReplaceAll(s[1:len(s)-1], "``", "`"), true +} + +func filterExistingIndexDDLs(createDDL string, indexDDLs []string) []string { + if len(indexDDLs) == 0 { + return nil + } + filtered := make([]string, 0, len(indexDDLs)) + for _, indexDDL := range indexDDLs { + name := generatedIndexDDLName(indexDDL) + if name != "" && createTableHasIndex(createDDL, name) { + continue + } + filtered = append(filtered, indexDDL) + } + return filtered +} + +func generatedIndexDDLName(indexDDL string) string { + upper := strings.ToUpper(indexDDL) + for _, marker := range []string{" ADD FULLTEXT KEY ", " ADD FULLTEXT INDEX ", " ADD UNIQUE KEY ", " ADD KEY ", " ADD INDEX "} { + i := strings.Index(upper, marker) + if i < 0 { + continue + } + name := strings.TrimSpace(indexDDL[i+len(marker):]) + if unquoted, ok := readLeadingSQLIdent(name); ok { + return unquoted + } + } + return "" +} + +func createTableHasIndex(createDDL string, name string) bool { + quoted := quoteSQLIdent(name) + upper := strings.ToUpper(createDDL) + for _, marker := range []string{"KEY " + quoted, "INDEX " + quoted, "CONSTRAINT " + quoted} { + if strings.Contains(upper, strings.ToUpper(marker)) { + return true + } + } + lowerDDL := strings.ToLower(createDDL) + lowerName := strings.ToLower(name) + for _, marker := range []string{"key " + lowerName, "index " + lowerName, "constraint " + lowerName} { + if strings.Contains(lowerDDL, marker) { + return true + } + } + return false +} + +func readLeadingSQLIdent(s string) (string, bool) { + s = strings.TrimSpace(s) + if s == "" { + return "", false + } + if s[0] == '`' { + end := 1 + for end < len(s) { + if s[end] == '`' { + if end+1 < len(s) && s[end+1] == '`' { + end += 2 + continue + } + return strings.ReplaceAll(s[1:end], "``", "`"), true + } + end++ + } + return "", false + } + end := 0 + for end < len(s) && isSQLIdentByte(s[end]) { + end++ + } + if end == 0 { + return "", false + } + return s[:end], true +} + +func createTableNameRange(sql string) (int, int, bool) { + i, ok := consumeSQLKeyword(sql, 0, "create") + if !ok { + return 0, 0, false + } + if next, ok := consumeSQLKeyword(sql, i, "or"); ok { + if next, ok = consumeSQLKeyword(sql, next, "replace"); ok { + i = next + } + } + for { + next, ok := consumeSQLKeyword(sql, i, "temporary") + if ok { + i = next + continue + } + next, ok = consumeSQLKeyword(sql, i, "cluster") + if ok { + i = next + continue + } + next, ok = consumeSQLKeyword(sql, i, "external") + if ok { + i = next + continue + } + break + } + isTable := false + if next, ok := consumeSQLKeyword(sql, i, "table"); ok { + i = next + isTable = true + } else if next, ok := consumeSQLKeyword(sql, i, "view"); ok { + i = next + } else { + return 0, 0, false + } + if isTable { + if next, ok := consumeSQLKeyword(sql, i, "if"); ok { + if next, ok = consumeSQLKeyword(sql, next, "not"); ok { + if next, ok = consumeSQLKeyword(sql, next, "exists"); ok { + i = next + } + } + } + } else if next, ok := consumeSQLKeyword(sql, i, "if"); ok { + if next, ok = consumeSQLKeyword(sql, next, "not"); ok { + if next, ok = consumeSQLKeyword(sql, next, "exists"); ok { + i = next + } + } + } + i = skipSQLSpace(sql, i) + nameStart := i + i, ok = consumeSQLIdentifier(sql, i) + if !ok { + return 0, 0, false + } + if j := skipSQLSpace(sql, i); j < len(sql) && sql[j] == '.' { + j = skipSQLSpace(sql, j+1) + if end, ok := consumeSQLIdentifier(sql, j); ok { + i = end + } + } + return nameStart, i, true +} + +func consumeSQLKeyword(sql string, i int, keyword string) (int, bool) { + i = skipSQLSpace(sql, i) + if len(sql)-i < len(keyword) || !strings.EqualFold(sql[i:i+len(keyword)], keyword) { + return i, false + } + end := i + len(keyword) + if end < len(sql) && isSQLIdentByte(sql[end]) { + return i, false + } + return end, true +} + +func skipSQLSpace(sql string, i int) int { + for i < len(sql) { + switch sql[i] { + case ' ', '\t', '\n', '\r': + i++ + default: + return i + } + } + return i +} + +func consumeSQLIdentifier(sql string, i int) (int, bool) { + if i >= len(sql) { + return i, false + } + if sql[i] == '`' { + i++ + for i < len(sql) { + if sql[i] != '`' { + i++ + continue + } + if i+1 < len(sql) && sql[i+1] == '`' { + i += 2 + continue + } + return i + 1, true + } + return i, false + } + start := i + for i < len(sql) && isSQLIdentByte(sql[i]) { + i++ + } + return i, i > start +} + +func isSQLIdentByte(b byte) bool { + return b == '_' || b == '$' || + (b >= '0' && b <= '9') || + (b >= 'a' && b <= 'z') || + (b >= 'A' && b <= 'Z') +} + +type tableDumpPlan struct { + table checkpointtool.TableCatalogEntry + data *checkpointtool.TableDumpData +} + +func dumpDataByTableID(plans []tableDumpPlan) map[uint64]*checkpointtool.TableDumpData { + if len(plans) == 0 { + return nil + } + byID := make(map[uint64]*checkpointtool.TableDumpData, len(plans)) + for _, plan := range plans { + byID[plan.table.TableID] = plan.data + } + return byID +} + +func prepareTableDumpPlans( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + tables []checkpointtool.TableCatalogEntry, + snapshotTS types.TS, +) ([]tableDumpPlan, error) { + tableIDs := make([]uint64, 0, len(tables)) + for _, table := range tables { + tableIDs = append(tableIDs, table.TableID) + } + dumpDataByTable, err := reader.PrepareTableDumpDataForTables(ctx, tableIDs, snapshotTS) + if err != nil { + return nil, err + } + + plans := make([]tableDumpPlan, 0, len(tables)) + for _, table := range tables { + data := dumpDataByTable[table.TableID] + if data == nil { + return nil, fmt.Errorf("prepare table %d (%s.%s): missing object list", table.TableID, table.DatabaseName, table.TableName) + } + plans = append(plans, tableDumpPlan{table: table, data: data}) + } + return plans, nil +} + +func dumpTablesConcurrently( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, + plans []tableDumpPlan, + snapshotTS types.TS, + outputDir string, + jobs int, + rowOrder checkpointtool.CSVRowOrder, + metaComments bool, + header bool, + out io.Writer, +) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + tableCh := make(chan tableDumpPlan) + var outMu sync.Mutex + var wg sync.WaitGroup + var errMu sync.Mutex + var dumpErr error + recordErr := func(err error) { + if err == nil { + return + } + errMu.Lock() + if !isContextCanceledOnly(err) { + dumpErr = preferRealError(dumpErr, err) + } + errMu.Unlock() + cancel() + } + + worker := func() { + defer wg.Done() + workerReader := reader.Fork(ctx) + for plan := range tableCh { + if err := dumpOneTable(ctx, workerReader, dumpOut, plan, snapshotTS, outputDir, rowOrder, metaComments, header, out, &outMu); err != nil { + recordErr(err) + return + } + } + } + for i := 0; i < jobs; i++ { + wg.Add(1) + go worker() + } +sendPlans: + for _, plan := range plans { + select { + case tableCh <- plan: + case <-ctx.Done(): + break sendPlans + } + } + close(tableCh) + wg.Wait() + errMu.Lock() + defer errMu.Unlock() + if dumpErr != nil { + return dumpErr + } + return ctx.Err() +} + +func tableCSVPath(outputDir string, table checkpointtool.TableCatalogEntry) string { + return outputPathJoin( + outputDir, + fmt.Sprintf("account_%d", table.AccountID), + fmt.Sprintf("db_%d", table.DatabaseID), + fmt.Sprintf("%s_%d.csv", safePathPart(table.TableName), table.TableID), + ) +} + +type loadDataPathResolver struct { + s3Args fileservice.ObjectStorageArguments + s3Backend string +} + +func newLoadDataPathResolver(storage toolfs.StorageOptions) (loadDataPathResolver, error) { + if storage.S3 == "" { + return loadDataPathResolver{}, nil + } + backend := strings.ToUpper(storage.Backend) + if backend == "" { + backend = "S3" + } + if backend != "S3" && backend != "MINIO" { + return loadDataPathResolver{}, nil + } + args, err := toolfs.ParseS3Arguments(storage.S3, storage.FSName) + if err != nil { + return loadDataPathResolver{}, fmt.Errorf("parse output S3 arguments: %w", err) + } + if args.Bucket == "" { + return loadDataPathResolver{}, fmt.Errorf("parse output S3 arguments: missing bucket") + } + return loadDataPathResolver{s3Args: args, s3Backend: backend}, nil +} + +func (r loadDataPathResolver) loadDataSource(outputDir string, table checkpointtool.TableCatalogEntry) string { + csvPath := tableCSVPath(outputDir, table) + if r.s3Args.Bucket == "" { + if absPath, err := filepath.Abs(csvPath); err == nil { + csvPath = absPath + } + return "LOAD DATA INFILE " + quoteSQLString(csvPath) + } + + options := []string{ + "bucket", r.s3Args.Bucket, + "filepath", outputS3ObjectKey(r.s3Args.KeyPrefix, csvPath), + } + if r.s3Args.Endpoint != "" { + options = append(options, "endpoint", r.s3Args.Endpoint) + } + if r.s3Args.Region != "" { + options = append(options, "region", r.s3Args.Region) + } + if r.s3Args.KeyID != "" { + options = append(options, "access_key_id", r.s3Args.KeyID) + } + if r.s3Args.KeySecret != "" { + options = append(options, "secret_access_key", r.s3Args.KeySecret) + } + if r.s3Args.RoleARN != "" { + options = append(options, "role_arn", r.s3Args.RoleARN) + } + if r.s3Args.ExternalID != "" { + options = append(options, "external_id", r.s3Args.ExternalID) + } + if r.s3Args.IsMinio || r.s3Backend == "MINIO" { + options = append(options, "provider", "minio") + } + return "LOAD DATA URL s3option{" + formatLoadDataOptions(options) + "}" +} + +func outputS3ObjectKey(keyPrefix string, csvPath string) string { + keyPrefix = strings.Trim(keyPrefix, "/") + csvPath = strings.TrimLeft(csvPath, "/") + if keyPrefix == "" { + return csvPath + } + return path.Join(keyPrefix, csvPath) +} + +func formatLoadDataOptions(options []string) string { + parts := make([]string, 0, len(options)/2) + for i := 0; i+1 < len(options); i += 2 { + parts = append(parts, quoteSQLString(options[i])+"="+quoteSQLString(options[i+1])) + } + return strings.Join(parts, ", ") +} + +func outputPathJoin(elem ...string) string { + return path.Join(elem...) +} + +func dumpOneTable( + ctx context.Context, + reader *checkpointtool.CheckpointReader, + dumpOut *dumpOutput, + plan tableDumpPlan, + snapshotTS types.TS, + outputDir string, + rowOrder checkpointtool.CSVRowOrder, + metaComments bool, + header bool, + out io.Writer, + outMu *sync.Mutex, +) error { + table := plan.table + if isViewRelation(table) { + outMu.Lock() + fmt.Fprintf(out, "View %d %s.%s skipped CSV dump\n", table.TableID, table.DatabaseName, table.TableName) + outMu.Unlock() + return nil + } + filePath := tableCSVPath(outputDir, table) + tableDir := path.Dir(filePath) + if err := dumpOut.MkdirAll(tableDir); err != nil { + return fmt.Errorf("create table output dir: %w", err) + } + outFile, err := dumpOut.Create(ctx, filePath) + if err != nil { + return fmt.Errorf("create output file for table %d: %w", table.TableID, err) + } + err = reader.DumpPreparedTableCSV( + ctx, + outFile, + plan.data, + snapshotTS, + checkpointtool.WithCSVMetaComments(metaComments), + checkpointtool.WithCSVHeader(header), + checkpointtool.WithCSVRowOrder(rowOrder), + ) + closeErr := outFile.Close() + if combinedErr := dumpTableError(err, closeErr); combinedErr != nil { + return fmt.Errorf("dump table %d (%s.%s): %w", table.TableID, table.DatabaseName, table.TableName, combinedErr) + } + outMu.Lock() + fmt.Fprintf(out, "Table %d %s.%s dumped to %s\n", table.TableID, table.DatabaseName, table.TableName, filePath) + outMu.Unlock() + return nil +} + +func dumpTableError(dumpErr, closeErr error) error { + if dumpErr != nil && closeErr != nil && dumpErr.Error() == closeErr.Error() { + return dumpErr + } + return preferRealError(dumpErr, closeErr) +} + +func preferRealError(current, next error) error { + switch { + case current == nil: + return next + case next == nil: + return current + case errors.Is(current, context.Canceled) && !errors.Is(next, context.Canceled): + return next + case errors.Is(next, context.Canceled) && !errors.Is(current, context.Canceled): + return current + default: + return errors.Join(current, next) + } +} + +func isContextCanceledOnly(err error) bool { + if err == nil || !errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + type multiUnwrapper interface { + Unwrap() []error + } + if unwrapped, ok := err.(multiUnwrapper); ok { + errs := unwrapped.Unwrap() + if len(errs) == 0 { + return false + } + for _, inner := range errs { + if !isContextCanceledOnly(inner) { + return false + } + } + return true + } + type unwrapper interface { + Unwrap() error + } + if unwrapped, ok := err.(unwrapper); ok { + return isContextCanceledOnly(unwrapped.Unwrap()) + } + return true +} + +func safePathPart(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "_" + } + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= 'A' && r <= 'Z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '_' || r == '-' || r == '.': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +// showCreateTableCommand implements the "ckp show-create-table" subcommand +// that prints a CREATE TABLE DDL for a table at a given checkpoint snapshot. +// +// Usage: +// +// mo-tool ckp show-create-table --table-id=12345 [--ts=...] [directory] +func showCreateTableCommand(storage *toolfs.StorageOptions) *cobra.Command { + var ( + tableID uint64 + tsStr string + ) + + cmd := &cobra.Command{ + Use: "show-create-table [directory]", + Short: "Show CREATE TABLE DDL for a table from checkpoint", + Long: `Display the CREATE TABLE SQL for a given table by reading the checkpoint's +mo_tables and mo_columns system tables (GCKP + following ICKPs). + +The DDL is reconstructed from mo_columns visible column definitions, with +hardcoded fallbacks for core built-in system tables. + +Examples: + mo-tool ckp show-create-table --table-id=12345 /path/to/ckp + mo-tool ckp show-create-table --table-id=2 --ts=1749001234567890:1 .`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + if tableID == 0 { + return fmt.Errorf("--table-id is required") + } + + logFile, err := setupLogFile() + if err != nil { + return fmt.Errorf("setup log file: %w", err) + } + defer logFile.Close() + + ctx := context.Background() + reader, err := openReader(ctx, dir, *storage) + if err != nil { + return fmt.Errorf("open checkpoint dir: %w", err) + } + defer reader.Close() + + snapshotTS, err := resolveSnapshotTS(ctx, reader, tsStr) + if err != nil { + return fmt.Errorf("resolve --ts: %w", err) + } + + ddl, err := reader.ShowCreateTable(ctx, tableID, snapshotTS) + if err != nil { + return fmt.Errorf("show create table %d: %w", tableID, err) + } + + indexDDLs, err := reader.ShowCreateIndexStatements(ctx, tableID, ddlTableName(ddl, tableID), snapshotTS) + if err != nil { + return fmt.Errorf("show create indexes for table %d: %w", tableID, err) + } + ddl, err = mergeCreateTableIndexDDLs(ddl, indexDDLs) + if err != nil { + return fmt.Errorf("merge create table indexes for table %d: %w", tableID, err) + } + fmt.Fprintln(cmd.OutOrStdout(), ddl) + return nil + }, + } + + cmd.Flags().Uint64Var(&tableID, "table-id", 0, "Table ID to show CREATE TABLE for (required)") + cmd.Flags().StringVar(&tsStr, "ts", "", "Snapshot timestamp: physical:logical, physical-logical, RFC3339, or local time (default: latest)") + + return cmd } diff --git a/cmd/mo-object-tool/ckp/checkpoint_test.go b/cmd/mo-object-tool/ckp/checkpoint_test.go new file mode 100644 index 0000000000000..75d1091e5ba6e --- /dev/null +++ b/cmd/mo-object-tool/ckp/checkpoint_test.go @@ -0,0 +1,484 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPreferRealErrorKeepsNonCanceledRootCause(t *testing.T) { + root := errors.New("s3 write failed") + + assert.ErrorIs(t, preferRealError(context.Canceled, root), root) + assert.ErrorIs(t, preferRealError(root, context.Canceled), root) + assert.ErrorIs(t, preferRealError(context.Canceled, context.Canceled), context.Canceled) + assert.NoError(t, preferRealError(nil, nil)) +} + +func TestDumpTableErrorDedupesPipeWriteAndCloseError(t *testing.T) { + err := errors.New("s3 write failed") + + assert.Equal(t, err, dumpTableError(err, errors.New("s3 write failed"))) +} + +func TestIsContextCanceledOnly(t *testing.T) { + root := errors.New("s3 write failed") + + assert.True(t, isContextCanceledOnly(context.Canceled)) + assert.True(t, isContextCanceledOnly(fmt.Errorf("dump table 1: %w", context.Canceled))) + assert.True(t, isContextCanceledOnly(errors.Join(context.Canceled, fmt.Errorf("worker: %w", context.Canceled)))) + assert.False(t, isContextCanceledOnly(root)) + assert.False(t, isContextCanceledOnly(context.DeadlineExceeded)) + assert.False(t, isContextCanceledOnly(errors.Join(context.Canceled, root))) +} + +func TestNormalizeCreateTableDDLName(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "compat_ckp", + TableName: "employees", + } + + tests := []struct { + name string + ddl string + want string + }{ + { + name: "plain table name", + ddl: "CREATE TABLE employees_copy_123 (id INT)", + want: "CREATE TABLE `compat_ckp`.`employees` (id INT)", + }, + { + name: "qualified table name", + ddl: "CREATE TABLE `compat_ckp`.`employees_copy_123` (id INT)", + want: "CREATE TABLE `compat_ckp`.`employees` (id INT)", + }, + { + name: "if not exists", + ddl: "CREATE TABLE IF NOT EXISTS old_name (id INT)", + want: "CREATE TABLE IF NOT EXISTS `compat_ckp`.`employees` (id INT)", + }, + { + name: "external table", + ddl: "CREATE EXTERNAL TABLE old_name (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", + want: "CREATE EXTERNAL TABLE `compat_ckp`.`employees` (id INT) INFILE {'filepath'='/tmp/data.csv','format'='csv'}", + }, + { + name: "view", + ddl: "CREATE VIEW old_view AS SELECT id FROM src", + want: "CREATE VIEW `compat_ckp`.`employees` AS SELECT id FROM src", + }, + { + name: "or replace view", + ddl: "CREATE OR REPLACE VIEW `old_db`.`old_view` AS SELECT id FROM src", + want: "CREATE OR REPLACE VIEW `compat_ckp`.`employees` AS SELECT id FROM src", + }, + { + name: "escaped target name", + ddl: "CREATE TABLE old_name (id INT)", + want: "CREATE TABLE `compat``ckp`.`employees` (id INT)", + }, + } + + tests[len(tests)-1].want = "CREATE TABLE `compat``ckp`.`employees` (id INT)" + tests[len(tests)-1].ddl = "CREATE TABLE old_name (id INT)" + tests[len(tests)-1].name = "escaped database name" + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "escaped database name" { + table.DatabaseName = "compat`ckp" + } else { + table.DatabaseName = "compat_ckp" + } + assert.Equal(t, tt.want, normalizeCreateTableDDLName(tt.ddl, table)) + }) + } +} + +func TestRestoreCreateTableDDLUsesExternalCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + paramJSON, err := json.Marshal(&tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Option: []string{"filepath", "/tmp/ext.csv", "format", "csv", "compression", "none"}, + Tail: &tree.TailParameter{ + Fields: &tree.Fields{ + Terminated: &tree.Terminated{Value: ","}, + EnclosedBy: &tree.EnclosedBy{Value: '"'}, + }, + Lines: &tree.Lines{ + TerminatedBy: &tree.Terminated{Value: "\n"}, + }, + IgnoredLines: 1, + }, + }, + }) + require.NoError(t, err) + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "stale_name", + CreateSQL: string(paramJSON), + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + ddl, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.NoError(t, err) + assert.Equal(t, "CREATE EXTERNAL TABLE `ckp_external`.`ext_csv_local` (\n `id` INT\n) INFILE {'filepath'='/tmp/ext.csv', 'compression'='none', 'format'='csv'} FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\\\\n' IGNORE 1 LINES", ddl) +} + +func TestRestoreCreateTableDDLExternalRequiresCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "ext_csv_local", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + _, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing external table parameter JSON") +} + +func TestRestoreCreateTableDDLUsesViewCreateSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_views", + TableName: "v_normal", + RelKind: "v", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "stale_name", + CreateSQL: "CREATE VIEW `old_db`.`old_view` AS SELECT id, name FROM `ckp_tables`.`t_normal`", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + ddl, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.NoError(t, err) + assert.Equal(t, "CREATE VIEW `ckp_views`.`v_normal` AS SELECT id, name FROM `ckp_tables`.`t_normal`", ddl) +} + +func TestRestoreCreateTableDDLViewRequiresCreateViewSQL(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_views", + TableName: "v_normal", + RelKind: "v", + } + dumpData := &checkpointtool.TableDumpData{ + Schema: &checkpointtool.TableSchema{ + TableName: "v_normal", + CreateSQL: "CREATE TABLE `v_normal` (`id` INT)", + Columns: []checkpointtool.TableColumn{{Name: "id", SQLType: "INT", Position: 1}}, + }, + } + + _, err := restoreCreateTableDDL(context.Background(), nil, table, dumpData, types.TS{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "rel_createsql is not CREATE VIEW") +} + +func TestShouldWriteLoadDataSkipsExternalAndViewRelations(t *testing.T) { + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "e"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "external"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "v"})) + assert.False(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "view"})) + assert.True(t, shouldWriteLoadData(true, checkpointtool.TableCatalogEntry{RelKind: "r"})) + assert.False(t, shouldWriteLoadData(false, checkpointtool.TableCatalogEntry{RelKind: "r"})) +} + +func TestFilterAccountRestoreScriptTablesSkipsSystemDatabases(t *testing.T) { + tables := []checkpointtool.TableCatalogEntry{ + {DatabaseName: "mo_catalog", TableName: "mo_user"}, + {DatabaseName: "mysql", TableName: "user"}, + {DatabaseName: "system", TableName: "statement_info"}, + {DatabaseName: "system_metrics", TableName: "metric"}, + {DatabaseName: "coverage", TableName: "t1"}, + {DatabaseName: "Coverage_Extra", TableName: "t2"}, + } + + filtered, skipped := filterAccountRestoreScriptTables(tables, true) + + require.Len(t, filtered, 2) + assert.Equal(t, 4, skipped) + assert.Equal(t, "coverage", filtered[0].DatabaseName) + assert.Equal(t, "t1", filtered[0].TableName) + assert.Equal(t, "Coverage_Extra", filtered[1].DatabaseName) + assert.Equal(t, "t2", filtered[1].TableName) +} + +func TestFilterAccountRestoreScriptTablesLeavesNonAccountDumpUnchanged(t *testing.T) { + tables := []checkpointtool.TableCatalogEntry{ + {DatabaseName: "mo_catalog", TableName: "mo_user"}, + {DatabaseName: "coverage", TableName: "t1"}, + } + + filtered, skipped := filterAccountRestoreScriptTables(tables, false) + + assert.Equal(t, tables, filtered) + assert.Zero(t, skipped) +} + +func TestPackageExternalTableSourceCopiesAndRewritesFilepath(t *testing.T) { + tmpDir := t.TempDir() + sourcePath := filepath.Join(tmpDir, "local_ext_people.csv") + require.NoError(t, os.WriteFile(sourcePath, []byte("id,name\n1,a\n"), 0o644)) + + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272731, + TableID: 272732, + DatabaseName: "ckp_external", + TableName: "ext_csv_local", + RelKind: "e", + } + ddl := "CREATE EXTERNAL TABLE ext_csv_local (id INT, name VARCHAR(50)) INFILE {'filepath'='" + sourcePath + "', 'format'='csv'}" + + got, err := packageExternalTableSource(context.Background(), &dumpOutput{}, filepath.Join(tmpDir, "dump"), table, ddl) + require.NoError(t, err) + assert.NotContains(t, got, sourcePath) + + copiedPath, _, _, ok := externalTableFilepathValueRange(got) + require.True(t, ok) + assert.True(t, filepath.IsAbs(copiedPath)) + assert.Contains(t, copiedPath, filepath.Join("external_sources", "account_0", "db_272731")) + data, err := os.ReadFile(copiedPath) + require.NoError(t, err) + assert.Equal(t, "id,name\n1,a\n", string(data)) +} + +func TestExternalTableFilepathValueRange(t *testing.T) { + ddl := "CREATE EXTERNAL TABLE t (id INT) INFILE {'format'='csv', 'filepath'='/tmp/a.csv'}" + value, start, end, ok := externalTableFilepathValueRange(ddl) + require.True(t, ok) + assert.Equal(t, "/tmp/a.csv", value) + assert.Equal(t, "CREATE EXTERNAL TABLE t (id INT) INFILE {'format'='csv', 'filepath'='/new.csv'}", ddl[:start]+quoteSQLString("/new.csv")+ddl[end:]) +} + +func TestFilterExistingIndexDDLs(t *testing.T) { + createDDL := "CREATE TABLE `items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists=2000 op_type 'vector_l2_ops'\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + "ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);", + } + + assert.Equal(t, []string{"ALTER TABLE `items_gist` ADD KEY `new_idx`(`id`);"}, filterExistingIndexDDLs(createDDL, indexDDLs)) +} + +func TestMergeCreateTableIndexDDLs(t *testing.T) { + createDDL := "CREATE TABLE `ann`.`items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + } + want := "CREATE TABLE `ann`.`items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops'\n" + + ")" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestMergeCreateTableIndexDDLsFullText(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " PRIMARY KEY (`id`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", + } + want := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " FULLTEXT KEY `idx_doc`(`doc`)\n" + + ")" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestFilterExistingIndexDDLsFullText(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`t_fulltext` (\n" + + " `id` BIGINT NOT NULL,\n" + + " `doc` TEXT DEFAULT NULL,\n" + + " FULLTEXT KEY `idx_doc`(`doc`)\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", + } + + assert.Empty(t, filterExistingIndexDDLs(createDDL, indexDDLs)) +} + +func TestMergeCreateTableIndexDDLsSingleLine(t *testing.T) { + createDDL := "CREATE TABLE `ann`.`items_sift` (id int primary key, embedding vecf32(128))" + indexDDLs := []string{ + "ALTER TABLE `items_sift` ADD KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops' ;", + } + want := "CREATE TABLE `ann`.`items_sift` (id int primary key, embedding vecf32(128), KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops')" + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestMergeCreateTableIndexDDLsSkipsExistingIndex(t *testing.T) { + createDDL := "CREATE TABLE `items_gist` (\n" + + " `id` int NOT NULL,\n" + + " `embedding` vecf32(960) DEFAULT NULL,\n" + + " PRIMARY KEY (`id`),\n" + + " KEY `ivf_2000` USING ivfflat(`embedding`) lists=2000 op_type 'vector_l2_ops'\n" + + ")" + indexDDLs := []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + assert.NoError(t, err) + assert.Equal(t, createDDL, got) +} + +func TestMergeCreateTableIndexDDLsWithConstraintsAndComments(t *testing.T) { + createDDL := "CREATE TABLE `ckp_constraints`.`parent` (\n" + + " `id` INT NOT NULL PRIMARY KEY,\n" + + " `code` VARCHAR(20) NOT NULL UNIQUE,\n" + + " `note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note'\n" + + ") COMMENT='parent table comment'" + indexDDLs := []string{ + "ALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Contains(t, got, "KEY `idx_parent_note`(`note`)") + assert.Contains(t, got, ") COMMENT='parent table comment'") +} + +func TestMergeCreateTableIndexDDLsWithAutoIncrementAndClusterTable(t *testing.T) { + createDDL := "CREATE CLUSTER TABLE `ckp_constraints`.`t_auto_inc` (\n" + + " `id` BIGINT NOT NULL AUTO_INCREMENT,\n" + + " `note` VARCHAR(64) COMMENT 'note (with parens)',\n" + + " PRIMARY KEY (`id`)\n" + + ") COMMENT='auto increment table'" + indexDDLs := []string{ + "ALTER TABLE `t_auto_inc` ADD KEY `idx_auto_inc_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Contains(t, got, "KEY `idx_auto_inc_note`(`note`)") + assert.Contains(t, got, "COMMENT='auto increment table'") +} + +func TestMergeCreateTableIndexDDLsFallsBackToSeparateAlter(t *testing.T) { + createDDL := "CREATE TABLE `parent` LIKE `parent_template`" + indexDDLs := []string{ + "ALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`);", + } + + got, err := mergeCreateTableIndexDDLs(createDDL, indexDDLs) + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE `parent` LIKE `parent_template`;\nALTER TABLE `parent` ADD KEY `idx_parent_note`(`note`)", got) +} + +func TestNormalizeCreateTableDDLNameWithCreateTableModifiers(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + DatabaseName: "ckp_constraints", + TableName: "parent", + } + + got := normalizeCreateTableDDLName("CREATE CLUSTER TABLE old_parent (id INT)", table) + assert.Equal(t, "CREATE CLUSTER TABLE `ckp_constraints`.`parent` (id INT)", got) +} + +func TestCleanObjectPath(t *testing.T) { + assert.Equal(t, "dump/account_1/t.csv", cleanObjectPath("dump/account_1/t.csv")) + assert.Equal(t, "tmp/dump/t.csv", cleanObjectPath("/tmp/dump/t.csv")) + assert.Equal(t, "dump/t.csv", cleanObjectPath("dump//nested/../t.csv")) +} + +func TestLoadDataPathResolverLocal(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272577, + TableID: 272578, + TableName: "bmsql_config", + } + + resolver, err := newLoadDataPathResolver(toolfs.StorageOptions{}) + require.NoError(t, err) + loadSource := resolver.loadDataSource("bmsql_config", table) + assert.True(t, strings.HasPrefix(loadSource, "LOAD DATA INFILE '")) + assert.Contains(t, loadSource, "/bmsql_config/account_0/db_272577/bmsql_config_272578.csv'") + assert.True(t, filepath.IsAbs(strings.TrimSuffix(strings.TrimPrefix(loadSource, "LOAD DATA INFILE '"), "'"))) +} + +func TestLoadDataPathResolverS3(t *testing.T) { + table := checkpointtool.TableCatalogEntry{ + AccountID: 0, + DatabaseID: 272577, + TableID: 272578, + TableName: "bmsql_config", + } + + resolver, err := newLoadDataPathResolver(toolfs.StorageOptions{ + S3: "bucket=mo-nightly-gz-1308875761,endpoint=https://cos.ap-guangzhou.myqcloud.com,region=ap-guangzhou," + + "key-prefix=/ckp-dump-test/tpcc_100_20260612_174916/bmsql_config/,key-id=xxx,key-secret=yyy", + Backend: "S3", + }) + require.NoError(t, err) + assert.Equal(t, + "LOAD DATA URL s3option{'bucket'='mo-nightly-gz-1308875761', 'filepath'='ckp-dump-test/tpcc_100_20260612_174916/bmsql_config/bmsql_config/account_0/db_272577/bmsql_config_272578.csv', 'endpoint'='https://cos.ap-guangzhou.myqcloud.com', 'region'='ap-guangzhou', 'access_key_id'='xxx', 'secret_access_key'='yyy'}", + resolver.loadDataSource("bmsql_config", table), + ) +} diff --git a/cmd/mo-object-tool/info.go b/cmd/mo-object-tool/info.go index b1906de20f007..3d65b48aa5fb8 100644 --- a/cmd/mo-object-tool/info.go +++ b/cmd/mo-object-tool/info.go @@ -21,10 +21,11 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) -func infoCommand() *cobra.Command { +func infoCommand(storage *toolfs.StorageOptions) *cobra.Command { cmd := &cobra.Command{ Use: "info ", Short: "Show object file information", @@ -32,19 +33,33 @@ func infoCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { path := args[0] - return showInfo(path) + return showInfo(path, *storage) }, } return cmd } -func showInfo(path string) error { +func showInfo(path string, storage toolfs.StorageOptions) error { ctx := context.Background() - reader, err := objecttool.Open(ctx, path) - if err != nil { - return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + var reader *objecttool.ObjectReader + if storage.IsRemote() { + fs, _, err := toolfs.Open(ctx, storage) + if err != nil { + return err + } + defer fs.Close(ctx) + reader, err = objecttool.OpenWithFS(ctx, fs, path, path) + if err != nil { + return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + } + } else { + var err error + reader, err = objecttool.Open(ctx, path) + if err != nil { + return moerr.NewInternalErrorf(ctx, "failed to open object: %v", err) + } } defer reader.Close() diff --git a/cmd/mo-object-tool/object.go b/cmd/mo-object-tool/object.go index 0223fee8451a0..d2c5442d5faf0 100644 --- a/cmd/mo-object-tool/object.go +++ b/cmd/mo-object-tool/object.go @@ -15,11 +15,15 @@ package object import ( + "context" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) func PrepareCommand() *cobra.Command { + var storage toolfs.StorageOptions cmd := &cobra.Command{ Use: "object [file]", Short: "Object file tools", @@ -28,15 +32,35 @@ func PrepareCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { // If file path is provided, enter view mode directly if len(args) == 1 { - return interactive.Run(args[0]) + return runObjectView(context.Background(), args[0], storage) } // Otherwise show help return cmd.Help() }, } + addStorageFlags(cmd, &storage) - cmd.AddCommand(viewCommand()) - cmd.AddCommand(infoCommand()) + cmd.AddCommand(viewCommand(&storage)) + cmd.AddCommand(infoCommand(&storage)) return cmd } + +func addStorageFlags(cmd *cobra.Command, storage *toolfs.StorageOptions) { + cmd.PersistentFlags().StringVar(&storage.FSConfig, "fs-config", "", "MO config TOML containing fileservice settings") + cmd.PersistentFlags().StringVar(&storage.FSName, "fs-name", "SHARED", "fileservice name to use from --fs-config") + cmd.PersistentFlags().StringVar(&storage.S3, "s3", "", "S3 arguments, for example bucket=...,endpoint=...,region=...,key-prefix=...,key-id=...,key-secret=...") + cmd.PersistentFlags().StringVar(&storage.Backend, "backend", "", "remote backend for --s3: S3 or MINIO") +} + +func runObjectView(ctx context.Context, path string, storage toolfs.StorageOptions) error { + if !storage.IsRemote() { + return interactive.Run(path) + } + fs, _, err := toolfs.Open(ctx, storage) + if err != nil { + return err + } + defer fs.Close(ctx) + return interactive.RunWithFS(ctx, fs, path) +} diff --git a/cmd/mo-object-tool/view.go b/cmd/mo-object-tool/view.go index aebe0927b7dab..12910c8efb26a 100644 --- a/cmd/mo-object-tool/view.go +++ b/cmd/mo-object-tool/view.go @@ -15,11 +15,13 @@ package object import ( - "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" + "context" + + "github.com/matrixorigin/matrixone/pkg/tools/toolfs" "github.com/spf13/cobra" ) -func viewCommand() *cobra.Command { +func viewCommand(storage *toolfs.StorageOptions) *cobra.Command { cmd := &cobra.Command{ Use: "view ", Short: "Interactive object file viewer", @@ -27,7 +29,7 @@ func viewCommand() *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { path := args[0] - return interactive.Run(path) + return runObjectView(context.Background(), path, *storage) }, } diff --git a/docs/design/checkpoint_tool_feature_analysis.md b/docs/design/checkpoint_tool_feature_analysis.md new file mode 100644 index 0000000000000..ef3f692b29039 --- /dev/null +++ b/docs/design/checkpoint_tool_feature_analysis.md @@ -0,0 +1,982 @@ +# Checkpoint CSV Dump 工具使用文档 + +## 概述 + +`mo-tool ckp` 是从 MatrixOne checkpoint 数据离线导出 CSV 的命令行工具。支持以表、database、租户（account）为单位导出，mo-data 可以是本地目录或远程 S3/MinIO，可指定时间戳导出历史快照数据。 + +--- + +## 一、构建 + +```bash +make mo-tool +``` + +构建产物为 `./mo-tool`。 + +### 1.1 构建注意事项 + +- **从 git worktree 构建**：如果从 `git worktree add --detach` 创建的 detached HEAD 状态构建，需要先在 Makefile 中给 `go build` 加上 `-buildvcs=false`，否则会报 `error obtaining VCS status: exit status 128`。具体修改：在 Makefile 的 `build` 目标中将 `go build` 改为 `go build -buildvcs=false`。 +- **构建 mo-service**：`make build` 生成 `./mo-service`，`make debug` 生成 race-detector 版本。 + +--- + +## 二、ckp list — 浏览 checkpoint 目录 + +`ckp list` 用于从 checkpoint 中列出数据库、表、租户等元数据，是 dump 前的关键查询命令。 + +### 2.1 命令 + +```bash +./mo-tool ckp list [directory] [flags] +``` + +### 2.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--type` | 否 | 列出类型：`tables`（默认）、`databases` 或 `accounts` | +| `--account-id` | 否 | 按租户 ID 过滤 | +| `--database-id` | 否 | 按数据库 ID 过滤 | +| `--ts` | 否 | 快照时间戳，默认使用 latest | +| `--include-views` | 否 | 包含视图（默认只列普通表） | + +### 2.3 示例 + +```bash +# 列出所有数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases + +# 列出指定数据库下的所有表 +./mo-tool ckp list /path/to/mo-data/shared --database-id 272528 + +# 列出所有租户 +./mo-tool ckp list /path/to/mo-data/shared --type accounts + +# 远程 S3/MinIO 下列出 +./mo-tool ckp list --fs-config etc/launch-minio-local/tn.toml --type databases +``` + +--- + +### 2.4 ckp show-create-table — 从 checkpoint 获取 DDL + +从 checkpoint 中获取指定表的完整 CREATE TABLE DDL，无需连接数据库实例即可查看表结构。 + +```bash +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id 272537 +``` + +输出示例： + +```sql +CREATE TABLE `t_all_wide` ( + `id` INT NOT NULL, + `c_tiny` TINYINT DEFAULT NULL, + ... + PRIMARY KEY (`id`) +); +``` + +支持远程访问：`--fs-config` / `--s3` 参数与 `dump` 命令相同。 + +--- + +## 三、以表为单位 dump + +### 3.1 命令 + +```bash +./mo-tool ckp dump --table-id= [选项] +``` + +### 3.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--table-id` | 是 | MatrixOne 内部 table_id，可从 `mo_tables` 系统表或 `ckp list` 中获取 | +| `--ts` | 否 | 快照时间戳。不指定则使用最新的 checkpoint 时间戳。支持格式：`physical:logical`（如 `1781084792256166694:1`）、`physical-logical`、RFC3339、或本地时间格式 | +| `-o` / `--output` | 否 | 输出路径。纯 CSV 模式指定文件路径，`--load-script` 模式指定目录（工具自动生成文件名）。不指定则输出到 stdout | +| `--header` | 否 | 在 CSV 第一行输出列名头行 | +| `--meta-comments` | 否 | 在 CSV 文件头部输出 DDL 和行数统计注释（以 `--` 开头） | +| `--row-order` | 否 | 行排序方式：`storage`（默认，流式按存储顺序）或 `lexical`（按可见列字典序排序） | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成包含 CREATE DATABASE + CREATE TABLE + LOAD DATA 的 SQL 文件 | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 3.3 示例 + +**导出到文件**： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/employees.csv /path/to/mo-data +``` + +**导出到 stdout**： + +```bash +./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data +``` + +**指定时间戳导出**： + +```bash +./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/employees.csv /path/to/mo-data +``` + +**生成 LOAD 脚本（含建库建表 + LOAD DATA）**： + +```bash +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data +``` + +**带 DDL 元数据注释导出**： + +```bash +./mo-tool ckp dump --table-id=272535 --header --meta-comments -o /tmp/employees.csv /path/to/mo-data +``` + +此时输出文件头部会包含： + +```text +-- CREATE TABLE employees (id INT PRIMARY KEY, name VARCHAR(100), ...) +-- Database: test_ckp +-- Table: employees +-- Visible rows: 103 (deleted: 0, physical: 103) +id,name,age,salary,department,hire_date,is_active +... +``` + +### 3.4 输出位置 + +| 模式 | `-o` 含义 | 输出文件 | +|------|----------|---------| +| 纯 CSV（默认） | 文件路径 | `-o /tmp/employees.csv` → `/tmp/employees.csv` | +| `--load-script` | 目录路径 | `-o /tmp/` → `/tmp/restore.sql` | + +未指定 `-o` 时输出到 stdout。 + +--- + +## 四、以 database 为单位 dump + +### 4.1 命令 + +```bash +./mo-tool ckp dump --database-id= --output-dir= [选项] +``` + +### 4.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--database-id` | 是 | Database ID（uint64），可从 `mo_database` 系统表或 `ckp list --type=databases` 获取 | +| `--output-dir` | 是 | 输出根目录。该目录会自动创建 | +| `--ts` | 否 | 快照时间戳，不指定则用最新 | +| `--header` | 否 | 在每个 CSV 文件第一行输出列名头行 | +| `--meta-comments` | 否 | 在每个 CSV 文件头部输出 DDL 和行数统计注释 | +| `--row-order` | 否 | 行排序方式 | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含 CREATE DATABASE + 所有表的 CREATE TABLE + LOAD DATA | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 4.3 示例 + +**导出指定 database ID 下的所有表**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data +``` + +**生成该 database 的 LOAD 恢复脚本**： + +```bash +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data +``` + +### 4.4 输出目录结构 + +以 database 为单位 dump 时，CSV 文件按以下目录结构组织： + +```text +/ +├── restore.sql # 如果指定了 --load-script，生成恢复脚本 +└── account_/ + └── db_/ + ├── _.csv + ├── _.csv + └── ... +``` + +具体示例： + +**dump CSV + LOAD 脚本**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header \ + --load-script -o ./dump_out/ /path/to/mo-data +``` + +```text +dump_out/ +├── restore.sql +└── account_7/ + └── db_9001/ + ├── employees_272535.csv + ├── departments_272536.csv + └── alter_compat_272538.csv +``` + +其中 `restore.sql` 的内容： + +```sql +CREATE DATABASE IF NOT EXISTS test_ckp; +USE test_ckp; + +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + ... +); + +LOAD DATA INFILE 'dump_out/account_7/db_9001/employees_272535.csv' +INTO TABLE employees +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; + +CREATE TABLE departments (...); + +LOAD DATA INFILE 'dump_out/account_7/db_9001/departments_272536.csv' +INTO TABLE departments +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + +**只 dump CSV（不含 LOAD 脚本）**： + +```bash +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data +``` + +```text +dump_out/ +└── account_7/ + └── db_9001/ + ├── employees_272535.csv + ├── departments_272536.csv + └── alter_compat_272538.csv +``` + +目录命名规则： + +- **account 目录**：`account_`，其中 `account_id` 为十进制数字。系统租户的 account_id 为 `0` +- **database 目录**：`db_`，其中 `database_id` 为十进制数字 +- **CSV 文件名**：`_.csv`，其中 `table_name` 为表名，`table_id` 为内部数字 ID。表名中的特殊字符会被替换为下划线 + +> **注意**：`relkind='v'` 的视图不会被导出。 + +### 4.5 命令输出 + +执行过程中，每成功导出一个表会打印一行信息： + +```text +Table 272535 test_ckp.employees dumped to dump_out/account_7/db_9001/employees_272535.csv +Table 272536 test_ckp.departments dumped to dump_out/account_7/db_9001/departments_272536.csv +Dumped 2 tables to dump_out +``` + +--- + +## 五、以租户（account）为单位 dump + +### 5.1 命令 + +```bash +./mo-tool ckp dump --account-id= --output-dir= [选项] +``` + +### 5.2 参数说明 + +| 参数 | 必需 | 说明 | +|------|------|------| +| `--account-id` | 是 | 租户 ID（uint32） | +| `--output-dir` | 是 | 输出根目录 | +| `--ts` | 否 | 快照时间戳 | +| `--header` | 否 | CSV 列名头行 | +| `--meta-comments` | 否 | DDL 和行数统计注释 | +| `--row-order` | 否 | 行排序方式 | +| `--load-script` | 否 | 切换为 LOAD 脚本输出模式：生成单个 SQL 文件，包含所有 database 的 CREATE DATABASE + CREATE TABLE + LOAD DATA | +| `--no-load` | 否 | 配合 `--load-script` 使用，跳过 LOAD DATA 语句，只输出 DDL | + +### 5.3 示例 + +**导出某个租户的所有表**： + +```bash +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data +``` + +**生成该租户的 LOAD 恢复脚本**： + +```bash +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data +``` + +### 5.4 输出目录结构 + +以租户为单位 dump 时，目录结构与 database 级别一致： + +```text +/ +├── restore.sql # 如果指定了 --load-script，生成恢复脚本 +└── account_/ + ├── db_/ + │ ├── _.csv + │ └── ... + ├── db_/ + │ └── ... + └── ... +``` + +具体示例： + +```bash +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header \ + --load-script -o ./dump_out/ /path/to/mo-data +``` + +```text +dump_out/ +├── restore.sql +└── account_7/ + ├── db_9001/ + │ ├── employees_272535.csv + │ └── alter_compat_272538.csv + └── db_9002/ + ├── orders_272540.csv + ├── lineitem_272541.csv + └── ... +``` + +`restore.sql` 包含该租户下所有 database 的建表和数据导入语句。 + +--- + +## 六、本地 mo-data + +直接传入本地 mo-data 目录路径即可，适用于 checkpoint 数据存储在本地磁盘的场景。 + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data +``` + +内部使用 `fileservice.NewLocalFS` 创建本地文件服务，直接读取 checkpoint 元数据和数据对象文件。 + +--- + +## 七、远程 S3/MinIO mo-data + +当 checkpoint 数据存储在 S3 或 MinIO 对象存储上时，支持两种远程访问方式。 + +### 7.1 方式一：通过 MO 配置文件 + +使用 MatrixOne 的 TOML 配置文件（如 `tn.toml`），其中的 `[fileservice]` 段包含对象存储的连接信息： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED +``` + +- `--fs-config`：MO 配置文件路径 +- `--fs-name`：使用的 fileservice 名称（默认为 `SHARED`），对应配置中 `[fileservice]` 段的 `name` 字段 + +### 7.2 方式二：直接指定 S3 参数 + +不依赖 MO 配置文件，直接传入对象存储的连接参数： + +```bash +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,key-prefix=server/data,key-id=minio,key-secret=minio123 +``` + +S3 参数格式（逗号分隔的 key=value 对）： + +| 参数 | 必需 | 说明 | +|------|------|------| +| `bucket` | 是 | S3 bucket 名称 | +| `endpoint` | 是 | S3/MinIO API 地址 | +| `key-prefix` | 是 | mo-data 在 bucket 中的路径前缀 | +| `key-id` | 是 | 访问密钥 ID | +| `key-secret` | 是 | 访问密钥 Secret | +| `region` | 否 | S3 region | + +`--backend` 可选值： +- `S3`：AWS S3 或兼容 S3 协议的服务（默认） +- `MINIO`：MinIO 对象存储 + +### 7.3 远程读取机制 + +远程读取使用 `lazyCacheFS` 实现按需缓存： + +1. 首次访问远程文件时，下载到本地临时目录 +2. 后续读取同一文件直接命中本地缓存 +3. 绕过 MO 内部的 checksum 格式验证，直接按 OS file range 读取 +4. 工具退出时自动清理临时缓存目录 + +checkpoint 元数据列表（`List` 操作）直接对远程服务执行，不下载全部文件。 + +### 7.4 远程 dump 示例 + +```bash +# 远程单表 dump +./mo-tool ckp dump --table-id=272535 --header -o /tmp/remote_table.csv \ + --fs-config etc/launch-minio-local/tn.toml \ + --fs-name SHARED + +# 远程 database dump +./mo-tool ckp dump --database-id=9001 --output-dir=./remote_dump --header \ + --fs-config etc/launch-minio-local/tn.toml + +# 远程 ckp info +./mo-tool ckp info --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED +``` + +### 7.5 远程访问已知限制 + +- **MINIO backend 数据文件不完整**：当 MatrixOne 以 MINIO 作为 SHARED fileservice 运行时，checkpoint 元数据文件（`ckp/meta_*.ckp`）正常写入，但部分数据对象文件可能因缓存策略未完全 flush 到 MINIO，导致 dump 时出现 `file is not found` 错误。 +- **建议**：对于本地开发/测试场景，使用 DISK backend（见下文 十五、checkpoint 配置建议），将 SHARED 改为本地目录，确保所有数据文件可被 mo-tool 直接读取。 +- **S3 region 参数**：使用 `--backend MINIO` 时必须显式指定 `region`，否则报 `Invalid region` 错误。例如 `region=us-east-1`。 + +--- + +## 八、CSV 格式说明 + +### 8.1 格式规范 + +导出的 CSV 文件兼容 MySQL/MariaDB/MatrixOne 的 `LOAD DATA` 语句，具体格式： + +| 特性 | 约定 | +|------|------| +| 字段分隔符 | `,`（逗号） | +| 字段引用符 | `"`（双引号） | +| 转义符 | `\`（反斜杠） | +| 行分隔符 | `\n`（换行） | +| NULL 表示 | `\N`（反斜杠+大写 N） | +| 字符串类型 | 总是用双引号括起（VARCHAR, CHAR, TEXT, BLOB, JSON, GEOMETRY, DATALINK 等） | +| 数值类型 | 不使用引号（INT, BIGINT, FLOAT, DOUBLE, DECIMAL, BOOL 等） | +| 日期时间类型 | 不使用引号（DATE, TIME, DATETIME, TIMESTAMP） | + +### 8.2 特殊值处理 + +| 场景 | CSV 输出 | +|------|----------| +| NULL 值 | `\N` | +| 包含双引号的字符串（如 `a"b`） | `"a""b"` | +| 包含反斜杠的字符串（如 `a\b`） | `"a\\b"` | +| 包含逗号的字符串（如 `a,b`） | `"a,b"` | +| 包含换行的字符串 | `"a\nb"` | + +### 8.3 CSV 输出示例 + +```csv +id,name,age,salary,department,hire_date,is_active +1,"Alice",30,75000.00,"Engineering",2020-01-15,true +2,"Bob",\N,\N,\N,\N,\N +3,"Charlie",\N,65000.00,\N,2021-03-10,false +4,"David""The Boss""",42,120000.00,"Management",2019-06-01,true +``` + +--- + +## 九、LOAD DATA 回灌 + +### 9.1 推荐方式：使用 `--load-script` + +`--load-script` 一步生成完整的 SQL 恢复脚本（CREATE DATABASE + CREATE TABLE + LOAD DATA），在目标 MO 实例中直接执行即可： + +```bash +# 生成恢复脚本 +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data + +# 在目标 MO 实例中执行 +mysql -h 127.0.0.1 -P 6001 -u root -p111 < /tmp/restore.sql +``` + +详见第十一章。 + +### 9.2 手动回灌（不使用 `--load-script`） + +如果不使用 `--load-script`，手动回灌流程如下： + +**步骤 1：建表** + +```sql +CREATE TABLE test_ckp.employees_load ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); +``` + +**步骤 2：LOAD DATA** + +```sql +LOAD DATA INFILE '/tmp/employees.csv' +INTO TABLE test_ckp.employees_load +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; -- 如果 dump 时使用了 --header +``` + +LOAD DATA 参数对应关系： + +| CSV 格式特性 | LOAD DATA 参数 | +|-------------|---------------| +| 逗号分隔字段 | `FIELDS TERMINATED BY ','` | +| 双引号括起字符串 | `ENCLOSED BY '"'` | +| 反斜杠转义 | 默认（MySQL/MO 默认转义符为 `\`） | +| 换行分隔 | `LINES TERMINATED BY '\n'` | +| NULL 为 `\N` | 默认（MySQL/MO 默认 NULL 表示即 `\N`） | +| 有 header 行 | `IGNORE 1 LINES` | +| 无 header 行 | 不需要 `IGNORE 1 LINES` | + +### 9.3 数据校验 + +```sql +-- 对比原始行数和导入行数 +SELECT (SELECT COUNT(*) FROM test_ckp.employees) AS original_rows, + (SELECT COUNT(*) FROM test_ckp.employees_load) AS loaded_rows; + +-- 对比数据差异 +SELECT COUNT(*) AS diff_rows +FROM ( + SELECT * FROM test_ckp.employees + EXCEPT + SELECT * FROM test_ckp.employees_load +) AS diff; +``` + +--- + +## 十、CSV 行排序选项 + +### 10.1 storage 顺序（默认） + +```bash +./mo-tool ckp dump --table-id=272535 --row-order=storage --header -o /tmp/table.csv /path/to/mo-data +``` + +- 按物理存储顺序输出，流式写入 +- 内存占用小，适合大表 +- 行顺序与存储布局一致，不可预测 + +### 10.2 lexical 顺序 + +```bash +./mo-tool ckp dump --table-id=272535 --row-order=lexical --header -o /tmp/table.csv /path/to/mo-data +``` + +- 按所有可见列的字典序排序后输出 +- 需要将全部行加载到内存中排序 +- 适合需要确定性输出顺序的场景 +- 不适合超大表 + +--- + +## 十一、交互式浏览器 + +除了命令行 dump，`mo-tool ckp` 还提供了交互式 TUI 浏览器，用于探索 checkpoint 内容： + +```bash +./mo-tool ckp view /path/to/mo-data +``` + +交互式浏览器支持以下操作： + +- **浏览 checkpoint 列表**：查看所有 GCKP/ICKP 条目及其时间范围 +- **浏览表列表**：选择一个 checkpoint 条目后，查看其中包含的所有表（可按 account 筛选） +- **浏览对象详情**：选择一个表后，查看其 data/tombstone 对象列表 +- **浏览逻辑表视图**：查看 tombstone 过滤后的实际数据行 +- **打开对象文件**：深入到单个数据对象中查看原始内容 + +远程 checkpoint 也可使用交互式浏览器： + +```bash +./mo-tool ckp view --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED +``` + +--- + +## 十二、checkpoint 信息查看 + +### 12.1 查看 checkpoint 摘要 + +```bash +./mo-tool ckp info /path/to/mo-data +``` + +输出 checkpoint 的条目类型统计和时间范围。 + +### 12.2 生成 LOAD 脚本（`--load-script`） + +dump 命令支持 `--load-script` 选项，生成一个可直接在目标 MatrixOne 实例中执行的 SQL 脚本，一行命令即可完成数据恢复。 + +脚本包含以下内容（按顺序）： + +1. **CREATE DATABASE** — 重建 database +2. **USE database** — 切换到目标 database +3. **CREATE TABLE**（每个表一条） — 重建表结构，DDL 从 checkpoint 的 `mo_tables.rel_createsql` 解析 +4. **LOAD DATA**（每个 CSV 文件一条，可选）— 导入数据 + +#### 单表生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的 `restore.sql` 示例： + +```sql +CREATE DATABASE IF NOT EXISTS test_ckp; +USE test_ckp; + +CREATE TABLE employees ( + id INT PRIMARY KEY, + name VARCHAR(100), + age INT, + salary DECIMAL(10,2), + department VARCHAR(50), + hire_date DATE, + is_active BOOL +); + +LOAD DATA INFILE '/tmp/employees_272535.csv' +INTO TABLE employees +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + +#### database 级别生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的脚本包含该 database 下所有表的 DDL 和 LOAD DATA 语句。CSV 文件输出到脚本同级目录。 + +#### 租户级别生成 LOAD 脚本 + +```bash +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data +``` + +生成的脚本包含该租户下所有 database 和表的 DDL 及 LOAD DATA 语句。 + +#### LOAD DATA 可选 + +```bash +# 只生成 DDL，不含 LOAD DATA（仅建库建表） +./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data +``` + +`--no-load` 跳过 LOAD DATA 语句，只输出 CREATE DATABASE + CREATE TABLE。 + +--- + +## 十三、常用命令速查 + +```bash +# 构建 +make mo-tool + +# === ckp info === +# 查看 checkpoint 摘要 +./mo-tool ckp info /path/to/mo-data/shared +./mo-tool ckp info --fs-config etc/launch-minio-local/tn.toml # 远程 + +# === ckp list === +# 列出所有数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases + +# 列出指定数据库的表 +./mo-tool ckp list /path/to/mo-data/shared --database-id + +# 列出所有租户 +./mo-tool ckp list /path/to/mo-data/shared --type accounts + +# 远程列出 +./mo-tool ckp list --fs-config etc/launch-minio-local/tn.toml --type databases + +# === ckp show-create-table === +# 查看 checkpoint 中表的 DDL +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id + +# === ckp dump === +# 单表 dump 到文件 +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv /path/to/mo-data/shared + +# 单表 dump 到 stdout +./mo-tool ckp dump --table-id=272535 --header /path/to/mo-data/shared + +# 指定时间戳单表 dump +./mo-tool ckp dump --table-id=272535 --ts=1781084792256166694:1 --header -o /tmp/table.csv /path/to/mo-data/shared + +# database 级别批量 dump +./mo-tool ckp dump --database-id=9001 --output-dir=./dump_out --header /path/to/mo-data/shared + +# 租户级别批量 dump +./mo-tool ckp dump --account-id=7 --output-dir=./dump_out --header /path/to/mo-data/shared + +# 生成 LOAD 脚本（单表/database/租户） +./mo-tool ckp dump --table-id=272535 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --database-id=9001 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --account-id=7 --load-script -o /tmp/ /path/to/mo-data/shared +./mo-tool ckp dump --database-id=9001 --load-script --no-load -o /tmp/ /path/to/mo-data/shared + +# dump 系统表 +./mo-tool ckp dump --table-id=2 -o /tmp/mo_tables.csv /path/to/mo-data/shared # mo_tables +./mo-tool ckp dump --table-id=3 -o /tmp/mo_columns.csv /path/to/mo-data/shared # mo_columns + +# 远程 S3/MinIO dump（方式一：配置文件） +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --fs-config etc/launch-minio-local/tn.toml --fs-name SHARED + +# 远程 S3/MinIO dump（方式二：直接指定参数） +./mo-tool ckp dump --table-id=272535 --header -o /tmp/table.csv \ + --backend MINIO \ + --s3 bucket=mo-test,endpoint=http://127.0.0.1:9000,region=us-east-1,key-prefix=server/data,key-id=minio,key-secret=minio123 + +# 交互式浏览器 +./mo-tool ckp view /path/to/mo-data/shared + +# === 测试数据准备 === +./prepare_ckp_dump_coverage_data.sh --host 127.0.0.1 --port 6001 --user dump --password 111 --db-prefix ckp --drop-existing +``` + +--- + +## 十四、恢复流程模板 + +### 推荐方式：`--load-script` 一键恢复 + +```bash +# 1. 从 checkpoint 生成恢复脚本 +./mo-tool ckp dump --database-id= --load-script --header -o ./ /path/to/mo-data + +# 2. 在目标 MO 实例执行 +mysql -h -P -u -p < restore.sql +``` + +生成的 `restore.sql` 结构： + +```sql +CREATE DATABASE IF NOT EXISTS ; +USE ; + +CREATE TABLE (...); +LOAD DATA INFILE '' INTO TABLE ...; + +CREATE TABLE (...); +LOAD DATA INFILE '' INTO TABLE ...; +``` + +### 手动方式 + +```sql +-- 建表 +CREATE TABLE . ( + ... +); + +-- 导入 CSV（dump 时使用了 --header） +LOAD DATA INFILE '' +INTO TABLE .
+FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + +--- + +## 十五、测试数据准备 + +### 15.1 prepare_ckp_dump_coverage_data.sh + +仓库根目录下的 `prepare_ckp_dump_coverage_data.sh` 脚本用于快速生成 checkpoint dump 的覆盖测试数据。 + +**功能**：创建 4 个精心设计的测试 database： + +| Database | 内容 | +|---|---| +| `_types` | 各种数据类型及边界值（有符号/无符号整数、浮点/定点数、bool/bit、字符串/JSON、binary/blob、时间类型、enum/set、uuid、vecf32、array、datalink） | +| `_constraints` | 约束场景（外键 RESTRICT/CASCADE/SET NULL、自增列、复合主键/唯一键、特殊标识符、全文索引、向量索引） | +| `_tables` | 表形态（普通表、空表、view、CTAS、LIKE、hash 分区、key 分区、cluster by、特殊表名） | +| `_mvcc_perf` | DML 历史（insert/update/delete）、truncate、alter add column、大规模行表（默认 10000 行）、32 列宽表 | + +**用法**： + +```bash +# 默认时间戳命名（每次生成不同的 database 名） +./prepare_ckp_dump_coverage_data.sh --host 127.0.0.1 --port 6001 --user dump --password 111 + +# 固定 database 名前缀 + 大规模数据，便于反复测试 +./prepare_ckp_dump_coverage_data.sh \ + --host 127.0.0.1 --port 6001 \ + --user dump --password 111 \ + --db-prefix ckp \ + --scale 10000 \ + --drop-existing +``` + +**主要参数**： + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--db-prefix` | `ckp_cov_` | database 名前缀 | +| `--scale` | `10000`（最大 `1000000`） | 大规模表的行数 | +| `--drop-existing` | false | 先删除同名 database | +| `--generate-only` | false | 只生成 SQL 文件，不连接数据库执行 | +| `--out-dir` | `/tmp/ckp_dump_coverage_` | SQL 文件输出目录 | + +### 15.2 已知兼容性问题 + +不同 MO 版本对 SQL 的支持不同，以下语法可能在特定版本上失败（脚本中 `/11_types_optional.sql` 和 `/21_indexes_optional.sql` 以 `--force` 模式执行，失败不中断）： + +| 语法 | 说明 | +|------|------| +| `ARRAY(VARCHAR(20))` | 3.0-dev 不支持 ARRAY 类型 | +| `LEAST()` | 3.0-dev 不支持 LEAST 函数 | +| `TEMPORARY TABLE` | 3.0-dev 不支持临时表 | +| `CHAR(n)` 函数 | 3.0-dev 不支持 `CHAR()` 函数，需用 `UNHEX()` 替代 | +| `COUNT(*) AS rows` | `rows` 是 3.0-dev 保留关键字，需改别名 | + +--- + +## 十六、checkpoint 配置建议 + +MO 的 checkpoint 行为由 TN 配置中的 `[tn.Ckp]` 段控制。**测试时**，默认配置可能导致 checkpoint 生成过慢，建议使用以下激进配置： + +```toml +# 默认配置（checkpoint 生成慢，不适合测试） +[tn.Ckp] +flush-interval = "60s" +min-count = 100 +scan-interval = "5s" +incremental-interval = "180s" +global-min-count = 60 + +# 测试推荐配置（checkpoint 快速生成） +[tn.Ckp] +flush-interval = "10s" +min-count = 1 +scan-interval = "2s" +incremental-interval = "30s" +global-min-count = 1 +``` + +**关键参数说明**： + +| 参数 | 说明 | +|------|------| +| `flush-interval` | checkpoint flush 间隔 | +| `min-count` | 触发 flush 的最小 log entry 数量 | +| `incremental-interval` | 增量 checkpoint 间隔 | +| `global-min-count` | 触发全局 checkpoint 的最小 log entry 数量。全局 checkpoint 包含完整的 catalog 快照（mo_tables/mo_columns），是 mo-tool dump 数据的前提 | + +**触发 checkpoint 的方式**： +- 自动触发：满足时间或 log count 阈值 +- 关闭触发：优雅关闭 MO（`SIGTERM`）会触发一次 shutdown checkpoint，将未持久化的数据写入 checkpoint + +**SHARED fileservice backend 选择**： + +| backend | 适用场景 | 说明 | +|---------|---------|------| +| `DISK` | 本地开发/测试 | mo-data 直接存储为本地文件，mo-tool 无需额外配置即可读取 | +| `MINIO` / `S3` | 生产环境 | mo-data 存储在对象存储上，需通过 `--fs-config` 或 `--s3` 参数访问（见 七、远程 S3/MinIO mo-data） | + +本地测试推荐使用 DISK backend 的配置： + +```toml +[[fileservice]] +backend = "DISK" +data-dir = "./etc/launch-minio-local/mo-data/shared" +name = "SHARED" +``` + +--- + +## 十七、版本兼容性 + +### 17.1 view_ckp mo-tool 读取 3.0-dev checkpoint + +| 功能 | 状态 | 说明 | +|------|------|------| +| `ckp info` | ✅ | checkpoint 条目统计和时间范围正常 | +| `ckp list --type databases` | ✅ | 系统和用户 database 全部列出 | +| `ckp list --type tables` | ✅ | 表元数据完整 | +| `ckp show-create-table` | ✅ | DDL 正确还原 | +| `ckp dump`（catalog 表） | ✅ | mo_tables/mo_columns 等系统表 dump 正常 | +| `ckp dump`（用户表） | ⚠️ | CSV header 正常生成，但数据行可能为空（取决于 checkpoint 是否完整包含数据对象） | +| `ckp dump --load-script` | ⚠️ | DDL 部分正常，数据导入可能无数据 | + +### 17.2 数据完整性检查 + +如果 dump 用户表得到 0 行数据（`visible_rows=0`），请检查： + +1. **checkpoint 是否包含数据对象**：在 `ckp info` 中确认 `Global ≥ 1`。如果只有 Incremental 条目且缺少对应的 Global checkpoint，数据对象可能未关联到 catalog。 +2. **checkpoint 时间戳**：使用 `--ts` 指定更早或更晚的 checkpoint 时间戳试试。 +3. **mo-data 文件完整性**：检查 `shared` 目录下是否有与 checkpoint meta 文件中引用的 UUID 对应的数据文件。 +4. **SHARED backend**：使用 MINIO 时可能存在缓存未 flush 问题，建议切换到 DISK backend（见 十六、checkpoint 配置建议）。 + +### 17.3 完整测试流程 + +```bash +# 1. 启动 MO（3.0-dev，使用 DISK SHARED backend + 激进 checkpoint 配置） +cd matrixone-3.0-dev +./mo-service -launch etc/launch-minio-local/launch.toml & + +# 2. 等待 MO 就绪 +mysql -h 127.0.0.1 -P 6001 -u dump -p111 -e "SELECT 1" + +# 3. 生成测试数据 +./prepare_ckp_dump_coverage_data.sh \ + --host 127.0.0.1 --port 6001 --user dump --password 111 \ + --db-prefix ckp --scale 10000 --drop-existing + +# 4. 等待 checkpoint 生成（约 1-2 分钟） +sleep 120 + +# 5. 优雅关闭 MO（触发 shutdown checkpoint） +pkill -TERM mo-service +sleep 10 + +# 6. 查看 checkpoint 信息 +./mo-tool ckp info /path/to/mo-data/shared + +# 7. 列出数据库 +./mo-tool ckp list /path/to/mo-data/shared --type databases + +# 8. 列出某个 database 的表 +./mo-tool ckp list /path/to/mo-data/shared --database-id + +# 9. 查看表 DDL +./mo-tool ckp show-create-table /path/to/mo-data/shared --table-id + +# 10. dump 单表 +./mo-tool ckp dump --table-id --header -o /tmp/table.csv /path/to/mo-data/shared + +# 11. dump 整个 database（含 LOAD 脚本） +./mo-tool ckp dump --database-id --load-script -o /tmp/dump_out /path/to/mo-data/shared +``` diff --git a/docs/design/checkpoint_tool_fragility.md b/docs/design/checkpoint_tool_fragility.md new file mode 100644 index 0000000000000..4d764bf8b0d8e --- /dev/null +++ b/docs/design/checkpoint_tool_fragility.md @@ -0,0 +1,263 @@ +# Checkpoint Tool 脆弱性分析：上游依赖与变更风险 + +本文档梳理 `mo-tool ckp` 离线 checkpoint 工具链中对 MatrixOne 内核私有 API / 数据格式的各项依赖，分析它们在不相关代码变更下被意外打破的风险。 + +--- + +## 1. 系统表 Schema 的硬编码副本 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:165-244` (`builtinColumnsForLayout`) + +**问题**：`mo_tables`（20 列）、`mo_columns`（27 列）、`mo_database`（10 列）三张系统表的完整列名、类型、Position 被逐列手写了一份，作为 checkpoint 数据中 catalog 元数据不可用时的 fallback。 + +```go +case catalog.MO_TABLES_ID: + cols := []builtinColumnDef{ + {Name: "rel_id", SQLType: "BIGINT", Position: 0}, + {Name: "relname", SQLType: "VARCHAR(5000)", Position: 1}, + // ... 共 20+ 列 + } +``` + +**触发条件**： +- `catalog` 包中 `MoTablesSchema` / `MoColumnsSchema` 增删或重命名任一列（例如新增 `rel_foo`、删除 `extra_info`） +- 该变更不会导致编译错误——因为 `builtinColumnsForLayout` 是手写的独立结构体，不引用 `catalog` 包的 schema 变量 + +**后果**：checkpoint 数据中如果 header 是泛型名（`col_0`, `col_1`），按位置取值时会拿到错误的列值；`builtinTableSchemaForLayout` 返回的 Column 列表与实际物理数据不对齐。 + +--- + +## 2. Catalog Layout 检测依赖列数 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:357-370` (`inferCatalogLayout`) + +**问题**：纯靠**列数**判断一份 checkpoint 是当前 layout 还是 3.0-dev 旧 layout： + +```go +switch dataWidth { +case len(schema): // 列数完全匹配 + return layout, 0 +case len(schema) + 1: // 多一列也接受，带 offset=1 + return layout, 1 +} +return currentCatalogLayout, 0 // 都不匹配 → 假设当前版本 +``` + +已知的两个 layout 差异： +| | `mo_tables` 列数 | `mo_columns` 列数 | +|---|---|---| +| current | 20 | 27 | +| 3.0-dev | 19（少 `rel_logical_id`） | 25（少 `attr_has_generated`, `attr_generated`） | + +**触发条件**： +- 未来的 layout 变体恰好与已知 layout 的列数相同（比如新版本加一列又删一列，列数不变但顺序变了） +- 列数对不上时静默 fallback 到 `currentCatalogLayout`，不报错 + +**后果**：`fallbackCatalogColIndex` 用 layout 的固定列顺序定位，列映射全错。对 `mo_tables` 的影响是取不到正确的 `rel_createsql`、`relname`；对 `mo_columns` 的影响是 `att_seqnum` / `att_is_hidden` 位置错误，导致 CSV 列全部错位或隐藏列未能过滤。 + +--- + +## 3. `logicalViewMetaCols = 3` 硬编码 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:34` + +**问题**：`LogicalTableView` 固定有 3 个 meta 列（`object`, `block`, `row`），所有数据列的偏移计算都依赖这个常量： + +```go +const logicalViewMetaCols = 3 + +// 在 table_dump.go 中大量使用： +dataRow := fullRow[logicalViewMetaCols:] +dataWidth := len(view.Headers) - logicalViewMetaCols +dataIdx := logicalViewMetaCols + pos +``` + +`LogicalTableView` 的 headers 在 `logical_table.go:43` 构造： +```go +view := &LogicalTableView{ + Headers: []string{"object", "block", "row"}, + // ... +} +``` + +**触发条件**：`BuildLogicalTableView` 的 header 构造逻辑变更（增减 meta 列，如新增 `segment` 列） + +**后果**：所有 `table_dump.go` 中的列偏移计算全部偏移，`buildSchemaFromMoTablesRow` 和 `buildColumnsFromMoColumnsRows` 拿到错误的数据列值。 + +--- + +## 4. 内核私有 API 直接依赖 + +### 4.1 `checkpoint.ReadEntriesFromMeta` + +**位置**：`pkg/vm/engine/tae/db/checkpoint/meta.go:45` + +工具调用方式： +```go +checkpoint.ReadEntriesFromMeta( + ctx, "", ioutil.GetCheckpointDir(), name, 0, nil, r.mp, r.fs, +) +``` + +**风险**： +- 7 参数函数，`sid=""`, `verbose=0`, `onEachEntry=nil` 对工具无意义 +- 内部绑定 meta 文件的二进制格式（注释中有明确的 attr/types 表格） +- meta 文件格式变更 → 函数行为改变，无需 deprecation + +### 4.2 `logtail.NewCKPReaderWithTableID_V2` + +**位置**：`pkg/vm/engine/tae/logtail/ckp_reader.go:237` + +**风险**： +- `_V2` 后缀说明已发生过一次破坏性 API 变更 +- 配套的 `ReadMeta()` 有隐式调用顺序要求（注释："must called after ReadMeta"） +- `ConsumeCheckpointWithTableID` 的回调签名如果改变，编译失败 + +### 4.3 `ioutil.GetCheckpointDir()` + +**位置**：`pkg/objectio/ioutil/` + +**风险**： +- 固定返回 `"ckp/"` 字符串 +- 如果 MO 调整 checkpoint 目录结构 → 工具找不到任何 checkpoint 文件 + +### 4.4 `ioutil.ListTSRangeFiles` / `ckputil.ListCKPMetaNames` + +**风险**： +- 依赖 checkpoint 目录下的文件命名约定 +- 如果文件命名规则变化（扩展名、前缀等），过滤逻辑失效 + +### 4.5 `ckputil.TableObjectsAttrs` + +**位置**：`pkg/vm/engine/ckputil/types.go:66` + +```go +var TableObjectsAttrs = []string{ + "account_id", "db_id", "table_id", "object_type", + "id", "create_ts", "delete_ts", ... +} +``` + +工具在 `checkpoint_reader.go:471` 直接引用： +```go +columns := ckputil.TableObjectsAttrs +``` + +**风险**： +- 包级 `var`（非 const），无编译期保护 +- 属性名或顺序变化 → 工具输出的列名不变但值对不上 + +### 4.6 `objectioutil.FindTombstonesOfObject` / `GetTombstonesByBlockId` + +**位置**：`pkg/objectio/ioutil/` + +工具在 `logical_table.go` 中使用： +```go +selection, err := objectioutil.FindTombstonesOfObject(ctx, objectID, tombstoneStats, r.fs) +// ... +err := objectioutil.GetTombstonesByBlockId(ctx, snapshotTS, &blockID, getTombstone, &mask, r.fs) +``` + +**风险**： +- 深度依赖 objectio 的 bitmap、ObjectId、ObjectStats 等内部类型 +- 如果 objectio 重构 bitmap 表示方式 → tombstone 过滤逻辑完全失效 +- `getTombstone` 回调闭包跨层传递 `tombstoneStats` 切片，控制流不直观 + +--- + +## 5. 字符串列名匹配的静默 Fallback + +**位置**：`pkg/tools/checkpointtool/table_dump.go:373-384` (`fallbackCatalogColIndex`) + +**问题**：列位置解析有两步 fallback，失败时静默退化为按位置猜测： + +``` +① view.Headers 中按字符串匹配列名（如 "relname", "attnum"） + ↓ 失败 +② inferCatalogLayout 判断 layout → 按 layout 固定顺序定位 + ↓ 仍找不到 +③ 返回 -1 → 上层以 0 作为默认值，拿到错误的列 +``` + +**触发条件**： +- `catalog` 包改了列名（如 `rel_createsql` → `rel_create_sql`）→ 步骤① 失败 +- 新版 checkpoint 的 header 变成了 `col_N` 泛型名（无字符串可匹配）→ 直接走步骤② +- 步骤② 的 layout 检测也失败（见第 2 节）→ 走步骤③ + +**后果**：不是报错而是**静默输出错误的 CSV**。用户在不知情的情况下拿到列值错位的导出结果。 + +--- + +## 6. `knownCatalogLayouts` 只覆盖两个版本 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:150-152` + +```go +func knownCatalogLayouts() []catalogLayout { + return []catalogLayout{currentCatalogLayout, legacy3CatalogLayout} +} +``` + +**问题**：只认识 "current" 和 "3.0-dev" 两种 layout。如果 MO 4.0 又改了一次 catalog schema，就需要手动加第三种。否则 `inferCatalogLayout` fallback 到 `currentCatalogLayout`，列位置全错。 + +--- + +## 7. Catalog Layout 假设差异列只在末尾 + +**位置**：`pkg/tools/checkpointtool/table_dump.go:53-57` + +```go +legacy3CatalogLayout = catalogLayout{ + moTablesSchema: catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1], // 切掉末尾 1 列 + moColumnsSchema: catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2], // 切掉末尾 2 列 +} +``` + +**前提假设**：新版本只在 `MoColumnsSchema` / `MoTablesSchema` **末尾**追加列。如果未来在**中间**插入列或调整列顺序，用"切末尾"方式构造的旧 layout schema 就会产生错位（前半段对齐、后半段全错）。 + +--- + +## 8. `ObjectEntryInfo` 的可见性判断依赖 `objectio.ObjectEntry.Visible(ts)` + +**位置**：`pkg/tools/checkpointtool/logical_table.go:218` + +```go +if obj.Visible(snapshotTS) { + visible = append(visible, entry) +} +``` + +**问题**：`objectio.ObjectEntry.Visible()` 的语义（基于 CreateTime / DeleteTime）是 MO 内部约定。如果可见性判断逻辑被调整（例如引入新的时间维度），工具的 tombstone 过滤和逻辑表输出就会不一致。 + +--- + +## 影响矩阵 + +| 上游变更 | 影响范围 | 是否静默 | +|---|---|---| +| `catalog` 包增删列 | schema 解析 → CSV 列全错 | ✅ 静默（编译通过） | +| 新增 catalog layout | 列位置偏移 | ✅ 静默（fallback 到 current） | +| `logical_table.go` 改 header | 所有数据列偏移 | ✅ 静默（`logicalViewMetaCols` 仍是 3） | +| `checkpoint` 包 meta 格式 | 无法加载 entry | ❌ 报错 | +| `logtail.NewCKPReader*` 改名 | 编译失败 | ❌ 编译错误 | +| `ioutil.GetCheckpointDir()` 改路径 | 找不到 checkpoint | ❌ 报错 | +| `ckputil.TableObjectsAttrs` 变动 | `ReadRangeData` 列名错误 | ✅ 静默 | +| `objectioutil` tombstone API 变更 | tombstone 过滤失效 | ❌ 可能编译或运行时错误 | +| `catalog` 列名修改 | 字符串匹配失败 → fallback 到手写位置 | ✅ 静默 | +| MO 列顺序调整（中间插入） | 旧 layout schema 错位 | ✅ 静默 | + +--- + +## 缓解建议 + +1. **消除 schema 硬编码**：`builtinColumnsForLayout` 应改为从 `catalog.MoTablesSchema` / `catalog.MoColumnsSchema` 动态生成，而非手写一份副本。 + +2. **在 checkpoint 数据中嵌入 layout 版本号**：让 `mo_tables` 数据中携带 catalog schema 版本标记，替代列数推断。 + +3. **将 `logicalViewMetaCols` 改为由 `LogicalTableView` 自身提供**：`LogicalTableView.MetaWidth()` 方法，而非在 `table_dump.go` 中硬编码常量 `3`。 + +4. **对所有内核 API 调用加一层抽象接口**：定义 `CheckpointSource` / `ObjectReader` 等接口，将 `checkpoint.ReadEntriesFromMeta`、`logtail.NewCKPReader*` 等调用隔离在 adapter 层内。 + +5. **在 `inferCatalogLayout` 无法匹配时发出 warning**：而非静默 fallback 到 `currentCatalogLayout`。 + +6. **对列名匹配失败增加告警**：`fallbackCatalogColIndex` 返回 `-1` 时记录日志，便于排查。 diff --git a/etc/launch-minio-local/log.toml b/etc/launch-minio-local/log.toml index 6753ed7d58772..0ff8d14184b26 100644 --- a/etc/launch-minio-local/log.toml +++ b/etc/launch-minio-local/log.toml @@ -1,4 +1,3 @@ -# service node type, [DN|CN|LOG] service-type = "LOG" data-dir = "./etc/launch-minio-local/mo-data" diff --git a/etc/launch-minio-local/tn.toml b/etc/launch-minio-local/tn.toml index 65fdbff27e59c..b455547cc6971 100644 --- a/etc/launch-minio-local/tn.toml +++ b/etc/launch-minio-local/tn.toml @@ -55,11 +55,11 @@ fileservice = "SHARED" log-backend = "logservice" [tn.Ckp] -flush-interval = "60s" -min-count = 100 -scan-interval = "5s" -incremental-interval = "180s" -global-min-count = 60 +flush-interval = "10s" +min-count = 5 +scan-interval = "2s" +incremental-interval = "30s" +global-min-count = 3 [tn.LogtailServer] rpc-max-message-size = "16KiB" diff --git a/pkg/common/moerr/error.go b/pkg/common/moerr/error.go index cebd286a6bcde..ddc748f1031e5 100644 --- a/pkg/common/moerr/error.go +++ b/pkg/common/moerr/error.go @@ -993,6 +993,10 @@ func NewFileNotFound(ctx context.Context, f string) *Error { return newError(ctx, ErrFileNotFound, f) } +func NewFileNotFoundErrorf(ctx context.Context, format string, args ...any) *Error { + return newError(ctx, ErrFileNotFound, fmt.Sprintf(format, args...)) +} + func NewResultFileNotFound(ctx context.Context, f string) *Error { return newError(ctx, ErrResultFileNotFound, f) } diff --git a/pkg/fileservice/aws_sdk_v2.go b/pkg/fileservice/aws_sdk_v2.go index 37ab5df3753b0..6217a6a281844 100644 --- a/pkg/fileservice/aws_sdk_v2.go +++ b/pkg/fileservice/aws_sdk_v2.go @@ -662,14 +662,9 @@ func (a *AwsSDKv2) WriteMultipartParallel( } } - if !sendJob(firstBufPtr, firstBuf, firstN) { - close(jobCh) - wg.Wait() - if firstErr != nil { - return firstErr - } - return ctx.Err() - } + pendingBufPtr := firstBufPtr + pendingBuf := firstBuf + pendingN := firstN for { nextBufPtr, nextBuf, nextN, readErr := readChunk() @@ -689,12 +684,57 @@ func (a *AwsSDKv2) WriteMultipartParallel( } break } - if !sendJob(nextBufPtr, nextBuf, nextN) { + if readErr != nil && errors.Is(readErr, io.EOF) { + if int64(pendingN)+int64(nextN) <= maxMultipartPartSize { + merged := make([]byte, pendingN+nextN) + copy(merged, pendingBuf[:pendingN]) + copy(merged[pendingN:], nextBuf[:nextN]) + bufPool.Put(pendingBufPtr) + bufPool.Put(nextBufPtr) + if !sendJob(nil, merged, len(merged)) { + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + break + } + } else { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + break + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + if !sendJob(nextBufPtr, nextBuf, nextN) { + break + } + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } - if readErr != nil && errors.Is(readErr, io.EOF) { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } + pendingBufPtr = nextBufPtr + pendingBuf = nextBuf + pendingN = nextN + } + + if pendingN > 0 { + _ = sendJob(pendingBufPtr, pendingBuf, pendingN) } close(jobCh) diff --git a/pkg/fileservice/error.go b/pkg/fileservice/error.go index f76ed61b2264e..afd7a2e79e71a 100644 --- a/pkg/fileservice/error.go +++ b/pkg/fileservice/error.go @@ -74,6 +74,8 @@ func IsRetryableError(err error) bool { strings.Contains(str, "connection reset by peer") || strings.Contains(str, "connection timed out") || strings.Contains(str, "dial tcp: lookup") || + strings.Contains(str, "server closed idle connection") || + strings.Contains(str, "server closed connection") || strings.Contains(str, "i/o timeout") || strings.Contains(str, "write: broken pipe") || strings.Contains(str, "TLS handshake timeout") || diff --git a/pkg/fileservice/parallel_sdk_test.go b/pkg/fileservice/parallel_sdk_test.go index 8ab11323d350f..ad1917fee4096 100644 --- a/pkg/fileservice/parallel_sdk_test.go +++ b/pkg/fileservice/parallel_sdk_test.go @@ -59,6 +59,28 @@ func (r *failAfterBytesReader) Read(p []byte) (int, error) { return n, err } +type waitAfterBytesReader struct { + r io.Reader + readSoFar int64 + waitAfter int64 + waitCh <-chan struct{} + timeout time.Duration +} + +func (r *waitAfterBytesReader) Read(p []byte) (int, error) { + if r.readSoFar >= r.waitAfter && r.waitCh != nil { + select { + case <-r.waitCh: + r.waitCh = nil + case <-time.After(r.timeout): + return 0, fmt.Errorf("timed out waiting after %d bytes", r.waitAfter) + } + } + n, err := r.r.Read(p) + r.readSoFar += int64(n) + return n, err +} + func newMockAWSServer(t *testing.T, failPart int32) (*httptest.Server, *awsServerState) { t.Helper() state := &awsServerState{ @@ -87,6 +109,9 @@ func newMockAWSServer(t *testing.T, failPart int32) (*httptest.Server, *awsServe pn, _ := strconv.Atoi(partStr) body, _ := io.ReadAll(r.Body) if state.failPart > 0 && int32(pn) == state.failPart { + if state.failPartSeen != nil { + state.failPartOnce.Do(func() { close(state.failPartSeen) }) + } w.WriteHeader(http.StatusInternalServerError) return } @@ -144,6 +169,8 @@ type awsServerState struct { parts map[int32][]byte completeBody []byte failPart int32 + failPartSeen chan struct{} + failPartOnce sync.Once failComplete bool failCreate bool uploadID string @@ -207,8 +234,11 @@ func TestAwsParallelMultipartSuccess(t *testing.T) { if err != nil { t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("complete body not recorded") @@ -304,6 +334,36 @@ func TestAwsMultipartUploadPartError(t *testing.T) { } } +func TestAwsMultipartSendJobFailureAfterPendingChunk(t *testing.T) { + server, state := newMockAWSServer(t, 1) + defer server.Close() + state.uploadID = "uid-pending-fail" + state.failPartSeen = make(chan struct{}) + + sdk := newTestAWSClient(t, server) + data := bytes.Repeat([]byte("q"), int(minMultipartPartSize*3)) + reader := &waitAfterBytesReader{ + r: bytes.NewReader(data), + waitAfter: minMultipartPartSize * 2, + waitCh: state.failPartSeen, + timeout: 3 * time.Second, + } + size := int64(len(data)) + err := sdk.WriteMultipartParallel(context.Background(), "object", reader, &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }) + if err == nil { + t.Fatalf("expected upload part error") + } + if !state.aborted.Load() { + t.Fatalf("expected abort request") + } + if len(state.completeBody) > 0 { + t.Fatalf("expected no complete body") + } +} + func TestAwsParallelMultipartUnknownSize(t *testing.T) { server, state := newMockAWSServer(t, 0) defer server.Close() @@ -317,8 +377,11 @@ func TestAwsParallelMultipartUnknownSize(t *testing.T) { }); err != nil { t.Fatalf("write failed: %v", err) } - if len(state.parts) != 2 { - t.Fatalf("expected multipart upload with unknown size") + if len(state.parts) != 1 { + t.Fatalf("expected multipart upload with merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } } @@ -376,8 +439,11 @@ func TestAwsWriteLargeNonSeekableFallsBackToMultipart(t *testing.T) { if state.putCount != 0 { t.Fatalf("expected multipart fallback instead of raw put, got %d put requests", state.putCount) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 multipart parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged multipart part, got %d", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("expected multipart complete request") @@ -523,6 +589,9 @@ func newMockCOSServer(t *testing.T, failPart int) (*httptest.Server, *cosServerS pn, _ := strconv.Atoi(partStr) body, _ := io.ReadAll(r.Body) if state.failPart > 0 && pn == state.failPart { + if state.failPartSeen != nil { + state.failPartOnce.Do(func() { close(state.failPartSeen) }) + } w.WriteHeader(state.failPartStatus) return } @@ -537,6 +606,15 @@ func newMockCOSServer(t *testing.T, failPart int) (*httptest.Server, *cosServerS } state.mu.Lock() state.completeBody = append([]byte{}, body...) + for partNum := 1; partNum < len(state.parts); partNum++ { + if len(state.parts[partNum]) < int(minMultipartPartSize) { + state.mu.Unlock() + w.WriteHeader(http.StatusBadRequest) + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(`EntityTooSmallYour proposed upload is smaller than the minimum allowed object size.`)) + return + } + } state.mu.Unlock() w.Header().Set("Content-Type", "application/xml") state.respBody = `locbucketobjectetag` @@ -557,6 +635,8 @@ type cosServerState struct { parts map[int][]byte completeBody []byte failPart int + failPartSeen chan struct{} + failPartOnce sync.Once failPartStatus int failComplete bool failCompleteStatus int @@ -604,8 +684,11 @@ func TestCOSParallelMultipartSuccess(t *testing.T) { if err != nil { t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) } - if len(state.parts) != 2 { - t.Fatalf("expected 2 parts, got %d", len(state.parts)) + if len(state.parts) != 1 { + t.Fatalf("expected merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) } if len(state.completeBody) == 0 { t.Fatalf("complete body not recorded") @@ -699,6 +782,97 @@ func TestCOSMultipartUploadPartError(t *testing.T) { } } +func TestCOSMultipartSendJobFailureAfterPendingChunk(t *testing.T) { + server, state := newMockCOSServer(t, 1) + defer server.Close() + state.uploadID = "cos-uid-pending-fail" + state.failPartSeen = make(chan struct{}) + + sdk := newTestCOSClient(t, server) + data := bytes.Repeat([]byte("s"), int(minMultipartPartSize*3)) + reader := &waitAfterBytesReader{ + r: bytes.NewReader(data), + waitAfter: minMultipartPartSize * 2, + waitCh: state.failPartSeen, + timeout: 3 * time.Second, + } + size := int64(len(data)) + err := sdk.WriteMultipartParallel(context.Background(), "object", reader, &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }) + if err == nil { + t.Fatalf("expected upload part error") + } + if !state.aborted.Load() { + t.Fatalf("expected abort request") + } + if state.completed.Load() || len(state.completeBody) > 0 { + t.Fatalf("expected no complete multipart upload") + } +} + +func TestCOSMultipartUploadPartRetriesServerClosedIdleConnection(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-uid-retry-idle-conn" + + baseClient := server.Client() + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + transport := &cosUploadPartIdleConnTransport{ + base: baseTransport, + part: "1", + } + baseClient.Transport = transport + + baseURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse url: %v", err) + } + client := costypes.NewClient( + &costypes.BaseURL{BucketURL: baseURL}, + baseClient, + ) + client.Conf.EnableCRC = false + sdk := &QCloudSDK{ + name: "cos-retry-idle-conn-test", + client: client, + } + + data := bytes.Repeat([]byte("r"), int(minMultipartPartSize+1)) + size := int64(len(data)) + if err := sdk.WriteMultipartParallel(context.Background(), "object", bytes.NewReader(data), &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 1, + }); err != nil { + t.Fatalf("write failed after transient upload-part error: %v", err) + } + if transport.uploadPartCalls.Load() < 2 { + t.Fatalf("expected upload part retry, got %d calls", transport.uploadPartCalls.Load()) + } + if !state.completed.Load() { + t.Fatalf("expected multipart upload complete") + } +} + +type cosUploadPartIdleConnTransport struct { + base http.RoundTripper + part string + uploadPartCalls atomic.Int32 +} + +func (t *cosUploadPartIdleConnTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPut && req.URL.Query().Get("partNumber") == t.part { + if t.uploadPartCalls.Add(1) == 1 { + return nil, fmt.Errorf("http: server closed idle connection") + } + } + return t.base.RoundTrip(req) +} + func TestCOSParallelMultipartUnknownSize(t *testing.T) { server, state := newMockCOSServer(t, 0) defer server.Close() @@ -712,8 +886,55 @@ func TestCOSParallelMultipartUnknownSize(t *testing.T) { }); err != nil { t.Fatalf("write failed: %v", err) } + if len(state.parts) != 1 { + t.Fatalf("expected multipart upload with merged final part, got %d parts", len(state.parts)) + } + if len(state.parts[1]) != len(data) { + t.Fatalf("expected final part size %d, got %d", len(data), len(state.parts[1])) + } +} + +func TestCOSParallelMultipartPipeSmallTail(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-pipe-small-tail" + + sdk := newTestCOSClient(t, server) + pr, pw := io.Pipe() + writeErrCh := make(chan error, 1) + go func() { + chunk := bytes.Repeat([]byte("x"), 1<<20) + for i := 0; i < 10; i++ { + if _, err := pw.Write(chunk); err != nil { + writeErrCh <- err + return + } + } + if _, err := pw.Write([]byte("tail")); err != nil { + writeErrCh <- err + return + } + writeErrCh <- pw.Close() + }() + + err := sdk.WriteMultipartParallel(context.Background(), "object", pr, nil, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 2, + }) + if writeErr := <-writeErrCh; writeErr != nil { + t.Fatalf("pipe writer failed: %v", writeErr) + } + if err != nil { + t.Fatalf("write failed: %v, parts=%d, complete=%s", err, len(state.parts), string(state.completeBody)) + } if len(state.parts) != 2 { - t.Fatalf("expected multipart upload with unknown size") + t.Fatalf("expected 2 parts, got %d", len(state.parts)) + } + if len(state.parts[1]) != int(minMultipartPartSize) { + t.Fatalf("expected first part size %d, got %d", minMultipartPartSize, len(state.parts[1])) + } + if len(state.parts[2]) != int(minMultipartPartSize)+len("tail") { + t.Fatalf("expected merged final part size %d, got %d", int(minMultipartPartSize)+len("tail"), len(state.parts[2])) } } @@ -755,6 +976,54 @@ func TestCOSMultipartCreateFail(t *testing.T) { } } +func TestCOSParallelMultipartDoesNotDeadlockOnTinyGlobalPool(t *testing.T) { + server, state := newMockCOSServer(t, 0) + defer server.Close() + state.uploadID = "cos-no-deadlock" + + sdk := newTestCOSClient(t, server) + data := bytes.Repeat([]byte("n"), int(minMultipartPartSize*2)) + size := int64(len(data)) + + oldPool := parallelUploadPool + tinyPool, err := ants.NewPool(1) + if err != nil { + t.Fatalf("create ants pool: %v", err) + } + parallelUploadPool = tinyPool + defer func() { + tinyPool.Release() + parallelUploadPool = oldPool + }() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { + done <- sdk.WriteMultipartParallel(ctx, "object", bytes.NewReader(data), &size, &ParallelMultipartOption{ + PartSize: minMultipartPartSize, + Concurrency: 2, + }) + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("write failed: %v", err) + } + case <-ctx.Done(): + t.Fatalf("multipart upload timed out, likely deadlocked") + } + + if len(state.parts) != 2 { + t.Fatalf("expected 2 parts, got %d", len(state.parts)) + } + if len(state.completeBody) == 0 { + t.Fatalf("expected multipart complete request") + } +} + // TestAwsParallelMultipartAbortOnReadError verifies that AbortMultipartUpload // is sent when a read error occurs after multipart upload has been initiated. // This covers the case where setErr cancels the derived context and the abort diff --git a/pkg/fileservice/parallel_upload_test.go b/pkg/fileservice/parallel_upload_test.go index e3e3fb435931b..bc45dc5ccff73 100644 --- a/pkg/fileservice/parallel_upload_test.go +++ b/pkg/fileservice/parallel_upload_test.go @@ -277,6 +277,72 @@ func TestS3FSWriteUnknownSizeDefaultNoParallel(t *testing.T) { } } +func TestS3FSWriteContextParallelModeOverridesFSMode(t *testing.T) { + storage := &mockParallelStorage{supports: true} + fs := &S3FS{ + name: "s3", + storage: storage, + ioMerger: NewIOMerger(), + asyncUpdate: true, + parallelMode: ParallelAuto, + } + + reader := strings.NewReader("streamed-data") + vector := IOVector{ + FilePath: "unknown-context-off", + Entries: []IOEntry{ + {Offset: 0, Size: -1, ReaderForWrite: reader}, + }, + } + + ctx := WithParallelMode(context.Background(), ParallelOff) + if err := fs.Write(ctx, vector); err != nil { + t.Fatalf("write failed: %v", err) + } + if storage.mpCalled != 0 { + t.Fatalf("parallel write should be disabled by context override") + } + if storage.writeCalled != 1 { + t.Fatalf("expected single write, got %d", storage.writeCalled) + } + if string(storage.lastData) != "streamed-data" { + t.Fatalf("unexpected data %q", string(storage.lastData)) + } +} + +func TestS3FSWriteContextParallelAutoEnablesUnknownSize(t *testing.T) { + storage := &mockParallelStorage{supports: true} + fs := &S3FS{ + name: "s3", + storage: storage, + ioMerger: NewIOMerger(), + asyncUpdate: true, + parallelMode: ParallelOff, + } + + reader := strings.NewReader("streamed-data") + vector := IOVector{ + FilePath: "unknown-context-auto", + Entries: []IOEntry{ + {Offset: 0, Size: -1, ReaderForWrite: reader}, + }, + } + + ctx := WithParallelMode(context.Background(), ParallelAuto) + if err := fs.Write(ctx, vector); err != nil { + t.Fatalf("write failed: %v", err) + } + if storage.mpCalled != 1 { + t.Fatalf("parallel write should be enabled by context auto override") + } + if storage.writeCalled != 0 { + t.Fatalf("unexpected single write") + } + if storage.lastSize != nil { + t.Fatalf("size hint should be nil for unknown-size stream") + } +} + func TestS3FSSmallSizeSkipsParallel(t *testing.T) { storage := &mockParallelStorage{supports: true} fs := &S3FS{ diff --git a/pkg/fileservice/qcloud_sdk.go b/pkg/fileservice/qcloud_sdk.go index 687788eb98f4d..9104d3e57d9b1 100644 --- a/pkg/fileservice/qcloud_sdk.go +++ b/pkg/fileservice/qcloud_sdk.go @@ -441,9 +441,11 @@ func (a *QCloudSDK) WriteMultipartParallel( jobCh := make(chan partJob, options.Concurrency*2) - startWorker := func() error { + startWorker := func() { wg.Add(1) - return getParallelUploadPool().Submit(func() { + // Use per-upload goroutines so concurrent multipart uploads do not starve + // each other on the small global parallelUploadPool. + go func() { defer wg.Done() for job := range jobCh { if ctx.Err() != nil { @@ -479,14 +481,11 @@ func (a *QCloudSDK) WriteMultipartParallel( }) partsLock.Unlock() } - }) + }() } for i := 0; i < options.Concurrency; i++ { - if submitErr := startWorker(); submitErr != nil { - setErr(submitErr) - break - } + startWorker() } sendJob := func(bufPtr *[]byte, buf []byte, n int) bool { @@ -516,14 +515,9 @@ func (a *QCloudSDK) WriteMultipartParallel( } } - if !sendJob(firstBufPtr, firstBuf, firstN) { - close(jobCh) - wg.Wait() - if firstErr != nil { - return firstErr - } - return ctx.Err() - } + pendingBufPtr := firstBufPtr + pendingBuf := firstBuf + pendingN := firstN for { nextBufPtr, nextBuf, nextN, readErr := readChunk() @@ -543,12 +537,57 @@ func (a *QCloudSDK) WriteMultipartParallel( } break } - if !sendJob(nextBufPtr, nextBuf, nextN) { + if readErr != nil && errors.Is(readErr, io.EOF) { + if int64(pendingN)+int64(nextN) <= maxMultipartPartSize { + merged := make([]byte, pendingN+nextN) + copy(merged, pendingBuf[:pendingN]) + copy(merged[pendingN:], nextBuf[:nextN]) + bufPool.Put(pendingBufPtr) + bufPool.Put(nextBufPtr) + if !sendJob(nil, merged, len(merged)) { + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + break + } + } else { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + break + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 + if !sendJob(nextBufPtr, nextBuf, nextN) { + break + } + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } - if readErr != nil && errors.Is(readErr, io.EOF) { + if !sendJob(pendingBufPtr, pendingBuf, pendingN) { + if nextBufPtr != nil { + bufPool.Put(nextBufPtr) + } + pendingBufPtr = nil + pendingBuf = nil + pendingN = 0 break } + pendingBufPtr = nextBufPtr + pendingBuf = nextBuf + pendingN = nextN + } + + if pendingN > 0 { + _ = sendJob(pendingBufPtr, pendingBuf, pendingN) } close(jobCh) diff --git a/pkg/fileservice/qcloud_sdk_test.go b/pkg/fileservice/qcloud_sdk_test.go index e7ab010ea80b7..62c3a5e70aa5d 100644 --- a/pkg/fileservice/qcloud_sdk_test.go +++ b/pkg/fileservice/qcloud_sdk_test.go @@ -183,6 +183,13 @@ func TestWrappedNetTimeoutRetryable(t *testing.T) { } } +func TestServerClosedIdleConnectionRetryable(t *testing.T) { + err := fmt.Errorf(`Put "https://bucket.cos.ap-guangzhou.myqcloud.com/object?partNumber=716&uploadId=id": http: server closed idle connection`) + if !IsRetryableError(err) { + t.Fatalf("expected server closed idle connection to be retryable") + } +} + type timeoutError struct{} func (timeoutError) Error() string { diff --git a/pkg/fileservice/s3_fs.go b/pkg/fileservice/s3_fs.go index 1a28ba7d52580..f47434c811622 100644 --- a/pkg/fileservice/s3_fs.go +++ b/pkg/fileservice/s3_fs.go @@ -462,7 +462,11 @@ func (s *S3FS) write(ctx context.Context, vector IOVector) (bytesWritten int, er } key := s.pathToKey(path.File) enableParallel := false - switch s.parallelMode { + parallelMode := s.parallelMode + if mode, ok := parallelModeFromContext(ctx); ok { + parallelMode = mode + } + switch parallelMode { case ParallelForce: enableParallel = true case ParallelAuto: @@ -470,6 +474,13 @@ func (s *S3FS) write(ctx context.Context, vector IOVector) (bytesWritten int, er enableParallel = true } } + if size == nil { + logutil.Info("s3 write unknown-size stream", + zap.String("key", key), + zap.Uint8("parallel-mode", uint8(parallelMode)), + zap.Bool("parallel-enabled", enableParallel), + ) + } if pmw, ok := s.storage.(ParallelMultipartWriter); ok && pmw.SupportsParallelMultipart() && enableParallel { diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index c87df71be0b00..6d5f566c0cecf 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -1442,7 +1442,7 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext return err } default: - return moerr.NewInternalErrorf(param.Ctx, "the value type %d is not support now", param.Cols[rowIdx].Typ.Id) + return moerr.NewInternalErrorf(param.Ctx, "the value type %d is not support now", param.Cols[colIdx].Typ.Id) } return nil } diff --git a/pkg/sql/colexec/external/external_test.go b/pkg/sql/colexec/external/external_test.go index 64802e08d7c5a..1c6c56680ef15 100644 --- a/pkg/sql/colexec/external/external_test.go +++ b/pkg/sql/colexec/external/external_test.go @@ -1261,6 +1261,71 @@ func Test_getMOCSVReader(t *testing.T) { require.Equal(t, nil, err) } +func TestGetColDataYear(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + + bat := batch.NewWithSize(1) + bat.Vecs[0] = vector.NewVec(types.T_year.ToType()) + defer bat.Clean(proc.Mp()) + + param := &ExternalParam{ + ExParamConst: ExParamConst{ + Cols: []*plan.ColDef{{ + Name: "c_year", + Typ: plan.Type{Id: int32(types.T_year)}, + }}, + Ctx: context.Background(), + Extern: &tree.ExternParam{}, + }, + } + + err := getColData( + bat, + []csvparser.Field{{Val: "2024"}}, + 0, + param, + proc.Mp(), + plan.ExternAttr{ColName: "c_year", ColIndex: 0, ColFieldIndex: 0}, + proc, + ) + require.NoError(t, err) + require.Equal(t, []types.MoYear{2024}, vector.MustFixedColWithTypeCheck[types.MoYear](bat.Vecs[0])) +} + +func TestGetColDataUnsupportedTypeReportsColumnIndex(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + + bat := batch.NewWithSize(2) + bat.Vecs[0] = vector.NewVec(types.T_int32.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_TS.ToType()) + defer bat.Clean(proc.Mp()) + + param := &ExternalParam{ + ExParamConst: ExParamConst{ + Cols: []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int32)}}, + {Name: "unsupported", Typ: plan.Type{Id: int32(types.T_TS)}}, + }, + Ctx: context.Background(), + Extern: &tree.ExternParam{}, + }, + } + + err := getColData( + bat, + []csvparser.Field{{Val: "1"}}, + 0, + param, + proc.Mp(), + plan.ExternAttr{ColName: "unsupported", ColIndex: 1, ColFieldIndex: 0}, + proc, + ) + require.Error(t, err) + require.Contains(t, err.Error(), "the value type 100 is not support now") +} + // Test_getColData_VecDimensionCheck tests that getColData correctly validates // vector dimension against the column definition vecf32(N)/vecf64(N) during LOAD DATA. // This is the regression test for the external.go part of https://github.com/matrixorigin/matrixone/issues/23872 diff --git a/pkg/tools/checkpointtool/checkpoint_reader.go b/pkg/tools/checkpointtool/checkpoint_reader.go index 628933f9b3e4a..47bf1c00001c8 100644 --- a/pkg/tools/checkpointtool/checkpoint_reader.go +++ b/pkg/tools/checkpointtool/checkpoint_reader.go @@ -17,6 +17,7 @@ package checkpointtool import ( "context" "fmt" + "os" "sort" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -25,9 +26,12 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logtail" ) // CheckpointReader reads checkpoint data from a directory @@ -37,6 +41,10 @@ type CheckpointReader struct { dir string entries []*checkpoint.CheckpointEntry mp *mpool.MPool + closeFS bool + + getTablesForTest func(*CheckpointReader, *checkpoint.CheckpointEntry) ([]*TableInfo, error) + getObjectEntriesForTest func(*CheckpointReader, *checkpoint.CheckpointEntry, uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) } // Option configures CheckpointReader @@ -49,6 +57,13 @@ func WithMPool(mp *mpool.MPool) Option { } } +// WithCloseFS closes the file service when the reader is closed. +func WithCloseFS() Option { + return func(r *CheckpointReader) { + r.closeFS = true + } +} + // Open opens checkpoint data from a directory func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, error) { fs, err := fileservice.NewLocalFS(ctx, "local", dir, fileservice.DisabledCacheConfig, nil) @@ -56,6 +71,11 @@ func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, e return nil, moerr.NewInternalErrorf(ctx, "create file service: %v", err) } + return OpenWithFS(ctx, fs, dir, append(opts, WithCloseFS())...) +} + +// OpenWithFS opens checkpoint data from an existing file service. +func OpenWithFS(ctx context.Context, fs fileservice.FileService, dir string, opts ...Option) (*CheckpointReader, error) { r := &CheckpointReader{ ctx: ctx, fs: fs, @@ -74,26 +94,87 @@ func Open(ctx context.Context, dir string, opts ...Option) (*CheckpointReader, e return r, nil } +// FS returns the file service used by this reader. +func (r *CheckpointReader) FS() fileservice.FileService { + return r.fs +} + +// Fork returns a lightweight reader that shares the file service and already +// loaded checkpoint entries, but uses an independent memory pool and does not +// own the file service. +func (r *CheckpointReader) Fork(ctx context.Context) *CheckpointReader { + if ctx == nil { + ctx = r.ctx + } + return &CheckpointReader{ + ctx: ctx, + fs: r.fs, + dir: r.dir, + entries: r.entries, + mp: mpool.MustNewZero(), + getTablesForTest: r.getTablesForTest, + getObjectEntriesForTest: r.getObjectEntriesForTest, + } +} + func (r *CheckpointReader) loadEntries() error { + ckpDir := ioutil.GetCheckpointDir() + logutil.Infof("[loadEntries] reading checkpoint dir: %s (display dir: %s)", ckpDir, r.dir) + + // Diagnostic: list all raw files in the checkpoint directory + rawEntries, listErr := fileservice.SortedList(r.fs.List(r.ctx, ckpDir)) + if listErr != nil { + logutil.Infof("[loadEntries] failed to list dir %s: %v", ckpDir, listErr) + } else { + logutil.Infof("[loadEntries] raw listing: total=%d dirs=%d files=%d", + len(rawEntries), countDirs(rawEntries), countFiles(rawEntries)) + for _, e := range rawEntries { + if !e.IsDir { + valid := "" + if f := ioutil.DecodeTSRangeFile(e.Name); f.IsValid() { + valid = " [valid tsrange]" + if f.IsMetadataFile() { + valid += " [meta]" + } else { + valid += " [non-meta ext=" + f.GetExt() + "]" + } + } + logutil.Infof("[loadEntries] file: %s%s", e.Name, valid) + } + } + } + names, err := ckputil.ListCKPMetaNames(r.ctx, r.fs) if err != nil { return err } if len(names) == 0 { + logutil.Infof("[loadEntries] no checkpoint meta files found in dir=%s", r.dir) return nil } + logutil.Infof("[loadEntries] found %d meta file(s) to read", len(names)) + totalRead := 0 for _, name := range names { entries, err := checkpoint.ReadEntriesFromMeta( - r.ctx, "", ioutil.GetCheckpointDir(), name, 0, nil, r.mp, r.fs, + r.ctx, "", ckpDir, name, 0, nil, r.mp, r.fs, ) if err != nil { + logutil.Infof("[loadEntries] failed to read meta file %s: %v", name, err) return err } + logutil.Infof("[loadEntries] meta file=%s entries=%d", name, len(entries)) + for _, e := range entries { + logutil.Infof("[loadEntries] entry: start=%s end=%s type=%s state=%d", + e.GetStart().ToString(), e.GetEnd().ToString(), e.GetType().String(), e.GetState()) + } r.entries = append(r.entries, entries...) + totalRead += len(entries) } + logutil.Infof("[loadEntries] total raw entries read: %d", totalRead) // Deduplicate by (start, end, type, location) + removed := 0 seen := make(map[string]bool) unique := make([]*checkpoint.CheckpointEntry, 0, len(r.entries)) for _, e := range r.entries { @@ -101,18 +182,42 @@ func (r *CheckpointReader) loadEntries() error { if !seen[key] { seen[key] = true unique = append(unique, e) + } else { + removed++ } } r.entries = unique + logutil.Infof("[loadEntries] dedup: removed=%d remaining=%d", removed, len(unique)) // Sort by end timestamp (newest first) sort.Slice(r.entries, func(i, j int) bool { ei, ej := r.entries[i].GetEnd(), r.entries[j].GetEnd() return ei.GT(&ej) }) + + // Log final usable entries + for _, e := range r.entries { + logutil.Infof("[loadEntries] usable ckp: start=%s end=%s type=%s", + e.GetStart().ToString(), e.GetEnd().ToString(), e.GetType().String()) + } + return nil } +func countDirs(entries []fileservice.DirEntry) int { + n := 0 + for _, e := range entries { + if e.IsDir { + n++ + } + } + return n +} + +func countFiles(entries []fileservice.DirEntry) int { + return len(entries) - countDirs(entries) +} + // Info returns checkpoint summary func (r *CheckpointReader) Info() *CheckpointInfo { info := &CheckpointInfo{Dir: r.dir, TotalEntries: len(r.entries)} @@ -127,11 +232,18 @@ func (r *CheckpointReader) Info() *CheckpointInfo { } start := e.GetStart() end := e.GetEnd() - if info.EarliestTS.IsEmpty() || start.LT(&info.EarliestTS) { - info.EarliestTS = start + if !start.IsEmpty() { + if info.EarliestTS.IsEmpty() || start.LT(&info.EarliestTS) { + info.EarliestTS = start + } } - if end.GT(&info.LatestTS) { - info.LatestTS = end + if !end.IsEmpty() { + if info.LatestTS.IsEmpty() || end.GT(&info.LatestTS) { + info.LatestTS = end + } + if info.EarliestTS.IsEmpty() || end.LT(&info.EarliestTS) { + info.EarliestTS = end + } } } return info @@ -185,32 +297,19 @@ func (r *CheckpointReader) GetTableRanges(entry *checkpoint.CheckpointEntry) ([] return nil, nil } - // Check if file exists before trying to read - fileName := loc.Name().String() - _, err := r.fs.StatFile(r.ctx, fileName) + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) if err != nil { - if moerr.IsMoErrCode(err, moerr.ErrFileNotFound) { - return nil, moerr.NewInternalErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", fileName) + if moerr.IsMoErrCode(err, moerr.ErrFileNotFound) || os.IsNotExist(err) { + return nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) } return nil, err } - reader, err := ioutil.NewFileReader(r.fs, fileName) - if err != nil { - return nil, err - } - - bats, release, err := reader.LoadAllColumns(r.ctx, nil, r.mp) - if err != nil { + reader := logtail.NewCKPReader(entry.GetVersion(), loc, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { return nil, err } - defer release() - - var ranges []ckputil.TableRange - for _, bat := range bats { - ranges = append(ranges, ckputil.ExportToTableRanges(bat)...) - } - return ranges, nil + return reader.GetTableRanges(r.ctx) } // GetAccounts extracts unique accounts from table ranges @@ -249,6 +348,9 @@ func (r *CheckpointReader) GetAccounts(entry *checkpoint.CheckpointEntry) ([]*Ac // GetTables extracts tables from an entry func (r *CheckpointReader) GetTables(entry *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + if r.getTablesForTest != nil { + return r.getTablesForTest(r, entry) + } ranges, err := r.GetTableRanges(entry) if err != nil { return nil, err @@ -323,28 +425,30 @@ func (r *CheckpointReader) ReadTableData(ctx context.Context, rng ckputil.TableR func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { view := &ComposedView{Timestamp: ts, Tables: make(map[uint64]*TableInfo)} - // Find latest GCKP <= ts + // Find latest GCKP <= ts that still has data on disk. + // GC may have cleaned up older GCKP data files; skip those and use the next one. var baseEntry *checkpoint.CheckpointEntry - for i := len(r.entries) - 1; i >= 0; i-- { - e := r.entries[i] + var baseEntryIdx int + for i, e := range r.entries { end := e.GetEnd() if e.IsGlobal() && end.LE(&ts) { + tables, err := r.GetTables(e) + if err != nil { + if isDataFileNotFound(err) { + continue // GC'd entry; try the next older GCKP + } + return nil, err + } baseEntry = e + baseEntryIdx = i + for _, t := range tables { + view.Tables[t.TableID] = t + } + view.BaseEntry = r.EntryInfo(baseEntryIdx, baseEntry) break } } - if baseEntry != nil { - view.BaseEntry = r.EntryInfo(0, baseEntry) - tables, err := r.GetTables(baseEntry) - if err != nil { - return nil, err - } - for _, t := range tables { - view.Tables[t.TableID] = t - } - } - // Add ICKPs after GCKP.end and <= ts baseEnd := types.TS{} if baseEntry != nil { @@ -353,12 +457,15 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { for i, e := range r.entries { start := e.GetStart() end := e.GetEnd() - if e.IsIncremental() && start.GT(&baseEnd) && end.LE(&ts) { - view.Incrementals = append(view.Incrementals, r.EntryInfo(i, e)) + if e.IsIncremental() && shouldIncludeIncrementalCheckpoint(start, end, baseEnd, ts, baseEntry != nil) { tables, err := r.GetTables(e) if err != nil { + if isDataFileNotFound(err) { + continue // GC'd entry; skip + } return nil, err } + view.Incrementals = append(view.Incrementals, r.EntryInfo(i, e)) for _, t := range tables { if existing, ok := view.Tables[t.TableID]; ok { existing.DataRanges = append(existing.DataRanges, t.DataRanges...) @@ -372,89 +479,153 @@ func (r *CheckpointReader) ComposeAt(ts types.TS) (*ComposedView, error) { return view, nil } +func shouldIncludeIncrementalCheckpoint(start, end, baseEnd, ts types.TS, hasBase bool) bool { + if !end.LE(&ts) { + return false + } + if hasBase { + return start.GT(&baseEnd) + } + return start.GE(&baseEnd) +} + +// ValidateSnapshot checks that ts can compose a catalog-backed checkpoint view. +func (r *CheckpointReader) ValidateSnapshot(ctx context.Context, ts types.TS) error { + view, err := r.ComposeAt(ts) + if err != nil { + return err + } + if len(view.Tables) == 0 { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: no tables are available", ts.ToString()) + } + if _, ok := view.Tables[moTablesID]; !ok { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: mo_tables catalog is not available; wait for a global checkpoint or choose a later timestamp", ts.ToString()) + } + if _, ok := view.Tables[moColumnsID]; !ok { + return moerr.NewInternalErrorf(ctx, "checkpoint snapshot %s is not usable: mo_columns catalog is not available; wait for a global checkpoint or choose a later timestamp", ts.ToString()) + } + return nil +} + +// isDataFileNotFound checks if err indicates a checkpoint data file was GC'd/missing. +func isDataFileNotFound(err error) bool { + if err == nil { + return false + } + return moerr.IsMoErrCode(err, moerr.ErrFileNotFound) || os.IsNotExist(err) +} + // Close releases resources func (r *CheckpointReader) Close() error { r.entries = nil + if r.closeFS && r.fs != nil { + r.fs.Close(r.ctx) + } return nil } // GetObjectEntries reads detailed object entries with timestamps for a table func (r *CheckpointReader) GetObjectEntries(entry *checkpoint.CheckpointEntry, tableID uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { - // First get all table ranges - allRanges, err := r.GetTableRanges(entry) - if err != nil { - return nil, nil, err + if r.getObjectEntriesForTest != nil { + return r.getObjectEntriesForTest(r, entry, tableID) } - - // Now read the checkpoint file again to get timestamps loc := entry.GetLocation() if loc.IsEmpty() { return nil, nil, nil } - fileName := loc.Name().String() - reader, err := ioutil.NewFileReader(r.fs, fileName) + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) if err != nil { + if isDataFileNotFound(err) { + return nil, nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) + } return nil, nil, err } - bats, release, err := reader.LoadAllColumns(r.ctx, nil, r.mp) + var dataEntries, tombEntries []*ObjectEntryInfo + + reader := logtail.NewCKPReaderWithTableID_V2(entry.GetVersion(), loc, tableID, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { + return nil, nil, err + } + + err = reader.ConsumeCheckpointWithTableID(r.ctx, func( + _ context.Context, + _ fileservice.FileService, + obj objectio.ObjectEntry, + isTombstone bool, + ) error { + info := &ObjectEntryInfo{ + ObjectStats: obj.ObjectStats, + CreateTime: obj.CreateTime, + DeleteTime: obj.DeleteTime, + } + if isTombstone { + tombEntries = append(tombEntries, info) + } else { + dataEntries = append(dataEntries, info) + } + return nil + }) if err != nil { return nil, nil, err } - defer release() + return dataEntries, tombEntries, nil +} - // Build a map of ranges with their timestamps - var dataEntries, tombEntries []*ObjectEntryInfo - rangeIdx := 0 +// GetObjectEntriesForTables reads detailed object entries for multiple tables +// from one checkpoint entry in a single pass. +func (r *CheckpointReader) GetObjectEntriesForTables( + entry *checkpoint.CheckpointEntry, + tableIDs map[uint64]struct{}, +) (map[uint64][]*ObjectEntryInfo, map[uint64][]*ObjectEntryInfo, error) { + loc := entry.GetLocation() + if loc.IsEmpty() { + return nil, nil, nil + } - for _, bat := range bats { - if bat.RowCount() == 0 { - continue + _, err := r.fs.StatFile(r.ctx, loc.Name().String()) + if err != nil { + if isDataFileNotFound(err) { + return nil, nil, moerr.NewFileNotFoundErrorf(r.ctx, "checkpoint data file not found (may have been GC'd): %s", loc.Name().String()) } + return nil, nil, err + } - // Check if timestamp columns exist - var createTSVec, deleteTSVec []types.TS - if len(bat.Vecs) > ckputil.TableObjectsAttr_CreateTS_Idx { - createTSVec = vector.MustFixedColWithTypeCheck[types.TS](bat.Vecs[ckputil.TableObjectsAttr_CreateTS_Idx]) + dataByTable := make(map[uint64][]*ObjectEntryInfo) + tombByTable := make(map[uint64][]*ObjectEntryInfo) + reader := logtail.NewCKPReader(entry.GetVersion(), loc, r.mp, r.fs) + if err := reader.ReadMeta(r.ctx); err != nil { + return nil, nil, err + } + err = reader.ForEachRow(r.ctx, func( + _ uint32, + _, tid uint64, + objectType int8, + objectStats objectio.ObjectStats, + create, delete types.TS, + _ types.Rowid, + ) error { + if _, ok := tableIDs[tid]; !ok { + return nil } - if len(bat.Vecs) > ckputil.TableObjectsAttr_DeleteTS_Idx { - deleteTSVec = vector.MustFixedColWithTypeCheck[types.TS](bat.Vecs[ckputil.TableObjectsAttr_DeleteTS_Idx]) + info := &ObjectEntryInfo{ + ObjectStats: objectStats, + CreateTime: create, + DeleteTime: delete, } - - for i := 0; i < bat.RowCount(); i++ { - if rangeIdx >= len(allRanges) { - break - } - - rng := allRanges[rangeIdx] - rangeIdx++ - - if rng.TableID != tableID { - continue - } - - entry := &ObjectEntryInfo{ - Range: rng, - } - - // Set timestamps if available - if createTSVec != nil && i < len(createTSVec) { - entry.CreateTime = createTSVec[i] - } - if deleteTSVec != nil && i < len(deleteTSVec) { - entry.DeleteTime = deleteTSVec[i] - } - - if rng.ObjectType == ckputil.ObjectType_Data { - dataEntries = append(dataEntries, entry) - } else if rng.ObjectType == ckputil.ObjectType_Tombstone { - tombEntries = append(tombEntries, entry) - } + switch objectType { + case ckputil.ObjectType_Data: + dataByTable[tid] = append(dataByTable[tid], info) + case ckputil.ObjectType_Tombstone: + tombByTable[tid] = append(tombByTable[tid], info) } + return nil + }) + if err != nil { + return nil, nil, err } - - return dataEntries, tombEntries, nil + return dataByTable, tombByTable, nil } // ReadRangeData reads actual data from a range and returns column names and row data as strings @@ -475,16 +646,7 @@ func (r *CheckpointReader) ReadRangeData(entry *checkpoint.CheckpointEntry, rng return nil, nil, nil } - // Use checkpoint schema for column names - columns := ckputil.TableObjectsAttrs - if len(columns) == 0 { - // Fallback: generate column names from vector count - bat := bats[0] - columns = make([]string, len(bat.Vecs)) - for i := range columns { - columns[i] = fmt.Sprintf("col_%d", i) - } - } + columns := checkpointRangeColumns(len(bats[0].Vecs)) // Extract rows within the range startRow := uint32(rng.Start.GetRowOffset()) @@ -503,77 +665,47 @@ func (r *CheckpointReader) ReadRangeData(entry *checkpoint.CheckpointEntry, rng } } - // Trim columns to match actual vector count - if len(bats) > 0 && len(columns) > len(bats[0].Vecs) { - columns = columns[:len(bats[0].Vecs)] - } - return columns, rows, nil } +func checkpointRangeColumns(width int) []string { + if width <= 0 { + return nil + } + columns := append([]string(nil), ckputil.TableObjectsAttrs...) + if len(columns) > width { + return columns[:width] + } + for len(columns) < width { + columns = append(columns, fmt.Sprintf("col_%d", len(columns))) + } + return columns +} + // vecValueToString converts a vector value at index to string func vecValueToString(vec *vector.Vector, idx int) string { if vec.IsNull(uint64(idx)) { return "NULL" } - switch vec.GetType().Oid { - case types.T_bool: - v := vector.MustFixedColWithTypeCheck[bool](vec) - return fmt.Sprintf("%v", v[idx]) - case types.T_int8: - v := vector.MustFixedColWithTypeCheck[int8](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int16: - v := vector.MustFixedColWithTypeCheck[int16](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int32: - v := vector.MustFixedColWithTypeCheck[int32](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_int64: - v := vector.MustFixedColWithTypeCheck[int64](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint8: - v := vector.MustFixedColWithTypeCheck[uint8](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint16: - v := vector.MustFixedColWithTypeCheck[uint16](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint32: - v := vector.MustFixedColWithTypeCheck[uint32](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_uint64: - v := vector.MustFixedColWithTypeCheck[uint64](vec) - return fmt.Sprintf("%d", v[idx]) - case types.T_float32: - v := vector.MustFixedColWithTypeCheck[float32](vec) - return fmt.Sprintf("%.4f", v[idx]) - case types.T_float64: - v := vector.MustFixedColWithTypeCheck[float64](vec) - return fmt.Sprintf("%.4f", v[idx]) - case types.T_char, types.T_varchar, types.T_text: - return vec.GetStringAt(idx) - case types.T_TS: - v := vector.MustFixedColWithTypeCheck[types.TS](vec) - return v[idx].ToString() - case types.T_Rowid: - v := vector.MustFixedColWithTypeCheck[types.Rowid](vec) - return fmt.Sprintf("%d-%d", v[idx].GetBlockOffset(), v[idx].GetRowOffset()) - case types.T_Blockid: - v := vector.MustFixedColWithTypeCheck[types.Blockid](vec) - return v[idx].String() - case types.T_Objectid: - v := vector.MustFixedColWithTypeCheck[types.Objectid](vec) - return v[idx].ShortStringEx() - default: - // For binary/blob types, show hex or truncated - if vec.GetType().IsVarlen() { - bs := vec.GetBytesAt(idx) - if len(bs) > 16 { - return fmt.Sprintf("%x...", bs[:16]) - } - return fmt.Sprintf("%x", bs) + if vec.GetType().Oid == types.T_time { + rowIdx := idx + if vec.IsConst() { + rowIdx = 0 + } + values := vector.MustFixedColWithTypeCheck[types.Time](vec) + if rowIdx >= 0 && rowIdx < len(values) { + return values[rowIdx].String2(vec.GetType().Scale) } - return fmt.Sprintf("<%s>", vec.GetType().String()) } + switch vec.GetType().Oid { + case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, + types.T_json, types.T_blob, types.T_text, types.T_datalink, types.T_geometry: + return string(vec.GetBytesAt(idx)) + } + value := vec.RowToString(idx) + if value == "null" { + return "NULL" + } + return value } diff --git a/pkg/tools/checkpointtool/checkpoint_reader_test.go b/pkg/tools/checkpointtool/checkpoint_reader_test.go new file mode 100644 index 0000000000000..8686f6f4a1c61 --- /dev/null +++ b/pkg/tools/checkpointtool/checkpoint_reader_test.go @@ -0,0 +1,90 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVecValueToString_SQLLikeFormatting(t *testing.T) { + mp := mpool.MustNewZero() + defer func() { + require.Equal(t, int64(0), mp.CurrNB()) + }() + + timeVec := vector.NewVec(types.New(types.T_time, 0, 6)) + defer timeVec.Free(mp) + timeValue, err := types.ParseTime("12:34:56.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(timeVec, timeValue, false, mp)) + require.Equal(t, "12:34:56.123456", vecValueToString(timeVec, 0)) + + datetimeVec := vector.NewVec(types.New(types.T_datetime, 0, 6)) + defer datetimeVec.Free(mp) + datetimeValue, err := types.ParseDatetime("2024-01-02 03:04:05.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(datetimeVec, datetimeValue, false, mp)) + require.Equal(t, "2024-01-02 03:04:05.123456", vecValueToString(datetimeVec, 0)) + + timestampVec := vector.NewVec(types.New(types.T_timestamp, 0, 6)) + defer timestampVec.Free(mp) + timestampValue, err := types.ParseTimestamp(time.Local, "2024-01-02 03:04:05.123456", 6) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(timestampVec, timestampValue, false, mp)) + require.Equal(t, timestampValue.String2(time.Local, 6), vecValueToString(timestampVec, 0)) + + decimalVec := vector.NewVec(types.New(types.T_decimal64, 18, 2)) + defer decimalVec.Free(mp) + decimalValue := types.Decimal64(12345) + require.NoError(t, vector.AppendFixed(decimalVec, decimalValue, false, mp)) + require.Equal(t, "123.45", vecValueToString(decimalVec, 0)) + + uuidVec := vector.NewVec(types.T_uuid.ToType()) + defer uuidVec.Free(mp) + uuidValue, err := types.ParseUuid("123e4567-e89b-12d3-a456-426614174000") + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(uuidVec, uuidValue, false, mp)) + require.Equal(t, "123e4567-e89b-12d3-a456-426614174000", vecValueToString(uuidVec, 0)) + + jsonVec := vector.NewVec(types.T_json.ToType()) + defer jsonVec.Free(mp) + require.NoError(t, vector.AppendBytes(jsonVec, []byte(`{"a":1}`), false, mp)) + require.Equal(t, `{"a":1}`, vecValueToString(jsonVec, 0)) + + nullVec := vector.NewVec(types.T_int64.ToType()) + defer nullVec.Free(mp) + require.NoError(t, vector.AppendFixed(nullVec, int64(0), true, mp)) + require.Equal(t, "NULL", vecValueToString(nullVec, 0)) +} + +func TestShouldIncludeIncrementalCheckpointWithoutBase(t *testing.T) { + zero := types.TS{} + ts1 := types.BuildTS(1, 0) + ts2 := types.BuildTS(2, 0) + + assert.True(t, shouldIncludeIncrementalCheckpoint(zero, ts1, zero, ts1, false)) + assert.True(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts2, false)) + assert.False(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts1, false)) + + assert.False(t, shouldIncludeIncrementalCheckpoint(zero, ts1, zero, ts1, true)) + assert.True(t, shouldIncludeIncrementalCheckpoint(ts1, ts2, zero, ts2, true)) +} diff --git a/pkg/tools/checkpointtool/interactive/bubbletea.go b/pkg/tools/checkpointtool/interactive/bubbletea.go index 2744b35abd2c8..cd11804d56f01 100644 --- a/pkg/tools/checkpointtool/interactive/bubbletea.go +++ b/pkg/tools/checkpointtool/interactive/bubbletea.go @@ -16,14 +16,10 @@ package interactive import ( "context" - "fmt" tea "github.com/charmbracelet/bubbletea" - "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" objectinteractive "github.com/matrixorigin/matrixone/pkg/tools/objecttool/interactive" - "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" ) // Run starts the interactive checkpoint viewer @@ -43,42 +39,10 @@ func Run(reader *checkpointtool.CheckpointReader) error { } // Check if we need to open an object file - if um.GetObjectToOpen() != "" && um.GetRangeToOpen() != nil { - rng := um.GetRangeToOpen() - opts := &objectinteractive.ViewOptions{ - StartRow: int64(rng.Start.GetRowOffset()), - EndRow: int64(rng.End.GetRowOffset()), - ColumnNames: map[uint16]string{ - 0: ckputil.TableObjectsAttr_Accout, - 1: ckputil.TableObjectsAttr_DB, - 2: ckputil.TableObjectsAttr_Table, - 3: ckputil.TableObjectsAttr_ObjectType, - 4: "object_name", - 5: "flags", - 6: "rows", - 7: "osize", - 8: "csize", - 9: ckputil.TableObjectsAttr_CreateTS, - 10: ckputil.TableObjectsAttr_DeleteTS, - 11: ckputil.TableObjectsAttr_Cluster, - }, - ColumnExpander: &objectinteractive.ColumnExpander{ - SourceCol: 4, // id column - NewCols: []string{"object_name", "flags", "rows", "osize", "csize"}, - NewTypes: []types.Type{ - types.T_varchar.ToType(), - types.T_varchar.ToType(), - types.T_uint32.ToType(), - types.T_varchar.ToType(), - types.T_varchar.ToType(), - }, - ExpandFunc: expandObjectStats, - }, - ObjectNameCol: 4, // object_name column after expansion - BaseDir: m.state.reader.Dir(), // Base directory for nested objects - CustomOverview: ckpDataOverview, - } - if err := objectinteractive.RunUnified(context.Background(), um.GetObjectToOpen(), opts); err != nil { + if um.GetObjectToOpen() != "" { + if err := objectinteractive.RunUnifiedWithFS( + context.Background(), m.state.reader.FS(), um.GetObjectToOpen(), nil, + ); err != nil { return err } // Clear the object to open flag and continue with current state @@ -89,116 +53,3 @@ func Run(reader *checkpointtool.CheckpointReader) error { return nil } } - -// ckpDataOverview creates custom overview for checkpoint data viewer -// Columns after expansion: account_id, db_id, table_id, object_type, object_name, flags, rows, osize, csize, create_ts, delete_ts, cluster -// Indices: 0 1 2 3 4 5 6 7 8 9 10 11 -func ckpDataOverview(rows [][]string) string { - if len(rows) == 0 { - return "No data" - } - - type objStats struct { - rows uint32 - osize uint32 - csize uint32 - } - dataObjs := make(map[string]*objStats) - tombObjs := make(map[string]*objStats) - - for _, row := range rows { - if len(row) < 9 { - continue - } - objType := row[3] // object_type: "1" = data, "2" = tombstone - objName := row[4] // object_name - rowsStr := row[6] // rows - osize := parseSize(row[7]) - csize := parseSize(row[8]) - - var rowCount uint32 - fmt.Sscanf(rowsStr, "%d", &rowCount) - - if objType == "1" { // Data - if _, exists := dataObjs[objName]; !exists { - dataObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} - } - } else if objType == "2" { // Tombstone - if _, exists := tombObjs[objName]; !exists { - tombObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} - } - } - } - - var totalRows, totalDeletes, totalOsize, totalCsize uint32 - for _, s := range dataObjs { - totalRows += s.rows - totalOsize += s.osize - totalCsize += s.csize - } - for _, s := range tombObjs { - totalDeletes += s.rows - } - - ratio := float64(0) - if totalOsize > 0 { - ratio = float64(totalCsize) / float64(totalOsize) * 100 - } - - return fmt.Sprintf("%d ranges │ %d data objs │ %d tomb objs │ %d rows │ %d deletes │ osize: %s │ csize: %s │ ratio: %.1f%%", - len(rows), len(dataObjs), len(tombObjs), totalRows, totalDeletes, - formatSize(totalOsize), formatSize(totalCsize), ratio) -} - -// parseSize parses size string like "279.6KB" back to bytes -func parseSize(s string) uint32 { - var val float64 - var unit string - fmt.Sscanf(s, "%f%s", &val, &unit) - switch unit { - case "KB": - return uint32(val * 1024) - case "MB": - return uint32(val * 1024 * 1024) - case "GB": - return uint32(val * 1024 * 1024 * 1024) - case "B": - return uint32(val) - default: - return uint32(val) - } -} - -// expandObjectStats expands ObjectStats bytes into multiple values -func expandObjectStats(value any) []any { - data, ok := value.([]byte) - if !ok || len(data) != objectio.ObjectStatsLen { - return []any{"", "", "", "", ""} - } - - var stats objectio.ObjectStats - stats.UnMarshal(data) - - // Format flags: A=Appendable, S=Sorted, C=CNCreated - flags := "" - if stats.GetAppendable() { - flags += "A" - } - if stats.GetSorted() { - flags += "S" - } - if stats.GetCNCreated() { - flags += "C" - } - if flags == "" { - flags = "-" - } - - return []any{ - stats.ObjectName().String(), - flags, - stats.Rows(), - formatSize(stats.OriginSize()), - formatSize(stats.Size()), - } -} diff --git a/pkg/tools/checkpointtool/interactive/helpers.go b/pkg/tools/checkpointtool/interactive/helpers.go index ee90f49b59d39..144594c9fa349 100644 --- a/pkg/tools/checkpointtool/interactive/helpers.go +++ b/pkg/tools/checkpointtool/interactive/helpers.go @@ -19,6 +19,7 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -66,3 +67,110 @@ func formatSize(bytes uint32) string { } return fmt.Sprintf("%.1f%cB", float64(bytes)/float64(div), "KMGTPE"[exp]) } + +func parseSize(s string) uint32 { + var val float64 + var unit string + fmt.Sscanf(s, "%f%s", &val, &unit) + switch unit { + case "KB": + return uint32(val * 1024) + case "MB": + return uint32(val * 1024 * 1024) + case "GB": + return uint32(val * 1024 * 1024 * 1024) + case "B": + return uint32(val) + default: + return uint32(val) + } +} + +func expandObjectStats(value any) []any { + data, ok := value.([]byte) + if !ok || len(data) != objectio.ObjectStatsLen { + return []any{"", "", "", "", ""} + } + + var stats objectio.ObjectStats + stats.UnMarshal(data) + + flags := "" + if stats.GetAppendable() { + flags += "A" + } + if stats.GetSorted() { + flags += "S" + } + if stats.GetCNCreated() { + flags += "C" + } + if flags == "" { + flags = "-" + } + + return []any{ + stats.ObjectName().String(), + flags, + stats.Rows(), + formatSize(stats.OriginSize()), + formatSize(stats.Size()), + } +} + +func ckpDataOverview(rows [][]string) string { + if len(rows) == 0 { + return "No data" + } + + type objStats struct { + rows uint32 + osize uint32 + csize uint32 + } + dataObjs := make(map[string]*objStats) + tombObjs := make(map[string]*objStats) + + for _, row := range rows { + if len(row) < 9 { + continue + } + objType := row[3] + objName := row[4] + rowsStr := row[6] + osize := parseSize(row[7]) + csize := parseSize(row[8]) + + var rowCount uint32 + fmt.Sscanf(rowsStr, "%d", &rowCount) + + if objType == "1" { + if _, exists := dataObjs[objName]; !exists { + dataObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} + } + } else if objType == "2" { + if _, exists := tombObjs[objName]; !exists { + tombObjs[objName] = &objStats{rows: rowCount, osize: osize, csize: csize} + } + } + } + + var totalRows, totalDeletes, totalOsize, totalCsize uint32 + for _, s := range dataObjs { + totalRows += s.rows + totalOsize += s.osize + totalCsize += s.csize + } + for _, s := range tombObjs { + totalDeletes += s.rows + } + + ratio := float64(0) + if totalOsize > 0 { + ratio = float64(totalCsize) / float64(totalOsize) * 100 + } + + return fmt.Sprintf("%d ranges │ %d data objs │ %d tomb objs │ %d rows │ %d deletes │ osize: %s │ csize: %s │ ratio: %.1f%%", + len(rows), len(dataObjs), len(tombObjs), totalRows, totalDeletes, + formatSize(totalOsize), formatSize(totalCsize), ratio) +} diff --git a/pkg/tools/checkpointtool/interactive/providers.go b/pkg/tools/checkpointtool/interactive/providers.go index 232a9476a23cb..b8af4dbc09d32 100644 --- a/pkg/tools/checkpointtool/interactive/providers.go +++ b/pkg/tools/checkpointtool/interactive/providers.go @@ -161,35 +161,37 @@ func (p *TableDetailProvider) GetRows() [][]string { rows := make([][]string, 0, len(dataEntries)+len(tombEntries)) for _, entry := range dataEntries { - objName := entry.Range.ObjectStats.ObjectName().String() - rangeStr := fmt.Sprintf("%d-%d~%d-%d", - entry.Range.Start.GetBlockOffset(), entry.Range.Start.GetRowOffset(), - entry.Range.End.GetBlockOffset(), entry.Range.End.GetRowOffset()) - rangeRows := entry.Range.End.GetRowOffset() - entry.Range.Start.GetRowOffset() + 1 + objName := entry.ObjectStats.ObjectName().String() + blockEnd := int(entry.ObjectStats.BlkCnt()) - 1 + if blockEnd < 0 { + blockEnd = 0 + } + blockSpan := fmt.Sprintf("0..%d", blockEnd) rows = append(rows, []string{ "Data", objName, - rangeStr, - fmt.Sprintf("%d", rangeRows), - formatSize(entry.Range.ObjectStats.Size()), + blockSpan, + fmt.Sprintf("%d", entry.ObjectStats.Rows()), + formatSize(entry.ObjectStats.Size()), formatTSShort(entry.CreateTime), }) } for _, entry := range tombEntries { - objName := entry.Range.ObjectStats.ObjectName().String() - rangeStr := fmt.Sprintf("%d-%d~%d-%d", - entry.Range.Start.GetBlockOffset(), entry.Range.Start.GetRowOffset(), - entry.Range.End.GetBlockOffset(), entry.Range.End.GetRowOffset()) - rangeRows := entry.Range.End.GetRowOffset() - entry.Range.Start.GetRowOffset() + 1 + objName := entry.ObjectStats.ObjectName().String() + blockEnd := int(entry.ObjectStats.BlkCnt()) - 1 + if blockEnd < 0 { + blockEnd = 0 + } + blockSpan := fmt.Sprintf("0..%d", blockEnd) rows = append(rows, []string{ "Tomb", objName, - rangeStr, - fmt.Sprintf("%d", rangeRows), - formatSize(entry.Range.ObjectStats.Size()), + blockSpan, + fmt.Sprintf("%d", entry.ObjectStats.Rows()), + formatSize(entry.ObjectStats.Size()), formatTSShort(entry.CreateTime), }) } @@ -217,16 +219,16 @@ func (p *TableDetailProvider) GetOverview() string { tombObjects := make(map[string]struct{}) for _, e := range dataEntries { - objName := e.Range.ObjectStats.ObjectName().String() + objName := e.ObjectStats.ObjectName().String() if _, exists := dataObjects[objName]; !exists { dataObjects[objName] = &objStats{ - originSize: e.Range.ObjectStats.OriginSize(), - compSize: e.Range.ObjectStats.Size(), + originSize: e.ObjectStats.OriginSize(), + compSize: e.ObjectStats.Size(), } } } for _, e := range tombEntries { - objName := e.Range.ObjectStats.ObjectName().String() + objName := e.ObjectStats.ObjectName().String() tombObjects[objName] = struct{}{} } @@ -242,6 +244,32 @@ func (p *TableDetailProvider) GetOverview() string { formatSize(totalOriginSize), formatSize(totalCompSize)) } +type LogicalTableProvider struct { + state *State +} + +func (p *LogicalTableProvider) GetRows() [][]string { + view := p.state.LogicalView() + if view == nil { + return nil + } + return view.Rows +} + +func (p *LogicalTableProvider) GetRowNums() []string { + return nil +} + +func (p *LogicalTableProvider) GetOverview() string { + view := p.state.LogicalView() + tbl := p.state.GetSelectedTable() + if view == nil || tbl == nil { + return "" + } + return fmt.Sprintf("Table %d logical view │ visible: %d │ deleted: %d │ physical: %d", + tbl.TableID, view.VisibleRows, view.DeletedRows, view.PhysicalRows) +} + // === Table Detail Handler === type tableDetailHandler struct { @@ -261,7 +289,7 @@ func (h *tableDetailHandler) OnSelect(rowIdx int) tea.Cmd { } if entry != nil { - return func() tea.Msg { return openObjectMsg{path: entry.Range.ObjectStats.ObjectName().String()} } + return func() tea.Msg { return openObjectMsg{path: entry.ObjectStats.ObjectName().String()} } } return nil } @@ -270,6 +298,13 @@ func (h *tableDetailHandler) OnBack() tea.Cmd { return func() tea.Msg { return goBackMsg{} } } +func (h *tableDetailHandler) OnCustomKey(key string) tea.Cmd { + if key == "L" { + return func() tea.Msg { return openLogicalTableMsg{} } + } + return nil +} + // FilterRow filters by type (Data/Tomb) func (h *tableDetailHandler) FilterRow(row []string, filter string) bool { if len(row) > 0 { @@ -278,6 +313,14 @@ func (h *tableDetailHandler) FilterRow(row []string, filter string) bool { return false } +type logicalTableHandler struct { + interactive.DefaultHandler +} + +func (h *logicalTableHandler) OnBack() tea.Cmd { + return func() tea.Msg { return goBackMsg{} } +} + // === Account List Provider === type AccountListProvider struct { diff --git a/pkg/tools/checkpointtool/interactive/state.go b/pkg/tools/checkpointtool/interactive/state.go index 3c7ac334cc7f8..fa60113cb54ee 100644 --- a/pkg/tools/checkpointtool/interactive/state.go +++ b/pkg/tools/checkpointtool/interactive/state.go @@ -15,6 +15,8 @@ package interactive import ( + "context" + "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -47,6 +49,7 @@ type State struct { info *checkpointtool.CheckpointInfo dataEntries []*checkpointtool.ObjectEntryInfo tombEntries []*checkpointtool.ObjectEntryInfo + logicalView *checkpointtool.LogicalTableView } // NewState creates a new state @@ -66,6 +69,7 @@ func (s *State) Mode() ViewMode { return s.mode func (s *State) Entries() []*checkpoint.CheckpointEntry { return s.entries } func (s *State) DataEntries() []*checkpointtool.ObjectEntryInfo { return s.dataEntries } func (s *State) TombEntries() []*checkpointtool.ObjectEntryInfo { return s.tombEntries } +func (s *State) LogicalView() *checkpointtool.LogicalTableView { return s.logicalView } func (s *State) HasAccountFilter() bool { return s.filterAccountID >= 0 } func (s *State) GetAccountFilter() int64 { return s.filterAccountID } @@ -129,6 +133,7 @@ func (s *State) SwitchToTables() error { return err } s.tables = tables + s.logicalView = nil s.mode = ViewModeTable return nil } @@ -142,6 +147,24 @@ func (s *State) SelectTable(tableID uint64) error { } s.dataEntries = dataEntries s.tombEntries = tombEntries + s.logicalView = nil + return nil +} + +// LoadLogicalView materializes the tombstone-applied logical rows for the selected table. +func (s *State) LoadLogicalView() error { + if s.logicalView != nil { + return nil + } + if s.selectedEntry < 0 || s.selectedEntry >= len(s.entries) { + return nil + } + snapshotTS := s.entries[s.selectedEntry].GetEnd() + view, err := s.reader.BuildLogicalTableView(context.Background(), snapshotTS, s.dataEntries, s.tombEntries) + if err != nil { + return err + } + s.logicalView = view return nil } diff --git a/pkg/tools/checkpointtool/interactive/unified_model.go b/pkg/tools/checkpointtool/interactive/unified_model.go index b85511152dc58..b7951327f8b85 100644 --- a/pkg/tools/checkpointtool/interactive/unified_model.go +++ b/pkg/tools/checkpointtool/interactive/unified_model.go @@ -15,18 +15,16 @@ package interactive import ( - "path/filepath" - tea "github.com/charmbracelet/bubbletea" "github.com/matrixorigin/matrixone/pkg/tools/checkpointtool" "github.com/matrixorigin/matrixone/pkg/tools/interactive" - "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" ) // Messages for page navigation type selectCheckpointMsg struct{ idx int } type selectTableMsg struct{ tableID uint64 } type openObjectMsg struct{ path string } +type openLogicalTableMsg struct{} type goBackMsg struct{} // UnifiedModel uses GenericPage for all views @@ -37,7 +35,6 @@ type UnifiedModel struct { // For opening objects objectToOpen string - rangeToOpen *ckputil.TableRange quitting bool } @@ -88,19 +85,40 @@ func (m *UnifiedModel) createTablesListPage() *interactive.GenericPage { func (m *UnifiedModel) createTableDetailPage() *interactive.GenericPage { config := interactive.PageConfig{ Title: "═══ Table Detail ═══", - Headers: []string{"Type", "Object", "Range", "Rows", "Size", "Created"}, + Headers: []string{"Type", "Object", "Blocks", "Rows", "Size", "Created"}, ShowRowNumber: true, EnableCursor: true, EnableSearch: true, EnableHScroll: true, EnableBack: true, MaxColWidth: 50, // Limit column width, use h/l to scroll + CustomHints: "[j/k] Navigate [Enter] Open object [L] Logical table [h/l] Scroll [/] Search [b/ESC] Back [q] Quit", } provider := &TableDetailProvider{state: m.state} handler := &tableDetailHandler{state: m.state} return interactive.NewGenericPage(config, provider, handler) } +func (m *UnifiedModel) createLogicalTablePage() *interactive.GenericPage { + headers := []string{"object", "block", "row"} + if view := m.state.LogicalView(); view != nil && len(view.Headers) > 0 { + headers = view.Headers + } + config := interactive.PageConfig{ + Title: "═══ Logical Table View ═══", + Headers: headers, + ShowRowNumber: true, + EnableCursor: false, + EnableSearch: true, + EnableHScroll: true, + EnableBack: true, + MaxColWidth: 40, + } + provider := &LogicalTableProvider{state: m.state} + handler := &logicalTableHandler{} + return interactive.NewGenericPage(config, provider, handler) +} + func (m *UnifiedModel) Init() tea.Cmd { return m.currentPage.Init() } @@ -131,18 +149,19 @@ func (m *UnifiedModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil case openObjectMsg: - m.objectToOpen = filepath.Join(m.state.reader.Dir(), msg.path) - idx := m.currentPage.GetCursor() - dataEntries := m.state.DataEntries() - tombEntries := m.state.TombEntries() - if idx < len(dataEntries) { - m.rangeToOpen = &dataEntries[idx].Range - } else if idx < len(dataEntries)+len(tombEntries) { - m.rangeToOpen = &tombEntries[idx-len(dataEntries)].Range - } + m.objectToOpen = msg.path m.quitting = true return m, tea.Quit + case openLogicalTableMsg: + if err := m.state.LoadLogicalView(); err != nil { + return m, nil + } + m.pageStack = append(m.pageStack, m.currentPage) + m.currentPage = m.createLogicalTablePage() + m.currentPage.Refresh() + return m, nil + case goBackMsg: if len(m.pageStack) > 0 { m.currentPage = m.pageStack[len(m.pageStack)-1] @@ -176,13 +195,7 @@ func (m *UnifiedModel) GetObjectToOpen() string { return m.objectToOpen } -// GetRangeToOpen returns the range to open (if any) -func (m *UnifiedModel) GetRangeToOpen() *ckputil.TableRange { - return m.rangeToOpen -} - // ClearObjectToOpen clears the object to open flag func (m *UnifiedModel) ClearObjectToOpen() { m.objectToOpen = "" - m.rangeToOpen = nil } diff --git a/pkg/tools/checkpointtool/interactive/unified_model_test.go b/pkg/tools/checkpointtool/interactive/unified_model_test.go index e212b65432dd6..33df08d71bbc9 100644 --- a/pkg/tools/checkpointtool/interactive/unified_model_test.go +++ b/pkg/tools/checkpointtool/interactive/unified_model_test.go @@ -221,6 +221,16 @@ func TestTableDetailHandler(t *testing.T) { assert.True(t, handler.MatchRow(row, "123")) assert.False(t, handler.MatchRow(row, "notfound")) }) + + t.Run("custom_key_logical_view", func(t *testing.T) { + handler := &tableDetailHandler{} + cmd := handler.OnCustomKey("L") + if assert.NotNil(t, cmd) { + msg := cmd() + _, ok := msg.(openLogicalTableMsg) + assert.True(t, ok) + } + }) } // TestTSFormatting tests timestamp formatting diff --git a/pkg/tools/checkpointtool/logical_table.go b/pkg/tools/checkpointtool/logical_table.go new file mode 100644 index 0000000000000..15d4a5c472037 --- /dev/null +++ b/pkg/tools/checkpointtool/logical_table.go @@ -0,0 +1,376 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/objectio" + objectioutil "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" +) + +type logicalTableStats struct { + PhysicalRows int + DeletedRows int + VisibleRows int +} + +// BuildLogicalTableView materializes a tombstone-applied logical table view. +func (r *CheckpointReader) BuildLogicalTableView( + ctx context.Context, + snapshotTS types.TS, + dataEntries []*ObjectEntryInfo, + tombEntries []*ObjectEntryInfo, +) (*LogicalTableView, error) { + view := newLogicalTableView() + stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, + func(cols []objecttool.ColInfo) error { + if len(view.Headers) != view.MetaWidth() { + return nil + } + for _, col := range cols { + view.Headers = append(view.Headers, fmt.Sprintf("col_%d", col.Idx)) + view.ColTypes = append(view.ColTypes, col.Type) + view.ColSeqNums = append(view.ColSeqNums, col.SeqNum) + } + return nil + }, + func(objShort string, blockIdx int, rowIdx int, values []string, _ []bool) error { + row := make([]string, 0, len(values)+view.MetaWidth()) + row = append(row, objShort, fmt.Sprintf("%d", blockIdx), fmt.Sprintf("%d", rowIdx)) + row = append(row, values...) + view.Rows = append(view.Rows, row) + return nil + }, + ) + if err != nil { + return nil, err + } + view.PhysicalRows = stats.PhysicalRows + view.DeletedRows = stats.DeletedRows + view.VisibleRows = stats.VisibleRows + return view, nil +} + +func (r *CheckpointReader) scanLogicalTable( + ctx context.Context, + snapshotTS types.TS, + dataEntries []*ObjectEntryInfo, + tombEntries []*ObjectEntryInfo, + onColumns func([]objecttool.ColInfo) error, + onRow func(objShort string, blockIdx int, rowIdx int, values []string, nulls []bool) error, +) (logicalTableStats, error) { + stats := logicalTableStats{} + if len(dataEntries) == 0 { + return stats, nil + } + + visibleDataEntries := visibleObjectEntries(dataEntries, snapshotTS) + visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) + tombstoneStats := dedupeObjectStats(visibleTombEntries) + columnsSent := false + var canonicalSeqNums []uint16 + + for _, entry := range visibleDataEntries { + objName := entry.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + if isDataFileNotFound(err) { + continue + } + return stats, err + } + debugLogicalObjectColumns(entry.ObjectStats, reader) + cols := reader.Columns() + + if !columnsSent && onColumns != nil { + canonicalSeqNums = columnSeqNums(cols) + if err := onColumns(cols); err != nil { + _ = reader.Close() + return stats, err + } + columnsSent = true + } + + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + if err != nil { + _ = reader.Close() + return stats, err + } + + for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { + bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) + if err != nil { + _ = reader.Close() + return stats, err + } + + if bat.RowCount() == 0 { + release() + continue + } + + stats.PhysicalRows += bat.RowCount() + + commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) + if err != nil { + release() + _ = reader.Close() + return stats, err + } + var commitTSs []types.TS + if commitTSVec != nil { + commitTSs = vector.MustFixedColWithTypeCheck[types.TS](commitTSVec) + } + + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, entry.ObjectStats, uint16(blockIdx), relevantTombstones) + if err != nil { + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + _ = reader.Close() + return stats, err + } + + for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { + if commitTSs != nil && rowIdx < len(commitTSs) && + !commitTSVec.IsNull(uint64(rowIdx)) && + commitTSs[rowIdx].GT(&snapshotTS) { + continue + } + if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { + stats.DeletedRows++ + continue + } + if onRow != nil { + values := make([]string, len(bat.Vecs)) + nulls := make([]bool, len(bat.Vecs)) + for i, vec := range bat.Vecs { + nulls[i] = vec.IsNull(uint64(rowIdx)) + values[i] = vecValueToString(vec, rowIdx) + } + if len(canonicalSeqNums) > 0 { + values, nulls = alignRowValuesBySeqNums(cols, canonicalSeqNums, values, nulls) + } + if err := onRow(entry.ObjectStats.ObjectName().Short().ShortString(), blockIdx, rowIdx, values, nulls); err != nil { + if deleteMask.IsValid() { + deleteMask.Release() + } + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + _ = reader.Close() + return stats, err + } + } + stats.VisibleRows++ + } + + if deleteMask.IsValid() { + deleteMask.Release() + } + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + } + + if err := reader.Close(); err != nil { + return stats, err + } + } + + return stats, nil +} + +func columnSeqNums(cols []objecttool.ColInfo) []uint16 { + seqNums := make([]uint16, len(cols)) + for i, col := range cols { + seqNums[i] = col.SeqNum + } + return seqNums +} + +func alignRowValuesBySeqNums( + cols []objecttool.ColInfo, + targetSeqNums []uint16, + values []string, + nulls []bool, +) ([]string, []bool) { + if len(cols) == 0 || len(targetSeqNums) == 0 { + return values, nulls + } + indexBySeqNum := make(map[uint16]int, len(cols)) + for idx, col := range cols { + indexBySeqNum[col.SeqNum] = idx + } + alignedValues := make([]string, len(targetSeqNums)) + alignedNulls := make([]bool, len(targetSeqNums)) + for targetIdx, seqNum := range targetSeqNums { + sourceIdx, ok := indexBySeqNum[seqNum] + if !ok || sourceIdx >= len(values) { + alignedNulls[targetIdx] = true + continue + } + alignedValues[targetIdx] = values[sourceIdx] + if sourceIdx < len(nulls) { + alignedNulls[targetIdx] = nulls[sourceIdx] + } + } + return alignedValues, alignedNulls +} + +func debugLogicalObjectColumns(stats objectio.ObjectStats, reader *objecttool.ObjectReader) { + if reader == nil { + return + } + info := reader.Info() + meta := reader.Meta() + header := meta.BlockHeader() + var cols strings.Builder + for i, col := range reader.Columns() { + if i > 0 { + cols.WriteString(",") + } + cols.WriteString(fmt.Sprintf("%d->seq%d:%s", col.Idx, col.SeqNum, col.Type.Oid.String())) + } + ckpDebugSchemaf( + "object columns object=%s stats_appendable=%v meta_appendable=%v blocks=%d rows=%d column_count=%d meta_column_count=%d max_seqnum=%d columns=[%s]", + stats.ObjectName().Short().ShortString(), + stats.GetAppendable(), + info != nil && info.IsAppendable, + meta.BlockCount(), + header.Rows(), + header.ColumnCount(), + header.MetaColumnCount(), + header.MaxSeqnum(), + cols.String(), + ) +} + +func dedupeObjectStats(entries []*ObjectEntryInfo) []objectio.ObjectStats { + seen := make(map[string]struct{}) + stats := make([]objectio.ObjectStats, 0, len(entries)) + for _, entry := range entries { + name := entry.ObjectStats.ObjectName().String() + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + stats = append(stats, entry.ObjectStats) + } + return stats +} + +func visibleObjectEntries(entries []*ObjectEntryInfo, snapshotTS types.TS) []*ObjectEntryInfo { + if len(entries) == 0 { + return nil + } + + merged := make(map[string]*ObjectEntryInfo, len(entries)) + for _, entry := range entries { + name := entry.ObjectStats.ObjectName().String() + existing, ok := merged[name] + if !ok { + copyEntry := *entry + merged[name] = ©Entry + continue + } + if existing.CreateTime.IsEmpty() || (!entry.CreateTime.IsEmpty() && entry.CreateTime.LT(&existing.CreateTime)) { + existing.CreateTime = entry.CreateTime + } + if existing.DeleteTime.IsEmpty() || (!entry.DeleteTime.IsEmpty() && existing.DeleteTime.LT(&entry.DeleteTime)) { + existing.DeleteTime = entry.DeleteTime + } + } + + visible := make([]*ObjectEntryInfo, 0, len(merged)) + for _, entry := range merged { + obj := objectio.ObjectEntry{ + ObjectStats: entry.ObjectStats, + CreateTime: entry.CreateTime, + DeleteTime: entry.DeleteTime, + } + if obj.Visible(snapshotTS) { + visible = append(visible, entry) + } + } + sort.Slice(visible, func(i, j int) bool { + return visible[i].ObjectStats.ObjectName().String() < visible[j].ObjectStats.ObjectName().String() + }) + return visible +} + +func (r *CheckpointReader) filterTombstonesForObject( + ctx context.Context, + objectID *objectio.ObjectId, + tombstoneStats []objectio.ObjectStats, +) ([]objectio.ObjectStats, error) { + if len(tombstoneStats) == 0 { + return nil, nil + } + selection, err := objectioutil.FindTombstonesOfObject(ctx, objectID, tombstoneStats, r.fs) + if err != nil { + return nil, err + } + relevant := make([]objectio.ObjectStats, 0, selection.Count()) + iter := selection.Iterator() + for iter.HasNext() { + idx := iter.Next() + if idx >= uint64(len(tombstoneStats)) { + return nil, moerr.NewInternalErrorf(ctx, "tombstone selection index out of range: %d", idx) + } + relevant = append(relevant, tombstoneStats[int(idx)]) + } + return relevant, nil +} + +func (r *CheckpointReader) buildDeleteMaskForBlock( + ctx context.Context, + snapshotTS *types.TS, + stats objectio.ObjectStats, + blockIdx uint16, + tombstoneStats []objectio.ObjectStats, +) (objectio.Bitmap, error) { + if len(tombstoneStats) == 0 { + return objectio.NullBitmap, nil + } + + mask := objectio.GetReusableBitmap() + blockID := stats.ConstructBlockId(blockIdx) + current := 0 + getTombstone := func() (*objectio.ObjectStats, error) { + if current >= len(tombstoneStats) { + return nil, nil + } + stat := &tombstoneStats[current] + current++ + return stat, nil + } + if err := objectioutil.GetTombstonesByBlockId(ctx, snapshotTS, &blockID, getTombstone, &mask, r.fs); err != nil { + mask.Release() + return objectio.NullBitmap, err + } + return mask, nil +} diff --git a/pkg/tools/checkpointtool/logical_table_test.go b/pkg/tools/checkpointtool/logical_table_test.go new file mode 100644 index 0000000000000..a0a844353112d --- /dev/null +++ b/pkg/tools/checkpointtool/logical_table_test.go @@ -0,0 +1,95 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildLogicalTableViewEmpty(t *testing.T) { + reader := &CheckpointReader{} + view, err := reader.BuildLogicalTableView(context.Background(), types.TS{}, nil, nil) + require.NoError(t, err) + assert.Equal(t, []string{"object", "block", "row"}, view.Headers) + assert.Empty(t, view.Rows) + assert.Equal(t, 0, view.PhysicalRows) + assert.Equal(t, 0, view.DeletedRows) + assert.Equal(t, 0, view.VisibleRows) +} + +func TestVisibleObjectEntriesFiltersAndDedupesBySnapshot(t *testing.T) { + snapshot := types.BuildTS(15, 0) + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(1, 10, 0), + newTestObjectEntryInfo(1, 12, 20), + newTestObjectEntryInfo(2, 16, 0), // created after snapshot + newTestObjectEntryInfo(3, 1, 14), // deleted before snapshot + } + + visible := visibleObjectEntries(entries, snapshot) + require.Len(t, visible, 1) + assert.Equal(t, uint32(1), visible[0].ObjectStats.Rows()) + assert.Equal(t, types.BuildTS(10, 0), visible[0].CreateTime) + assert.Equal(t, types.BuildTS(20, 0), visible[0].DeleteTime) +} + +func TestVisibleObjectEntriesDeletedAfterSnapshotBecomesInvisibleLater(t *testing.T) { + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(7, 5, 0), + newTestObjectEntryInfo(7, 8, 30), + } + + visibleAt20 := visibleObjectEntries(entries, types.BuildTS(20, 0)) + require.Len(t, visibleAt20, 1) + + visibleAt30 := visibleObjectEntries(entries, types.BuildTS(30, 0)) + assert.Empty(t, visibleAt30) +} + +func TestDedupeObjectStats(t *testing.T) { + entries := []*ObjectEntryInfo{ + newTestObjectEntryInfo(1, 1, 0), + newTestObjectEntryInfo(1, 2, 0), + newTestObjectEntryInfo(2, 3, 0), + } + + stats := dedupeObjectStats(entries) + require.Len(t, stats, 2) + assert.NotEqual(t, stats[0].ObjectName().String(), stats[1].ObjectName().String()) +} + +func newTestObjectEntryInfo(idByte byte, createPhysical int64, deletePhysical int64) *ObjectEntryInfo { + var objectID objectio.ObjectId + objectID[0] = idByte + stats := objectio.NewObjectStats() + _ = objectio.SetObjectStatsObjectName(stats, objectio.BuildObjectNameWithObjectID(&objectID)) + _ = objectio.SetObjectStatsRowCnt(stats, uint32(idByte)) + _ = objectio.SetObjectStatsBlkCnt(stats, 1) + + entry := &ObjectEntryInfo{ + ObjectStats: *stats, + CreateTime: types.BuildTS(createPhysical, 0), + } + if deletePhysical > 0 { + entry.DeleteTime = types.BuildTS(deletePhysical, 0) + } + return entry +} diff --git a/pkg/tools/checkpointtool/table_dump.go b/pkg/tools/checkpointtool/table_dump.go new file mode 100644 index 0000000000000..ed3491f6bca6f --- /dev/null +++ b/pkg/tools/checkpointtool/table_dump.go @@ -0,0 +1,5208 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/matrixorigin/matrixone/pkg/vm/engine" +) + +const ( + // Meta columns in LogicalTableView (object, block, row) + logicalViewMetaCols = 3 + + // System table IDs + moDatabaseID = uint64(catalog.MO_DATABASE_ID) + moTablesID = uint64(catalog.MO_TABLES_ID) + moColumnsID = uint64(catalog.MO_COLUMNS_ID) +) + +const ( + csvPipelineQueueCapacity = 2 + csvPipelineReaderMax = 4 + csvPipelineReportEvery = 10 * time.Second + csvPipelineMinFreeMemory = 1 << 30 + csvPipelineFreeRatio = 10 + csvPipelineMemoryPoll = 200 * time.Millisecond + csvPipelineWorkerMemory = 256 << 20 +) + +type catalogLayout struct { + name string + moDatabaseSchema []string + moTablesSchema []string + moColumnsSchema []string +} + +type catalogLayoutMatch struct { + layout catalogLayout + offset int +} + +var ( + currentCatalogLayout = catalogLayout{ + name: "current", + moDatabaseSchema: append([]string(nil), catalog.MoDatabaseSchema...), + moTablesSchema: append([]string(nil), catalog.MoTablesSchema...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema...), + } + preCPKLayout = catalogLayout{ + name: "pre-cpk", + moDatabaseSchema: catalogSchemaWithout(catalog.MoDatabaseSchema, catalog.SystemDBAttr_CPKey), + moTablesSchema: catalogSchemaWithout( + catalog.MoTablesSchema, + catalog.SystemRelAttr_ExtraInfo, + catalog.SystemRelAttr_CPKey, + ), + moColumnsSchema: catalogSchemaWithout( + catalog.MoColumnsSchema, + catalog.SystemColAttr_CPKey, + ), + } + legacy3CatalogLayout = catalogLayout{ + name: "3.0-dev", + moDatabaseSchema: append([]string(nil), catalog.MoDatabaseSchema...), + moTablesSchema: append([]string(nil), catalog.MoTablesSchema[:len(catalog.MoTablesSchema)-1]...), + moColumnsSchema: append([]string(nil), catalog.MoColumnsSchema[:len(catalog.MoColumnsSchema)-2]...), + } +) + +// TableColumn describes one column in a user table schema. +type TableColumn struct { + Name string // SQL column name + SQLType string // SQL type string (e.g. "BIGINT", "VARCHAR(100)") + Position int // SQL ordinal position + PhysicalPosition int // physical/object column position + Unsigned bool + NotNull bool + Default string + HasDefault bool + OnUpdate string + Generated string + GeneratedStored bool + Comment string + ConstraintType string + AutoIncrement bool + ClusterBy bool + EnumValues string +} + +type TableUniqueKey struct { + Name string + Columns []string + Unique bool + Algo string + AlgoParams string + Comment string +} + +type TableForeignKey struct { + Name string + Columns []string + ReferDatabase string + ReferTable string + ReferColumns []string + OnDelete plan.ForeignKeyDef_RefAction + OnUpdate plan.ForeignKeyDef_RefAction +} + +// TableSchema holds the decoded schema for one user table. +type TableSchema struct { + TableName string + DatabaseName string + AccountID uint32 + Columns []TableColumn // sorted by Position + CreateSQL string // raw CREATE TABLE from mo_tables.rel_createsql + Comment string + UniqueKeys []TableUniqueKey + PrimaryKey []string + ForeignKeys []TableForeignKey + Partition string + ClusterBy []string +} + +type TableCatalogEntry struct { + TableID uint64 + AccountID uint32 + DatabaseID uint64 + DatabaseName string + TableName string + RelKind string +} + +type TableListOptions struct { + AccountID *uint32 + DatabaseID *uint64 + IncludeViews bool +} + +type CSVRowOrder string + +const ( + CSVRowOrderStorage CSVRowOrder = "storage" + CSVRowOrderLexical CSVRowOrder = "lexical" +) + +type CSVExportOptions struct { + IncludeMetadata bool + IncludeHeader bool + RowOrder CSVRowOrder +} + +type exportedCSVRow struct { + values []string + nulls []bool +} + +// TableDumpData contains the checkpoint metadata needed to dump one table. +type TableDumpData struct { + TableID uint64 + Schema *TableSchema + DataEntries []*ObjectEntryInfo + TombEntries []*ObjectEntryInfo +} + +var moIndexesHeaders = []string{ + "id", + "table_id", + "database_id", + "name", + "type", + catalog.IndexAlgoName, + catalog.IndexAlgoTableType, + catalog.IndexAlgoParams, + "is_visible", + "hidden", + "comment", + "column_name", + "ordinal_position", + "options", + "index_table_name", +} + +var moPartitionMetadataHeaders = []string{ + "table_id", + "table_name", + "database_name", + "partition_method", + "partition_description", + "partition_count", +} + +var moPartitionTablesHeaders = []string{ + "partition_id", + "partition_table_name", + "primary_table_id", + "partition_name", + "partition_ordinal_position", + "partition_expression_str", + "partition_expression", +} + +type indexDDLColumn struct { + name string + ordinal int +} + +type indexDDLInfo struct { + name string + indexType string + algo string + algoParams string + comment string + columns map[string]indexDDLColumn +} + +type CSVExportOption func(*CSVExportOptions) + +func defaultCSVExportOptions() CSVExportOptions { + return CSVExportOptions{ + IncludeMetadata: true, + IncludeHeader: true, + RowOrder: CSVRowOrderStorage, + } +} + +func WithCSVMetaComments(include bool) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.IncludeMetadata = include + } +} + +func WithCSVHeader(include bool) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.IncludeHeader = include + } +} + +func WithCSVRowOrder(order CSVRowOrder) CSVExportOption { + return func(opts *CSVExportOptions) { + opts.RowOrder = order + } +} + +func ParseCSVRowOrder(s string) (CSVRowOrder, error) { + switch CSVRowOrder(strings.ToLower(strings.TrimSpace(s))) { + case "", CSVRowOrderStorage: + return CSVRowOrderStorage, nil + case CSVRowOrderLexical: + return CSVRowOrderLexical, nil + default: + return "", moerr.NewInvalidInputf(context.Background(), "unsupported row order %q (supported: %s, %s)", s, CSVRowOrderStorage, CSVRowOrderLexical) + } +} + +func resolveCSVExportOptions(opts []CSVExportOption) CSVExportOptions { + resolved := defaultCSVExportOptions() + for _, opt := range opts { + if opt != nil { + opt(&resolved) + } + } + return resolved +} + +type builtinColumnDef struct { + Name string + SQLType string + Position int + Hidden bool +} + +func knownCatalogLayouts() []catalogLayout { + return []catalogLayout{preCPKLayout, currentCatalogLayout, legacy3CatalogLayout} +} + +func schemaForLayout(layout catalogLayout, tableID uint64) []string { + switch tableID { + case moDatabaseID: + return layout.moDatabaseSchema + case moTablesID: + return layout.moTablesSchema + case moColumnsID: + return layout.moColumnsSchema + default: + return nil + } +} + +func catalogSchemaWithout(schema []string, names ...string) []string { + excluded := make(map[string]struct{}, len(names)) + for _, name := range names { + excluded[name] = struct{}{} + } + filtered := make([]string, 0, len(schema)) + for _, name := range schema { + if _, ok := excluded[name]; ok { + continue + } + filtered = append(filtered, name) + } + return filtered +} + +func builtinColumnsForLayout(layout catalogLayout, tableID uint64) []builtinColumnDef { + switch tableID { + case catalog.MO_DATABASE_ID: + return []builtinColumnDef{ + {Name: "dat_id", SQLType: "BIGINT", Position: 0}, + {Name: "datname", SQLType: "VARCHAR(5000)", Position: 1}, + {Name: "dat_catalog_name", SQLType: "VARCHAR(5000)", Position: 2}, + {Name: "dat_createsql", SQLType: "VARCHAR(5000)", Position: 3}, + {Name: "owner", SQLType: "INT UNSIGNED", Position: 4}, + {Name: "creator", SQLType: "INT UNSIGNED", Position: 5}, + {Name: "created_time", SQLType: "TIMESTAMP", Position: 6}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 7}, + {Name: "dat_type", SQLType: "VARCHAR(32)", Position: 8}, + {Name: catalog.SystemDBAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 9, Hidden: true}, + } + case catalog.MO_TABLES_ID: + cols := []builtinColumnDef{ + {Name: "rel_id", SQLType: "BIGINT", Position: 0}, + {Name: "relname", SQLType: "VARCHAR(5000)", Position: 1}, + {Name: "reldatabase", SQLType: "VARCHAR(5000)", Position: 2}, + {Name: "reldatabase_id", SQLType: "BIGINT", Position: 3}, + {Name: "relpersistence", SQLType: "VARCHAR(5000)", Position: 4}, + {Name: "relkind", SQLType: "VARCHAR(5000)", Position: 5}, + {Name: "rel_comment", SQLType: "VARCHAR(5000)", Position: 6}, + {Name: "rel_createsql", SQLType: "TEXT", Position: 7}, + {Name: "created_time", SQLType: "TIMESTAMP", Position: 8}, + {Name: "creator", SQLType: "INT UNSIGNED", Position: 9}, + {Name: "owner", SQLType: "INT UNSIGNED", Position: 10}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 11}, + {Name: "partitioned", SQLType: "TINYINT", Position: 12}, + {Name: "partition_info", SQLType: "BLOB", Position: 13}, + {Name: "viewdef", SQLType: "VARCHAR(5000)", Position: 14}, + {Name: "constraint", SQLType: "VARCHAR(5000)", Position: 15}, + {Name: "schema_version", SQLType: "INT UNSIGNED", Position: 16}, + {Name: "schema_catalog_version", SQLType: "INT UNSIGNED", Position: 17}, + {Name: "extra_info", SQLType: "VARCHAR", Position: 18, Hidden: true}, + {Name: catalog.SystemRelAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 19, Hidden: true}, + } + if layout.name != legacy3CatalogLayout.name { + cols = append(cols, builtinColumnDef{Name: "rel_logical_id", SQLType: "BIGINT", Position: 20}) + } + return cols + case catalog.MO_COLUMNS_ID: + cols := []builtinColumnDef{ + {Name: "att_uniq_name", SQLType: "VARCHAR(256)", Position: 0}, + {Name: "account_id", SQLType: "INT UNSIGNED", Position: 1}, + {Name: "att_database_id", SQLType: "BIGINT", Position: 2}, + {Name: "att_database", SQLType: "VARCHAR(256)", Position: 3}, + {Name: "att_relname_id", SQLType: "BIGINT", Position: 4}, + {Name: "att_relname", SQLType: "VARCHAR(256)", Position: 5}, + {Name: "attname", SQLType: "VARCHAR(256)", Position: 6}, + {Name: "atttyp", SQLType: "VARCHAR(256)", Position: 7}, + {Name: "attnum", SQLType: "INT", Position: 8}, + {Name: "att_length", SQLType: "INT", Position: 9}, + {Name: "attnotnull", SQLType: "TINYINT", Position: 10}, + {Name: "atthasdef", SQLType: "TINYINT", Position: 11}, + {Name: "att_default", SQLType: "VARCHAR(2048)", Position: 12}, + {Name: "attisdropped", SQLType: "TINYINT", Position: 13}, + {Name: "att_constraint_type", SQLType: "CHAR(1)", Position: 14}, + {Name: "att_is_unsigned", SQLType: "TINYINT", Position: 15}, + {Name: "att_is_auto_increment", SQLType: "TINYINT", Position: 16}, + {Name: "att_comment", SQLType: "VARCHAR(2048)", Position: 17}, + {Name: "att_is_hidden", SQLType: "TINYINT", Position: 18}, + {Name: catalog.SystemColAttr_HasUpdate, SQLType: "TINYINT", Position: 19}, + {Name: catalog.SystemColAttr_Update, SQLType: "VARCHAR(2048)", Position: 20}, + {Name: catalog.SystemColAttr_IsClusterBy, SQLType: "TINYINT", Position: 21}, + {Name: catalog.SystemColAttr_Seqnum, SQLType: "SMALLINT UNSIGNED", Position: 22}, + {Name: catalog.SystemColAttr_EnumValues, SQLType: "VARCHAR", Position: 23}, + {Name: catalog.SystemColAttr_CPKey, SQLType: "VARCHAR(65535)", Position: 24, Hidden: true}, + } + if layout.name != legacy3CatalogLayout.name { + cols = append(cols, + builtinColumnDef{Name: "attr_has_generated", SQLType: "TINYINT", Position: 25}, + builtinColumnDef{Name: "attr_generated", SQLType: "VARCHAR(2048)", Position: 26}, + ) + } + return cols + default: + return nil + } +} + +func builtinTableSchemaForLayout(layout catalogLayout, tableID uint64) *TableSchema { + cols := builtinColumnsForLayout(layout, tableID) + if len(cols) == 0 { + return nil + } + + schema := &TableSchema{ + DatabaseName: "mo_catalog", + } + switch tableID { + case catalog.MO_DATABASE_ID: + schema.TableName = "mo_database" + case catalog.MO_TABLES_ID: + schema.TableName = "mo_tables" + case catalog.MO_COLUMNS_ID: + schema.TableName = "mo_columns" + } + for _, col := range cols { + if col.Hidden { + continue + } + schema.Columns = append(schema.Columns, TableColumn{ + Name: col.Name, + SQLType: col.SQLType, + Position: col.Position, + PhysicalPosition: col.Position, + }) + } + schema.CreateSQL = renderCreateTableDDL(schema.TableName, schema.Columns) + return schema +} + +func renderCreateTableDDL(tableName string, cols []TableColumn) string { + return renderCreateTableDDLWithComment(tableName, cols, "") +} + +func renderCreateTableDDLWithComment(tableName string, cols []TableColumn, comment string, uniqueKeys ...TableUniqueKey) string { + return renderCreateTableDDLFull(tableName, cols, comment, "", nil, uniqueKeys...) +} + +func renderCreateTableDDLFull(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys ...TableUniqueKey) string { + return renderCreateTableDDLFullWithForeignKeys(tableName, cols, comment, partition, primaryKey, uniqueKeys, nil) +} + +func renderCreateTableDDLFullWithForeignKeys(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey) string { + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName, cols, comment, partition, primaryKey, uniqueKeys, foreignKeys, nil) +} + +func renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName string, cols []TableColumn, comment string, partition string, primaryKey []string, uniqueKeys []TableUniqueKey, foreignKeys []TableForeignKey, clusterBy []string) string { + if tableName == "" || len(cols) == 0 { + return "" + } + + var sb strings.Builder + primaryCols := primaryKeyColumns(cols, primaryKey) + uniqueKeys = normalizedUniqueKeys(uniqueKeys) + foreignKeys = normalizedForeignKeys(foreignKeys) + extraClauses := 0 + if len(primaryCols) > 0 { + extraClauses++ + } + extraClauses += len(uniqueKeys) + len(foreignKeys) + sb.WriteString("CREATE TABLE ") + sb.WriteString(quoteDDLIdent(tableName)) + sb.WriteString(" (\n") + for i, col := range cols { + sb.WriteString(" ") + sb.WriteString(quoteDDLIdent(col.Name)) + if sqlType := renderColumnSQLType(col); sqlType != "" { + sb.WriteString(" ") + sb.WriteString(sqlType) + if col.Unsigned && !strings.Contains(strings.ToUpper(sqlType), "UNSIGNED") { + sb.WriteString(" UNSIGNED") + } + } + appendColumnDDLAttributes(&sb, col) + if i < len(cols)-1 || extraClauses > 0 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + if len(primaryCols) > 0 { + sb.WriteString(" PRIMARY KEY (") + for i, col := range primaryCols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.Name)) + } + if len(uniqueKeys) > 0 || len(foreignKeys) > 0 { + sb.WriteString("),\n") + } else { + sb.WriteString(")\n") + } + } + for i, key := range uniqueKeys { + if len(key.Columns) == 0 { + continue + } + sb.WriteString(" ") + fullText := isFullTextKey(key) + if fullText { + sb.WriteString("FULLTEXT ") + } else if key.Unique { + sb.WriteString("UNIQUE ") + } + sb.WriteString("KEY ") + sb.WriteString(quoteDDLIdent(key.Name)) + if !fullText { + appendIndexAlgorithmDDL(&sb, key.Algo) + } + sb.WriteString("(") + for i, col := range key.Columns { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col)) + } + sb.WriteString(")") + if err := appendIndexTrailingOptionsDDL(&sb, key.AlgoParams, key.Comment); err != nil { + ckpDebugSchemaf("mo_tables constraint index options skipped index=%s algo=%q params=%q err=%v", key.Name, key.Algo, key.AlgoParams, err) + } + if i < len(uniqueKeys)-1 || len(foreignKeys) > 0 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + for i, key := range foreignKeys { + sb.WriteString(" ") + sb.WriteString(renderForeignKeyClause(key)) + if i < len(foreignKeys)-1 { + sb.WriteString(",\n") + } else { + sb.WriteString("\n") + } + } + sb.WriteString(")") + if comment != "" { + sb.WriteString(" COMMENT=") + sb.WriteString(quoteDDLString(comment)) + } + if partition != "" { + sb.WriteString(" ") + sb.WriteString(partition) + } + if len(clusterBy) > 0 { + sb.WriteString(" CLUSTER BY (") + appendDDLIdentList(&sb, clusterBy) + sb.WriteString(")") + } else if clusterCols := clusterByColumns(cols); len(clusterCols) > 0 { + sb.WriteString(" CLUSTER BY (") + for i, col := range clusterCols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.Name)) + } + sb.WriteString(")") + } + sb.WriteString(";") + return sb.String() +} + +func renderColumnSQLType(col TableColumn) string { + sqlType := strings.TrimSpace(col.SQLType) + upperType := strings.ToUpper(sqlType) + enumValues := strings.TrimSpace(col.EnumValues) + if enumValues == "" { + return sqlType + } + switch { + case upperType == "JSON": + if arrayType := renderTypedArraySQLType(enumValues); arrayType != "" { + return arrayType + } + return sqlType + case upperType == "ENUM": + return renderEnumSetSQLType("ENUM", enumValues) + case upperType == "SET", upperType == "BIGINT UNSIGNED": + return renderEnumSetSQLType("SET", enumValues) + default: + return sqlType + } +} + +func renderTypedArraySQLType(enumValues string) string { + values := strings.TrimSpace(enumValues) + if len(values) < len("array(") || !strings.EqualFold(values[:len("array(")], "array(") { + return "" + } + return "ARRAY" + values[len("array"):] +} + +func renderEnumSetSQLType(kind string, enumValues string) string { + values := strings.TrimSpace(enumValues) + if values == "" { + return kind + } + upper := strings.ToUpper(values) + if strings.HasPrefix(upper, kind+"(") { + return values + } + if strings.HasPrefix(values, "(") { + return kind + renderEnumSetValueList(values) + } + return kind + renderEnumSetValueList(values) +} + +func renderEnumSetValueList(values string) string { + parts := splitEnumSetValues(stripEnumSetValueParens(values)) + quoted := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if isQuotedSQLString(part) { + quoted = append(quoted, part) + continue + } + quoted = append(quoted, quoteDDLString(part)) + } + return "(" + strings.Join(quoted, ",") + ")" +} + +func stripEnumSetValueParens(values string) string { + values = strings.TrimSpace(values) + if len(values) >= 2 && values[0] == '(' && values[len(values)-1] == ')' { + return strings.TrimSpace(values[1 : len(values)-1]) + } + return values +} + +func splitEnumSetValues(values string) []string { + var parts []string + start := 0 + inString := byte(0) + for i := 0; i < len(values); i++ { + ch := values[i] + if inString != 0 { + if ch == '\\' && i+1 < len(values) { + i++ + continue + } + if ch == inString { + if i+1 < len(values) && values[i+1] == inString { + i++ + continue + } + inString = 0 + } + continue + } + switch ch { + case '\'', '"': + inString = ch + case ',': + parts = append(parts, values[start:i]) + start = i + 1 + } + } + parts = append(parts, values[start:]) + return parts +} + +func isQuotedSQLString(s string) bool { + s = strings.TrimSpace(s) + return len(s) >= 2 && ((s[0] == '\'' && s[len(s)-1] == '\'') || (s[0] == '"' && s[len(s)-1] == '"')) +} + +func appendColumnDDLAttributes(sb *strings.Builder, col TableColumn) { + if col.Generated != "" { + if col.NotNull { + sb.WriteString(" NOT NULL") + } + sb.WriteString(" GENERATED ALWAYS AS (") + sb.WriteString(col.Generated) + sb.WriteString(")") + if col.GeneratedStored { + sb.WriteString(" STORED") + } else { + sb.WriteString(" VIRTUAL") + } + } else { + if col.NotNull || col.AutoIncrement { + sb.WriteString(" NOT NULL") + } + if col.AutoIncrement { + sb.WriteString(" AUTO_INCREMENT") + } + if col.HasDefault && !col.AutoIncrement { + if strings.TrimSpace(col.Default) == "" { + sb.WriteString(" DEFAULT NULL") + } else { + sb.WriteString(" DEFAULT ") + sb.WriteString(formatDDLDefault(col.Default)) + } + } + if col.OnUpdate != "" { + sb.WriteString(" ON UPDATE ") + sb.WriteString(formatDDLDefault(col.OnUpdate)) + } + } + if col.Comment != "" { + sb.WriteString(" COMMENT ") + sb.WriteString(quoteDDLString(col.Comment)) + } +} + +func primaryKeyColumns(cols []TableColumn, primaryKey []string) []TableColumn { + if len(primaryKey) == 0 { + return nil + } + byName := make(map[string]TableColumn, len(cols)) + for _, col := range cols { + byName[strings.ToLower(col.Name)] = col + } + primary := make([]TableColumn, 0, 1) + for i, name := range primaryKey { + col, ok := byName[strings.ToLower(name)] + if !ok { + col = TableColumn{Name: name, Position: i + 1} + } + primary = append(primary, col) + } + return primary +} + +func clusterByColumns(cols []TableColumn) []TableColumn { + clusterBy := make([]TableColumn, 0, 1) + for _, col := range cols { + if col.ClusterBy { + clusterBy = append(clusterBy, col) + } + } + sort.Slice(clusterBy, func(i, j int) bool { + return clusterBy[i].Position < clusterBy[j].Position + }) + return clusterBy +} + +func normalizedUniqueKeys(keys []TableUniqueKey) []TableUniqueKey { + seen := make(map[string]struct{}, len(keys)) + out := make([]TableUniqueKey, 0, len(keys)) + for _, key := range keys { + cols := make([]string, 0, len(key.Columns)) + for _, col := range key.Columns { + col = strings.TrimSpace(col) + if col == "" { + continue + } + cols = append(cols, col) + } + if len(cols) == 0 { + continue + } + name := strings.TrimSpace(key.Name) + if name == "" { + name = cols[0] + } + signature := strconv.FormatBool(key.Unique) + "\x00" + strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) + if _, ok := seen[signature]; ok { + continue + } + seen[signature] = struct{}{} + out = append(out, TableUniqueKey{ + Name: name, + Columns: cols, + Unique: key.Unique, + Algo: strings.TrimSpace(key.Algo), + AlgoParams: strings.TrimSpace(key.AlgoParams), + Comment: strings.TrimSpace(key.Comment), + }) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Name != out[j].Name { + return out[i].Name < out[j].Name + } + return strings.Join(out[i].Columns, "\x00") < strings.Join(out[j].Columns, "\x00") + }) + return out +} + +func normalizedForeignKeys(keys []TableForeignKey) []TableForeignKey { + seen := make(map[string]struct{}, len(keys)) + out := make([]TableForeignKey, 0, len(keys)) + for _, key := range keys { + cols := normalizedNameList(key.Columns) + referCols := normalizedNameList(key.ReferColumns) + if len(cols) == 0 || len(cols) != len(referCols) || strings.TrimSpace(key.ReferTable) == "" { + continue + } + name := strings.TrimSpace(key.Name) + if name == "" { + name = cols[0] + } + signature := strings.ToLower(name) + "\x00" + strings.ToLower(strings.Join(cols, "\x00")) + "\x00" + + strings.ToLower(key.ReferDatabase) + "\x00" + strings.ToLower(key.ReferTable) + "\x00" + + strings.ToLower(strings.Join(referCols, "\x00")) + if _, ok := seen[signature]; ok { + continue + } + seen[signature] = struct{}{} + out = append(out, TableForeignKey{ + Name: name, + Columns: cols, + ReferDatabase: strings.TrimSpace(key.ReferDatabase), + ReferTable: strings.TrimSpace(key.ReferTable), + ReferColumns: referCols, + OnDelete: key.OnDelete, + OnUpdate: key.OnUpdate, + }) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Name != out[j].Name { + return out[i].Name < out[j].Name + } + return strings.Join(out[i].Columns, "\x00") < strings.Join(out[j].Columns, "\x00") + }) + return out +} + +func normalizedNameList(names []string) []string { + out := make([]string, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + if name != "" && !catalog.IsAlias(name) { + out = append(out, name) + } + } + return out +} + +func renderForeignKeyClause(key TableForeignKey) string { + var sb strings.Builder + sb.WriteString("CONSTRAINT ") + sb.WriteString(quoteDDLIdent(key.Name)) + sb.WriteString(" FOREIGN KEY (") + appendDDLIdentList(&sb, key.Columns) + sb.WriteString(") REFERENCES ") + sb.WriteString(quoteDDLIdent(key.ReferTable)) + sb.WriteString(" (") + appendDDLIdentList(&sb, key.ReferColumns) + sb.WriteString(")") + sb.WriteString(" ON DELETE ") + sb.WriteString(renderFKAction(key.OnDelete)) + sb.WriteString(" ON UPDATE ") + sb.WriteString(renderFKAction(key.OnUpdate)) + return sb.String() +} + +func appendDDLIdentList(sb *strings.Builder, names []string) { + for i, name := range names { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(name)) + } +} + +func renderFKAction(action plan.ForeignKeyDef_RefAction) string { + switch action { + case plan.ForeignKeyDef_CASCADE: + return "CASCADE" + case plan.ForeignKeyDef_SET_NULL: + return "SET NULL" + case plan.ForeignKeyDef_SET_DEFAULT: + return "SET DEFAULT" + case plan.ForeignKeyDef_NO_ACTION: + return "NO ACTION" + default: + return "RESTRICT" + } +} + +func formatDDLDefault(defaultExpr string) string { + defaultExpr = strings.TrimSpace(defaultExpr) + if defaultExpr == "" { + return "NULL" + } + if strings.EqualFold(defaultExpr, "null") { + return "NULL" + } + if len(defaultExpr) >= 2 && defaultExpr[0] == '\'' && defaultExpr[len(defaultExpr)-1] == '\'' { + return "'" + strings.ReplaceAll(defaultExpr[1:len(defaultExpr)-1], "'", "''") + "'" + } + return defaultExpr +} + +// RenderCreateTableDDLFromSchema renders a CREATE TABLE statement from resolved +// checkpoint column metadata. +func RenderCreateTableDDLFromSchema(schema *TableSchema) string { + if schema == nil { + return "" + } + if schema.TableName == "" || len(schema.Columns) == 0 { + return "" + } + if schema.Comment != "" && !isPrintableSQLText(schema.Comment) { + return "" + } + for _, col := range schema.Columns { + if col.Name == "" || !isPrintableSQLType(col.SQLType) { + return "" + } + if col.Default != "" && !isPrintableDDLExpression(col.Default) { + return "" + } + if col.OnUpdate != "" && !isPrintableDDLExpression(col.OnUpdate) { + return "" + } + if col.Generated != "" && !isPrintableDDLExpression(col.Generated) { + return "" + } + if col.Comment != "" && !isPrintableSQLText(col.Comment) { + return "" + } + } + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(schema.TableName, schema.Columns, schema.Comment, schema.Partition, schema.PrimaryKey, schema.UniqueKeys, schema.ForeignKeys, schema.ClusterBy) +} + +func inferBuiltinCatalogLayout( + tableID uint64, + moTablesView *LogicalTableView, + moColumnsView *LogicalTableView, +) catalogLayout { + switch tableID { + case moTablesID: + if moTablesView != nil { + if layout, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + return layout + } + } + case moColumnsID: + if moColumnsView != nil { + if layout, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + return layout + } + } + case catalog.MO_DATABASE_ID: + if moTablesView != nil { + if layout, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + return layout + } + } + if moColumnsView != nil { + if layout, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + return layout + } + } + } + return currentCatalogLayout +} + +func mergeBuiltinSchemaFallback(schema *TableSchema, builtin *TableSchema, tableID uint64) *TableSchema { + if builtin == nil { + return schema + } + if schema == nil { + schema = &TableSchema{TableName: fmt.Sprintf("%d", tableID)} + } + if schema.TableName == "" || schema.TableName == fmt.Sprintf("%d", tableID) { + schema.TableName = builtin.TableName + } + if schema.DatabaseName == "" { + schema.DatabaseName = builtin.DatabaseName + } + if schema.CreateSQL == "" { + schema.CreateSQL = builtin.CreateSQL + } + if len(schema.Columns) == 0 { + schema.Columns = builtin.Columns + } + return schema +} + +func inferCatalogLayout(dataWidth int, tableID uint64) (catalogLayout, int, bool) { + for _, offset := range []int{0, 1, 2} { + for _, layout := range knownCatalogLayouts() { + schema := schemaForLayout(layout, tableID) + if len(schema) == 0 { + continue + } + if dataWidth == len(schema)+offset { + return layout, offset, true + } + } + } + return catalogLayout{}, 0, false +} + +func catalogLayoutMatches(dataWidth int, tableID uint64) []catalogLayoutMatch { + var matches []catalogLayoutMatch + for _, layout := range knownCatalogLayouts() { + schema := schemaForLayout(layout, tableID) + if len(schema) == 0 { + continue + } + for _, offset := range []int{0, 1, 2} { + if dataWidth >= len(schema)+offset { + matches = append(matches, catalogLayoutMatch{ + layout: layout, + offset: offset, + }) + } + } + } + return matches +} + +func fallbackCatalogColIndex(view *LogicalTableView, tableID uint64, colName string) int { + if idx := view.columnDataIndex(colName); idx >= 0 { + return idx + } + dataOffset := logicalViewDataOffset(view) + layout, offset, ok := inferCatalogLayout(len(view.Headers)-dataOffset, tableID) + if !ok { + return -1 + } + for i, name := range schemaForLayout(layout, tableID) { + if name == colName { + return i + offset + } + } + return -1 +} + +func catalogColIndexForLayout(layout catalogLayout, tableID uint64, colName string, offset int) int { + for i, name := range schemaForLayout(layout, tableID) { + if name == colName { + return i + offset + } + } + return -1 +} + +// ReadTableSchema reads the schema for a user table by reading mo_tables and mo_columns +// from the checkpoint. +// +// It composes the logical view at snapshotTS, reads system table data for mo_tables (table 2) +// and mo_columns (table 3), and extracts the schema for the given user tableID. +// +// IMPORTANT: LogicalTableView includes hidden columns (e.g. __mo_fake_pk_col) in its data +// rows, so we cannot use catalog index constants directly. Instead, column positions are +// resolved by name from the view headers. +// +// If the schema cannot be resolved from checkpoint catalog metadata, the returned schema +// will not contain visible columns. Callers that need an exact user-facing schema must +// treat that as an error instead of falling back to raw physical object columns. +func (r *CheckpointReader) ReadTableSchema( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, + dataView *LogicalTableView, +) *TableSchema { + _ = dataView + schema := &TableSchema{TableName: fmt.Sprintf("%d", tableID)} + + // Try to read mo_tables from the checkpoint (table 2) + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil { + // Don't log - expected for system tables or when mo_tables not in checkpoint + } else if moTablesView != nil { + if moTablesSchema := findTableSchemaFromMoTables(moTablesView, tableID); moTablesSchema != nil { + schema = moTablesSchema + } + } + + // Try to read mo_columns from the checkpoint (table 3) + moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) + if err == nil && moColumnsView != nil { + cols := buildColumnsFromMoColumnsRows(moColumnsView, tableID) + if len(cols) > 0 { + schema.Columns = cols + } + schema.ClusterBy = buildClusterByFromMoColumnsRows(moColumnsView, tableID) + if moTablesView != nil { + schema.ForeignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) + } + } + + if len(schema.Columns) == 0 { + layout := inferBuiltinCatalogLayout(tableID, moTablesView, moColumnsView) + schema = mergeBuiltinSchemaFallback(schema, builtinTableSchemaForLayout(layout, tableID), tableID) + } + if schema.Partition == "" { + if partitionClause := r.readPartitionClause(ctx, tableID, snapshotTS, schema.AccountID); partitionClause != "" { + schema.Partition = partitionClause + } + } + + return schema +} + +// getTableLogicalView composes the checkpoint view at snapshotTS and builds a logical +// table view for the given tableID by aggregating data/tomb entries across all +// relevant checkpoint entries (GCKP + ICKPs). +func (r *CheckpointReader) getTableLogicalView( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (*LogicalTableView, error) { + allData, allTomb, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + if err != nil { + return nil, err + } + view, err := r.BuildLogicalTableView(ctx, snapshotTS, allData, allTomb) + if err != nil { + return nil, err + } + applyCatalogColumnHeaders(view, tableID) + return view, nil +} + +func applyCatalogColumnHeaders(view *LogicalTableView, tableID uint64) { + if view == nil || len(view.ColSeqNums) == 0 { + return + } + schema := schemaForLayout(currentCatalogLayout, tableID) + if len(schema) == 0 { + return + } + offset := logicalViewDataOffset(view) + for dataIdx, seqNum := range view.ColSeqNums { + headerIdx := offset + dataIdx + if headerIdx >= len(view.Headers) || int(seqNum) >= len(schema) { + continue + } + view.Headers[headerIdx] = schema[seqNum] + } +} + +func (r *CheckpointReader) getTableEntriesAt( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { + composed, err := r.ComposeAt(snapshotTS) + if err != nil { + return nil, nil, err + } + + tbl, ok := composed.Tables[tableID] + if !ok || (len(tbl.DataRanges) == 0 && len(tbl.TombRanges) == 0) { + return nil, nil, moerr.NewInternalErrorf(ctx, "table %d not found in checkpoint at ts %s", tableID, snapshotTS.ToString()) + } + + type entryRef struct { + entry *EntryInfo + } + entryRefs := make([]entryRef, 0, 1+len(composed.Incrementals)) + if composed.BaseEntry != nil { + entryRefs = append(entryRefs, entryRef{entry: composed.BaseEntry}) + } + for _, incr := range composed.Incrementals { + entryRefs = append(entryRefs, entryRef{entry: incr}) + } + + var allData, allTomb []*ObjectEntryInfo + for _, ref := range entryRefs { + e := r.entries[ref.entry.Index] + dataEntries, tombEntries, err := r.GetObjectEntries(e, tableID) + if err != nil { + if isDataFileNotFound(err) { + continue + } + return nil, nil, err + } + allData = append(allData, dataEntries...) + allTomb = append(allTomb, tombEntries...) + } + + if len(allData) == 0 { + return nil, nil, moerr.NewInternalErrorf(ctx, "no data entries for table %d", tableID) + } + return allData, allTomb, nil +} + +// DumpTableCSV reads a user table's logical view and writes it as CSV to w. +// +// It first resolves the table schema from mo_tables/mo_columns, then writes the +// CSV with proper column headers and data rows. +func (r *CheckpointReader) DumpTableCSV( + ctx context.Context, + w io.Writer, + tableID uint64, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, + opts ...CSVExportOption, +) error { + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + return r.streamTableCSV(ctx, w, schema, snapshotTS, dataEntries, tombEntries, resolveCSVExportOptions(opts)) +} + +// DumpTableCSVComposed dumps a table to CSV by composing the full checkpoint view +// at snapshotTS (aggregating across GCKP + all ICKPs). This is equivalent to calling +// DumpTableCSV with entries from ComposeAt. +func (r *CheckpointReader) DumpTableCSVComposed( + ctx context.Context, + w io.Writer, + tableID uint64, + snapshotTS types.TS, + opts ...CSVExportOption, +) error { + data, err := r.PrepareTableDumpData(ctx, tableID, snapshotTS) + if err != nil { + return err + } + return r.DumpPreparedTableCSV(ctx, w, data, snapshotTS, opts...) +} + +// PrepareTableDumpData resolves the table schema and object lists once so batch +// dumps can avoid repeatedly composing checkpoint metadata per table worker. +func (r *CheckpointReader) PrepareTableDumpData( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (*TableDumpData, error) { + dataEntries, tombEntries, err := r.getTableEntriesAt(ctx, tableID, snapshotTS) + if err != nil { + if !isTableDataUnavailable(err) { + return nil, err + } + dataEntries = nil + tombEntries = nil + } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + partitionTableIDs, err := r.readPartitionTableIDs(ctx, snapshotTS, tableID, schema.AccountID) + if err != nil { + return nil, err + } + for _, partitionTableID := range partitionTableIDs { + partitionData, partitionTomb, err := r.getTableEntriesAt(ctx, partitionTableID, snapshotTS) + if err != nil { + if !isTableDataUnavailable(err) { + return nil, err + } + ckpDebugSchemaf("partition table data unavailable primary=%d partition=%d err=%v", tableID, partitionTableID, err) + continue + } + dataEntries = append(dataEntries, partitionData...) + tombEntries = append(tombEntries, partitionTomb...) + } + if len(partitionTableIDs) > 0 { + ckpDebugSchemaf("partition table dump data resolved primary=%d partitions=%v data_entries=%d tomb_entries=%d", tableID, partitionTableIDs, len(dataEntries), len(tombEntries)) + } + return &TableDumpData{ + TableID: tableID, + Schema: cloneTableSchema(schema), + DataEntries: dataEntries, + TombEntries: tombEntries, + }, nil +} + +// PrepareTableDumpDataForTables resolves schemas and object lists for multiple +// tables by composing the checkpoint once and scanning each selected checkpoint +// entry once. +func (r *CheckpointReader) PrepareTableDumpDataForTables( + ctx context.Context, + tableIDs []uint64, + snapshotTS types.TS, +) (map[uint64]*TableDumpData, error) { + composed, err := r.ComposeAt(snapshotTS) + if err != nil { + return nil, err + } + + tableSet := make(map[uint64]struct{}, len(tableIDs)) + accountByPrimary := make(map[uint64]uint32, len(tableIDs)) + result := make(map[uint64]*TableDumpData, len(tableIDs)) + partitionToPrimary := make(map[uint64]uint64) + partitionIDsByPrimary := make(map[uint64][]uint64) + for _, tableID := range tableIDs { + if _, ok := tableSet[tableID]; ok { + continue + } + schema := r.ReadTableSchema(ctx, tableID, snapshotTS, nil) + if len(schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf( + ctx, + "cannot resolve visible columns for table %d from checkpoint metadata; mo_columns data is unavailable or incomplete", + tableID, + ) + } + tableSet[tableID] = struct{}{} + accountByPrimary[tableID] = schema.AccountID + result[tableID] = &TableDumpData{ + TableID: tableID, + Schema: cloneTableSchema(schema), + } + } + partitionTableMap, err := r.readPartitionTableIDsForPrimaries(ctx, snapshotTS, tableSet, accountByPrimary) + if err != nil { + return nil, err + } + for primaryID, partitionIDs := range partitionTableMap { + for _, partitionID := range partitionIDs { + tableSet[partitionID] = struct{}{} + partitionToPrimary[partitionID] = primaryID + } + partitionIDsByPrimary[primaryID] = append(partitionIDsByPrimary[primaryID], partitionIDs...) + } + + entryRefs := make([]*EntryInfo, 0, len(composed.Incrementals)+1) + if composed.BaseEntry != nil { + entryRefs = append(entryRefs, composed.BaseEntry) + } + entryRefs = append(entryRefs, composed.Incrementals...) + + for _, ref := range entryRefs { + e := r.entries[ref.Index] + dataByTable, tombByTable, err := r.GetObjectEntriesForTables(e, tableSet) + if err != nil { + if isDataFileNotFound(err) { + continue + } + return nil, err + } + for tableID, entries := range dataByTable { + targetID := tableID + if primaryID, ok := partitionToPrimary[tableID]; ok { + targetID = primaryID + } + if result[targetID] == nil { + continue + } + result[targetID].DataEntries = append(result[targetID].DataEntries, entries...) + } + for tableID, entries := range tombByTable { + targetID := tableID + if primaryID, ok := partitionToPrimary[tableID]; ok { + targetID = primaryID + } + if result[targetID] == nil { + continue + } + result[targetID].TombEntries = append(result[targetID].TombEntries, entries...) + } + } + + for tableID, data := range result { + if data.Schema == nil || len(data.Schema.Columns) == 0 { + return nil, moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", tableID) + } + if partitionIDs := partitionIDsByPrimary[tableID]; len(partitionIDs) > 0 { + ckpDebugSchemaf("partition table dump data resolved primary=%d partitions=%v data_entries=%d tomb_entries=%d", tableID, partitionIDs, len(data.DataEntries), len(data.TombEntries)) + } + } + return result, nil +} + +func isTableDataUnavailable(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "not found in checkpoint") || + strings.Contains(msg, "no data entries for table") +} + +// DumpPreparedTableCSV writes a table using metadata previously resolved by +// PrepareTableDumpData. +func (r *CheckpointReader) DumpPreparedTableCSV( + ctx context.Context, + w io.Writer, + data *TableDumpData, + snapshotTS types.TS, + opts ...CSVExportOption, +) error { + if data == nil { + return moerr.NewInternalError(ctx, "missing prepared table dump data") + } + if data.Schema == nil || len(data.Schema.Columns) == 0 { + return moerr.NewInternalErrorf(ctx, "cannot resolve visible columns for table %d from checkpoint metadata", data.TableID) + } + return r.streamTableCSV(ctx, w, data.Schema, snapshotTS, data.DataEntries, data.TombEntries, resolveCSVExportOptions(opts)) +} + +func cloneTableSchema(schema *TableSchema) *TableSchema { + if schema == nil { + return nil + } + clone := &TableSchema{ + TableName: strings.Clone(schema.TableName), + DatabaseName: strings.Clone(schema.DatabaseName), + AccountID: schema.AccountID, + CreateSQL: strings.Clone(schema.CreateSQL), + Comment: strings.Clone(schema.Comment), + Partition: strings.Clone(schema.Partition), + } + if len(schema.PrimaryKey) > 0 { + clone.PrimaryKey = make([]string, len(schema.PrimaryKey)) + for i, col := range schema.PrimaryKey { + clone.PrimaryKey[i] = strings.Clone(col) + } + } + if len(schema.ClusterBy) > 0 { + clone.ClusterBy = make([]string, len(schema.ClusterBy)) + for i, col := range schema.ClusterBy { + clone.ClusterBy[i] = strings.Clone(col) + } + } + if len(schema.Columns) > 0 { + clone.Columns = make([]TableColumn, len(schema.Columns)) + for i, col := range schema.Columns { + clone.Columns[i] = TableColumn{ + Name: strings.Clone(col.Name), + SQLType: strings.Clone(col.SQLType), + Position: col.Position, + PhysicalPosition: col.PhysicalPosition, + Unsigned: col.Unsigned, + NotNull: col.NotNull, + Default: strings.Clone(col.Default), + HasDefault: col.HasDefault, + OnUpdate: strings.Clone(col.OnUpdate), + Generated: strings.Clone(col.Generated), + GeneratedStored: col.GeneratedStored, + Comment: strings.Clone(col.Comment), + ConstraintType: strings.Clone(col.ConstraintType), + AutoIncrement: col.AutoIncrement, + ClusterBy: col.ClusterBy, + EnumValues: strings.Clone(col.EnumValues), + } + } + } + if len(schema.UniqueKeys) > 0 { + clone.UniqueKeys = make([]TableUniqueKey, len(schema.UniqueKeys)) + for i, key := range schema.UniqueKeys { + clone.UniqueKeys[i].Name = strings.Clone(key.Name) + clone.UniqueKeys[i].Unique = key.Unique + clone.UniqueKeys[i].Algo = strings.Clone(key.Algo) + clone.UniqueKeys[i].AlgoParams = strings.Clone(key.AlgoParams) + clone.UniqueKeys[i].Comment = strings.Clone(key.Comment) + if len(key.Columns) > 0 { + clone.UniqueKeys[i].Columns = make([]string, len(key.Columns)) + for j, col := range key.Columns { + clone.UniqueKeys[i].Columns[j] = strings.Clone(col) + } + } + } + } + if len(schema.ForeignKeys) > 0 { + clone.ForeignKeys = make([]TableForeignKey, len(schema.ForeignKeys)) + for i, key := range schema.ForeignKeys { + clone.ForeignKeys[i] = TableForeignKey{ + Name: strings.Clone(key.Name), + ReferDatabase: strings.Clone(key.ReferDatabase), + ReferTable: strings.Clone(key.ReferTable), + OnDelete: key.OnDelete, + OnUpdate: key.OnUpdate, + } + if len(key.Columns) > 0 { + clone.ForeignKeys[i].Columns = make([]string, len(key.Columns)) + for j, col := range key.Columns { + clone.ForeignKeys[i].Columns[j] = strings.Clone(col) + } + } + if len(key.ReferColumns) > 0 { + clone.ForeignKeys[i].ReferColumns = make([]string, len(key.ReferColumns)) + for j, col := range key.ReferColumns { + clone.ForeignKeys[i].ReferColumns[j] = strings.Clone(col) + } + } + } + } + return clone +} + +func (r *CheckpointReader) ListCatalogTables( + ctx context.Context, + snapshotTS types.TS, + opts TableListOptions, +) ([]TableCatalogEntry, error) { + tables, projectedErr := r.listCatalogTablesFromProjectedMoTables(ctx, snapshotTS) + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil && projectedErr != nil { + return nil, err + } + if projectedErr != nil || len(tables) == 0 { + tables = nil + } + if moTablesView != nil { + tables = mergeCatalogTableEntries(tables, buildCatalogTablesFromMoTablesRows(moTablesView)) + if schema := r.ReadTableSchema(ctx, moTablesID, snapshotTS, moTablesView); len(schema.Columns) > 0 { + projected := MergeLogicalViewWithSchema(moTablesView, schema) + tables = mergeCatalogTableEntries(tables, buildCatalogTablesFromMoTablesRows(projected)) + } + } + return filterCatalogTablesForList(tables, opts), nil +} + +func (r *CheckpointReader) ListCatalogDatabases( + ctx context.Context, + snapshotTS types.TS, + opts TableListOptions, +) ([]TableCatalogEntry, error) { + moDatabaseView, err := r.getTableLogicalView(ctx, moDatabaseID, snapshotTS) + + // Primary path: read databases directly from mo_database. + // This covers databases that have no tables (e.g. empty subscription databases), + // which the mo_tables-based fallback would miss. + if err == nil && moDatabaseView != nil { + databases := buildCatalogDatabasesFromMoDatabaseRows(moDatabaseView) + if schema := r.ReadTableSchema(ctx, moDatabaseID, snapshotTS, moDatabaseView); len(schema.Columns) > 0 { + projected := MergeLogicalViewWithSchema(moDatabaseView, schema) + databases = mergeCatalogDatabaseEntries(databases, buildCatalogDatabasesFromMoDatabaseRows(projected)) + } + // Merge databases derived from mo_tables to fill any gaps that + // mo_database alone may not cover (e.g. when mo_database checkpoint + // data is incomplete). + if moTablesDBs, tablesErr := r.listDatabasesFromMoTables(ctx, snapshotTS); tablesErr == nil { + databases = mergeCatalogDatabaseEntries(databases, moTablesDBs) + } + return filterCatalogTablesForList(databases, opts), nil + } + + // Fallback: when mo_database is not available in the checkpoint (e.g. + // the table has not been flushed to a checkpoint entry yet), derive + // databases from mo_tables. This is the pre-mo_database behavior and + // works whenever mo_tables checkpoint data exists. + databases, tablesErr := r.listDatabasesFromMoTables(ctx, snapshotTS) + if tablesErr != nil { + return nil, err // return the original mo_database error, not the mo_tables one + } + return filterCatalogTablesForList(databases, opts), nil +} + +// listDatabasesFromMoTables derives database catalog entries from mo_tables +// rows. It returns database-level entries (account_id, database_id, database_name) +// with empty TableName and zero TableID. +func (r *CheckpointReader) listDatabasesFromMoTables( + ctx context.Context, + snapshotTS types.TS, +) ([]TableCatalogEntry, error) { + tables, err := r.ListCatalogTables(ctx, snapshotTS, TableListOptions{}) + if err != nil { + return nil, err + } + seen := make(map[string]struct{}) + databases := make([]TableCatalogEntry, 0, len(tables)) + for _, t := range tables { + key := fmt.Sprintf("%d\x00%d\x00%s", t.AccountID, t.DatabaseID, strings.ToLower(t.DatabaseName)) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + if !validCatalogName(t.DatabaseName) { + continue + } + databases = append(databases, TableCatalogEntry{ + AccountID: t.AccountID, + DatabaseID: t.DatabaseID, + DatabaseName: t.DatabaseName, + }) + } + return databases, nil +} + +func filterCatalogTablesForList(tables []TableCatalogEntry, opts TableListOptions) []TableCatalogEntry { + filtered := make([]TableCatalogEntry, 0, len(tables)) + for _, table := range tables { + if opts.AccountID != nil && table.AccountID != *opts.AccountID { + continue + } + if opts.DatabaseID != nil && table.DatabaseID != *opts.DatabaseID { + continue + } + filtered = append(filtered, table) + } + sort.Slice(filtered, func(i, j int) bool { + if filtered[i].AccountID != filtered[j].AccountID { + return filtered[i].AccountID < filtered[j].AccountID + } + if filtered[i].DatabaseName != filtered[j].DatabaseName { + return filtered[i].DatabaseName < filtered[j].DatabaseName + } + if filtered[i].TableName != filtered[j].TableName { + return filtered[i].TableName < filtered[j].TableName + } + return filtered[i].TableID < filtered[j].TableID + }) + return filtered +} + +func buildCatalogDatabasesFromMoDatabaseRows(view *LogicalTableView) []TableCatalogEntry { + seen := make(map[string]struct{}) + merged := make([]TableCatalogEntry, 0) + try := func(datIDCol, datNameCol, accountIDCol int) { + databases := buildCatalogDatabasesFromMoDatabaseRowsAt(view, datIDCol, datNameCol, accountIDCol) + for _, database := range databases { + key := catalogDatabaseEntryKey(database) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + merged = append(merged, database) + } + } + + try( + fallbackCatalogColIndex(view, moDatabaseID, "dat_id"), + fallbackCatalogColIndex(view, moDatabaseID, "datname"), + fallbackCatalogColIndex(view, moDatabaseID, "account_id"), + ) + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moDatabaseID) { + try( + catalogColIndexForLayout(match.layout, moDatabaseID, "dat_id", match.offset), + catalogColIndexForLayout(match.layout, moDatabaseID, "datname", match.offset), + catalogColIndexForLayout(match.layout, moDatabaseID, "account_id", match.offset), + ) + } + return merged +} + +func buildCatalogDatabasesFromMoDatabaseRowsAt( + view *LogicalTableView, + datIDCol int, + datNameCol int, + accountIDCol int, +) []TableCatalogEntry { + if datIDCol < 0 || datNameCol < 0 { + return nil + } + + seen := make(map[string]struct{}) + databases := make([]TableCatalogEntry, 0) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if datIDCol >= len(row) || datNameCol >= len(row) { + continue + } + databaseID, err := strconv.ParseUint(row[datIDCol], 10, 64) + if err != nil || databaseID == 0 { + continue + } + entry := TableCatalogEntry{ + AccountID: 0, + DatabaseID: databaseID, + DatabaseName: strings.Clone(row[datNameCol]), + } + if !validCatalogName(entry.DatabaseName) { + continue + } + if accountIDCol >= 0 && accountIDCol < len(row) { + if accountID, err := strconv.ParseUint(row[accountIDCol], 10, 32); err == nil { + entry.AccountID = uint32(accountID) + } + } + key := catalogDatabaseEntryKey(entry) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + databases = append(databases, entry) + } + return databases +} + +func mergeCatalogDatabaseEntries(base []TableCatalogEntry, extra []TableCatalogEntry) []TableCatalogEntry { + if len(extra) == 0 { + return base + } + seen := make(map[string]struct{}, len(base)+len(extra)) + for _, database := range base { + seen[catalogDatabaseEntryKey(database)] = struct{}{} + } + for _, database := range extra { + key := catalogDatabaseEntryKey(database) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + base = append(base, database) + } + return base +} + +func catalogDatabaseEntryKey(entry TableCatalogEntry) string { + return fmt.Sprintf("%d\x00%d\x00%s", entry.AccountID, entry.DatabaseID, strings.ToLower(entry.DatabaseName)) +} + +func (r *CheckpointReader) listCatalogTablesFromProjectedMoTables( + ctx context.Context, + snapshotTS types.TS, +) ([]TableCatalogEntry, error) { + var buf bytes.Buffer + if err := r.DumpTableCSVComposed(ctx, &buf, moTablesID, snapshotTS, WithCSVHeader(true), WithCSVMetaComments(false)); err != nil { + return nil, err + } + reader := csv.NewReader(bytes.NewReader(buf.Bytes())) + reader.FieldsPerRecord = -1 + records, err := reader.ReadAll() + if err != nil { + return nil, err + } + if len(records) == 0 { + return nil, nil + } + return buildCatalogTablesFromCSVRecords(records[0], records[1:]), nil +} + +func buildCatalogTablesFromCSVRecords(header []string, rows [][]string) []TableCatalogEntry { + index := make(map[string]int, len(header)) + for i, name := range header { + index[name] = i + } + col := func(name string) int { + if idx, ok := index[name]; ok { + return idx + } + return -1 + } + return buildCatalogTablesFromMoTablesRowsAt( + &LogicalTableView{ + Headers: header, + Rows: rows, + }, + col("rel_id"), + col("relname"), + col("reldatabase"), + col("reldatabase_id"), + col("relkind"), + col("account_id"), + ) +} + +func mergeCatalogTableEntries(base []TableCatalogEntry, extra []TableCatalogEntry) []TableCatalogEntry { + if len(extra) == 0 { + return base + } + seen := make(map[uint64]int, len(base)+len(extra)) + for i, table := range base { + seen[table.TableID] = i + } + for _, table := range extra { + if idx, ok := seen[table.TableID]; ok { + if betterCatalogTableEntry(table, base[idx]) { + base[idx] = table + } + continue + } + seen[table.TableID] = len(base) + base = append(base, table) + } + return base +} + +func betterCatalogTableEntry(candidate, current TableCatalogEntry) bool { + if !validCatalogName(current.TableName) && validCatalogName(candidate.TableName) { + return true + } + if !validCatalogName(current.DatabaseName) && validCatalogName(candidate.DatabaseName) { + return true + } + if current.DatabaseID == 0 && candidate.DatabaseID != 0 { + return true + } + if current.RelKind == "" && candidate.RelKind != "" { + return true + } + return false +} + +func (r *CheckpointReader) streamTableCSV( + ctx context.Context, + w io.Writer, + schema *TableSchema, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, + options CSVExportOptions, +) error { + if !options.IncludeMetadata && options.RowOrder == CSVRowOrderStorage { + return r.streamTableCSVPipeline(ctx, w, schema, snapshotTS, dataEntries, tombEntries, options) + } + + needsBuffer := options.IncludeMetadata || options.RowOrder == CSVRowOrderLexical + output := w + var tmpFile *os.File + if needsBuffer { + var err error + tmpFile, err = os.CreateTemp("", "mo-tool-table-dump-*.csv") + if err != nil { + return err + } + tmpName := tmpFile.Name() + defer os.Remove(tmpName) + defer tmpFile.Close() + output = tmpFile + } + + header := make([]string, 0, len(schema.Columns)) + columnSeqNums := make([]int, 0, len(schema.Columns)) + for _, col := range schema.Columns { + header = append(header, col.Name) + seqNum := col.PhysicalPosition + if seqNum < 0 { + seqNum = col.Position + } + columnSeqNums = append(columnSeqNums, seqNum) + } + physicalPositions := append([]int(nil), columnSeqNums...) + var projectedTypes []types.Type + if options.IncludeHeader { + // header is emitted after we know output mode but before rows + if err := writeSQLLoadCSVRow(output, nil, header, nil); err != nil { + return err + } + } + + var lexicalRows []exportedCSVRow + onRow := func(_ string, _ int, _ int, values []string, nulls []bool) error { + row := projectCSVRow(values, physicalPositions) + rowNulls := projectCSVNulls(nulls, physicalPositions) + if options.RowOrder == CSVRowOrderLexical { + lexicalRows = append(lexicalRows, exportedCSVRow{values: row, nulls: rowNulls}) + return nil + } + return writeSQLLoadCSVRow(output, projectedTypes, row, rowNulls) + } + + stats, err := r.scanLogicalTable(ctx, snapshotTS, dataEntries, tombEntries, + func(cols []objecttool.ColInfo) error { + physicalPositions = dataIndexesForSeqNums(cols, columnSeqNums) + projectedTypes = buildProjectedTypes(cols, physicalPositions) + return nil + }, + onRow, + ) + if err != nil { + return err + } + if options.RowOrder == CSVRowOrderLexical { + sortCSVRowsLexical(lexicalRows) + for _, row := range lexicalRows { + if err := writeSQLLoadCSVRow(output, projectedTypes, row.values, row.nulls); err != nil { + return err + } + } + } + + if !needsBuffer { + return nil + } + if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { + return err + } + if options.IncludeMetadata { + if err := writeCSVMetadata(w, schema, stats); err != nil { + return err + } + } + _, err = io.Copy(w, tmpFile) + return err +} + +type csvPipelineChunk struct { + objectIdx int + blockIdx int + rows int + data []byte + objectDone bool +} + +type csvPipelineObjectJob struct { + objectIdx int + entry *ObjectEntryInfo +} + +type csvPipelineBlock struct { + objectIdx int + blockIdx int + entry *ObjectEntryInfo + projectedTypes []types.Type + relevantTombstones []objectio.ObjectStats + bat *batch.Batch + release func() + commitTSVec *vector.Vector + releaseCommitTS func() +} + +func (b *csvPipelineBlock) releaseBlock() { + if b.releaseCommitTS != nil { + b.releaseCommitTS() + b.releaseCommitTS = nil + } + if b.release != nil { + b.release() + b.release = nil + } +} + +type csvPipelineCounters struct { + totalObjects int64 + processedObjects atomic.Int64 + readQueuedBlocks atomic.Int64 + readQueuedRows atomic.Int64 + readBatches atomic.Int64 + readRows atomic.Int64 + processedBatches atomic.Int64 + visibleRows atomic.Int64 + physicalRows atomic.Int64 + queuedBatches atomic.Int64 + queuedBytes atomic.Int64 + writtenBatches atomic.Int64 + writtenBytes atomic.Int64 + memoryWaits atomic.Int64 + memoryAvailable atomic.Int64 + memoryFloor atomic.Int64 + readNanos atomic.Int64 + processNanos atomic.Int64 + writeNanos atomic.Int64 +} + +type csvPipelineWorkerPlan struct { + readerWorkers int + processorWorkers int + readQueueCapacity int + cpuLimit int + memoryLimit int + memoryTotal uint64 + memoryFree uint64 + memoryFloor uint64 + memoryPerWork uint64 +} + +func (r *CheckpointReader) streamTableCSVPipeline( + ctx context.Context, + w io.Writer, + schema *TableSchema, + snapshotTS types.TS, + dataEntries, tombEntries []*ObjectEntryInfo, + options CSVExportOptions, +) error { + header := make([]string, 0, len(schema.Columns)) + columnSeqNums := make([]int, 0, len(schema.Columns)) + for _, col := range schema.Columns { + header = append(header, col.Name) + seqNum := col.PhysicalPosition + if seqNum < 0 { + seqNum = col.Position + } + columnSeqNums = append(columnSeqNums, seqNum) + } + if options.IncludeHeader { + if err := writeSQLLoadCSVRow(w, nil, header, nil); err != nil { + return err + } + } + + visibleDataEntries := visibleObjectEntries(dataEntries, snapshotTS) + visibleTombEntries := visibleObjectEntries(tombEntries, snapshotTS) + tombstoneStats := dedupeObjectStats(visibleTombEntries) + counters := &csvPipelineCounters{totalObjects: int64(len(visibleDataEntries))} + workerPlan := csvPipelineWorkerCount(len(visibleDataEntries)) + counters.memoryFloor.Store(int64(workerPlan.memoryFloor)) + counters.memoryAvailable.Store(int64(workerPlan.memoryFree)) + + fmt.Fprintf(os.Stderr, + "csv pipeline: start objects=%d reader_mode=in_processor processor_workers=%d writer_workers=1 cpu_limit=%d memory_limit=%d memory_total=%d memory_free=%d memory_floor=%d memory_per_worker=%d write_queue_capacity=%d report_every=%s\n", + len(visibleDataEntries), + workerPlan.processorWorkers, + workerPlan.cpuLimit, + workerPlan.memoryLimit, + workerPlan.memoryTotal, + workerPlan.memoryFree, + workerPlan.memoryFloor, + workerPlan.memoryPerWork, + csvPipelineQueueCapacity, + csvPipelineReportEvery, + ) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + chunks := make(chan csvPipelineChunk, csvPipelineQueueCapacity) + writerDone := make(chan error, 1) + reportDone := make(chan struct{}) + defer close(reportDone) + go reportCSVPipeline(ctx, reportDone, counters) + go writeCSVChunks(ctx, w, chunks, counters, cancel, writerDone) + + producerErr := r.produceCSVChunks(ctx, chunks, counters, snapshotTS, visibleDataEntries, tombstoneStats, columnSeqNums, workerPlan) + close(chunks) + writerErr := <-writerDone + printCSVPipelineReport("finish", counters) + return csvPipelineError(producerErr, writerErr) +} + +func csvPipelineError(producerErr, writerErr error) error { + switch { + case producerErr == nil: + return writerErr + case writerErr == nil: + return producerErr + case errors.Is(producerErr, context.Canceled) && !errors.Is(writerErr, context.Canceled): + return writerErr + case errors.Is(writerErr, context.Canceled) && !errors.Is(producerErr, context.Canceled): + return producerErr + default: + return errors.Join(producerErr, writerErr) + } +} + +func (r *CheckpointReader) produceCSVChunks( + ctx context.Context, + chunks chan<- csvPipelineChunk, + counters *csvPipelineCounters, + snapshotTS types.TS, + visibleDataEntries []*ObjectEntryInfo, + tombstoneStats []objectio.ObjectStats, + columnSeqNums []int, + workerPlan csvPipelineWorkerPlan, +) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + objectJobs := make(chan csvPipelineObjectJob, workerPlan.processorWorkers) + var processorWG sync.WaitGroup + var errOnce sync.Once + var workerErr error + setErr := func(err error) { + if err == nil { + return + } + errOnce.Do(func() { + workerErr = err + cancel() + }) + } + + processorWG.Add(workerPlan.processorWorkers) + for workerID := 0; workerID < workerPlan.processorWorkers; workerID++ { + go func() { + defer processorWG.Done() + for job := range objectJobs { + if err := ctx.Err(); err != nil { + setErr(err) + return + } + if err := r.processCSVObjectChunks(ctx, chunks, counters, snapshotTS, tombstoneStats, columnSeqNums, job); err != nil { + setErr(err) + return + } + } + }() + } + + for objectIdx, entry := range visibleDataEntries { + select { + case objectJobs <- csvPipelineObjectJob{objectIdx: objectIdx, entry: entry}: + case <-ctx.Done(): + close(objectJobs) + processorWG.Wait() + if workerErr != nil { + return workerErr + } + return ctx.Err() + } + } + close(objectJobs) + processorWG.Wait() + if workerErr != nil { + return workerErr + } + return ctx.Err() +} + +func (r *CheckpointReader) processCSVObjectChunks( + ctx context.Context, + chunks chan<- csvPipelineChunk, + counters *csvPipelineCounters, + snapshotTS types.TS, + tombstoneStats []objectio.ObjectStats, + columnSeqNums []int, + job csvPipelineObjectJob, +) error { + sendChunk := func(chunk csvPipelineChunk) error { + select { + case chunks <- chunk: + return nil + case <-ctx.Done(): + if len(chunk.data) > 0 { + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + } + return ctx.Err() + } + } + + entry := job.entry + if err := ctx.Err(); err != nil { + return err + } + objName := entry.ObjectStats.ObjectName().String() + reader, err := objecttool.OpenWithFS(ctx, r.fs, objName, objName) + if err != nil { + if isDataFileNotFound(err) { + counters.processedObjects.Add(1) + return sendChunk(csvPipelineChunk{objectIdx: job.objectIdx, objectDone: true}) + } + return err + } + defer reader.Close() + + physicalPositions := dataIndexesForSeqNums(reader.Columns(), columnSeqNums) + projectedTypes := buildProjectedTypes(reader.Columns(), physicalPositions) + relevantTombstones, err := r.filterTombstonesForObject(ctx, entry.ObjectStats.ObjectName().ObjectId(), tombstoneStats) + if err != nil { + return err + } + + for blockIdx := 0; blockIdx < int(entry.ObjectStats.BlkCnt()); blockIdx++ { + if err := ctx.Err(); err != nil { + return err + } + if err := waitForCSVMemory(ctx, counters); err != nil { + return err + } + + start := time.Now() + bat, release, err := reader.ReadBlock(ctx, uint32(blockIdx)) + if err != nil { + return err + } + commitTSVec, releaseCommitTS, err := reader.ReadBlockCommitTS(ctx, uint32(blockIdx)) + counters.readNanos.Add(time.Since(start).Nanoseconds()) + if err != nil { + release() + return err + } + if bat.RowCount() == 0 { + if releaseCommitTS != nil { + releaseCommitTS() + } + release() + continue + } + + block := csvPipelineBlock{ + objectIdx: job.objectIdx, + blockIdx: blockIdx, + entry: entry, + projectedTypes: projectedTypes, + relevantTombstones: relevantTombstones, + bat: bat, + release: release, + commitTSVec: commitTSVec, + releaseCommitTS: releaseCommitTS, + } + counters.readBatches.Add(1) + counters.readRows.Add(int64(bat.RowCount())) + chunk, err := r.buildCSVChunkForBlock(ctx, block, snapshotTS, physicalPositions, counters) + if err != nil { + return err + } + if len(chunk.data) == 0 { + continue + } + counters.queuedBatches.Add(1) + counters.queuedBytes.Add(int64(len(chunk.data))) + if err := sendChunk(chunk); err != nil { + return err + } + } + + counters.processedObjects.Add(1) + return sendChunk(csvPipelineChunk{objectIdx: job.objectIdx, objectDone: true}) +} + +func csvPipelineWorkerCount(objects int) csvPipelineWorkerPlan { + cpuLimit := runtime.GOMAXPROCS(0) + if cpuLimit < 1 { + cpuLimit = 1 + } + + total, free, ok := readSystemMemory() + floor := csvPipelineMemoryFloorFromTotal(total, ok) + memoryLimit := cpuLimit + if ok { + memoryLimit = 1 + if free > floor { + budget := free - floor + memoryLimit = int(budget / csvPipelineWorkerMemory) + if memoryLimit < 1 { + memoryLimit = 1 + } + } + } + processorWorkers := cpuLimit + if memoryLimit < processorWorkers { + processorWorkers = memoryLimit + } + if objects > 0 && objects < processorWorkers { + processorWorkers = objects + } + if processorWorkers < 1 { + processorWorkers = 1 + } + + readerWorkers := csvPipelineReaderMax + if objects > 0 && objects < readerWorkers { + readerWorkers = objects + } + if processorWorkers < readerWorkers { + readerWorkers = processorWorkers + } + if readerWorkers < 1 { + readerWorkers = 1 + } + readQueueCapacity := readerWorkers * 2 + if processorWorkers < readQueueCapacity { + readQueueCapacity = processorWorkers + } + if readQueueCapacity < 1 { + readQueueCapacity = 1 + } + return csvPipelineWorkerPlan{ + readerWorkers: readerWorkers, + processorWorkers: processorWorkers, + readQueueCapacity: readQueueCapacity, + cpuLimit: cpuLimit, + memoryLimit: memoryLimit, + memoryTotal: total, + memoryFree: free, + memoryFloor: floor, + memoryPerWork: csvPipelineWorkerMemory, + } +} + +func csvPipelineMemoryFloorFromTotal(total uint64, ok bool) uint64 { + if !ok || total == 0 { + return csvPipelineMinFreeMemory + } + floor := total / csvPipelineFreeRatio + if floor < csvPipelineMinFreeMemory { + return csvPipelineMinFreeMemory + } + return floor +} + +func waitForCSVMemory(ctx context.Context, counters *csvPipelineCounters) error { + floor := uint64(counters.memoryFloor.Load()) + for { + _, available, ok := readSystemMemory() + if !ok { + return nil + } + counters.memoryAvailable.Store(int64(available)) + if available >= floor { + return nil + } + counters.memoryWaits.Add(1) + timer := time.NewTimer(csvPipelineMemoryPoll) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } +} + +func readSystemMemory() (total uint64, available uint64, ok bool) { + data, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, 0, false + } + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + value, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + continue + } + bytes := value * 1024 + switch fields[0] { + case "MemTotal:": + total = bytes + case "MemAvailable:": + available = bytes + } + } + return total, available, total > 0 && available > 0 +} + +func (r *CheckpointReader) buildCSVChunkForBlock( + ctx context.Context, + block csvPipelineBlock, + snapshotTS types.TS, + physicalPositions []int, + counters *csvPipelineCounters, +) (csvPipelineChunk, error) { + start := time.Now() + defer func() { + counters.processNanos.Add(time.Since(start).Nanoseconds()) + }() + defer block.releaseBlock() + + bat := block.bat + if bat.RowCount() == 0 { + return csvPipelineChunk{}, nil + } + counters.physicalRows.Add(int64(bat.RowCount())) + + var commitTSs []types.TS + if block.commitTSVec != nil { + commitTSs = vector.MustFixedColWithTypeCheck[types.TS](block.commitTSVec) + } + + deleteMask, err := r.buildDeleteMaskForBlock(ctx, &snapshotTS, block.entry.ObjectStats, uint16(block.blockIdx), block.relevantTombstones) + if err != nil { + return csvPipelineChunk{}, err + } + if deleteMask.IsValid() { + defer deleteMask.Release() + } + + var buf bytes.Buffer + rows := 0 + for rowIdx := 0; rowIdx < bat.RowCount(); rowIdx++ { + if commitTSs != nil && rowIdx < len(commitTSs) && + !block.commitTSVec.IsNull(uint64(rowIdx)) && + commitTSs[rowIdx].GT(&snapshotTS) { + continue + } + if deleteMask.IsValid() && deleteMask.Contains(uint64(rowIdx)) { + continue + } + if err := writeProjectedCSVRowFromVecs(&buf, block.projectedTypes, bat.Vecs, physicalPositions, rowIdx); err != nil { + return csvPipelineChunk{}, err + } + rows++ + } + + if rows == 0 { + return csvPipelineChunk{}, nil + } + counters.processedBatches.Add(1) + counters.visibleRows.Add(int64(rows)) + return csvPipelineChunk{ + objectIdx: block.objectIdx, + blockIdx: block.blockIdx, + rows: rows, + data: buf.Bytes(), + }, nil +} + +func writeCSVChunks( + ctx context.Context, + w io.Writer, + chunks <-chan csvPipelineChunk, + counters *csvPipelineCounters, + cancel context.CancelFunc, + done chan<- error, +) { + type chunkKey struct { + objectIdx int + blockIdx int + } + pending := make(map[chunkKey]csvPipelineChunk) + objectDone := make(map[int]bool) + nextObjectIdx := 0 + nextBlockIdx := 0 + + writeChunk := func(chunk csvPipelineChunk) error { + start := time.Now() + if _, err := w.Write(chunk.data); err != nil { + cancel() + return err + } + counters.writeNanos.Add(time.Since(start).Nanoseconds()) + counters.queuedBatches.Add(-1) + counters.queuedBytes.Add(-int64(len(chunk.data))) + counters.writtenBatches.Add(1) + counters.writtenBytes.Add(int64(len(chunk.data))) + return nil + } + + nextPendingBlock := func(objectIdx, minBlockIdx int) (int, bool) { + found := false + next := 0 + for key := range pending { + if key.objectIdx != objectIdx || key.blockIdx < minBlockIdx { + continue + } + if !found || key.blockIdx < next { + found = true + next = key.blockIdx + } + } + return next, found + } + + flush := func() error { + for { + key := chunkKey{objectIdx: nextObjectIdx, blockIdx: nextBlockIdx} + chunk, ok := pending[key] + if !ok && objectDone[nextObjectIdx] { + if blockIdx, found := nextPendingBlock(nextObjectIdx, nextBlockIdx); found { + nextBlockIdx = blockIdx + key.blockIdx = blockIdx + chunk = pending[key] + ok = true + } + } + if ok { + delete(pending, key) + if err := writeChunk(chunk); err != nil { + return err + } + nextBlockIdx++ + continue + } + if objectDone[nextObjectIdx] { + delete(objectDone, nextObjectIdx) + nextObjectIdx++ + nextBlockIdx = 0 + continue + } + return nil + } + } + + for chunk := range chunks { + if err := ctx.Err(); err != nil { + done <- err + return + } + if chunk.objectDone { + objectDone[chunk.objectIdx] = true + } else { + pending[chunkKey{objectIdx: chunk.objectIdx, blockIdx: chunk.blockIdx}] = chunk + } + if err := flush(); err != nil { + done <- err + return + } + } + if err := flush(); err != nil { + done <- err + return + } + if len(pending) > 0 { + done <- moerr.NewInternalErrorNoCtxf("csv pipeline finished with %d unordered chunks still pending", len(pending)) + return + } + done <- nil +} + +func reportCSVPipeline(ctx context.Context, done <-chan struct{}, counters *csvPipelineCounters) { + ticker := time.NewTicker(csvPipelineReportEvery) + defer ticker.Stop() + for { + select { + case <-ticker.C: + printCSVPipelineReport("progress", counters) + case <-done: + return + case <-ctx.Done(): + return + } + } +} + +func printCSVPipelineReport(stage string, counters *csvPipelineCounters) { + fmt.Fprintf(os.Stderr, + "csv pipeline: %s objects=%d/%d read_batches=%d read_queue_blocks=%d read_queue_rows=%d processed_batches=%d write_queue_batches=%d write_queue_bytes=%d visible_rows=%d physical_rows=%d written_batches=%d written_bytes=%d read_ms=%d process_ms=%d write_ms=%d mem_available=%d mem_floor=%d mem_waits=%d\n", + stage, + counters.processedObjects.Load(), + counters.totalObjects, + counters.readBatches.Load(), + counters.readQueuedBlocks.Load(), + counters.readQueuedRows.Load(), + counters.processedBatches.Load(), + counters.queuedBatches.Load(), + counters.queuedBytes.Load(), + counters.visibleRows.Load(), + counters.physicalRows.Load(), + counters.writtenBatches.Load(), + counters.writtenBytes.Load(), + counters.readNanos.Load()/int64(time.Millisecond), + counters.processNanos.Load()/int64(time.Millisecond), + counters.writeNanos.Load()/int64(time.Millisecond), + counters.memoryAvailable.Load(), + counters.memoryFloor.Load(), + counters.memoryWaits.Load(), + ) +} + +// WriteCSV writes a LogicalTableView with the given schema as CSV to w. +func WriteCSV(w io.Writer, schema *TableSchema, view *LogicalTableView, opts ...CSVExportOption) error { + options := resolveCSVExportOptions(opts) + + if options.IncludeMetadata { + if err := writeCSVMetadata(w, schema, logicalTableStats{ + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + }); err != nil { + return err + } + } + + // Merge schema column names into headers + merged := MergeLogicalViewWithSchema(view, schema) + projectedTypes := make([]types.Type, len(schema.Columns)) + for i, col := range schema.Columns { + projectedTypes[i] = sqlTypeStringToType(col.SQLType) + } + if options.IncludeHeader { + if err := writeSQLLoadCSVRow(w, nil, merged.Headers, nil); err != nil { + return err + } + } + + rows := merged.Rows + if options.RowOrder == CSVRowOrderLexical { + rows = make([][]string, len(merged.Rows)) + for i, row := range merged.Rows { + rows[i] = append([]string(nil), row...) + } + sort.SliceStable(rows, func(i, j int) bool { + return compareCSVRowsLexical(rows[i], rows[j]) < 0 + }) + } + for _, row := range rows { + if err := writeSQLLoadCSVRow(w, projectedTypes, row, nil); err != nil { + return err + } + } + return nil +} + +func projectCSVRow(values []string, physicalPositions []int) []string { + row := make([]string, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(values) { + row[i] = values[pos] + } + } + return row +} + +func projectCSVNulls(values []bool, physicalPositions []int) []bool { + row := make([]bool, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(values) { + row[i] = values[pos] + } + } + return row +} + +func sortCSVRowsLexical(rows []exportedCSVRow) { + sort.SliceStable(rows, func(i, j int) bool { + return compareCSVRowsLexical(rows[i].values, rows[j].values) < 0 + }) +} + +func buildProjectedTypes(cols []objecttool.ColInfo, physicalPositions []int) []types.Type { + projected := make([]types.Type, len(physicalPositions)) + for i, pos := range physicalPositions { + if pos >= 0 && pos < len(cols) { + projected[i] = cols[pos].Type + } + } + return projected +} + +func dataIndexesForSeqNums(cols []objecttool.ColInfo, seqNums []int) []int { + indexes := make([]int, len(seqNums)) + for i, seqNum := range seqNums { + indexes[i] = seqNum + for dataIdx, col := range cols { + if int(col.SeqNum) == seqNum { + indexes[i] = dataIdx + break + } + } + } + return indexes +} + +func writeSQLLoadCSVRow(w io.Writer, colTypes []types.Type, fields []string, nulls []bool) error { + for i, field := range fields { + if i > 0 { + if _, err := io.WriteString(w, ","); err != nil { + return err + } + } + typ := types.Type{} + if i < len(colTypes) { + typ = colTypes[i] + } + isNull := i < len(nulls) && nulls[i] + if err := writeSQLLoadCSVField(w, typ, field, isNull); err != nil { + return err + } + } + _, err := io.WriteString(w, "\n") + return err +} + +func writeProjectedCSVRowFromVecs( + w *bytes.Buffer, + colTypes []types.Type, + vecs []*vector.Vector, + physicalPositions []int, + rowIdx int, +) error { + for i, pos := range physicalPositions { + if i > 0 { + if _, err := io.WriteString(w, ","); err != nil { + return err + } + } + typ := types.Type{} + if i < len(colTypes) { + typ = colTypes[i] + } + if err := writeSQLLoadCSVFieldFromVec(w, typ, vecs, pos, rowIdx); err != nil { + return err + } + } + w.WriteByte('\n') + return nil +} + +func writeSQLLoadCSVFieldFromVec( + w *bytes.Buffer, + typ types.Type, + vecs []*vector.Vector, + pos int, + rowIdx int, +) error { + if pos < 0 || pos >= len(vecs) || vecs[pos] == nil || vecs[pos].IsNull(uint64(rowIdx)) { + _, err := io.WriteString(w, `\N`) + return err + } + vec := vecs[pos] + rowIdx = vectorRowIndex(vec, rowIdx) + if shouldQuoteSQLLoadType(typ) { + w.WriteByte('"') + appendCSVQuotedVecValue(w, vec, rowIdx) + w.WriteByte('"') + return nil + } + appendCSVUnquotedVecValue(w, typ, vec, rowIdx) + return nil +} + +func vectorRowIndex(vec *vector.Vector, rowIdx int) int { + if vec.IsConst() { + return 0 + } + return rowIdx +} + +func appendCSVQuotedVecValue(w *bytes.Buffer, vec *vector.Vector, rowIdx int) { + switch vec.GetType().Oid { + case types.T_char, types.T_varchar, types.T_blob, types.T_text, + types.T_binary, types.T_varbinary, types.T_datalink: + appendEscapedSQLLoadBytes(w, vec.GetBytesAt(rowIdx), '"') + case types.T_json: + appendEscapedSQLLoadString(w, types.DecodeJson(vec.GetBytesAt(rowIdx)).String(), '"') + case types.T_bit: + appendEscapedSQLLoadBytes(w, bitValueToLoadBytes(vector.MustFixedColWithTypeCheck[uint64](vec)[rowIdx], vec.GetType().Width), '"') + default: + appendEscapedSQLLoadString(w, vecValueToString(vec, rowIdx), '"') + } +} + +func appendCSVUnquotedVecValue(w *bytes.Buffer, typ types.Type, vec *vector.Vector, rowIdx int) { + switch vec.GetType().Oid { + case types.T_bool: + if vector.MustFixedColWithTypeCheck[bool](vec)[rowIdx] { + w.WriteString("true") + } else { + w.WriteString("false") + } + case types.T_int8: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int8](vec)[rowIdx])) + case types.T_int16: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int16](vec)[rowIdx])) + case types.T_int32: + appendCSVInt(w, int64(vector.MustFixedColWithTypeCheck[int32](vec)[rowIdx])) + case types.T_int64: + appendCSVInt(w, vector.MustFixedColWithTypeCheck[int64](vec)[rowIdx]) + case types.T_uint8: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint8](vec)[rowIdx])) + case types.T_uint16: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint16](vec)[rowIdx])) + case types.T_uint32: + appendCSVUint(w, uint64(vector.MustFixedColWithTypeCheck[uint32](vec)[rowIdx])) + case types.T_uint64: + appendCSVUint(w, vector.MustFixedColWithTypeCheck[uint64](vec)[rowIdx]) + case types.T_float32: + appendCSVFloat(w, float64(vector.MustFixedColWithTypeCheck[float32](vec)[rowIdx]), 32) + case types.T_float64: + appendCSVFloat(w, vector.MustFixedColWithTypeCheck[float64](vec)[rowIdx], 64) + case types.T_decimal64: + appendDecimal64(w, vector.MustFixedColWithTypeCheck[types.Decimal64](vec)[rowIdx], vec.GetType().Scale) + case types.T_decimal128: + appendDecimal128(w, vector.MustFixedColWithTypeCheck[types.Decimal128](vec)[rowIdx], vec.GetType().Scale) + case types.T_date: + appendDate(w, vector.MustFixedColWithTypeCheck[types.Date](vec)[rowIdx]) + case types.T_time: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Time](vec)[rowIdx].String2(vec.GetType().Scale)) + case types.T_datetime: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Datetime](vec)[rowIdx].String2(vec.GetType().Scale)) + case types.T_timestamp: + w.WriteString(vector.MustFixedColWithTypeCheck[types.Timestamp](vec)[rowIdx].String2(time.Local, vec.GetType().Scale)) + default: + w.WriteString(vecValueToString(vec, rowIdx)) + } +} + +func appendEscapedSQLLoadString(w *bytes.Buffer, s string, enclosed byte) { + for i := 0; i < len(s); i++ { + switch s[i] { + case '\\': + w.WriteByte('\\') + w.WriteByte('\\') + case enclosed: + if enclosed != 0 && enclosed != '\\' { + w.WriteByte(enclosed) + w.WriteByte(enclosed) + } else { + w.WriteByte(s[i]) + } + default: + w.WriteByte(s[i]) + } + } +} + +func appendEscapedSQLLoadBytes(w *bytes.Buffer, data []byte, enclosed byte) { + for _, b := range data { + switch b { + case 0: + w.WriteByte('\\') + w.WriteByte('0') + case '\b': + w.WriteByte('\\') + w.WriteByte('b') + case '\n': + w.WriteByte('\\') + w.WriteByte('n') + case '\r': + w.WriteByte('\\') + w.WriteByte('r') + case '\t': + w.WriteByte('\\') + w.WriteByte('t') + case 0x1a: + w.WriteByte('\\') + w.WriteByte('Z') + case '\\': + w.WriteByte('\\') + w.WriteByte('\\') + case enclosed: + if enclosed != 0 && enclosed != '\\' { + w.WriteByte(enclosed) + w.WriteByte(enclosed) + } else { + w.WriteByte(b) + } + default: + w.WriteByte(b) + } + } +} + +func bitValueToLoadBytes(value uint64, width int32) []byte { + byteLen := int((width + 7) / 8) + if byteLen <= 0 { + byteLen = 1 + } + if byteLen > 8 { + byteLen = 8 + } + out := make([]byte, byteLen) + for i := byteLen - 1; i >= 0; i-- { + out[i] = byte(value) + value >>= 8 + } + return out +} + +func appendDecimal64(w *bytes.Buffer, value types.Decimal64, scale int32) { + if value.Sign() { + w.WriteByte('-') + value = value.Minus() + } + v := uint64(value) + if scale <= 0 { + appendCSVUint(w, v) + return + } + pow := uint64(1) + for i := int32(0); i < scale; i++ { + pow *= 10 + } + intPart := v / pow + fracPart := v % pow + appendCSVUint(w, intPart) + w.WriteByte('.') + var scratch [32]byte + frac := strconv.AppendUint(scratch[:0], fracPart, 10) + for i := len(frac); i < int(scale); i++ { + w.WriteByte('0') + } + w.Write(frac) +} + +func appendDecimal128(w *bytes.Buffer, value types.Decimal128, scale int32) { + if value.Sign() { + w.WriteByte('-') + value = value.Minus() + } + var scratch [80]byte + i := len(scratch) + ten := types.Decimal128{B0_63: types.Pow10[1]} + one := types.Decimal128{B0_63: 1} + for value.B64_127 != 0 || value.B0_63 != 0 { + digit, _ := value.Mod128(ten) + i-- + scratch[i] = byte(digit.B0_63) + '0' + value, _ = value.Div128(ten) + if digit.B0_63 >= 5 { + value, _ = value.Sub128(one) + } + scale-- + if scale == 0 { + i-- + scratch[i] = '.' + } + } + for scale > 0 { + i-- + scratch[i] = '0' + scale-- + if scale == 0 { + i-- + scratch[i] = '.' + } + } + if scale == 0 { + i-- + scratch[i] = '0' + } + w.Write(scratch[i:]) +} + +func appendDate(w *bytes.Buffer, value types.Date) { + year, month, day, _ := value.Calendar(true) + appendZeroPaddedInt(w, int64(year), 4) + w.WriteByte('-') + appendZeroPaddedInt(w, int64(month), 2) + w.WriteByte('-') + appendZeroPaddedInt(w, int64(day), 2) +} + +func appendZeroPaddedInt(w *bytes.Buffer, value int64, width int) { + var scratch [32]byte + buf := strconv.AppendInt(scratch[:0], value, 10) + for i := len(buf); i < width; i++ { + w.WriteByte('0') + } + w.Write(buf) +} + +func appendCSVInt(w *bytes.Buffer, value int64) { + var scratch [32]byte + w.Write(strconv.AppendInt(scratch[:0], value, 10)) +} + +func appendCSVUint(w *bytes.Buffer, value uint64) { + var scratch [32]byte + w.Write(strconv.AppendUint(scratch[:0], value, 10)) +} + +func appendCSVFloat(w *bytes.Buffer, value float64, bitSize int) { + var scratch [32]byte + w.Write(strconv.AppendFloat(scratch[:0], value, 'g', -1, bitSize)) +} + +func writeSQLLoadCSVField(w io.Writer, typ types.Type, field string, isNull bool) error { + if isNull { + _, err := io.WriteString(w, `\N`) + return err + } + + if shouldQuoteSQLLoadType(typ) { + if _, err := io.WriteString(w, `"`); err != nil { + return err + } + escaped := escapeSQLLoadString(field, '"') + if _, err := io.WriteString(w, escaped); err != nil { + return err + } + _, err := io.WriteString(w, `"`) + return err + } + + _, err := io.WriteString(w, field) + return err +} + +func escapeSQLLoadString(s string, enclosed byte) string { + s = strings.ReplaceAll(s, `\`, `\\`) + if enclosed != 0 && enclosed != '\\' { + s = strings.ReplaceAll(s, string(enclosed), string([]byte{enclosed, enclosed})) + } + return s +} + +func shouldQuoteSQLLoadType(typ types.Type) bool { + switch typ.Oid { + case types.T_char, types.T_varchar, types.T_blob, types.T_text, + types.T_binary, types.T_varbinary, types.T_datalink, types.T_json, + types.T_geometry, types.T_array_float32, types.T_array_float64, types.T_bit: + return true + default: + return false + } +} + +func sqlTypeStringToType(sqlType string) types.Type { + s := strings.ToUpper(strings.TrimSpace(sqlType)) + switch { + case strings.Contains(s, "CHAR"), strings.Contains(s, "TEXT"), strings.Contains(s, "BLOB"), strings.Contains(s, "DATALINK"): + return types.T_varchar.ToType() + case strings.Contains(s, "JSON"): + return types.T_json.ToType() + case strings.Contains(s, "GEOMETRY"): + return types.T_geometry.ToType() + case strings.Contains(s, "DECIMAL"): + return types.New(types.T_decimal128, 0, 0) + case strings.Contains(s, "TIMESTAMP"): + return types.T_timestamp.ToType() + case strings.Contains(s, "DATETIME"): + return types.T_datetime.ToType() + case strings.HasPrefix(s, "TIME"): + return types.T_time.ToType() + case strings.Contains(s, "DATE"): + return types.T_date.ToType() + case strings.Contains(s, "DOUBLE"): + return types.T_float64.ToType() + case strings.Contains(s, "FLOAT"): + return types.T_float32.ToType() + case strings.Contains(s, "BOOL"): + return types.T_bool.ToType() + case strings.Contains(s, "BIGINT"): + return types.T_int64.ToType() + case strings.Contains(s, "INT"): + return types.T_int32.ToType() + default: + return types.Type{} + } +} + +func compareCSVRowsLexical(left, right []string) int { + limit := len(left) + if len(right) < limit { + limit = len(right) + } + for i := 0; i < limit; i++ { + if cmp := strings.Compare(left[i], right[i]); cmp != 0 { + return cmp + } + } + switch { + case len(left) < len(right): + return -1 + case len(left) > len(right): + return 1 + default: + return 0 + } +} + +func writeCSVMetadata(w io.Writer, schema *TableSchema, stats logicalTableStats) error { + if schema.CreateSQL != "" { + if _, err := fmt.Fprintf(w, "-- %s\n", schema.CreateSQL); err != nil { + return err + } + } + if schema.DatabaseName != "" { + if _, err := fmt.Fprintf(w, "-- Database: %s\n", schema.DatabaseName); err != nil { + return err + } + } + if schema.TableName != "" { + if _, err := fmt.Fprintf(w, "-- Table: %s\n", schema.TableName); err != nil { + return err + } + } + _, err := fmt.Fprintf(w, "-- Visible rows: %d (deleted: %d, physical: %d)\n", + stats.VisibleRows, stats.DeletedRows, stats.PhysicalRows) + return err +} + +// MergeLogicalViewWithSchema replaces col_N headers with real column names from the schema +// and filters data rows to only include visible (non-hidden) columns by their physical position. +// Returns a new LogicalTableView with merged headers and filtered data rows. +func MergeLogicalViewWithSchema(view *LogicalTableView, schema *TableSchema) *LogicalTableView { + // Without schema columns we cannot safely distinguish visible columns from hidden ones. + if len(schema.Columns) == 0 { + return &LogicalTableView{ + Headers: nil, + Rows: nil, + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + } + } + + // Build visible column headers and their physical data positions + newHeaders := make([]string, 0, len(schema.Columns)) + colMap := make([]int, 0, len(schema.Columns)) // visibleIdx → physical data column index + + for _, col := range schema.Columns { + newHeaders = append(newHeaders, col.Name) + dataPos := dataIndexForSeqNum(view, col.PhysicalPosition) + if dataPos < 0 { + dataPos = col.PhysicalPosition + } + if dataPos < 0 { + dataPos = col.Position + } + colMap = append(colMap, dataPos) + } + + // Extract data rows: pick only visible columns by their physical position + newRows := make([][]string, len(view.Rows)) + dataOffset := logicalViewDataOffset(view) + for i, row := range view.Rows { + newRow := make([]string, len(colMap)) + for j, pos := range colMap { + dataIdx := dataOffset + pos + if dataIdx < len(row) { + newRow[j] = row[dataIdx] + } + } + newRows[i] = newRow + } + + return &LogicalTableView{ + Headers: newHeaders, + Rows: newRows, + VisibleRows: view.VisibleRows, + DeletedRows: view.DeletedRows, + PhysicalRows: view.PhysicalRows, + } +} + +func dataIndexForSeqNum(view *LogicalTableView, seqNum int) int { + if view == nil || seqNum < 0 { + return -1 + } + for idx, candidate := range view.ColSeqNums { + if int(candidate) == seqNum { + return idx + } + } + return -1 +} + +// buildSchemaFromMoTablesRow extracts a TableSchema from a mo_tables data row. +// Uses column name lookup from view headers to handle hidden column offsets. +// mo_tables columns: rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql +func buildSchemaFromMoTablesRow(view *LogicalTableView, fullRow []string) *TableSchema { + return buildSchemaFromMoTablesRowAt( + view, + fullRow, + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + fallbackCatalogColIndex(view, moTablesID, "account_id"), + ) +} + +func buildSchemaFromMoTablesRowAt( + view *LogicalTableView, + fullRow []string, + relNameIdx int, + relDBIdx int, + createSQLIdx int, + commentIdx int, + constraintIdx int, + accountIDIdx int, +) *TableSchema { + schema := &TableSchema{} + dataRow := fullRow[logicalViewDataOffset(view):] + + if relNameIdx >= 0 && relNameIdx < len(dataRow) { + schema.TableName = strings.Clone(dataRow[relNameIdx]) + } + + if relDBIdx >= 0 && relDBIdx < len(dataRow) { + schema.DatabaseName = strings.Clone(dataRow[relDBIdx]) + } + + if createSQLIdx >= 0 && createSQLIdx < len(dataRow) { + if isPrintableCreateTableSQL(dataRow[createSQLIdx]) || isPrintableExternalParamJSON(dataRow[createSQLIdx]) { + schema.CreateSQL = strings.Clone(dataRow[createSQLIdx]) + } + } + if commentIdx >= 0 && commentIdx < len(dataRow) && isPrintableSQLText(dataRow[commentIdx]) { + schema.Comment = strings.Clone(dataRow[commentIdx]) + } + if constraintIdx >= 0 && constraintIdx < len(dataRow) { + schema.UniqueKeys = decodeUniqueKeysFromMoTablesConstraint(dataRow[constraintIdx]) + schema.PrimaryKey = decodePrimaryKeyFromMoTablesConstraint(dataRow[constraintIdx]) + } + if accountIDIdx >= 0 && accountIDIdx < len(dataRow) { + if accountID, err := strconv.ParseUint(dataRow[accountIDIdx], 10, 32); err == nil { + schema.AccountID = uint32(accountID) + } + } + return schema +} + +func findTableSchemaFromMoTables(view *LogicalTableView, tableID uint64) *TableSchema { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol, accountIDCol int) *TableSchema { + if relIDCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || row[relIDCol] != tableIDStr { + continue + } + return buildSchemaFromMoTablesRowAt(view, fullRow, relNameCol, relDBCol, createSQLCol, commentCol, constraintCol, accountIDCol) + } + return nil + } + + if schema := try( + fallbackCatalogColIndex(view, moTablesID, "rel_id"), + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "rel_createsql"), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + fallbackCatalogColIndex(view, moTablesID, "account_id"), + ); schema != nil { + return schema + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + if schema := try( + catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "rel_createsql", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Comment, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Constraint, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "account_id", match.offset), + ); schema != nil { + return schema + } + } + return nil +} + +func findTableNameFromMoTables(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, relNameCol int) string { + if relIDCol < 0 || relNameCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || relNameCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr && row[relNameCol] != "" { + return strings.Clone(row[relNameCol]) + } + } + return "" + } + + relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") + relNameCol := fallbackCatalogColIndex(view, moTablesID, "relname") + if name := try(relIDCol, relNameCol); name != "" { + return name + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + relIDCol = catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset) + relNameCol = catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset) + if name := try(relIDCol, relNameCol); name != "" { + return name + } + } + return "" +} + +func findTableCommentFromMoTables(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, "rel_id") + commentCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Comment) + if relIDCol < 0 || commentCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || commentCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr && isPrintableSQLText(row[commentCol]) { + return strings.Clone(row[commentCol]) + } + } + return "" +} + +func findUniqueKeysFromMoTables(view *LogicalTableView, tableID uint64) []TableUniqueKey { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID) + constraintCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) + if relIDCol < 0 || constraintCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return decodeUniqueKeysFromMoTablesConstraint(row[constraintCol]) + } + } + return nil +} + +func findPrimaryKeyFromMoTables(view *LogicalTableView, tableID uint64) []string { + tableIDStr := fmt.Sprintf("%d", tableID) + relIDCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID) + constraintCol := fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint) + if relIDCol < 0 || constraintCol < 0 { + return nil + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return decodePrimaryKeyFromMoTablesConstraint(row[constraintCol]) + } + } + return nil +} + +func findForeignKeysFromCatalogViews(moTablesView, moColumnsView *LogicalTableView, tableID uint64) []TableForeignKey { + if moTablesView == nil || moColumnsView == nil { + return nil + } + tableByID := mapTablesByID(moTablesView) + colNamesByTableID := mapColumnNamesByTableIDAndColID(moColumnsView) + constraint := findMoTablesConstraint(moTablesView, tableID) + return decodeForeignKeysFromMoTablesConstraint(constraint, tableID, tableByID, colNamesByTableID) +} + +func mapTablesByID(view *LogicalTableView) map[uint64]TableCatalogEntry { + entries := buildCatalogTablesFromMoTablesRows(view) + out := make(map[uint64]TableCatalogEntry, len(entries)) + for _, entry := range entries { + out[entry.TableID] = entry + } + return out +} + +func mapColumnNamesByTableIDAndColID(view *LogicalTableView) map[uint64]map[uint64]string { + result := make(map[uint64]map[uint64]string) + add := func(relIDCol, uniqNameCol, numCol, nameCol, hiddenCol int) { + if relIDCol < 0 || nameCol < 0 { + return + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || nameCol >= len(row) { + continue + } + if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { + continue + } + tableID, ok := parseUintCell(row[relIDCol]) + if !ok { + continue + } + name := strings.TrimSpace(row[nameCol]) + if name == "" || catalog.IsAlias(name) { + continue + } + if result[tableID] == nil { + result[tableID] = make(map[uint64]string) + } + addColumnNameID := func(col string) { + colID, err := strconv.ParseUint(strings.TrimSpace(col), 10, 64) + if err == nil { + result[tableID][colID] = name + } + } + if uniqNameCol >= 0 && uniqNameCol < len(row) { + addColumnNameID(row[uniqNameCol]) + } + if numCol >= 0 && numCol < len(row) { + addColumnNameID(row[numCol]) + } + } + } + + add( + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_RelID), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_UniqName), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Num), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Name), + fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsHidden), + ) + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + add( + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_RelID, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_UniqName, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Num, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Name, match.offset), + catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_IsHidden, match.offset), + ) + } + return result +} + +func findMoTablesConstraint(view *LogicalTableView, tableID uint64) string { + tableIDStr := fmt.Sprintf("%d", tableID) + try := func(relIDCol, constraintCol int) string { + if relIDCol < 0 || constraintCol < 0 { + return "" + } + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || constraintCol >= len(row) { + continue + } + if row[relIDCol] == tableIDStr { + return row[constraintCol] + } + } + return "" + } + if raw := try( + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_ID), + fallbackCatalogColIndex(view, moTablesID, catalog.SystemRelAttr_Constraint), + ); raw != "" { + return raw + } + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + if raw := try( + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_ID, match.offset), + catalogColIndexForLayout(match.layout, moTablesID, catalog.SystemRelAttr_Constraint, match.offset), + ); raw != "" { + return raw + } + } + return "" +} + +func createTableDDLFromCatalogViews(tableID uint64, moTablesView, moColumnsView *LogicalTableView, partitionMetadataView ...*LogicalTableView) string { + tableName := "" + tableComment := "" + uniqueKeys := []TableUniqueKey(nil) + primaryKey := []string(nil) + foreignKeys := []TableForeignKey(nil) + clusterBy := []string(nil) + partition := "" + if moTablesView != nil { + tableName = findTableNameFromMoTables(moTablesView, tableID) + tableComment = findTableCommentFromMoTables(moTablesView, tableID) + uniqueKeys = findUniqueKeysFromMoTables(moTablesView, tableID) + primaryKey = findPrimaryKeyFromMoTables(moTablesView, tableID) + if moColumnsView != nil { + foreignKeys = findForeignKeysFromCatalogViews(moTablesView, moColumnsView, tableID) + } + } + if moColumnsView != nil { + clusterBy = buildClusterByFromMoColumnsRows(moColumnsView, tableID) + } + if len(partitionMetadataView) > 0 && partitionMetadataView[0] != nil { + partition = buildPartitionClauseFromMetadata(partitionMetadataView[0], tableID) + } + if moColumnsView != nil { + if ddl := buildCreateTableFromMoColumnsWithOptions(moColumnsView, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy); ddl != "" { + return ddl + } + } + return "" +} + +func buildCatalogTablesFromMoTablesRows(view *LogicalTableView) []TableCatalogEntry { + seen := make(map[uint64]struct{}) + merged := make([]TableCatalogEntry, 0) + try := func(relIDCol, relNameCol, relDBCol, relDBIDCol, relKindCol, accountIDCol int) { + tables := buildCatalogTablesFromMoTablesRowsAt( + view, + relIDCol, + relNameCol, + relDBCol, + relDBIDCol, + relKindCol, + accountIDCol, + ) + for _, table := range tables { + if _, ok := seen[table.TableID]; ok { + continue + } + seen[table.TableID] = struct{}{} + merged = append(merged, table) + } + } + + try( + fallbackCatalogColIndex(view, moTablesID, "rel_id"), + fallbackCatalogColIndex(view, moTablesID, "relname"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase"), + fallbackCatalogColIndex(view, moTablesID, "reldatabase_id"), + fallbackCatalogColIndex(view, moTablesID, "relkind"), + fallbackCatalogColIndex(view, moTablesID, "account_id"), + ) + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moTablesID) { + try( + catalogColIndexForLayout(match.layout, moTablesID, "rel_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relname", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "reldatabase_id", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "relkind", match.offset), + catalogColIndexForLayout(match.layout, moTablesID, "account_id", match.offset), + ) + } + return merged +} + +func buildCatalogTablesFromMoTablesRowsAt( + view *LogicalTableView, + relIDCol int, + relNameCol int, + relDBCol int, + relDBIDCol int, + relKindCol int, + accountIDCol int, +) []TableCatalogEntry { + if relIDCol < 0 || relNameCol < 0 || relDBCol < 0 { + return nil + } + + seen := make(map[uint64]struct{}) + tables := make([]TableCatalogEntry, 0) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relIDCol >= len(row) || relNameCol >= len(row) || relDBCol >= len(row) { + continue + } + tableID, err := strconv.ParseUint(row[relIDCol], 10, 64) + if err != nil || tableID == 0 { + continue + } + if _, ok := seen[tableID]; ok { + continue + } + entry := TableCatalogEntry{ + TableID: tableID, + AccountID: 0, + DatabaseName: strings.Clone(row[relDBCol]), + TableName: strings.Clone(row[relNameCol]), + } + if !validCatalogName(entry.DatabaseName) || !validCatalogName(entry.TableName) { + continue + } + if relDBIDCol >= 0 && relDBIDCol < len(row) { + if dbID, err := strconv.ParseUint(row[relDBIDCol], 10, 64); err == nil { + entry.DatabaseID = dbID + } + } + if relKindCol >= 0 && relKindCol < len(row) { + entry.RelKind = strings.Clone(row[relKindCol]) + if !validRelKind(entry.RelKind) { + continue + } + } + if accountIDCol >= 0 && accountIDCol < len(row) { + if accountID, err := strconv.ParseUint(row[accountIDCol], 10, 32); err == nil { + entry.AccountID = uint32(accountID) + } + } + entry.TableName = displayTableName(entry.DatabaseName, entry.TableName) + seen[tableID] = struct{}{} + tables = append(tables, entry) + } + return tables +} + +func displayTableName(databaseName, tableName string) string { + if !defines.IsTempTableName(tableName) { + return tableName + } + prefix := defines.TempTableNamePrefix + rest := strings.TrimPrefix(tableName, prefix) + sessionEnd := strings.IndexByte(rest, '_') + if sessionEnd <= 0 || sessionEnd+1 >= len(rest) { + return tableName + } + afterSession := rest[sessionEnd+1:] + dbPrefix := databaseName + "_" + if strings.HasPrefix(afterSession, dbPrefix) && len(afterSession) > len(dbPrefix) { + return afterSession[len(dbPrefix):] + } + return tableName +} + +func validCatalogName(s string) bool { + if s == "" { + return false + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z': + case r >= 'A' && r <= 'Z': + case r >= '0' && r <= '9': + case r == '_' || r == '-' || r == '$': + default: + return false + } + } + return true +} + +func validRelKind(s string) bool { + switch s { + case "", "r", "v", "e", "cluster", "external", "view": + return true + default: + return false + } +} + +// buildColumnsFromMoColumnsRows builds a sorted column list from mo_columns data rows +// filtered for a specific tableID and excluding hidden columns. +// Uses column name lookup from view headers to handle hidden column offsets. +// mo_columns columns: att_relname_id, att_relname, attname, atttyp, attnum, ..., att_is_hidden +func buildColumnsFromMoColumnsRows(view *LogicalTableView, tableID uint64) []TableColumn { + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") + nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") + typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") + numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") + hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") + seqNumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Seqnum) + + ckpDebugSchemaf( + "mo_columns schema resolve start table=%d rows=%d headers=%d data_width=%d fallback_cols relname_id=%d name=%d typ=%d num=%d hidden=%d seqnum=%d", + tableID, + len(view.Rows), + len(view.Headers), + len(view.Headers)-logicalViewDataOffset(view), + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + + if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { + if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + ckpDebugSchemaf("mo_columns schema resolve success table=%d source=fallback cols=%d", tableID, len(cols)) + return cols + } + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) + typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) + numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) + hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) + seqNumCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Seqnum, match.offset) + ckpDebugSchemaf( + "mo_columns schema try table=%d layout=%s offset=%d relname_id=%d name=%d typ=%d num=%d hidden=%d seqnum=%d", + tableID, + match.layout.name, + match.offset, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + if cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol); len(cols) > 0 { + ckpDebugSchemaf("mo_columns schema resolve success table=%d source=%s offset=%d cols=%d", tableID, match.layout.name, match.offset, len(cols)) + return cols + } + } + + ckpDebugSchemaf("mo_columns schema resolve failed table=%d", tableID) + return nil +} + +func buildColumnsFromMoColumnsRowsAt( + view *LogicalTableView, + tableID uint64, + relnameIDCol int, + nameCol int, + typCol int, + numCol int, + hiddenCol int, + seqNumCol int, +) []TableColumn { + tableIDStr := fmt.Sprintf("%d", tableID) + cols := make([]TableColumn, 0, len(view.Rows)) + matchedRows := 0 + typeDecodeFailures := 0 + notNullCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_NullAbility) + hasDefaultCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasExpr) + defaultCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_DefaultExpr) + constraintCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_ConstraintType) + unsignedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsUnsigned) + autoIncrementCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsAutoIncrement) + commentCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Comment) + hasUpdateCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasUpdate) + updateCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Update) + clusterByCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsClusterBy) + enumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_EnumValues) + hasGeneratedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_HasGenerated) + generatedCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Generated) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + + // Skip hidden columns + if hiddenCol >= 0 && hiddenCol < len(row) { + if row[hiddenCol] == "1" || row[hiddenCol] == "true" { + continue + } + } + + // Filter by table ID + if relnameIDCol < 0 || relnameIDCol >= len(row) { + continue + } + if row[relnameIDCol] != tableIDStr { + continue + } + matchedRows++ + + pos := len(cols) + if numCol >= 0 && numCol < len(row) { + if n, err := strconv.Atoi(row[numCol]); err == nil { + pos = n + } + } + + physicalPos := pos - 1 + if seqNumCol >= 0 && seqNumCol < len(row) { + if n, err := strconv.Atoi(row[seqNumCol]); err == nil { + physicalPos = n + } + } + + col := TableColumn{ + Position: pos, + PhysicalPosition: physicalPos, + } + if nameCol >= 0 && nameCol < len(row) { + col.Name = row[nameCol] + } + if typCol >= 0 && typCol < len(row) { + if sqlType, ok := decodeMoColumnSQLType(row[typCol]); ok { + col.SQLType = sqlType + } else { + typeDecodeFailures++ + ckpDebugSchemaf( + "mo_columns atttyp decode failed table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d column=%q %s", + tableID, + matchedRows, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + col.Name, + debugMoColumnTypeCell(row[typCol], fullRow), + ) + } + } + if notNullCol >= 0 && notNullCol < len(row) { + col.NotNull = isTruthyCatalogValue(row[notNullCol]) + } + if hasDefaultCol >= 0 && hasDefaultCol < len(row) { + col.HasDefault = isTruthyCatalogValue(row[hasDefaultCol]) + } + if defaultCol >= 0 && defaultCol < len(row) { + if defaultExpr := decodeMoColumnDefault(row[defaultCol]); defaultExpr != "" { + col.Default = defaultExpr + } + } + if constraintCol >= 0 && constraintCol < len(row) { + col.ConstraintType = row[constraintCol] + } + if unsignedCol >= 0 && unsignedCol < len(row) { + col.Unsigned = isTruthyCatalogValue(row[unsignedCol]) + } + if autoIncrementCol >= 0 && autoIncrementCol < len(row) { + col.AutoIncrement = isTruthyCatalogValue(row[autoIncrementCol]) + } + if commentCol >= 0 && commentCol < len(row) { + col.Comment = row[commentCol] + } + if hasUpdateCol >= 0 && hasUpdateCol < len(row) && isTruthyCatalogValue(row[hasUpdateCol]) && + updateCol >= 0 && updateCol < len(row) { + col.OnUpdate = decodeMoColumnOnUpdate(row[updateCol]) + } + if clusterByCol >= 0 && clusterByCol < len(row) { + col.ClusterBy = isTruthyCatalogValue(row[clusterByCol]) + } + if enumCol >= 0 && enumCol < len(row) { + col.EnumValues = row[enumCol] + } + if hasGeneratedCol >= 0 && hasGeneratedCol < len(row) && isTruthyCatalogValue(row[hasGeneratedCol]) && + generatedCol >= 0 && generatedCol < len(row) { + col.Generated, col.GeneratedStored = decodeMoColumnGenerated(row[generatedCol]) + } + cols = append(cols, col) + } + + if typeDecodeFailures > 0 { + ckpDebugSchemaf( + "mo_columns schema rejected table=%d matched_rows=%d type_decode_failures=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d", + tableID, + matchedRows, + typeDecodeFailures, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + return nil + } + + // Sort by position + sort.Slice(cols, func(i, j int) bool { + return cols[i].Position < cols[j].Position + }) + + if matchedRows > 0 && len(cols) == 0 { + ckpDebugSchemaf( + "mo_columns matched rows but produced no visible columns table=%d matched_rows=%d relname_id_col=%d name_col=%d typ_col=%d num_col=%d hidden_col=%d seqnum_col=%d", + tableID, + matchedRows, + relnameIDCol, + nameCol, + typCol, + numCol, + hiddenCol, + seqNumCol, + ) + } + return cols +} + +func buildClusterByFromMoColumnsRows(view *LogicalTableView, tableID uint64) []string { + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_RelID) + nameCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Name) + clusterByCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_IsClusterBy) + if relnameIDCol < 0 || nameCol < 0 || clusterByCol < 0 { + return nil + } + if clusterBy := buildClusterByFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, clusterByCol); len(clusterBy) > 0 { + return clusterBy + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_RelID, match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_Name, match.offset) + clusterByCol = catalogColIndexForLayout(match.layout, moColumnsID, catalog.SystemColAttr_IsClusterBy, match.offset) + if clusterBy := buildClusterByFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, clusterByCol); len(clusterBy) > 0 { + return clusterBy + } + } + return nil +} + +func buildClusterByFromMoColumnsRowsAt( + view *LogicalTableView, + tableID uint64, + relnameIDCol int, + nameCol int, + clusterByCol int, +) []string { + if relnameIDCol < 0 || nameCol < 0 || clusterByCol < 0 { + return nil + } + tableIDStr := fmt.Sprintf("%d", tableID) + for _, fullRow := range view.Rows { + row := fullRow[logicalViewDataOffset(view):] + if relnameIDCol >= len(row) || nameCol >= len(row) || clusterByCol >= len(row) { + continue + } + if row[relnameIDCol] != tableIDStr || !isTruthyCatalogValue(row[clusterByCol]) { + continue + } + name := strings.TrimSpace(row[nameCol]) + if name == "" { + continue + } + if names := splitCompositeClusterByColumnName(name); len(names) > 0 { + return names + } + return []string{name} + } + return nil +} + +func splitCompositeClusterByColumnName(name string) []string { + if !strings.HasPrefix(name, catalog.PrefixCBColName) { + return nil + } + var names []string + for next := len(catalog.PrefixCBColName); next < len(name); { + if next+3 > len(name) { + return nil + } + strLen, err := strconv.Atoi(name[next : next+3]) + if err != nil || strLen <= 0 || next+3+strLen > len(name) { + return nil + } + names = append(names, name[next+3:next+3+strLen]) + next += strLen + 3 + } + return names +} + +// getDataColumnsFromView extracts only the data columns (skipping object/block/row meta columns). +// ShowCreateTable returns the CREATE TABLE DDL for a given tableID by reading +// the checkpoint's mo_tables and mo_columns system tables (GCKP + following ICKPs). +// +// Priority: +// 1. Reconstructed from mo_columns (attname, decoded atttyp, attnum, att_is_hidden) +// 2. Hardcoded built-in table schemas (for core system tables like mo_tables, mo_columns, etc.) +// +// ALTER TABLE updates mo_columns but does not rewrite mo_tables.rel_createsql, so +// mo_columns is the authoritative source for the current visible column set. +// +// NOTE: +// We intentionally do not synthesize a CREATE TABLE from raw physical object column +// types when mo_tables/mo_columns metadata is unavailable. Physical object columns may +// include hidden system columns, and without mo_columns we cannot safely distinguish +// visible columns from hidden ones. +func (r *CheckpointReader) ShowCreateTable( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, +) (string, error) { + // Read mo_tables first so the mo_columns path can still render the table name + // while taking the current column set from mo_columns. + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil { + moTablesView = nil + } + var accountID uint32 + if moTablesView != nil { + for _, table := range buildCatalogTablesFromMoTablesRows(moTablesView) { + if table.TableID == tableID { + accountID = table.AccountID + break + } + } + } + + // 1. Reconstruct from mo_columns. + moColumnsView, err := r.getTableLogicalView(ctx, moColumnsID, snapshotTS) + if err != nil { + moColumnsView = nil + } + + var partitionMetadataView *LogicalTableView + if view, err := r.getPartitionMetadataView(ctx, snapshotTS, accountID); err == nil { + partitionMetadataView = view + } + + if ddl := createTableDDLFromCatalogViews(tableID, moTablesView, moColumnsView, partitionMetadataView); ddl != "" { + return ddl, nil + } + + // 3. Hardcoded built-in table schemas + layout := currentCatalogLayout + switch tableID { + case moTablesID: + if moTablesView != nil { + if inferred, _, ok := inferCatalogLayout(len(moTablesView.Headers)-logicalViewMetaCols, moTablesID); ok { + layout = inferred + } + } + case moColumnsID: + if moColumnsView != nil { + if inferred, _, ok := inferCatalogLayout(len(moColumnsView.Headers)-logicalViewMetaCols, moColumnsID); ok { + layout = inferred + } + } + } + if ddl := hardcodedCreateTableForLayout(tableID, layout); ddl != "" { + return ddl, nil + } + + return "", moerr.NewInternalErrorf( + ctx, + "cannot resolve exact schema for table %d from checkpoint metadata; mo_tables/mo_columns data is unavailable or incomplete", + tableID, + ) +} + +// ShowCreateIndexStatements returns ALTER TABLE statements for secondary indexes +// recorded in mo_catalog.mo_indexes. CREATE TABLE reconstruction from mo_columns +// cannot see these rows, so callers that need complete DDL should merge or apply +// these definitions. +func (r *CheckpointReader) ShowCreateIndexStatements( + ctx context.Context, + tableID uint64, + tableName string, + snapshotTS types.TS, +) ([]string, error) { + moIndexesTableID, ok, err := r.findCatalogTableID(ctx, snapshotTS, catalog.MO_INDEXES) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + + view, err := r.dumpCatalogTableViewWithHeaders(ctx, moIndexesTableID, snapshotTS, moIndexesHeaders) + if err != nil { + return nil, err + } + return buildCreateIndexStatementsFromMoIndexes(view, tableID, tableName) +} + +func (r *CheckpointReader) getPartitionMetadataView( + ctx context.Context, + snapshotTS types.TS, + accountID uint32, +) (*LogicalTableView, error) { + partitionMetadataTableID, ok, err := r.findCatalogTableIDForAccount(ctx, snapshotTS, catalog.MOPartitionMetadata, accountID) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + return r.dumpCatalogTableViewWithHeaders(ctx, partitionMetadataTableID, snapshotTS, moPartitionMetadataHeaders) +} + +func (r *CheckpointReader) getPartitionTablesView( + ctx context.Context, + snapshotTS types.TS, + accountID uint32, +) (*LogicalTableView, error) { + partitionTablesID, ok, err := r.findCatalogTableIDForAccount(ctx, snapshotTS, catalog.MOPartitionTables, accountID) + if err != nil { + return nil, err + } + if !ok { + return nil, nil + } + return r.dumpCatalogTableViewWithHeaders(ctx, partitionTablesID, snapshotTS, moPartitionTablesHeaders) +} + +func (r *CheckpointReader) readPartitionTableIDs( + ctx context.Context, + snapshotTS types.TS, + primaryTableID uint64, + accountID uint32, +) ([]uint64, error) { + view, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) + if err != nil || view == nil { + return nil, err + } + primarySet := map[uint64]struct{}{primaryTableID: {}} + return buildPartitionTableIDMap(view, primarySet)[primaryTableID], nil +} + +func (r *CheckpointReader) readPartitionTableIDsForPrimaries( + ctx context.Context, + snapshotTS types.TS, + primaryTableIDs map[uint64]struct{}, + accountByPrimary map[uint64]uint32, +) (map[uint64][]uint64, error) { + if len(primaryTableIDs) == 0 { + return nil, nil + } + result := make(map[uint64][]uint64) + primaryByAccount := make(map[uint32]map[uint64]struct{}) + for primaryID := range primaryTableIDs { + accountID := accountByPrimary[primaryID] + if primaryByAccount[accountID] == nil { + primaryByAccount[accountID] = make(map[uint64]struct{}) + } + primaryByAccount[accountID][primaryID] = struct{}{} + } + for accountID, primaries := range primaryByAccount { + view, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) + if err != nil { + return nil, err + } + if view == nil { + continue + } + for primaryID, partitionIDs := range buildPartitionTableIDMap(view, primaries) { + result[primaryID] = append(result[primaryID], partitionIDs...) + } + } + return result, nil +} + +func (r *CheckpointReader) readPartitionClause( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, + accountID uint32, +) string { + metadataView, err := r.getPartitionMetadataView(ctx, snapshotTS, accountID) + if err != nil || metadataView == nil { + return "" + } + tablesView, err := r.getPartitionTablesView(ctx, snapshotTS, accountID) + if err != nil { + tablesView = nil + } + return buildPartitionClauseFromMetadata(metadataView, tableID, tablesView) +} + +func (r *CheckpointReader) dumpCatalogTableViewWithHeaders( + ctx context.Context, + tableID uint64, + snapshotTS types.TS, + headers []string, +) (*LogicalTableView, error) { + view, err := r.getTableLogicalView(ctx, tableID, snapshotTS) + if err != nil { + if isMissingCheckpointTableError(err, tableID) { + return &LogicalTableView{}, nil + } + return nil, err + } + if view == nil { + return &LogicalTableView{}, nil + } + applyCatalogHeadersBySeqNums(view, headers) + return view, nil +} + +func isMissingCheckpointTableError(err error, tableID uint64) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), fmt.Sprintf("table %d not found in checkpoint", tableID)) +} + +func applyCatalogHeadersBySeqNums(view *LogicalTableView, headers []string) { + if view == nil { + return + } + dataOffset := logicalViewDataOffset(view) + if dataOffset == 0 && len(view.Headers) < logicalViewMetaCols { + dataOffset = logicalViewMetaCols + } + fixedHeaders := make([]string, len(view.Headers)) + if len(fixedHeaders) < dataOffset { + fixedHeaders = make([]string, dataOffset) + } + copy(fixedHeaders, logicalTableViewMetaHeaders) + for i := dataOffset; i < len(fixedHeaders); i++ { + fixedHeaders[i] = fmt.Sprintf("col_%d", i-dataOffset) + } + + mapped := 0 + for dataIdx, seqNum := range view.ColSeqNums { + headerIdx := dataOffset + dataIdx + if headerIdx >= len(fixedHeaders) || int(seqNum) >= len(headers) { + continue + } + fixedHeaders[headerIdx] = headers[seqNum] + mapped++ + } + if mapped == 0 { + for i, header := range headers { + headerIdx := dataOffset + i + if headerIdx >= len(fixedHeaders) { + fixedHeaders = append(fixedHeaders, header) + continue + } + fixedHeaders[headerIdx] = header + } + } + view.Headers = fixedHeaders +} + +func (r *CheckpointReader) findCatalogTableID( + ctx context.Context, + snapshotTS types.TS, + tableName string, +) (uint64, bool, error) { + return r.findCatalogTableIDForAccount(ctx, snapshotTS, tableName, 0) +} + +func (r *CheckpointReader) findCatalogTableIDForAccount( + ctx context.Context, + snapshotTS types.TS, + tableName string, + accountID uint32, +) (uint64, bool, error) { + moTablesView, err := r.getTableLogicalView(ctx, moTablesID, snapshotTS) + if err != nil { + return 0, false, moerr.NewInternalErrorf(ctx, "read mo_tables: %v", err) + } + var fallback uint64 + tables := buildCatalogTablesFromMoTablesRows(moTablesView) + for _, table := range tables { + if table.TableName != tableName { + continue + } + if table.DatabaseName != "" && table.DatabaseName != catalog.MO_CATALOG { + continue + } + if table.AccountID == accountID { + return table.TableID, true, nil + } + if accountID == 0 && fallback == 0 { + fallback = table.TableID + } + } + if fallback != 0 { + return fallback, true, nil + } + return 0, false, nil +} + +func buildCreateIndexStatementsFromMoIndexes( + view *LogicalTableView, + tableID uint64, + tableName string, +) ([]string, error) { + if view == nil { + return nil, nil + } + tableIDCol := view.columnDataIndex("table_id") + nameCol := view.columnDataIndex("name") + typeCol := view.columnDataIndex("type") + algoCol := view.columnDataIndex(catalog.IndexAlgoName) + paramsCol := view.columnDataIndex(catalog.IndexAlgoParams) + commentCol := view.columnDataIndex("comment") + columnNameCol := view.columnDataIndex("column_name") + ordinalCol := view.columnDataIndex("ordinal_position") + hiddenCol := view.columnDataIndex("hidden") + if tableIDCol < 0 || nameCol < 0 || columnNameCol < 0 { + ckpDebugSchemaf( + "mo_indexes headers unavailable table=%d headers=%v table_id_col=%d name_col=%d column_name_col=%d", + tableID, + view.Headers, + tableIDCol, + nameCol, + columnNameCol, + ) + return nil, nil + } + + byName := make(map[string]*indexDDLInfo) + tableIDStr := strconv.FormatUint(tableID, 10) + matchedRows := 0 + hiddenRows := 0 + for _, row := range view.Rows { + if tableIDCol >= len(row) || row[tableIDCol] != tableIDStr { + continue + } + matchedRows++ + if hiddenCol >= 0 && hiddenCol < len(row) && isTruthyCatalogValue(row[hiddenCol]) { + hiddenRows++ + } + if nameCol >= len(row) || columnNameCol >= len(row) { + continue + } + name := row[nameCol] + colName := row[columnNameCol] + if name == "" || strings.EqualFold(name, "PRIMARY") || colName == "" || catalog.IsAlias(colName) { + continue + } + info := byName[name] + if info == nil { + info = &indexDDLInfo{name: name, columns: make(map[string]indexDDLColumn)} + byName[name] = info + } + if typeCol >= 0 && typeCol < len(row) && info.indexType == "" { + info.indexType = row[typeCol] + } + if algoCol >= 0 && algoCol < len(row) && info.algo == "" { + info.algo = row[algoCol] + } + if paramsCol >= 0 && paramsCol < len(row) && info.algoParams == "" { + info.algoParams = row[paramsCol] + } + if commentCol >= 0 && commentCol < len(row) && info.comment == "" { + info.comment = row[commentCol] + } + ordinal := len(info.columns) + 1 + if ordinalCol >= 0 && ordinalCol < len(row) { + if parsed, err := strconv.Atoi(row[ordinalCol]); err == nil { + ordinal = parsed + } + } + if existing, ok := info.columns[colName]; !ok || ordinal < existing.ordinal { + info.columns[colName] = indexDDLColumn{name: colName, ordinal: ordinal} + } + } + + names := make([]string, 0, len(byName)) + for name, info := range byName { + if len(info.columns) > 0 { + names = append(names, name) + } + } + sort.Strings(names) + + statements := make([]string, 0, len(names)) + for _, name := range names { + stmt, err := renderCreateIndexStatement(tableName, byName[name]) + if err != nil { + return nil, err + } + if stmt != "" { + statements = append(statements, stmt) + } + } + ckpDebugSchemaf( + "mo_indexes resolved table=%d rows=%d hidden_rows=%d indexes=%d names=%v statements=%d", + tableID, + matchedRows, + hiddenRows, + len(byName), + names, + len(statements), + ) + return statements, nil +} + +func buildPartitionClauseFromMetadata(view *LogicalTableView, tableID uint64, partitionTablesView ...*LogicalTableView) string { + if view == nil { + return "" + } + var tablesView *LogicalTableView + if len(partitionTablesView) > 0 { + tablesView = partitionTablesView[0] + } + tableIDCol := view.columnDataIndex("table_id") + methodCol := view.columnDataIndex("partition_method") + descriptionCol := view.columnDataIndex("partition_description") + countCol := view.columnDataIndex("partition_count") + if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { + ckpDebugSchemaf( + "partition metadata headers unavailable table=%d headers=%v table_id_col=%d method_col=%d description_col=%d count_col=%d", + tableID, + view.Headers, + tableIDCol, + methodCol, + descriptionCol, + countCol, + ) + } + tableIDStr := strconv.FormatUint(tableID, 10) + if clause := buildPartitionClauseFromMetadataAt(view, tableIDStr, tableIDCol, methodCol, descriptionCol, countCol, tablesView); clause != "" { + return clause + } + if clause := buildPartitionClauseFromMetadataByRowShape(view, tableIDStr, tablesView); clause != "" { + return clause + } + ckpDebugSchemaf("partition metadata not found table=%d rows=%d headers=%v", tableID, len(view.Rows), view.Headers) + return "" +} + +type partitionTableRef struct { + id uint64 + ordinal int + name string + partitionName string + expressionText string +} + +func buildPartitionTableIDMap(view *LogicalTableView, primaryTableIDs map[uint64]struct{}) map[uint64][]uint64 { + result := make(map[uint64][]uint64) + if view == nil || len(primaryTableIDs) == 0 { + return result + } + partitionIDCol := view.columnDataIndex("partition_id") + partitionTableNameCol := view.columnDataIndex("partition_table_name") + primaryIDCol := view.columnDataIndex("primary_table_id") + partitionNameCol := view.columnDataIndex("partition_name") + ordinalCol := view.columnDataIndex("partition_ordinal_position") + expressionCol := view.columnDataIndex("partition_expression_str") + byPrimary := make(map[uint64][]partitionTableRef) + if partitionIDCol >= 0 && primaryIDCol >= 0 { + addPartitionTableRefsAt(view, primaryTableIDs, byPrimary, partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol) + } + if len(byPrimary) == 0 { + addPartitionTableRefsByRowShape(view, primaryTableIDs, byPrimary) + } + for primaryID, refs := range byPrimary { + sort.Slice(refs, func(i, j int) bool { + if refs[i].ordinal != refs[j].ordinal { + return refs[i].ordinal < refs[j].ordinal + } + if refs[i].name != refs[j].name { + return refs[i].name < refs[j].name + } + return refs[i].id < refs[j].id + }) + ids := make([]uint64, 0, len(refs)) + seen := make(map[uint64]struct{}, len(refs)) + for _, ref := range refs { + if ref.id == 0 { + continue + } + if _, ok := seen[ref.id]; ok { + continue + } + seen[ref.id] = struct{}{} + ids = append(ids, ref.id) + } + if len(ids) > 0 { + result[primaryID] = ids + ckpDebugSchemaf("partition table ids resolved primary=%d partitions=%v", primaryID, ids) + } + } + return result +} + +func addPartitionTableRefsAt( + view *LogicalTableView, + primaryTableIDs map[uint64]struct{}, + out map[uint64][]partitionTableRef, + partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol int, +) { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + if partitionIDCol >= len(dataRow) || primaryIDCol >= len(dataRow) { + continue + } + partitionID, ok := parseUintCell(dataRow[partitionIDCol]) + if !ok { + continue + } + primaryID, ok := parseUintCell(dataRow[primaryIDCol]) + if !ok { + continue + } + if _, wanted := primaryTableIDs[primaryID]; !wanted { + continue + } + out[primaryID] = append(out[primaryID], partitionTableRef{ + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionTableNameCol), + partitionName: cellAt(dataRow, partitionNameCol), + expressionText: cellAt(dataRow, expressionCol), + }) + } +} + +func addPartitionTableRefsByRowShape(view *LogicalTableView, primaryTableIDs map[uint64]struct{}, out map[uint64][]partitionTableRef) { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + for primaryIDCol, cell := range dataRow { + primaryID, ok := parseUintCell(cell) + if !ok { + continue + } + if _, wanted := primaryTableIDs[primaryID]; !wanted { + continue + } + partitionIDCol := primaryIDCol - 2 + partitionTableNameCol := primaryIDCol - 1 + partitionNameCol := primaryIDCol + 1 + ordinalCol := primaryIDCol + 2 + expressionCol := primaryIDCol + 3 + partitionID, ok := parseUintCell(cellAt(dataRow, partitionIDCol)) + if !ok { + continue + } + out[primaryID] = append(out[primaryID], partitionTableRef{ + id: partitionID, + ordinal: parseIntCellDefault(cellAt(dataRow, ordinalCol), len(out[primaryID])), + name: cellAt(dataRow, partitionTableNameCol), + partitionName: cellAt(dataRow, partitionNameCol), + expressionText: cellAt(dataRow, expressionCol), + }) + } + } +} + +func buildPartitionClauseFromMetadataAt(view *LogicalTableView, tableIDStr string, tableIDCol, methodCol, descriptionCol, countCol int, partitionTablesView *LogicalTableView) string { + if tableIDCol < 0 || methodCol < 0 || descriptionCol < 0 { + return "" + } + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + if tableIDCol >= len(dataRow) || dataRow[tableIDCol] != tableIDStr { + continue + } + if methodCol >= len(dataRow) || descriptionCol >= len(dataRow) { + continue + } + return renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(dataRow[methodCol]), strings.TrimSpace(dataRow[descriptionCol]), cellAt(dataRow, countCol), partitionTablesView) + } + return "" +} + +func buildPartitionClauseFromMetadataByRowShape(view *LogicalTableView, tableIDStr string, partitionTablesView *LogicalTableView) string { + dataOffset := logicalViewDataOffset(view) + for _, row := range view.Rows { + if len(row) < dataOffset { + continue + } + dataRow := row[dataOffset:] + for tableIDCol, cell := range dataRow { + if cell != tableIDStr { + continue + } + methodCol := tableIDCol + 3 + descriptionCol := tableIDCol + 4 + countCol := tableIDCol + 5 + if descriptionCol >= len(dataRow) { + continue + } + clause := renderPartitionClauseFromMetadataCells(tableIDStr, strings.TrimSpace(cellAt(dataRow, methodCol)), strings.TrimSpace(cellAt(dataRow, descriptionCol)), cellAt(dataRow, countCol), partitionTablesView) + if clause != "" { + ckpDebugSchemaf("partition metadata resolved by row shape table=%s table_id_col=%d clause=%q", tableIDStr, tableIDCol, clause) + return clause + } + } + } + return "" +} + +func renderPartitionClauseFromMetadataCells(tableIDStr, method, description, countText string, partitionTablesView *LogicalTableView) string { + if description == "" || !isPrintableSQLText(description) { + return "" + } + clause := "partition by " + description + if isAutoPartitionCountMethod(method) { + if count, err := strconv.Atoi(strings.TrimSpace(countText)); err == nil && count > 0 { + clause += fmt.Sprintf(" partitions %d", count) + } + } + if definitions := renderPartitionDefinitions(tableIDStr, method, partitionTablesView); definitions != "" { + clause += definitions + } + ckpDebugSchemaf("partition metadata resolved table=%s method=%q description=%q clause=%q", tableIDStr, method, description, clause) + return clause +} + +func renderPartitionDefinitions(tableIDStr, method string, view *LogicalTableView) string { + if view == nil || !needsExplicitPartitionDefinitions(method) { + return "" + } + primaryID, err := strconv.ParseUint(strings.TrimSpace(tableIDStr), 10, 64) + if err != nil { + return "" + } + refsByPrimary := make(map[uint64][]partitionTableRef) + primarySet := map[uint64]struct{}{primaryID: {}} + partitionIDCol := view.columnDataIndex("partition_id") + partitionTableNameCol := view.columnDataIndex("partition_table_name") + primaryIDCol := view.columnDataIndex("primary_table_id") + partitionNameCol := view.columnDataIndex("partition_name") + ordinalCol := view.columnDataIndex("partition_ordinal_position") + expressionCol := view.columnDataIndex("partition_expression_str") + if partitionIDCol >= 0 && primaryIDCol >= 0 { + addPartitionTableRefsAt(view, primarySet, refsByPrimary, partitionIDCol, primaryIDCol, partitionTableNameCol, partitionNameCol, ordinalCol, expressionCol) + } + if len(refsByPrimary) == 0 { + addPartitionTableRefsByRowShape(view, primarySet, refsByPrimary) + } + refs := refsByPrimary[primaryID] + if len(refs) == 0 { + return "" + } + sort.Slice(refs, func(i, j int) bool { + if refs[i].ordinal != refs[j].ordinal { + return refs[i].ordinal < refs[j].ordinal + } + if refs[i].partitionName != refs[j].partitionName { + return refs[i].partitionName < refs[j].partitionName + } + return refs[i].id < refs[j].id + }) + parts := make([]string, 0, len(refs)) + for _, ref := range refs { + name := strings.TrimSpace(ref.partitionName) + expr := strings.TrimSpace(ref.expressionText) + if name == "" || expr == "" || !isPrintableSQLText(name) || !isPrintableSQLText(expr) { + continue + } + parts = append(parts, fmt.Sprintf("partition %s %s", quoteDDLIdent(name), expr)) + } + if len(parts) == 0 { + return "" + } + return " (\n " + strings.Join(parts, ",\n ") + "\n)" +} + +func cellAt(row []string, idx int) string { + if idx < 0 || idx >= len(row) { + return "" + } + return row[idx] +} + +func parseUintCell(cell string) (uint64, bool) { + value, err := strconv.ParseUint(strings.TrimSpace(cell), 10, 64) + return value, err == nil && value != 0 +} + +func parseIntCellDefault(cell string, fallback int) int { + value, err := strconv.Atoi(strings.TrimSpace(cell)) + if err != nil { + return fallback + } + return value +} + +func isAutoPartitionCountMethod(method string) bool { + switch strings.ToLower(strings.TrimSpace(method)) { + case "key", "linearkey", "hash", "linearhash": + return true + default: + return false + } +} + +func needsExplicitPartitionDefinitions(method string) bool { + switch strings.ToLower(strings.TrimSpace(method)) { + case "range", "list": + return true + default: + return false + } +} + +func renderCreateIndexStatement(tableName string, info *indexDDLInfo) (string, error) { + if info == nil || tableName == "" || len(info.columns) == 0 { + return "", nil + } + cols := make([]indexDDLColumn, 0, len(info.columns)) + for _, col := range info.columns { + cols = append(cols, col) + } + sort.Slice(cols, func(i, j int) bool { + if cols[i].ordinal != cols[j].ordinal { + return cols[i].ordinal < cols[j].ordinal + } + return cols[i].name < cols[j].name + }) + + var sb strings.Builder + sb.WriteString("ALTER TABLE ") + sb.WriteString(quoteDDLIdent(tableName)) + sb.WriteString(" ADD ") + fullText := isFullTextIndex(info) + if fullText { + sb.WriteString("FULLTEXT ") + } else if strings.EqualFold(info.indexType, "UNIQUE") { + sb.WriteString("UNIQUE ") + } + sb.WriteString("KEY ") + sb.WriteString(quoteDDLIdent(info.name)) + if !fullText { + appendIndexAlgorithmDDL(&sb, info.algo) + } + sb.WriteString("(") + for i, col := range cols { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(quoteDDLIdent(col.name)) + } + sb.WriteString(")") + if err := appendIndexTrailingOptionsDDL(&sb, info.algoParams, info.comment); err != nil { + return "", err + } + sb.WriteString(";") + return sb.String(), nil +} + +func isFullTextIndex(info *indexDDLInfo) bool { + if info == nil { + return false + } + return catalog.IsFullTextIndexAlgo(info.algo) || strings.EqualFold(info.indexType, "FULLTEXT") +} + +func isFullTextKey(key TableUniqueKey) bool { + return catalog.IsFullTextIndexAlgo(key.Algo) +} + +func appendIndexAlgorithmDDL(sb *strings.Builder, algo string) { + algo = strings.TrimSpace(algo) + if !catalog.IsNullIndexAlgo(algo) { + sb.WriteString(" USING ") + sb.WriteString(algo) + } +} + +func appendIndexTrailingOptionsDDL(sb *strings.Builder, algoParams, comment string) error { + if strings.TrimSpace(algoParams) != "" { + params, err := catalog.IndexParamsToStringList(algoParams) + if err != nil { + return err + } + if strings.TrimSpace(params) != "" { + sb.WriteString(params) + } + } + if comment != "" { + sb.WriteString(" COMMENT ") + sb.WriteString(quoteDDLString(comment)) + } + return nil +} + +func decodeUniqueKeysFromMoTablesConstraint(raw string) (keys []TableUniqueKey) { + if raw == "" { + return nil + } + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables constraint decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + for _, ct := range c.Cts { + indexDef, ok := ct.(*engine.IndexDef) + if !ok || indexDef == nil { + continue + } + for _, index := range indexDef.Indexes { + if index == nil || strings.EqualFold(index.IndexName, "PRIMARY") { + continue + } + cols := make([]string, 0, len(index.Parts)) + for _, part := range index.Parts { + part = strings.TrimSpace(part) + if part == "" || catalog.IsAlias(part) { + continue + } + cols = append(cols, part) + } + if len(cols) == 0 { + continue + } + name := index.IndexName + if name == "" { + name = cols[0] + } + keys = append(keys, TableUniqueKey{ + Name: name, + Columns: cols, + Unique: index.Unique, + Algo: index.IndexAlgo, + AlgoParams: index.IndexAlgoParams, + Comment: index.Comment, + }) + } + } + keys = normalizedUniqueKeys(keys) + ckpDebugSchemaf("mo_tables constraint indexes decoded count=%d", len(keys)) + return keys +} + +func decodePrimaryKeyFromMoTablesConstraint(raw string) (cols []string) { + if raw == "" { + return nil + } + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables primary key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + for _, ct := range c.Cts { + pkDef, ok := ct.(*engine.PrimaryKeyDef) + if !ok || pkDef == nil || pkDef.Pkey == nil { + continue + } + pk := pkDef.Pkey + if catalog.IsFakePkName(pk.PkeyColName) { + continue + } + for _, name := range pk.Names { + name = strings.TrimSpace(name) + if name != "" && !catalog.IsAlias(name) && !catalog.IsFakePkName(name) && name != catalog.CPrimaryKeyColName { + cols = append(cols, name) + } + } + if len(cols) == 0 { + name := strings.TrimSpace(pk.PkeyColName) + if name != "" && !catalog.IsAlias(name) && !catalog.IsFakePkName(name) && name != catalog.CPrimaryKeyColName { + cols = append(cols, name) + } + } + if len(cols) > 0 { + ckpDebugSchemaf("mo_tables primary key decoded cols=%v", cols) + return cols + } + } + return nil +} + +func decodeForeignKeysFromMoTablesConstraint( + raw string, + tableID uint64, + tableByID map[uint64]TableCatalogEntry, + colNamesByTableID map[uint64]map[uint64]string, +) (keys []TableForeignKey) { + if raw == "" { + return nil + } + c := &engine.ConstraintDef{} + if err := c.UnmarshalBinary([]byte(raw)); err != nil { + ckpDebugSchemaf("mo_tables foreign key decode failed len=%d hex=%s err=%v", len(raw), debugHexPrefix(raw, 64), err) + return nil + } + childColsByID := colNamesByTableID[tableID] + for _, ct := range c.Cts { + fkDef, ok := ct.(*engine.ForeignKeyDef) + if !ok || fkDef == nil { + ckpDebugSchemaf("mo_tables foreign key skip table=%d constraint_type=%T", tableID, ct) + continue + } + ckpDebugSchemaf("mo_tables foreign key scan table=%d fkeys=%d", tableID, len(fkDef.Fkeys)) + for _, fk := range fkDef.Fkeys { + if fk == nil || len(fk.Cols) == 0 || len(fk.Cols) != len(fk.ForeignCols) { + ckpDebugSchemaf("mo_tables foreign key skip table=%d invalid fk=%v", tableID, fk) + continue + } + parent, ok := tableByID[fk.ForeignTbl] + if !ok || parent.TableName == "" { + ckpDebugSchemaf("mo_tables foreign key skip table=%d missing parent=%d", tableID, fk.ForeignTbl) + continue + } + cols := namesForColumnIDs(childColsByID, fk.Cols) + referCols := namesForColumnIDs(colNamesByTableID[fk.ForeignTbl], fk.ForeignCols) + if len(cols) == 0 || len(cols) != len(referCols) { + ckpDebugSchemaf("mo_tables foreign key skip table=%d cols=%v resolved=%v foreign_table=%d foreign_cols=%v resolved_foreign=%v child_cols_by_id=%v foreign_cols_by_id=%v", tableID, fk.Cols, cols, fk.ForeignTbl, fk.ForeignCols, referCols, childColsByID, colNamesByTableID[fk.ForeignTbl]) + continue + } + name := strings.TrimSpace(fk.Name) + if name == "" { + name = cols[0] + } + keys = append(keys, TableForeignKey{ + Name: name, + Columns: cols, + ReferDatabase: parent.DatabaseName, + ReferTable: parent.TableName, + ReferColumns: referCols, + OnDelete: fk.OnDelete, + OnUpdate: fk.OnUpdate, + }) + } + } + keys = normalizedForeignKeys(keys) + ckpDebugSchemaf("mo_tables foreign keys decoded count=%d", len(keys)) + return keys +} + +func namesForColumnIDs(byID map[uint64]string, ids []uint64) []string { + if len(byID) == 0 || len(ids) == 0 { + return nil + } + names := make([]string, 0, len(ids)) + for _, id := range ids { + name := strings.TrimSpace(byID[id]) + if name == "" { + return nil + } + names = append(names, name) + } + return names +} + +func quoteDDLIdent(s string) string { + return "`" + strings.ReplaceAll(s, "`", "``") + "`" +} + +func quoteDDLString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `''`) + return "'" + s + "'" +} + +func isTruthyCatalogValue(s string) bool { + return s == "1" || strings.EqualFold(s, "true") +} + +// buildCreateTableFromMoColumns reconstructs a CREATE TABLE DDL from mo_columns data. +func buildCreateTableFromMoColumns(view *LogicalTableView, tableID uint64, tableNames ...string) string { + tableName := fmt.Sprintf("%d", tableID) + if len(tableNames) > 0 && tableNames[0] != "" { + tableName = tableNames[0] + } + tableComment := "" + if len(tableNames) > 1 && tableNames[1] != "" { + tableComment = tableNames[1] + } + return buildCreateTableFromMoColumnsWithOptions(view, tableID, tableName, tableComment, "", nil, nil, nil, nil) +} + +func buildCreateTableFromMoColumnsWithOptions( + view *LogicalTableView, + tableID uint64, + tableName string, + tableComment string, + partition string, + primaryKey []string, + uniqueKeys []TableUniqueKey, + foreignKeys []TableForeignKey, + clusterBy []string, +) string { + if clusterBy == nil { + clusterBy = buildClusterByFromMoColumnsRows(view, tableID) + } + relnameIDCol := fallbackCatalogColIndex(view, moColumnsID, "att_relname_id") + nameCol := fallbackCatalogColIndex(view, moColumnsID, "attname") + typCol := fallbackCatalogColIndex(view, moColumnsID, "atttyp") + numCol := fallbackCatalogColIndex(view, moColumnsID, "attnum") + hiddenCol := fallbackCatalogColIndex(view, moColumnsID, "att_is_hidden") + + if relnameIDCol >= 0 && nameCol >= 0 && typCol >= 0 && numCol >= 0 { + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + return ddl + } + } + + dataWidth := len(view.Headers) - logicalViewDataOffset(view) + for _, match := range catalogLayoutMatches(dataWidth, moColumnsID) { + relnameIDCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_relname_id", match.offset) + nameCol = catalogColIndexForLayout(match.layout, moColumnsID, "attname", match.offset) + typCol = catalogColIndexForLayout(match.layout, moColumnsID, "atttyp", match.offset) + numCol = catalogColIndexForLayout(match.layout, moColumnsID, "attnum", match.offset) + hiddenCol = catalogColIndexForLayout(match.layout, moColumnsID, "att_is_hidden", match.offset) + if ddl := buildCreateTableFromMoColumnsAt(view, tableID, tableName, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy, relnameIDCol, nameCol, typCol, numCol, hiddenCol); ddl != "" { + return ddl + } + } + + return "" +} + +func buildCreateTableFromMoColumnsAt( + view *LogicalTableView, + tableID uint64, + tableName string, + tableComment string, + partition string, + primaryKey []string, + uniqueKeys []TableUniqueKey, + foreignKeys []TableForeignKey, + clusterBy []string, + relnameIDCol int, + nameCol int, + typCol int, + numCol int, + hiddenCol int, +) string { + seqNumCol := fallbackCatalogColIndex(view, moColumnsID, catalog.SystemColAttr_Seqnum) + cols := buildColumnsFromMoColumnsRowsAt(view, tableID, relnameIDCol, nameCol, typCol, numCol, hiddenCol, seqNumCol) + if len(cols) == 0 { + return "" + } + return renderCreateTableDDLFullWithForeignKeysAndClusterBy(tableName, cols, tableComment, partition, primaryKey, uniqueKeys, foreignKeys, clusterBy) +} + +func isPrintableSQLType(sqlType string) bool { + sqlType = strings.TrimSpace(sqlType) + if sqlType == "" { + return false + } + hasLetter := false + for _, r := range sqlType { + switch { + case r >= 'A' && r <= 'Z': + hasLetter = true + case r >= 'a' && r <= 'z': + hasLetter = true + case r >= '0' && r <= '9': + case strings.ContainsRune(" ()_,'+-.", r): + default: + return false + } + } + return hasLetter +} + +func decodeMoColumnSQLType(raw string) (string, bool) { + if raw == "" { + return "", false + } + if sqlType, ok := decodeMoColumnEncodedSQLType(raw); ok { + return sqlType, true + } + if isPrintableSQLType(raw) { + return raw, true + } + return "", false +} + +func decodeMoColumnDefault(raw string) string { + if raw == "" { + return "" + } + var def plan.Default + if err := types.Decode([]byte(raw), &def); err == nil { + return strings.TrimSpace(def.OriginString) + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw) + } + return "" +} + +func decodeMoColumnOnUpdate(raw string) string { + if raw == "" { + return "" + } + var update plan.OnUpdate + if err := types.Decode([]byte(raw), &update); err == nil { + return strings.TrimSpace(update.OriginString) + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw) + } + return "" +} + +func decodeMoColumnGenerated(raw string) (string, bool) { + if raw == "" { + return "", false + } + var generated plan.GeneratedCol + if err := types.Decode([]byte(raw), &generated); err == nil { + return strings.TrimSpace(generated.OriginString), generated.IsStored + } + if isPrintableDDLExpression(raw) { + return strings.TrimSpace(raw), false + } + return "", false +} + +func decodeMoColumnEncodedSQLType(raw string) (string, bool) { + var typ types.Type + if len(raw) < typ.ProtoSize() { + return "", false + } + if err := types.Decode([]byte(raw), &typ); err == nil && typ.Oid != types.T_any { + sqlType := sqlTypeFromMOType(typ) + if isPrintableSQLType(sqlType) { + return sqlType, true + } + } + return "", false +} + +func sqlTypeFromMOType(typ types.Type) string { + switch typ.Oid { + case types.T_time: + return scaledSQLType("TIME", typ.Scale) + case types.T_datetime: + return scaledSQLType("DATETIME", typ.Scale) + case types.T_timestamp: + return scaledSQLType("TIMESTAMP", typ.Scale) + default: + return typ.DescString() + } +} + +func scaledSQLType(name string, scale int32) string { + if scale <= 0 { + return name + } + return fmt.Sprintf("%s(%d)", name, scale) +} + +func isPrintableDDLExpression(expr string) bool { + expr = strings.TrimSpace(expr) + if expr == "" { + return true + } + return isPrintableSQLText(expr) +} + +func isPrintableSQLText(s string) bool { + for _, r := range s { + switch { + case r == '\n' || r == '\r' || r == '\t': + case r >= 0x20 && r <= 0x7e: + default: + return false + } + } + return true +} + +func ckpDebugSchemaf(format string, args ...any) { + if os.Getenv("MO_TOOL_CKP_DEBUG_SCHEMA") == "" { + return + } + fmt.Fprintf(os.Stderr, "ckp schema debug: "+format+"\n", args...) +} + +func debugHexPrefix(s string, limit int) string { + if limit > 0 && len(s) > limit { + s = s[:limit] + } + return hex.EncodeToString([]byte(s)) +} + +func debugMoColumnTypeCell(raw string, fullRow []string) string { + var typ types.Type + decodeErr := "" + oid := "" + width := int32(0) + scale := int32(0) + desc := "" + if len(raw) >= typ.ProtoSize() { + if err := types.Decode([]byte(raw), &typ); err != nil { + decodeErr = err.Error() + } else { + decodeErr = "" + oid = typ.Oid.String() + width = typ.Width + scale = typ.Scale + desc = typ.DescString() + } + } + objectName := "" + blockIdx := "" + rowIdx := "" + if len(fullRow) > 0 { + objectName = fullRow[0] + } + if len(fullRow) > 1 { + blockIdx = fullRow[1] + } + if len(fullRow) > 2 { + rowIdx = fullRow[2] + } + return fmt.Sprintf( + "raw_len=%d raw_hex=%s decode_err=%s oid=%s width=%d scale=%d desc=%q object=%s block=%s row=%s", + len(raw), + debugHexPrefix(raw, 64), + decodeErr, + oid, + width, + scale, + desc, + objectName, + blockIdx, + rowIdx, + ) +} + +func isPrintableCreateTableSQL(ddl string) bool { + ddl = strings.TrimSpace(ddl) + if ddl == "" { + return false + } + for _, r := range ddl { + switch { + case r == '\n' || r == '\r' || r == '\t': + case r >= 0x20 && r <= 0x7e: + default: + return false + } + } + upper := strings.ToUpper(ddl) + return strings.HasPrefix(upper, "CREATE TABLE ") || + strings.HasPrefix(upper, "CREATE TEMPORARY TABLE ") || + strings.HasPrefix(upper, "CREATE CLUSTER TABLE ") || + strings.HasPrefix(upper, "CREATE EXTERNAL TABLE ") || + strings.HasPrefix(upper, "CREATE VIEW ") || + strings.HasPrefix(upper, "CREATE OR REPLACE VIEW ") +} + +func isPrintableExternalParamJSON(raw string) bool { + raw = strings.TrimSpace(raw) + if raw == "" || raw[0] != '{' || raw[len(raw)-1] != '}' { + return false + } + if !isPrintableSQLText(raw) { + return false + } + return strings.Contains(raw, `"ExParamConst"`) || strings.Contains(raw, `"Filepath"`) || strings.Contains(raw, `"Option"`) +} + +// hardcodedCreateTable returns the CREATE TABLE DDL for core built-in system tables. +// These tables' schemas are known at compile time and may not appear in the checkpoint's +// mo_tables/mo_columns (due to minimal deployments). +func hardcodedCreateTableForLayout(tableID uint64, layout catalogLayout) string { + schema := builtinTableSchemaForLayout(layout, tableID) + if schema == nil { + return "" + } + return renderCreateTableDDL(schema.TableName, schema.Columns) +} + +// columnDataIndex returns the data-column index (0-based, after stripping meta columns) +// for the named column, or -1 if not found. +func (v *LogicalTableView) columnDataIndex(colName string) int { + dataOffset := logicalViewDataOffset(v) + for i := dataOffset; i < len(v.Headers); i++ { + if v.Headers[i] == colName { + return i - dataOffset + } + } + return -1 +} + +func logicalViewDataOffset(view *LogicalTableView) int { + if view == nil || len(view.Headers) < logicalViewMetaCols { + return 0 + } + for i, h := range logicalTableViewMetaHeaders { + if view.Headers[i] != h { + return 0 + } + } + return logicalViewMetaCols +} diff --git a/pkg/tools/checkpointtool/table_dump_test.go b/pkg/tools/checkpointtool/table_dump_test.go new file mode 100644 index 0000000000000..ddf8de5b06a16 --- /dev/null +++ b/pkg/tools/checkpointtool/table_dump_test.go @@ -0,0 +1,1787 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkpointtool + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/util/csvparser" + "github.com/matrixorigin/matrixone/pkg/vm/engine" + "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCSVPipelineErrorKeepsNonCanceledRootCause(t *testing.T) { + root := errors.New("remote read failed") + + assert.ErrorIs(t, csvPipelineError(context.Canceled, root), root) + assert.ErrorIs(t, csvPipelineError(root, context.Canceled), root) + assert.ErrorIs(t, csvPipelineError(context.Canceled, context.Canceled), context.Canceled) + assert.NoError(t, csvPipelineError(nil, nil)) +} + +func TestComposeAtUsesLatestUsableGlobalCheckpoint(t *testing.T) { + older := checkpoint.NewCheckpointEntry("", types.BuildTS(1, 0), types.BuildTS(10, 0), checkpoint.ET_Global) + newer := checkpoint.NewCheckpointEntry("", types.BuildTS(11, 0), types.BuildTS(20, 0), checkpoint.ET_Global) + reader := &CheckpointReader{ + entries: []*checkpoint.CheckpointEntry{newer, older}, + getTablesForTest: func(_ *CheckpointReader, entry *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + return []*TableInfo{{TableID: uint64(entry.GetEnd().Physical())}}, nil + }, + } + + view, err := reader.ComposeAt(types.BuildTS(30, 0)) + require.NoError(t, err) + require.NotNil(t, view.BaseEntry) + assert.Equal(t, 0, view.BaseEntry.Index) + assert.Contains(t, view.Tables, uint64(20)) + assert.NotContains(t, view.Tables, uint64(10)) +} + +func TestGetTableEntriesAtReturnsNonMissingObjectEntryError(t *testing.T) { + readErr := errors.New("decode checkpoint entry failed") + entry := checkpoint.NewCheckpointEntry("", types.BuildTS(1, 0), types.BuildTS(10, 0), checkpoint.ET_Global) + reader := &CheckpointReader{ + entries: []*checkpoint.CheckpointEntry{entry}, + getTablesForTest: func(_ *CheckpointReader, _ *checkpoint.CheckpointEntry) ([]*TableInfo, error) { + return []*TableInfo{{TableID: 42, DataRanges: []ckputil.TableRange{{TableID: 42}}}}, nil + }, + getObjectEntriesForTest: func(_ *CheckpointReader, _ *checkpoint.CheckpointEntry, _ uint64) ([]*ObjectEntryInfo, []*ObjectEntryInfo, error) { + return nil, nil, readErr + }, + } + + _, _, err := reader.getTableEntriesAt(context.Background(), 42, types.BuildTS(10, 0)) + require.ErrorIs(t, err, readErr) +} + +func TestWriteCSVChunksPreservesStorageOrder(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + chunks := make(chan csvPipelineChunk, 5) + done := make(chan error, 1) + counters := &csvPipelineCounters{} + var buf bytes.Buffer + + go writeCSVChunks(ctx, &buf, chunks, counters, cancel, done) + chunks <- csvPipelineChunk{objectIdx: 1, blockIdx: 0, data: []byte("c\n")} + chunks <- csvPipelineChunk{objectIdx: 0, blockIdx: 1, data: []byte("b\n")} + chunks <- csvPipelineChunk{objectIdx: 0, blockIdx: 0, data: []byte("a\n")} + chunks <- csvPipelineChunk{objectIdx: 0, objectDone: true} + chunks <- csvPipelineChunk{objectIdx: 1, objectDone: true} + close(chunks) + + require.NoError(t, <-done) + assert.Equal(t, "a\nb\nc\n", buf.String()) +} + +func encodedSQLType(t *testing.T, typ types.Type) string { + t.Helper() + data, err := types.Encode(&typ) + require.NoError(t, err) + return string(data) +} + +func encodedDefault(t *testing.T, origin string, nullable bool) string { + t.Helper() + data, err := types.Encode(&plan.Default{OriginString: origin, NullAbility: nullable}) + require.NoError(t, err) + return string(data) +} + +func encodedConstraint(t *testing.T, constraints ...engine.Constraint) string { + t.Helper() + def := &engine.ConstraintDef{ + Cts: constraints, + } + data, err := def.MarshalBinary() + require.NoError(t, err) + return string(data) +} + +func encodedPrimaryKeyConstraint(t *testing.T, names ...string) engine.Constraint { + t.Helper() + pkeyColName := "" + if len(names) == 1 { + pkeyColName = names[0] + } else if len(names) > 1 { + pkeyColName = catalog.CPrimaryKeyColName + } + return &engine.PrimaryKeyDef{ + Pkey: &plan.PrimaryKeyDef{ + PkeyColName: pkeyColName, + Names: names, + }, + } +} + +// TestWriteCSV_empty tests writing an empty logical view to CSV. +func TestWriteCSV_empty(t *testing.T) { + schema := &TableSchema{ + TableName: "test_table", + DatabaseName: "test_db", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{}, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + lines := strings.Split(strings.TrimSpace(output), "\n") + // Skip comment lines to find the CSV header + csvStart := 0 + for i, line := range lines { + if !strings.HasPrefix(line, "--") { + csvStart = i + break + } + } + require.GreaterOrEqual(t, len(lines), csvStart+1, "expected at least a header line") + assert.Equal(t, "id,name", lines[csvStart]) +} + +func TestDumpPreparedTableCSV_EmptyTableWritesHeader(t *testing.T) { + reader := &CheckpointReader{} + data := &TableDumpData{ + TableID: 334019, + Schema: &TableSchema{ + TableName: "t_empty", + DatabaseName: "ckp_tables", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(64)", Position: 2, PhysicalPosition: 1}, + }, + }, + } + + var buf bytes.Buffer + err := reader.DumpPreparedTableCSV( + context.Background(), + &buf, + data, + types.TS{}, + WithCSVMetaComments(false), + WithCSVHeader(true), + ) + require.NoError(t, err) + assert.Equal(t, "id,name\n", buf.String()) +} + +func TestIsTableDataUnavailable(t *testing.T) { + assert.False(t, isTableDataUnavailable(assert.AnError)) + assert.False(t, isTableDataUnavailable(nil)) + assert.True(t, isTableDataUnavailable(fmt.Errorf("internal error: table 334019 not found in checkpoint at ts 1-0"))) + assert.True(t, isTableDataUnavailable(fmt.Errorf("internal error: no data entries for table 334019"))) + assert.False(t, isTableDataUnavailable(fmt.Errorf("compose checkpoint failed"))) +} + +// TestWriteCSV_withData tests writing data rows to CSV. +func TestWriteCSV_withData(t *testing.T) { + schema := &TableSchema{ + TableName: "users", + DatabaseName: "mydb", + CreateSQL: "CREATE TABLE users (id BIGINT, name VARCHAR(100), age INT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 1}, + {Name: "age", SQLType: "INT", Position: 3, PhysicalPosition: 2}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "Alice", "30"}, + {"obj1", "0", "1", "2", "Bob", "25"}, + {"obj2", "1", "0", "3", "Charlie, Jr.", "35"}, + {"obj2", "1", "1", "4", "NULL", "NULL"}, + }, + VisibleRows: 4, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + lines := strings.Split(strings.TrimSpace(output), "\n") + + // Skip comment lines + var dataLines []string + for _, line := range lines { + if !strings.HasPrefix(line, "--") { + dataLines = append(dataLines, line) + } + } + + assert.Equal(t, 5, len(dataLines)) // header + 4 rows + assert.Equal(t, "id,name,age", dataLines[0]) + assert.Equal(t, `1,"Alice",30`, dataLines[1]) + assert.Equal(t, `2,"Bob",25`, dataLines[2]) + assert.Equal(t, `3,"Charlie, Jr.",35`, dataLines[3]) + assert.Equal(t, `4,"NULL",NULL`, dataLines[4]) +} + +func TestWriteProjectedCSVRowFromVecsFastPath(t *testing.T) { + mp := mpool.MustNewZero() + vecInt := vector.NewVec(types.T_int64.ToType()) + vecDecimal := vector.NewVec(types.T_decimal64.ToTypeWithScale(2)) + vecDecimal128 := vector.NewVec(types.T_decimal128.ToTypeWithScale(5)) + vecDate := vector.NewVec(types.T_date.ToType()) + vecString := vector.NewVec(types.T_varchar.ToType()) + vecNull := vector.NewVec(types.T_int32.ToType()) + defer vecInt.Free(mp) + defer vecDecimal.Free(mp) + defer vecDecimal128.Free(mp) + defer vecDate.Free(mp) + defer vecString.Free(mp) + defer vecNull.Free(mp) + + require.NoError(t, vector.AppendFixed(vecInt, int64(-42), false, mp)) + decimal, err := types.ParseDecimal64("-123.40", 18, 2) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(vecDecimal, decimal, false, mp)) + decimal128, err := types.ParseDecimal128("123456789012345.67890", 30, 5) + require.NoError(t, err) + require.NoError(t, vector.AppendFixed(vecDecimal128, decimal128, false, mp)) + require.NoError(t, vector.AppendFixed(vecDate, types.DateFromCalendar(2024, 6, 1), false, mp)) + require.NoError(t, vector.AppendBytes(vecString, []byte(`a"b\c`), false, mp)) + require.NoError(t, vector.AppendFixed(vecNull, int32(0), true, mp)) + + var buf bytes.Buffer + err = writeProjectedCSVRowFromVecs( + &buf, + []types.Type{ + types.T_int64.ToType(), + types.T_decimal64.ToTypeWithScale(2), + types.T_decimal128.ToTypeWithScale(5), + types.T_date.ToType(), + types.T_varchar.ToType(), + types.T_int32.ToType(), + }, + []*vector.Vector{vecInt, vecDecimal, vecDecimal128, vecDate, vecString, vecNull}, + []int{0, 1, 2, 3, 4, 5}, + 0, + ) + require.NoError(t, err) + assert.Equal(t, "-42,-123.40,123456789012345.67890,2024-06-01,\"a\"\"b\\\\c\",\\N\n", buf.String()) +} + +// TestWriteCSV_withCreateSQLHeader tests the header comment from CreateSQL. +func TestWriteCSV_withCreateSQLHeader(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id BIGINT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0"}, + Rows: [][]string{ + {"obj1", "0", "0", "42"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "-- CREATE TABLE t1 (id BIGINT)") + assert.Contains(t, output, "-- Database: db1") + assert.Contains(t, output, "-- Table: t1") +} + +func TestWriteCSV_WithoutMetadataComments(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id BIGINT)", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0"}, + Rows: [][]string{ + {"obj1", "0", "0", "42"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view, WithCSVMetaComments(false)) + require.NoError(t, err) + + output := strings.TrimSpace(buf.String()) + assert.Equal(t, "id\n42", output) + assert.NotContains(t, output, "-- CREATE TABLE") +} + +// TestWriteCSV_mismatchedColumns verifies that position-based column mapping +// only includes columns matching the schema positions. +func TestWriteCSV_mismatchedColumns(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + Columns: []TableColumn{ + {Name: "a", SQLType: "INT", Position: 1, PhysicalPosition: 0}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "extra"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + // Skip comment lines + var dataLines []string + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if !strings.HasPrefix(line, "--") { + dataLines = append(dataLines, line) + } + } + // Position-based: only column at position 0 (a) is included, extra columns dropped + assert.Equal(t, "a", dataLines[0]) + assert.Equal(t, "1", dataLines[1]) +} + +// TestBuildSchemaFromMoTablesRow tests extracting a TableSchema from mo_tables row data. +func TestBuildSchemaFromMoTablesRow(t *testing.T) { + // mo_tables physical layout: rel_id, relname, reldatabase, ... + // After meta cols stripped, data cols = [rel_id, relname, reldatabase, reldatabase_id, ..., rel_createsql] + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + } + + // Full row with meta + data: rel_id="12345", relname="my_table", reldatabase="my_db", rel_createsql="CREATE TABLE my_table (x INT)" + fullRow := []string{ + "obj1", "0", "0", // meta cols + "12345", "my_table", "my_db", "100", "x", "x", "x", "CREATE TABLE my_table (x INT)", + } + + schema := buildSchemaFromMoTablesRow(view, fullRow) + assert.Equal(t, "my_table", schema.TableName) + assert.Equal(t, "my_db", schema.DatabaseName) + assert.Equal(t, "CREATE TABLE my_table (x INT)", schema.CreateSQL) +} + +// TestBuildColumnsFromMoColumnsRows tests building column list from mo_columns rows. +func TestBuildColumnsFromMoColumnsRows(t *testing.T) { + // mo_columns physical layout: ..., att_relname_id, att_relname, attname, atttyp, attnum, ..., att_is_hidden + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + "col_19", "col_20", "col_21", "attr_seqnum", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "id", "BIGINT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "name", "VARCHAR(100)", "2", "", "", "", "", "", "", "", "", "", "0", "", "", "", "1"}, + {"obj1", "0", "2", "", "", "", "", "12345", "my_table", "_hidden_col", "INT", "3", "", "", "", "", "", "", "", "", "", "1", "", "", "", "2"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(view, 12345) + require.Len(t, cols, 2) // hidden column filtered + + assert.Equal(t, "id", cols[0].Name) + assert.Equal(t, "BIGINT", cols[0].SQLType) + assert.Equal(t, 1, cols[0].Position) + assert.Equal(t, 0, cols[0].PhysicalPosition) + + assert.Equal(t, "name", cols[1].Name) + assert.Equal(t, "VARCHAR(100)", cols[1].SQLType) + assert.Equal(t, 2, cols[1].Position) + assert.Equal(t, 1, cols[1].PhysicalPosition) +} + +func TestBuildColumnsFromMoColumnsRows_FallbackToAttnumWhenSeqnumMissing(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "id", "BIGINT", "1", "", "", "", "", "", "", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "name", "VARCHAR(100)", "2", "", "", "", "", "", "", "", "", "", "0"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(view, 12345) + require.Len(t, cols, 2) + assert.Equal(t, 0, cols[0].PhysicalPosition) + assert.Equal(t, 1, cols[1].PhysicalPosition) +} + +func TestBuildColumnsFromMoColumnsRows_GenericPreCPKWithPhysicalPrefix(t *testing.T) { + const ( + tableID = uint64(12345) + offset = 1 + ) + + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moColumnsSchema)+offset; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + + row := func(metaRow, name, typ, attnum, hidden, seqnum string) []string { + data := make([]string, len(preCPKLayout.moColumnsSchema)+offset) + setCatalogValue := func(colName, value string) { + idx := catalogColIndexForLayout(preCPKLayout, moColumnsID, colName, offset) + require.GreaterOrEqual(t, idx, 0) + data[idx] = value + } + data[0] = "physical-prefix" + setCatalogValue("att_relname_id", fmt.Sprintf("%d", tableID)) + setCatalogValue("attname", name) + setCatalogValue("atttyp", typ) + setCatalogValue("attnum", attnum) + setCatalogValue("att_is_hidden", hidden) + setCatalogValue("attr_seqnum", seqnum) + return append([]string{"obj1", "0", metaRow}, data...) + } + + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("0", "id", "INT", "1", "0", "0"), + row("1", "name", "VARCHAR(100)", "2", "0", "1"), + row("2", "__mo_rowid", "ROWID", "0", "1", "2"), + }, + } + + cols := buildColumnsFromMoColumnsRows(view, tableID) + require.Len(t, cols, 2) + assert.Equal(t, "id", cols[0].Name) + assert.Equal(t, 0, cols[0].PhysicalPosition) + assert.Equal(t, "name", cols[1].Name) + assert.Equal(t, 1, cols[1].PhysicalPosition) + + ddl := buildCreateTableFromMoColumns(view, tableID) + assert.Contains(t, ddl, "`id` INT") + assert.Contains(t, ddl, "`name` VARCHAR(100)") + assert.NotContains(t, ddl, "__mo_rowid") +} + +func TestCreateTableDDLFromCatalogViews_PrefersMoColumnsOverStaleCreateSQL(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "12345", "t", "db", "100", + "", "", "", "CREATE TABLE t (a INT, b VARCHAR(100))", + }, + }, + } + moColumnsView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"obj1", "0", "0", "12345", "a", encodedSQLType(t, types.T_int32.ToType()), "1", "0"}, + {"obj1", "0", "1", "12345", "c", encodedSQLType(t, types.T_int64.ToType()), "2", "0"}, + }, + } + + ddl := createTableDDLFromCatalogViews(12345, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `t`") + assert.Contains(t, ddl, "`a` INT") + assert.Contains(t, ddl, "`c` BIGINT") + assert.NotContains(t, ddl, "`b`") +} + +func TestCreateTableDDLFromCatalogViews_DoesNotFallbackToRelCreateSQL(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "12345", "employees", "db", "100", + "", "", "", "CREATE TABLE employees (id INT)", + }, + }, + } + + assert.Empty(t, createTableDDLFromCatalogViews(12345, moTablesView, nil)) +} + +func TestCreateTableDDLFromCatalogViews_DecodesMoColumnTypes(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "col_4", "col_5", "col_6", "rel_createsql", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "333997", + "", "", "", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", + }, + }, + } + moColumnsView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"obj1", "0", "0", "333999", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "0"}, + {"obj1", "0", "1", "333999", "code", encodedSQLType(t, types.New(types.T_varchar, 20, 0)), "2", "0"}, + }, + } + + ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `parent`") + assert.Contains(t, ddl, "`id` INT") + assert.Contains(t, ddl, "`code` VARCHAR(20)") + assert.NotContains(t, ddl, "`old`") +} + +func TestCreateTableDDLFromCatalogViews_IncludesColumnAndTableAttributes(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", "rel_createsql", "constraint", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "333997", + "", "r", "parent table comment", "CREATE TABLE `ckp_constraints`.`parent` (`old` INT)", encodedConstraint( + t, + encodedPrimaryKeyConstraint(t, "id"), + &engine.IndexDef{Indexes: []*plan.IndexDef{ + {IndexName: "code", Parts: []string{"code"}, Unique: true}, + {IndexName: "idx_parent_note", Parts: []string{"note"}}, + {IndexName: "ivf_500", Parts: []string{"embedding"}, IndexAlgo: "ivfflat", IndexAlgoParams: `{"lists":"500","op_type":"vector_l2_ops"}`}, + {IndexName: "ivf_2000", Parts: []string{"embedding"}, IndexAlgo: "ivfflat", IndexAlgoParams: `{"lists":"2000","op_type":"vector_l2_ops"}`}, + }}, + ), + }, + }, + } + + headers := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + row := func(name, typ, attnum, notNull, hasDefault, defaultExpr, constraintType, comment, seqnum string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, "333999") + set(catalog.SystemColAttr_RelName, "parent") + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_HasExpr, hasDefault) + set(catalog.SystemColAttr_DefaultExpr, defaultExpr) + set(catalog.SystemColAttr_ConstraintType, constraintType) + set(catalog.SystemColAttr_Comment, comment) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + return append([]string{"obj1", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("id", encodedSQLType(t, types.T_int32.ToType()), "1", "1", "0", "", "p", "", "0"), + row("code", encodedSQLType(t, types.New(types.T_varchar, 20, 0)), "2", "1", "0", "", "", "", "1"), + row("note", encodedSQLType(t, types.New(types.T_varchar, 100, 0)), "3", "0", "1", encodedDefault(t, "'parent-default'", true), "", "parent note", "2"), + row("embedding", encodedSQLType(t, types.New(types.T_array_float32, 128, 0)), "4", "0", "0", "", "", "", "3"), + }, + } + partitionMetadataView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "table_id", "table_name", "database_name", "partition_method", "partition_description", "partition_count", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "333999", "parent", "ckp_constraints", "Key", "key algorithm = 2 (`id`, `tenant_id`)", "4", + }, + }, + } + + ddl := createTableDDLFromCatalogViews(333999, moTablesView, moColumnsView, partitionMetadataView) + assert.Contains(t, ddl, "CREATE TABLE `parent`") + assert.Contains(t, ddl, "`id` INT NOT NULL") + assert.Contains(t, ddl, "`code` VARCHAR(20) NOT NULL") + assert.Contains(t, ddl, "`note` VARCHAR(100) DEFAULT 'parent-default' COMMENT 'parent note'") + assert.Contains(t, ddl, "PRIMARY KEY (`id`)") + assert.Contains(t, ddl, "UNIQUE KEY `code`(`code`)") + assert.Contains(t, ddl, "KEY `idx_parent_note`(`note`)") + assert.Contains(t, ddl, "KEY `ivf_500` USING ivfflat(`embedding`) lists = 500 op_type 'vector_l2_ops'") + assert.Contains(t, ddl, "KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops'") + assert.Contains(t, ddl, "COMMENT='parent table comment'") + assert.Contains(t, ddl, "partition by key algorithm = 2 (`id`, `tenant_id`) partitions 4") + assert.NotContains(t, ddl, "`old`") +} + +func TestCreateTableDDLFromCatalogViews_PreservesCompositeClusterByFromMoColumns(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "272528", "statement_info", "system", "272500", + "", "r", "record each statement and stats info[mo_no_del_hint]", + }, + }, + } + + moColumnsView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...), + Rows: [][]string{ + moColumnsTestRow(t, "272528", "statement_info", "request_at", encodedSQLType(t, types.New(types.T_datetime, 0, 6)), "1", "1", "0"), + moColumnsTestRow(t, "272528", "statement_info", "account_id", encodedSQLType(t, types.T_uint32.ToType()), "2", "0", "1"), + moColumnsTestRow(t, "272528", "statement_info", "__mo_cbkey_010request_at010account_id", encodedSQLType(t, types.T_varchar.ToType()), "3", "0", "2", catalog.SystemColAttr_IsHidden, "1", catalog.SystemColAttr_IsClusterBy, "1"), + }, + } + + ddl := createTableDDLFromCatalogViews(272528, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CREATE TABLE `statement_info`") + assert.Contains(t, ddl, "`request_at` DATETIME(6) NOT NULL") + assert.Contains(t, ddl, "`account_id` INT UNSIGNED") + assert.NotContains(t, ddl, "__mo_cbkey_010request_at010account_id") + assert.Contains(t, ddl, "COMMENT='record each statement and stats info[mo_no_del_hint]' CLUSTER BY (`request_at`, `account_id`)") +} + +func moColumnsTestRow(t *testing.T, relID, relName, name, typ, attnum, notNull, seqnum string, extra ...string) []string { + t.Helper() + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, relID) + set(catalog.SystemColAttr_RelName, relName) + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + for i := 0; i+1 < len(extra); i += 2 { + set(extra[i], extra[i+1]) + } + return append([]string{"obj1", "0", attnum}, data...) +} + +func TestCreateTableDDLFromCatalogViews_IncludesForeignKeys(t *testing.T) { + moTablesHeaders := append([]string{"object", "block", "row"}, catalog.MoTablesSchema...) + tableRow := func(relID, relName, dbName, dbID, constraint string) []string { + data := make([]string, len(catalog.MoTablesSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoTablesSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_tables header %s", colName) + } + set(catalog.SystemRelAttr_ID, relID) + set(catalog.SystemRelAttr_Name, relName) + set(catalog.SystemRelAttr_DBName, dbName) + set(catalog.SystemRelAttr_DBID, dbID) + set(catalog.SystemRelAttr_Kind, "r") + set(catalog.SystemRelAttr_Constraint, constraint) + return append([]string{"obj", "0", relID}, data...) + } + moTablesView := &LogicalTableView{ + Headers: moTablesHeaders, + Rows: [][]string{ + tableRow("100", "parent", "ckp_constraints", "10", encodedConstraint(t, encodedPrimaryKeyConstraint(t, "id"))), + tableRow("101", "child_cascade", "ckp_constraints", "10", encodedConstraint(t, + &engine.ForeignKeyDef{Fkeys: []*plan.ForeignKeyDef{{ + Name: "fk_child_cascade_parent", + Cols: []uint64{2}, + ForeignTbl: 100, + ForeignCols: []uint64{1}, + OnDelete: plan.ForeignKeyDef_CASCADE, + OnUpdate: plan.ForeignKeyDef_RESTRICT, + }}}, + )), + }, + } + + moColumnsHeaders := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + columnRow := func(relID, relName, colID, name, typ, attnum, notNull string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_UniqName, colID) + set(catalog.SystemColAttr_RelID, relID) + set(catalog.SystemColAttr_RelName, relName) + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, attnum) + return append([]string{"obj", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: moColumnsHeaders, + Rows: [][]string{ + columnRow("100", "parent", "100-id", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "101-id", "id", encodedSQLType(t, types.T_int32.ToType()), "1", "1"), + columnRow("101", "child_cascade", "101-parent_id", "parent_id", encodedSQLType(t, types.T_int32.ToType()), "2", "0"), + }, + } + + ddl := createTableDDLFromCatalogViews(101, moTablesView, moColumnsView) + assert.Contains(t, ddl, "CONSTRAINT `fk_child_cascade_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT") +} + +func TestCloneTableSchemaCopiesForeignKeys(t *testing.T) { + schema := &TableSchema{ + TableName: "child_restrict", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1, NotNull: true}, + {Name: "parent_id", SQLType: "INT", Position: 2, NotNull: true}, + }, + PrimaryKey: []string{"id"}, + ForeignKeys: []TableForeignKey{{ + Name: "fk_child_restrict_parent", + Columns: []string{"parent_id"}, + ReferTable: "parent", + ReferColumns: []string{"id"}, + OnDelete: plan.ForeignKeyDef_RESTRICT, + OnUpdate: plan.ForeignKeyDef_RESTRICT, + }}, + } + + ddl := RenderCreateTableDDLFromSchema(cloneTableSchema(schema)) + assert.Contains(t, ddl, "CONSTRAINT `fk_child_restrict_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT") +} + +func TestBuildPartitionClauseFromMetadata_UsesSeqNumsWithHiddenColumn(t *testing.T) { + view := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"), + ColSeqNums: []uint16{ + 6, // hidden/cpkey column before visible catalog columns + 0, 1, 2, 3, 4, 5, + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "hidden-key", + "334023", "t_hash_partition", "ckp_tables", "Hash", "hash (`id`)", "4", + }, + }, + } + applyCatalogHeadersBySeqNums(view, moPartitionMetadataHeaders) + + assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 334023)) +} + +func TestBuildPartitionClauseFromMetadata_FallsBackToRowShape(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + { + "obj1", "0", "0", + "272577", "t_hash_partition", "ckp_tables", "Hash", "hash (`id`)", "4", + }, + }, + } + + assert.Equal(t, "partition by hash (`id`) partitions 4", buildPartitionClauseFromMetadata(view, 272577)) +} + +func TestBuildPartitionClauseFromMetadata_IncludesRangePartitionDefinitions(t *testing.T) { + metadataView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionMetadataHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272882", "t_range_partition", "ckp_partition_options", "Range", "range (`id`)", "4"}, + }, + } + tablesView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272883", "%!%p0%!%t_range_partition", "272882", "p0", "0", "values less than (100)", ""}, + {"obj1", "0", "1", "272884", "%!%p1%!%t_range_partition", "272882", "p1", "1", "values less than (200)", ""}, + {"obj1", "0", "2", "272885", "%!%pmax%!%t_range_partition", "272882", "pmax", "2", "values less than MAXVALUE", ""}, + }, + } + + assert.Equal(t, "partition by range (`id`) (\n partition `p0` values less than (100),\n partition `p1` values less than (200),\n partition `pmax` values less than MAXVALUE\n)", buildPartitionClauseFromMetadata(metadataView, 272882, tablesView)) +} + +func TestBuildPartitionClauseFromMetadata_IncludesListPartitionDefinitions(t *testing.T) { + metadataView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionMetadataHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272895", "t_list_columns_partition", "ckp_partition_options", "List", "list columns (`category`)", "3"}, + }, + } + tablesView := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "1", "272897", "%!%p_b%!%t_list_columns_partition", "272895", "p_b", "1", "values in ('b')", ""}, + {"obj1", "0", "0", "272896", "%!%p_a%!%t_list_columns_partition", "272895", "p_a", "0", "values in ('a')", ""}, + }, + } + + assert.Equal(t, "partition by list columns (`category`) (\n partition `p_a` values in ('a'),\n partition `p_b` values in ('b')\n)", buildPartitionClauseFromMetadata(metadataView, 272895, tablesView)) +} + +func TestBuildPartitionTableIDMap(t *testing.T) { + view := &LogicalTableView{ + Headers: append([]string{"object", "block", "row"}, moPartitionTablesHeaders...), + Rows: [][]string{ + {"obj1", "0", "0", "272578", "%!%p0%!%t_hash_partition", "272577", "p0", "0", "", ""}, + {"obj1", "0", "1", "272579", "%!%p1%!%t_hash_partition", "272577", "p1", "1", "", ""}, + {"obj1", "0", "2", "272580", "%!%p0%!%other", "999999", "p0", "0", "", ""}, + }, + } + + got := buildPartitionTableIDMap(view, map[uint64]struct{}{272577: {}}) + assert.Equal(t, []uint64{272578, 272579}, got[272577]) +} + +func TestBuildPartitionTableIDMap_FallsBackToRowShape(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + {"obj1", "0", "1", "334040", "%!%p1%!%t_hash_partition", "334039", "p1", "1", "", ""}, + {"obj1", "0", "0", "334041", "%!%p0%!%t_hash_partition", "334039", "p0", "0", "", ""}, + }, + } + + got := buildPartitionTableIDMap(view, map[uint64]struct{}{334039: {}}) + assert.Equal(t, []uint64{334041, 334040}, got[334039]) +} + +func TestCreateTableDDLFromCatalogViews_IgnoresMoColumnsPrimaryWithoutConstraint(t *testing.T) { + moTablesView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "rel_id", "relname", "reldatabase", "reldatabase_id", + "relpersistence", "relkind", "rel_comment", "rel_createsql", "constraint", + }, + Rows: [][]string{ + { + "obj1", "0", "0", + "334100", "t_int_signed", "ckp_types", "334099", + "", "r", "", "", "", + }, + }, + } + + headers := append([]string{"object", "block", "row"}, catalog.MoColumnsSchema...) + row := func(name, typ, attnum, notNull, constraintType, seqnum string) []string { + data := make([]string, len(catalog.MoColumnsSchema)) + set := func(colName, value string) { + for i, header := range catalog.MoColumnsSchema { + if header == colName { + data[i] = value + return + } + } + t.Fatalf("missing mo_columns header %s", colName) + } + set(catalog.SystemColAttr_RelID, "334100") + set(catalog.SystemColAttr_RelName, "t_int_signed") + set(catalog.SystemColAttr_Name, name) + set(catalog.SystemColAttr_Type, typ) + set(catalog.SystemColAttr_Num, attnum) + set(catalog.SystemColAttr_NullAbility, notNull) + set(catalog.SystemColAttr_ConstraintType, constraintType) + set(catalog.SystemColAttr_IsHidden, "0") + set(catalog.SystemColAttr_Seqnum, seqnum) + return append([]string{"obj1", "0", attnum}, data...) + } + moColumnsView := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("id", encodedSQLType(t, types.T_int32.ToType()), "1", "1", "p", "0"), + row("c_tiny", encodedSQLType(t, types.T_int8.ToType()), "2", "0", "", "1"), + row("c_small", encodedSQLType(t, types.T_int16.ToType()), "3", "0", "", "2"), + row("c_int", encodedSQLType(t, types.T_int32.ToType()), "4", "0", "", "3"), + row("c_big", encodedSQLType(t, types.T_int64.ToType()), "5", "0", "", "4"), + }, + } + + ddl := createTableDDLFromCatalogViews(334100, moTablesView, moColumnsView) + assert.Contains(t, ddl, "`id` INT NOT NULL") + assert.NotContains(t, ddl, "PRIMARY KEY") +} + +func TestRenderCreateTableDDLFromSchema_PrefersColumnsOverCreateSQL(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t", + CreateSQL: "CREATE TABLE t (a INT, b VARCHAR(100))", + Columns: []TableColumn{ + {Name: "a", SQLType: "INT", Position: 1}, + {Name: "c", SQLType: "BIGINT", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`a` INT") + assert.Contains(t, ddl, "`c` BIGINT") + assert.NotContains(t, ddl, "`b`") +} + +func TestRenderCreateTableDDLFromSchema_DoesNotFallbackToCreateSQLWhenColumnTypesAreBinary(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t", + CreateSQL: "CREATE TABLE t (a INT)", + Columns: []TableColumn{ + {Name: "a", SQLType: string([]byte{'A', 0, 0xff})}, + }, + }) + + assert.Empty(t, ddl) +} + +func TestRenderCreateTableDDLFromSchema_PreservesArrayType(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_array", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1}, + {Name: "tags", SQLType: "ARRAY(VARCHAR(20))", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`tags` ARRAY(VARCHAR(20))") + assert.NotContains(t, ddl, "`tags` JSON") +} + +func TestRenderCreateTableDDLFromSchema_PreservesTypedArrayMetadata(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_array", + Columns: []TableColumn{ + {Name: "id", SQLType: "INT", Position: 1}, + {Name: "tags", SQLType: "JSON", EnumValues: "array(varchar(20))", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`tags` ARRAY(varchar(20))") + assert.NotContains(t, ddl, "`tags` JSON") +} + +func TestIsPrintableCreateTableSQLAcceptsExternalTable(t *testing.T) { + assert.True(t, isPrintableCreateTableSQL("CREATE EXTERNAL TABLE ext_csv (id INT) INFILE {'filepath'='/tmp/ext.csv','format'='csv'}")) +} + +func TestIsPrintableCreateTableSQLAcceptsView(t *testing.T) { + assert.True(t, isPrintableCreateTableSQL("CREATE VIEW `ckp_views`.`v_normal` AS SELECT id FROM `ckp_tables`.`t_normal`")) +} + +func TestIsPrintableExternalParamJSON(t *testing.T) { + assert.True(t, isPrintableExternalParamJSON(`{"ExParamConst":{"Option":["filepath","/tmp/ext.csv","format","csv"]}}`)) + assert.False(t, isPrintableExternalParamJSON(`CREATE EXTERNAL TABLE ext_csv (id INT)`)) +} + +func TestFindTableSchemaFromMoTablesUsesCatalogLayoutFallback(t *testing.T) { + schema := schemaForLayout(currentCatalogLayout, moTablesID) + headers := append([]string{}, logicalTableViewMetaHeaders...) + for range schema { + headers = append(headers, "") + } + row := make([]string, len(headers)) + row[0], row[1], row[2] = "obj", "0", "0" + data := row[logicalViewMetaCols:] + for i, name := range schema { + switch name { + case "rel_id": + data[i] = "272746" + case "relname": + data[i] = "ext_csv_local" + case "reldatabase": + data[i] = "ckp25010_external" + case "rel_createsql": + data[i] = `{"ExParamConst":{"Option":["filepath","/tmp/ext.csv","format","csv"]}}` + } + } + + got := findTableSchemaFromMoTables(&LogicalTableView{Headers: headers, Rows: [][]string{row}}, 272746) + require.NotNil(t, got) + assert.Equal(t, "ext_csv_local", got.TableName) + assert.Equal(t, "ckp25010_external", got.DatabaseName) + assert.Contains(t, got.CreateSQL, "ExParamConst") + assert.Contains(t, got.CreateSQL, "filepath") +} + +func TestRenderCreateTableDDLFromSchema_EnumSetValues(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_enum_set", + Columns: []TableColumn{ + {Name: "c_enum", SQLType: "ENUM", EnumValues: "red,green,blue", Position: 1}, + {Name: "c_set", SQLType: "BIGINT UNSIGNED", EnumValues: "a,b,c", Position: 2}, + }, + }) + + assert.Contains(t, ddl, "`c_enum` ENUM('red','green','blue')") + assert.Contains(t, ddl, "`c_set` SET('a','b','c')") +} + +func TestRenderCreateTableDDLFromSchema_AutoIncrementSkipsDefaultNull(t *testing.T) { + ddl := RenderCreateTableDDLFromSchema(&TableSchema{ + TableName: "t_auto", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT UNSIGNED", Position: 1, AutoIncrement: true, HasDefault: true}, + }, + }) + + assert.Contains(t, ddl, "`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT") + assert.NotContains(t, ddl, "AUTO_INCREMENT DEFAULT NULL") +} + +func TestDecodeMoColumnEncodedSQLType_TemporalScale(t *testing.T) { + sqlType, ok := decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_time, 0, 6))) + require.True(t, ok) + assert.Equal(t, "TIME(6)", sqlType) + + sqlType, ok = decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_datetime, 0, 3))) + require.True(t, ok) + assert.Equal(t, "DATETIME(3)", sqlType) + + sqlType, ok = decodeMoColumnEncodedSQLType(encodedSQLType(t, types.New(types.T_timestamp, 0, 6))) + require.True(t, ok) + assert.Equal(t, "TIMESTAMP(6)", sqlType) +} + +func TestBuildCatalogTablesFromMoTablesRows_GenericWithTrailingColumns(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moTablesSchema)+2; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + row := func(relID, relName, dbName, dbID, relKind, accountID string) []string { + data := make([]string, len(preCPKLayout.moTablesSchema)+2) + data[0] = relID + data[1] = relName + data[2] = dbName + data[3] = dbID + data[5] = relKind + data[11] = accountID + return append([]string{"obj1", "0", relID}, data...) + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("1001", "orders", "tpch_10g", "9001", "r", "7"), + row("1002", "revenue0", "tpch_10g", "9001", "v", "7"), + }, + } + + tables := buildCatalogTablesFromMoTablesRows(view) + require.Len(t, tables, 2) + assert.Equal(t, uint64(1001), tables[0].TableID) + assert.Equal(t, uint32(7), tables[0].AccountID) + assert.Equal(t, uint64(9001), tables[0].DatabaseID) + assert.Equal(t, "tpch_10g", tables[0].DatabaseName) + assert.Equal(t, "orders", tables[0].TableName) + assert.Equal(t, "r", tables[0].RelKind) + assert.Equal(t, "revenue0", tables[1].TableName) + assert.Equal(t, "v", tables[1].RelKind) +} + +func TestBuildCatalogDatabasesFromMoDatabaseRows_IncludesDatabaseWithoutTables(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(currentCatalogLayout.moDatabaseSchema); i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + row := func(datID, datName, accountID, datType string) []string { + data := make([]string, len(currentCatalogLayout.moDatabaseSchema)) + data[0] = datID + data[1] = datName + data[7] = accountID + data[8] = datType + return append([]string{"obj1", "0", datID}, data...) + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{ + row("335900", "ckp_pubsub_sub_all", "415", catalog.SystemDBTypeSubscription), + }, + } + + databases := buildCatalogDatabasesFromMoDatabaseRows(view) + require.Len(t, databases, 1) + assert.Equal(t, uint32(415), databases[0].AccountID) + assert.Equal(t, uint64(335900), databases[0].DatabaseID) + assert.Equal(t, "ckp_pubsub_sub_all", databases[0].DatabaseName) + assert.Empty(t, databases[0].TableName) + assert.Zero(t, databases[0].TableID) +} + +func TestBuildCatalogTablesFromMoTablesRows_UsesTemporaryDisplayName(t *testing.T) { + headers := []string{"object", "block", "row"} + for i := 0; i < len(preCPKLayout.moTablesSchema); i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + } + data := make([]string, len(preCPKLayout.moTablesSchema)) + data[0] = "1003" + data[1] = "__mo_tmp_123e4567e89b12d3a456426614174000_ckp_temp_t_session_only" + data[2] = "ckp_temp" + data[3] = "9002" + data[5] = "r" + data[11] = "7" + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{append([]string{"obj1", "0", "0"}, data...)}, + } + + tables := buildCatalogTablesFromMoTablesRows(view) + require.Len(t, tables, 1) + assert.Equal(t, "t_session_only", tables[0].TableName) +} + +func TestDisplayTableName(t *testing.T) { + assert.Equal(t, "orders", displayTableName("db1", "orders")) + assert.Equal(t, "t1", displayTableName("db1", "__mo_tmp_123e4567e89b12d3a456426614174000_db1_t1")) + assert.Equal(t, "__mo_tmp_123e4567e89b12d3a456426614174000_other_t1", displayTableName("db1", "__mo_tmp_123e4567e89b12d3a456426614174000_other_t1")) + assert.Equal(t, "__mo_tmp_bad", displayTableName("db1", "__mo_tmp_bad")) +} + +func TestListCatalogTablesDoesNotFilterRelKinds(t *testing.T) { + tables := []TableCatalogEntry{ + {TableID: 1, TableName: "orders", DatabaseName: "db1", RelKind: "r"}, + {TableID: 2, TableName: "ext_orders", DatabaseName: "db1", RelKind: "e"}, + {TableID: 3, TableName: "orders_v", DatabaseName: "db1", RelKind: "v"}, + {TableID: 4, TableName: "cluster_orders", DatabaseName: "db1", RelKind: "cluster"}, + } + filtered := filterCatalogTablesForList(tables, TableListOptions{}) + + require.Len(t, filtered, 4) + names := make([]string, 0, len(filtered)) + for _, table := range filtered { + names = append(names, table.TableName) + } + assert.ElementsMatch(t, []string{"orders", "ext_orders", "orders_v", "cluster_orders"}, names) +} + +func TestIsMissingCheckpointTableError(t *testing.T) { + err := moerr.NewInternalErrorf(context.Background(), "table %d not found in checkpoint at ts %s", uint64(272419), "1-0") + assert.True(t, isMissingCheckpointTableError(err, 272419)) + assert.False(t, isMissingCheckpointTableError(err, 272420)) + assert.False(t, isMissingCheckpointTableError(nil, 272419)) +} + +func TestInferCatalogLayout(t *testing.T) { + tests := []struct { + name string + dataWidth int + tableID uint64 + layout string + offset int + ok bool + }{ + {name: "current_tables", dataWidth: len(catalog.MoTablesSchema), tableID: moTablesID, layout: currentCatalogLayout.name, offset: 0, ok: true}, + {name: "current_tables_fakepk", dataWidth: len(catalog.MoTablesSchema) + 1, tableID: moTablesID, layout: currentCatalogLayout.name, offset: 1, ok: true}, + {name: "current_columns_fakepk", dataWidth: len(catalog.MoColumnsSchema) + 1, tableID: moColumnsID, layout: currentCatalogLayout.name, offset: 1, ok: true}, + {name: "pre_cpk_tables", dataWidth: len(preCPKLayout.moTablesSchema), tableID: moTablesID, layout: preCPKLayout.name, offset: 0, ok: true}, + {name: "pre_cpk_columns", dataWidth: len(preCPKLayout.moColumnsSchema), tableID: moColumnsID, layout: preCPKLayout.name, offset: 0, ok: true}, + {name: "legacy3_tables", dataWidth: len(legacy3CatalogLayout.moTablesSchema), tableID: moTablesID, layout: legacy3CatalogLayout.name, offset: 0, ok: true}, + {name: "legacy3_columns", dataWidth: len(legacy3CatalogLayout.moColumnsSchema), tableID: moColumnsID, layout: legacy3CatalogLayout.name, offset: 0, ok: true}, + {name: "unknown_tables", dataWidth: 7, tableID: moTablesID, ok: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + layout, offset, ok := inferCatalogLayout(tt.dataWidth, tt.tableID) + assert.Equal(t, tt.ok, ok) + if !tt.ok { + return + } + assert.Equal(t, tt.layout, layout.name) + assert.Equal(t, tt.offset, offset) + }) + } +} + +func TestFallbackCatalogColIndex_UnknownLayoutDoesNotGuessCurrent(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5", "col_6"}, + } + assert.Equal(t, -1, fallbackCatalogColIndex(view, moTablesID, "rel_createsql")) +} + +// TestLogicalTableViewWithSchema_mergeHeaders merges schema column names using position-based mapping. +func TestLogicalTableViewWithSchema_mergeHeaders(t *testing.T) { + schema := &TableSchema{ + TableName: "t1", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 0}, + {Name: "val", SQLType: "INT", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "42", "100"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + assert.Equal(t, []string{"id", "val"}, merged.Headers) + // Data rows only include columns matching schema positions + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"42", "100"}, merged.Rows[0]) +} + +func TestMergeLogicalViewWithSchema_requiresResolvedSchema(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj1", "0", "0", "42", "100"}, + }, + VisibleRows: 1, + DeletedRows: 0, + PhysicalRows: 1, + } + + merged := MergeLogicalViewWithSchema(view, &TableSchema{}) + assert.Empty(t, merged.Headers) + assert.Empty(t, merged.Rows) + assert.Equal(t, 1, merged.VisibleRows) + assert.Equal(t, 1, merged.PhysicalRows) +} + +func TestMergeLogicalViewWithSchema_UsesPhysicalPosition(t *testing.T) { + schema := &TableSchema{ + TableName: "shifted", + Columns: []TableColumn{ + {Name: "id", SQLType: "VARCHAR(36)", Position: 1, PhysicalPosition: 4}, + {Name: "task_id", SQLType: "VARCHAR(36)", Position: 2, PhysicalPosition: 5}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3", "col_4", "col_5"}, + Rows: [][]string{ + {"obj1", "0", "0", "ignored0", "ignored1", "ignored2", "ignored3", "case-001", "task-001"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"case-001", "task-001"}, merged.Rows[0]) +} + +// TestMergeLogicalViewWithSchema_hiddenColumns verifies that hidden columns (e.g. __mo_fake_pk_col, +// __mo_cpkey_col) are excluded from the merged view using position-based column mapping. +func TestMergeLogicalViewWithSchema_hiddenColumns(t *testing.T) { + // Simulate: physical columns = [__mo_fake_pk_col(hidden,pos=0), id(pos=1), name(pos=2), __mo_cpkey_col(hidden,pos=3)] + // Schema only has visible columns: id(pos=1), name(pos=2) + schema := &TableSchema{ + TableName: "users", + Columns: []TableColumn{ + {Name: "id", SQLType: "BIGINT", Position: 1, PhysicalPosition: 1}, + {Name: "name", SQLType: "VARCHAR(100)", Position: 2, PhysicalPosition: 2}, + }, + } + + // The LogicalTableView has all 4 physical columns + 3 meta columns = 7 total + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2", "col_3"}, + Rows: [][]string{ + {"obj1", "0", "0", "fake_pk_123", "1", "Alice", "cpkey_456"}, + {"obj1", "0", "1", "fake_pk_789", "2", "Bob", "cpkey_999"}, + }, + } + + merged := MergeLogicalViewWithSchema(view, schema) + + // Headers should only contain visible columns + assert.Equal(t, []string{"id", "name"}, merged.Headers) + + // Data rows: only columns at positions 1 and 2 (skip pos 0 and pos 3) + require.Len(t, merged.Rows, 2) + assert.Equal(t, []string{"1", "Alice"}, merged.Rows[0]) + assert.Equal(t, []string{"2", "Bob"}, merged.Rows[1]) + + // Full CSV round-trip + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "id,name") + assert.Contains(t, output, `1,"Alice"`) + assert.Contains(t, output, `2,"Bob"`) + assert.NotContains(t, output, "fake_pk") // hidden columns excluded + assert.NotContains(t, output, "cpkey") // hidden columns excluded +} + +func TestMergeLogicalViewWithSchema_LargeSparsePhysicalPositions(t *testing.T) { + const physicalCols = 128 + headers := []string{"object", "block", "row"} + row := []string{"obj1", "0", "0"} + for i := 0; i < physicalCols; i++ { + headers = append(headers, fmt.Sprintf("col_%d", i)) + row = append(row, fmt.Sprintf("v%d", i)) + } + + schema := &TableSchema{ + TableName: "wide_table", + Columns: []TableColumn{ + {Name: "first", Position: 1, PhysicalPosition: 0}, + {Name: "middle", Position: 2, PhysicalPosition: 63}, + {Name: "last", Position: 3, PhysicalPosition: 127}, + }, + } + view := &LogicalTableView{ + Headers: headers, + Rows: [][]string{row}, + } + + merged := MergeLogicalViewWithSchema(view, schema) + require.Len(t, merged.Rows, 1) + assert.Equal(t, []string{"v0", "v63", "v127"}, merged.Rows[0]) +} + +func TestBuildCreateIndexStatementsFromMoIndexes_IVFFlat(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "table_id", "name", "type", "algo", "algo_params", "comment", + "column_name", "ordinal_position", "hidden", + }, + Rows: [][]string{ + {"272535", "PRIMARY", "PRIMARY", "", "", "", "id", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "embedding", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "embedding", "1", "0"}, + {"272535", "ivf_2000", "MULTIPLE", "ivfflat", `{"lists":"2000","op_type":"vector_l2_ops"}`, "", "__mo_alias_embedding", "1", "0"}, + {"999999", "other_idx", "MULTIPLE", "", "", "", "v", "1", "0"}, + }, + } + + stmts, err := buildCreateIndexStatementsFromMoIndexes(view, 272535, "items_gist") + require.NoError(t, err) + require.Equal(t, []string{ + "ALTER TABLE `items_gist` ADD KEY `ivf_2000` USING ivfflat(`embedding`) lists = 2000 op_type 'vector_l2_ops' ;", + }, stmts) +} + +func TestBuildCreateIndexStatementsFromMoIndexes_FullText(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{ + "table_id", "name", "type", "algo", "algo_params", "comment", + "column_name", "ordinal_position", "hidden", + }, + Rows: [][]string{ + {"272535", "idx_doc", "MULTIPLE", "fulltext", "", "", "doc", "1", "0"}, + }, + } + + stmts, err := buildCreateIndexStatementsFromMoIndexes(view, 272535, "t_fulltext") + require.NoError(t, err) + require.Equal(t, []string{ + "ALTER TABLE `t_fulltext` ADD FULLTEXT KEY `idx_doc`(`doc`);", + }, stmts) +} + +func TestWriteCSV_LexicalRowOrder(t *testing.T) { + schema := &TableSchema{ + TableName: "sorted", + Columns: []TableColumn{ + {Name: "id", Position: 1, PhysicalPosition: 0}, + {Name: "name", Position: 2, PhysicalPosition: 1}, + }, + } + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1"}, + Rows: [][]string{ + {"obj2", "0", "1", "2", "zeta"}, + {"obj1", "0", "0", "1", "beta"}, + {"obj3", "0", "0", "1", "alpha"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, view, WithCSVMetaComments(false), WithCSVRowOrder(CSVRowOrderLexical)) + require.NoError(t, err) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Equal(t, []string{ + "id,name", + "1,alpha", + "1,beta", + "2,zeta", + }, lines) +} + +// TestColumnSchemaRoundTrip tests the full pipeline: rows → columns → schema → CSV header. +func TestColumnSchemaRoundTrip(t *testing.T) { + moColumnsView := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "col_0", "col_1", "col_2", "col_3", + "att_relname_id", "att_relname", "attname", "atttyp", "attnum", + "col_9", "col_10", "col_11", "col_12", "col_13", "col_14", "col_15", "col_16", "col_17", + "att_is_hidden", + "col_19", "col_20", "col_21", "attr_seqnum", + }, + Rows: [][]string{ + {"obj1", "0", "0", "", "", "", "", "12345", "my_table", "a", "INT", "1", "", "", "", "", "", "", "", "", "", "0", "", "", "", "0"}, + {"obj1", "0", "1", "", "", "", "", "12345", "my_table", "b", "TEXT", "2", "", "", "", "", "", "", "", "", "", "0", "", "", "", "1"}, + {"obj1", "0", "2", "", "", "", "", "12345", "my_table", "c", "FLOAT", "3", "", "", "", "", "", "", "", "", "", "0", "", "", "", "2"}, + }, + } + + cols := buildColumnsFromMoColumnsRows(moColumnsView, 12345) + require.Len(t, cols, 3) + + schema := &TableSchema{ + TableName: "roundtrip", + DatabaseName: "test", + Columns: cols, + CreateSQL: "CREATE TABLE roundtrip (a INT, b TEXT, c FLOAT)", + } + + dataView := &LogicalTableView{ + Headers: []string{"object", "block", "row", "col_0", "col_1", "col_2"}, + Rows: [][]string{ + {"obj1", "0", "0", "1", "hello", "1.500000"}, + {"obj1", "0", "1", "2", "world", "2.000000"}, + }, + } + + var buf bytes.Buffer + err := WriteCSV(&buf, schema, dataView) + require.NoError(t, err) + + output := buf.String() + assert.Contains(t, output, "a,b,c") + assert.Contains(t, output, `1,"hello",1.500000`) + + // Verify comments are present + assert.Contains(t, output, "-- CREATE TABLE roundtrip") + assert.Contains(t, output, "-- Database: test") + fmt.Println(output) +} + +func TestBuildCreateTableFromMoColumns(t *testing.T) { + // Simulate mo_columns data for a user table (id=100) with 3 columns + view := &LogicalTableView{ + Headers: []string{ + "object", "block", "row", + "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden", + }, + Rows: [][]string{ + // id BIGINT, pos=0 + {"", "0", "0", "100", "id", "BIGINT", "1", "0"}, + // name VARCHAR(100), pos=1 + {"", "0", "1", "100", "name", "VARCHAR(100)", "2", "0"}, + // __mo_fake_pk_col, pos=2, hidden -> skipped + {"", "0", "2", "100", "__mo_fake_pk_col", "VARCHAR(65535)", "3", "1"}, + }, + } + + ddl := buildCreateTableFromMoColumns(view, 100) + assert.NotEmpty(t, ddl) + assert.Contains(t, ddl, "BIGINT") + assert.Contains(t, ddl, "VARCHAR(100)") + assert.NotContains(t, ddl, "__mo_fake_pk_col") + assert.Contains(t, ddl, "CREATE TABLE") +} + +func TestHardcodedCreateTableForLegacy3Dev(t *testing.T) { + ddl := hardcodedCreateTableForLayout(catalog.MO_TABLES_ID, legacy3CatalogLayout) + assert.Contains(t, ddl, "CREATE TABLE `mo_tables`") + assert.NotContains(t, ddl, "rel_logical_id") + assert.NotContains(t, ddl, "extra_info") + assert.NotContains(t, ddl, catalog.SystemRelAttr_CPKey) + + ddl = hardcodedCreateTableForLayout(catalog.MO_COLUMNS_ID, legacy3CatalogLayout) + assert.Contains(t, ddl, "CREATE TABLE `mo_columns`") + assert.NotContains(t, ddl, "attr_generated") + assert.NotContains(t, ddl, "attr_has_generated") + assert.NotContains(t, ddl, catalog.SystemColAttr_CPKey) +} + +func TestBuildCreateTableFromMoColumns_empty(t *testing.T) { + view := &LogicalTableView{ + Headers: []string{"object", "block", "row", "att_relname_id", "attname", "atttyp", "attnum", "att_is_hidden"}, + Rows: [][]string{ + {"", "0", "0", "200", "x", "INT", "1", "0"}, + }, + } + ddl := buildCreateTableFromMoColumns(view, 999) // different table id + assert.Empty(t, ddl) +} + +func TestHardcodedCreateTable(t *testing.T) { + tests := []struct { + tableID uint64 + wantName string + }{ + {catalog.MO_TABLES_ID, "mo_tables"}, + {catalog.MO_COLUMNS_ID, "mo_columns"}, + {catalog.MO_DATABASE_ID, "mo_database"}, + {999999, ""}, // unknown table should return empty + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("table-%d", tt.tableID), func(t *testing.T) { + ddl := hardcodedCreateTableForLayout(tt.tableID, currentCatalogLayout) + if tt.wantName == "" { + assert.Empty(t, ddl) + } else { + assert.Contains(t, ddl, tt.wantName) + assert.Contains(t, ddl, "CREATE TABLE") + } + }) + } +} + +func TestBuiltinTableSchemaForLayout_CurrentVisibleColumnsOnly(t *testing.T) { + schema := builtinTableSchemaForLayout(currentCatalogLayout, catalog.MO_TABLES_ID) + require.NotNil(t, schema) + assert.Equal(t, "mo_tables", schema.TableName) + assert.Equal(t, "mo_catalog", schema.DatabaseName) + assert.NotEmpty(t, schema.CreateSQL) + + names := make([]string, 0, len(schema.Columns)) + for _, col := range schema.Columns { + names = append(names, col.Name) + assert.Equal(t, col.Position, col.PhysicalPosition) + } + assert.Contains(t, names, "rel_logical_id") + assert.NotContains(t, names, "extra_info") + assert.NotContains(t, names, catalog.SystemRelAttr_CPKey) +} + +func TestBuiltinTableSchemaForLayout_Legacy3Compatibility(t *testing.T) { + tablesSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_TABLES_ID) + require.NotNil(t, tablesSchema) + tableNames := make([]string, 0, len(tablesSchema.Columns)) + for _, col := range tablesSchema.Columns { + tableNames = append(tableNames, col.Name) + } + assert.NotContains(t, tableNames, "rel_logical_id") + assert.NotContains(t, tableNames, "extra_info") + assert.NotContains(t, tableNames, catalog.SystemRelAttr_CPKey) + + columnsSchema := builtinTableSchemaForLayout(legacy3CatalogLayout, catalog.MO_COLUMNS_ID) + require.NotNil(t, columnsSchema) + columnNames := make([]string, 0, len(columnsSchema.Columns)) + for _, col := range columnsSchema.Columns { + columnNames = append(columnNames, col.Name) + } + assert.NotContains(t, columnNames, "attr_has_generated") + assert.NotContains(t, columnNames, "attr_generated") + assert.NotContains(t, columnNames, catalog.SystemColAttr_CPKey) +} + +func TestReadTableSchema_BuiltinFallbackForSystemTable(t *testing.T) { + reader := &CheckpointReader{} + schema := reader.ReadTableSchema(context.Background(), catalog.MO_TABLES_ID, types.TS{}, nil) + + require.NotNil(t, schema) + assert.Equal(t, "mo_tables", schema.TableName) + assert.Equal(t, "mo_catalog", schema.DatabaseName) + require.NotEmpty(t, schema.Columns) + + names := make([]string, 0, len(schema.Columns)) + for _, col := range schema.Columns { + names = append(names, col.Name) + } + assert.Contains(t, names, "relname") + assert.NotContains(t, names, "extra_info") + assert.NotContains(t, names, catalog.SystemRelAttr_CPKey) +} + +func TestWriteCSVMetadata(t *testing.T) { + var buf bytes.Buffer + err := writeCSVMetadata(&buf, &TableSchema{ + TableName: "t1", + DatabaseName: "db1", + CreateSQL: "CREATE TABLE t1 (id INT)", + }, logicalTableStats{ + VisibleRows: 10, + DeletedRows: 3, + PhysicalRows: 13, + }) + require.NoError(t, err) + out := buf.String() + assert.Contains(t, out, "-- CREATE TABLE t1 (id INT)") + assert.Contains(t, out, "-- Database: db1") + assert.Contains(t, out, "-- Table: t1") + assert.Contains(t, out, "-- Visible rows: 10 (deleted: 3, physical: 13)") +} + +func TestWriteSQLLoadCSVRow_NullAndEscaping(t *testing.T) { + var buf bytes.Buffer + types := []types.Type{types.T_varchar.ToType(), types.T_int64.ToType(), types.T_json.ToType()} + err := writeSQLLoadCSVRow(&buf, types, []string{`a"b\c`, "42", `{"x":"y"}`}, []bool{false, true, false}) + require.NoError(t, err) + assert.Equal(t, "\"a\"\"b\\\\c\",\\N,\"{\"\"x\"\":\"\"y\"\"}\"\n", buf.String()) +} + +func TestWriteSQLLoadCSVRow_RoundTripsThroughCSVParser(t *testing.T) { + var buf bytes.Buffer + colTypes := []types.Type{types.T_varchar.ToType(), types.T_int64.ToType(), types.T_json.ToType()} + err := writeSQLLoadCSVRow(&buf, colTypes, []string{`a"b\c`, "42", `{"x":"y"}`}, []bool{false, true, false}) + require.NoError(t, err) + + parser, err := csvparser.NewCSVParser(&csvparser.CSVConfig{ + FieldsTerminatedBy: ",", + FieldsEnclosedBy: `"`, + FieldsEscapedBy: `\`, + LinesTerminatedBy: "\n", + Null: []string{`\N`}, + UnescapedQuote: true, + }, strings.NewReader(buf.String()), csvparser.ReadBlockSize, false) + require.NoError(t, err) + + row, err := parser.Read(nil) + require.NoError(t, err) + require.Len(t, row, 3) + assert.Equal(t, `a"b\c`, row[0].Val) + assert.False(t, row[0].IsNull) + assert.True(t, row[0].HasStringQuote) + assert.True(t, row[1].IsNull) + assert.Equal(t, `{"x":"y"}`, row[2].Val) + assert.False(t, row[2].IsNull) + assert.True(t, row[2].HasStringQuote) +} + +func TestWriteSQLLoadCSVFieldFromVec_BitUsesPackedBytes(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.New(types.T_bit, 1, 0)) + require.NoError(t, vector.AppendFixed(vec, uint64(1), false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, "\"\x01\"", buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_JSONUsesVisibleText(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.T_json.ToType()) + bj, err := types.ParseStringToByteJson(`{"name":"mo","n":1}`) + require.NoError(t, err) + stored, err := types.EncodeJson(bj) + require.NoError(t, err) + require.NoError(t, vector.AppendBytes(vec, stored, false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, `"{""n"": 1, ""name"": ""mo""}"`, buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_TemporalKeepsScale(t *testing.T) { + mp := mpool.MustNewZero() + timeVec := vector.NewVec(types.New(types.T_time, 0, 6)) + datetimeVec := vector.NewVec(types.New(types.T_datetime, 0, 3)) + timestampVec := vector.NewVec(types.New(types.T_timestamp, 0, 6)) + require.NoError(t, vector.AppendFixed(timeVec, types.TimeFromClock(false, 11, 22, 33, 123456), false, mp)) + require.NoError(t, vector.AppendFixed(datetimeVec, types.DatetimeFromClock(2024, 1, 2, 3, 4, 5, 123000), false, mp)) + require.NoError(t, vector.AppendFixed(timestampVec, types.FromClockZone(time.Local, 2024, 1, 2, 3, 4, 5, 123456), false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *timeVec.GetType(), []*vector.Vector{timeVec}, 0, 0)) + assert.Equal(t, "11:22:33.123456", buf.String()) + buf.Reset() + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *datetimeVec.GetType(), []*vector.Vector{datetimeVec}, 0, 0)) + assert.Equal(t, "2024-01-02 03:04:05.123", buf.String()) + buf.Reset() + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *timestampVec.GetType(), []*vector.Vector{timestampVec}, 0, 0)) + assert.Equal(t, "2024-01-02 03:04:05.123456", buf.String()) +} + +func TestWriteSQLLoadCSVFieldFromVec_BinaryEscapesControlBytes(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(types.T_blob.ToType()) + require.NoError(t, vector.AppendBytes(vec, []byte{'a', '\n', 0, '\r', '\t', 0x1a, '"', '\\'}, false, mp)) + + var buf bytes.Buffer + require.NoError(t, writeSQLLoadCSVFieldFromVec(&buf, *vec.GetType(), []*vector.Vector{vec}, 0, 0)) + assert.Equal(t, `"a\n\0\r\t\Z""\\"`, buf.String()) +} + +func TestParseCSVRowOrder(t *testing.T) { + order, err := ParseCSVRowOrder("storage") + require.NoError(t, err) + assert.Equal(t, CSVRowOrderStorage, order) + + order, err = ParseCSVRowOrder("lexical") + require.NoError(t, err) + assert.Equal(t, CSVRowOrderLexical, order) + + _, err = ParseCSVRowOrder("unknown") + require.Error(t, err) +} diff --git a/pkg/tools/checkpointtool/types.go b/pkg/tools/checkpointtool/types.go index cc3ee2f09de88..0d9882059d9b1 100644 --- a/pkg/tools/checkpointtool/types.go +++ b/pkg/tools/checkpointtool/types.go @@ -16,6 +16,7 @@ package checkpointtool import ( "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/vm/engine/ckputil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/db/checkpoint" ) @@ -68,9 +69,9 @@ type TableInfo struct { // ObjectEntryInfo contains detailed object entry information with timestamps type ObjectEntryInfo struct { - Range ckputil.TableRange - CreateTime types.TS - DeleteTime types.TS + ObjectStats objectio.ObjectStats + CreateTime types.TS + DeleteTime types.TS } // ComposedView represents logical checkpoint view at a timestamp @@ -80,3 +81,58 @@ type ComposedView struct { Incrementals []*EntryInfo Tables map[uint64]*TableInfo } + +// LogicalTableView contains a tombstone-applied table view for a checkpoint table. +type LogicalTableView struct { + Headers []string + Rows [][]string + ColTypes []types.Type // column types for data columns (after meta cols) + ColSeqNums []uint16 // object seqnums for data columns (after meta cols) + PhysicalRows int + DeletedRows int + VisibleRows int +} + +var logicalTableViewMetaHeaders = []string{"object", "block", "row"} + +func newLogicalTableView() *LogicalTableView { + return &LogicalTableView{ + Headers: append([]string(nil), logicalTableViewMetaHeaders...), + Rows: make([][]string, 0), + } +} + +func (v *LogicalTableView) MetaWidth() int { + if v == nil { + return len(logicalTableViewMetaHeaders) + } + limit := len(logicalTableViewMetaHeaders) + if len(v.Headers) < limit { + limit = len(v.Headers) + } + for i := 0; i < limit; i++ { + if v.Headers[i] != logicalTableViewMetaHeaders[i] { + return i + } + } + return limit +} + +func (v *LogicalTableView) DataWidth() int { + if v == nil { + return 0 + } + width := len(v.Headers) - v.MetaWidth() + if width < 0 { + return 0 + } + return width +} + +func (v *LogicalTableView) DataRow(fullRow []string) []string { + metaWidth := v.MetaWidth() + if len(fullRow) < metaWidth { + return nil + } + return fullRow[metaWidth:] +} diff --git a/pkg/tools/objecttool/interactive/repl.go b/pkg/tools/objecttool/interactive/repl.go index 14354a8447bba..8b68bf3903b76 100644 --- a/pkg/tools/objecttool/interactive/repl.go +++ b/pkg/tools/objecttool/interactive/repl.go @@ -14,9 +14,18 @@ package interactive -import "context" +import ( + "context" + + "github.com/matrixorigin/matrixone/pkg/fileservice" +) // Run runs the interactive interface func Run(path string) error { return RunUnified(context.Background(), path, nil) } + +// RunWithFS runs the interactive interface using an existing file service. +func RunWithFS(ctx context.Context, fs fileservice.FileService, path string) error { + return RunUnifiedWithFS(ctx, fs, path, nil) +} diff --git a/pkg/tools/objecttool/interactive/state.go b/pkg/tools/objecttool/interactive/state.go index 16bf45c4d15d8..0bb280b0bc7f3 100644 --- a/pkg/tools/objecttool/interactive/state.go +++ b/pkg/tools/objecttool/interactive/state.go @@ -1154,8 +1154,9 @@ func (s *State) Columns() []objecttool.ColInfo { cols := make([]objecttool.ColInfo, len(s.metaCols)) for i, col := range s.metaCols { cols[i] = objecttool.ColInfo{ - Idx: col.Idx, - Type: col.Type, + Idx: col.Idx, + SeqNum: col.Idx, + Type: col.Type, } } return cols @@ -1170,8 +1171,9 @@ func (s *State) expandColumns(cols []objecttool.ColInfo) []objecttool.ColInfo { // Replace source column with expanded columns for j := range exp.NewCols { result = append(result, objecttool.ColInfo{ - Idx: uint16(len(result)), - Type: exp.NewTypes[j], + Idx: uint16(len(result)), + SeqNum: uint16(len(result)), + Type: exp.NewTypes[j], }) } } else { diff --git a/pkg/tools/objecttool/interactive/unified_model.go b/pkg/tools/objecttool/interactive/unified_model.go index 2f1f0a6697b48..582aac292ceba 100644 --- a/pkg/tools/objecttool/interactive/unified_model.go +++ b/pkg/tools/objecttool/interactive/unified_model.go @@ -18,6 +18,7 @@ import ( "context" tea "github.com/charmbracelet/bubbletea" + "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/tools/interactive" "github.com/matrixorigin/matrixone/pkg/tools/objecttool" ) @@ -32,6 +33,10 @@ type ObjectUnifiedModel struct { cmdInput string } +var newObjectProgram = func(m tea.Model) *tea.Program { + return tea.NewProgram(m, tea.WithAltScreen()) +} + // NewObjectUnifiedModel creates a new unified model for object viewing func NewObjectUnifiedModel(ctx context.Context, reader *objecttool.ObjectReader, opts *ViewOptions) *ObjectUnifiedModel { state := NewState(ctx, reader) @@ -199,10 +204,34 @@ func RunUnified(ctx context.Context, path string, opts *ViewOptions) error { } defer reader.Close() + return runUnifiedWithReader(ctx, reader, opts, func(nestedPath string) error { + return RunUnified(ctx, nestedPath, nil) + }) +} + +// RunUnifiedWithFS runs the object viewer using an existing file service. +func RunUnifiedWithFS(ctx context.Context, fs fileservice.FileService, path string, opts *ViewOptions) error { + reader, err := objecttool.OpenWithFS(ctx, fs, path, path) + if err != nil { + return err + } + defer reader.Close() + + return runUnifiedWithReader(ctx, reader, opts, func(nestedPath string) error { + return RunUnifiedWithFS(ctx, fs, nestedPath, nil) + }) +} + +func runUnifiedWithReader( + ctx context.Context, + reader *objecttool.ObjectReader, + opts *ViewOptions, + openNested func(string) error, +) error { m := NewObjectUnifiedModel(ctx, reader, opts) for { - p := tea.NewProgram(m, tea.WithAltScreen()) + p := newObjectProgram(m) finalModel, err := p.Run() if err != nil { return err @@ -218,7 +247,7 @@ func RunUnified(ctx context.Context, path string, opts *ViewOptions) error { nestedPath := um.GetObjectToOpen() um.ClearObjectToOpen() // Open nested object with no special options - if err := RunUnified(ctx, nestedPath, nil); err != nil { + if err := openNested(nestedPath); err != nil { return err } // Continue with current viewer after returning diff --git a/pkg/tools/objecttool/interactive/unified_model_test.go b/pkg/tools/objecttool/interactive/unified_model_test.go new file mode 100644 index 0000000000000..025da5a66367a --- /dev/null +++ b/pkg/tools/objecttool/interactive/unified_model_test.go @@ -0,0 +1,189 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interactive + +import ( + "context" + "io" + "os" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/tools/objecttool" + "github.com/stretchr/testify/require" +) + +func TestObjectUnifiedModelUpdateAndCommandMode(t *testing.T) { + dir := t.TempDir() + objectPath := createTestObjectFileWithRowsAndCols(t, dir, "unified_model.obj", 3, 3) + + reader, err := objecttool.Open(context.Background(), objectPath) + require.NoError(t, err) + defer reader.Close() + + model := NewObjectUnifiedModel(context.Background(), reader, &ViewOptions{ + StartRow: 1, + EndRow: 2, + ColumnNames: map[uint16]string{0: "id"}, + ColumnExpander: &ColumnExpander{ + SourceCol: 2, + NewCols: []string{"left", "right"}, + NewTypes: []types.Type{types.T_varchar.ToType(), types.T_varchar.ToType()}, + ExpandFunc: func(any) []any { + return []any{"l", "r"} + }, + }, + ObjectNameCol: 0, + BaseDir: dir, + CustomOverview: func([][]string) string { return "overview" }, + }) + _ = model.Init() + + updated, cmd := model.Update(tea.WindowSizeMsg{Width: 100, Height: 30}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("m")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeBlkMeta, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("M")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeObjMeta, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("d")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, ViewModeData, model.state.viewMode) + + updated, cmd = model.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(":")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.True(t, model.cmdMode) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("2")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Equal(t, "2", model.cmdInput) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyBackspace}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.Empty(t, model.cmdInput) + + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("1")}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + updated, cmd = model.handleCommandMode(tea.KeyMsg{Type: tea.KeyEnter}) + require.Nil(t, cmd) + model = updated.(*ObjectUnifiedModel) + require.False(t, model.cmdMode) + require.Equal(t, int64(1), model.state.GlobalRowOffset()) + + model.state.objectToOpen = "nested.obj" + require.Equal(t, dir+"/nested.obj", model.GetObjectToOpen()) + model.ClearObjectToOpen() + require.Empty(t, model.GetObjectToOpen()) + require.Contains(t, model.View(), "overview") +} + +func TestObjectUnifiedEntrypointsReturnOpenErrors(t *testing.T) { + ctx := context.Background() + require.Error(t, Run("/path/that/does/not/exist.obj")) + + fs, err := fileservice.NewMemoryFS(defines.LocalFileServiceName, fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + require.Error(t, RunWithFS(ctx, fs, "missing.obj")) + require.Error(t, RunUnifiedWithFS(ctx, fs, "missing.obj", nil)) +} + +func TestRunUnifiedWithFSSuccessQuitsFromInput(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + createTestObjectFileWithRowsAndCols(t, dir, "run_unified.obj", 2, 2) + objectPath := dir + string(os.PathSeparator) + "run_unified.obj" + + oldNewObjectProgram := newObjectProgram + t.Cleanup(func() { newObjectProgram = oldNewObjectProgram }) + newObjectProgram = func(m tea.Model) *tea.Program { + return tea.NewProgram(m, tea.WithInput(strings.NewReader("q")), tea.WithOutput(io.Discard)) + } + + fs, err := fileservice.NewFileService(ctx, fileservice.Config{ + Name: defines.LocalFileServiceName, + Backend: "DISK", + DataDir: dir, + Cache: fileservice.DisabledCacheConfig, + }, nil) + require.NoError(t, err) + defer fs.Close(ctx) + + require.NoError(t, RunUnifiedWithFS(ctx, fs, "run_unified.obj", &ViewOptions{ + StartRow: 0, + EndRow: 1, + })) + require.NoError(t, RunUnified(ctx, objectPath, nil)) +} + +func TestStateColumnsForMetadataAndExpandedData(t *testing.T) { + state := &State{ + viewMode: ViewModeObjMeta, + metaCols: []ColInfo{ + {Idx: 0, Name: "Property", Type: types.T_varchar.ToType()}, + {Idx: 1, Name: "Value", Type: types.T_varchar.ToType()}, + }, + } + require.Len(t, state.Columns(), 2) + + state.viewMode = ViewModeData + state.colExpander = &ColumnExpander{ + SourceCol: 1, + NewCols: []string{"a", "b"}, + NewTypes: []types.Type{types.T_int32.ToType(), types.T_varchar.ToType()}, + } + cols := state.expandColumns([]objecttool.ColInfo{ + {Idx: 0, SeqNum: 0, Type: types.T_int32.ToType()}, + {Idx: 1, SeqNum: 1, Type: types.T_varchar.ToType()}, + {Idx: 2, SeqNum: 2, Type: types.T_bool.ToType()}, + }) + require.Len(t, cols, 4) + require.Equal(t, uint16(2), cols[2].Idx) + require.Equal(t, types.T_varchar, cols[2].Type.Oid) + require.Equal(t, uint16(3), cols[3].Idx) +} + +func TestParseNumber(t *testing.T) { + var n int64 + rest, err := parseNumber("123abc", &n) + require.NoError(t, err) + require.Equal(t, int64(123), n) + require.Equal(t, "abc", rest) + + rest, err = parseNumber("", &n) + require.NoError(t, err) + require.Empty(t, rest) + require.Equal(t, int64(0), n) +} + +func TestRunUnifiedOpenErrorForDirectory(t *testing.T) { + dir := t.TempDir() + require.Error(t, RunUnified(context.Background(), dir+string(os.PathSeparator)+"missing.obj", nil)) +} diff --git a/pkg/tools/objecttool/object_reader.go b/pkg/tools/objecttool/object_reader.go index 1f5897f395936..c46403918a957 100644 --- a/pkg/tools/objecttool/object_reader.go +++ b/pkg/tools/objecttool/object_reader.go @@ -47,8 +47,9 @@ type ObjectInfo struct { // ColInfo contains column information type ColInfo struct { - Idx uint16 - Type types.Type + Idx uint16 + SeqNum uint16 + Type types.Type } // ObjectReader reads object files @@ -81,8 +82,12 @@ func Open(ctx context.Context, path string) (*ObjectReader, error) { return nil, moerr.NewInternalErrorf(ctx, "create file service: %v", err) } - // 3. Create reader - objReader, err := objectio.NewObjectReaderWithStr(filename, fs, + return OpenWithFS(ctx, fs, filename, path) +} + +// OpenWithFS opens an object file from an existing file service. +func OpenWithFS(ctx context.Context, fs fileservice.FileService, fileName string, displayPath string) (*ObjectReader, error) { + objReader, err := objectio.NewObjectReaderWithStr(fileName, fs, objectio.WithMetaCachePolicyOption(fileservice.SkipMemoryCache|fileservice.SkipFullFilePreloads)) if err != nil { return nil, moerr.NewInternalErrorf(ctx, "create object reader: %v", err) @@ -99,7 +104,7 @@ func Open(ctx context.Context, path string) (*ObjectReader, error) { dataMeta := meta.MustDataMeta() // 5. Build info - info := buildObjectInfo(path, dataMeta) + info := buildObjectInfo(displayPath, dataMeta) cols := buildColInfo(dataMeta) return &ObjectReader{ @@ -115,9 +120,10 @@ func Open(ctx context.Context, path string) (*ObjectReader, error) { func buildObjectInfo(path string, meta objectio.ObjectDataMeta) *ObjectInfo { info := &ObjectInfo{ - Path: path, - BlockCount: meta.BlockCount(), - ColCount: meta.BlockHeader().ColumnCount(), + Path: path, + BlockCount: meta.BlockCount(), + ColCount: meta.BlockHeader().ColumnCount(), + IsAppendable: meta.BlockHeader().Appendable(), } // Calculate total row count @@ -131,16 +137,35 @@ func buildObjectInfo(path string, meta objectio.ObjectDataMeta) *ObjectInfo { func buildColInfo(meta objectio.ObjectDataMeta) []ColInfo { colCount := meta.BlockHeader().ColumnCount() cols := make([]ColInfo, colCount) + filled := make([]bool, colCount) // Get column types from first block if meta.BlockCount() > 0 { blockMeta := meta.GetBlockMeta(0) - for i := uint16(0); i < colCount; i++ { - colMeta := blockMeta.ColumnMeta(i) - cols[i] = ColInfo{ - Idx: i, - Type: types.T(colMeta.DataType()).ToType(), + metaColCount := blockMeta.GetMetaColumnCount() + for seqNum := uint16(0); seqNum < metaColCount; seqNum++ { + colMeta := blockMeta.ColumnMeta(seqNum) + idx := colMeta.Idx() + if idx >= colCount || colMeta.Location().OriginSize() == 0 { + continue } + cols[idx] = ColInfo{ + Idx: idx, + SeqNum: seqNum, + Type: types.T(colMeta.DataType()).ToType(), + } + filled[idx] = true + } + } + + for i := uint16(0); i < colCount; i++ { + if filled[i] { + continue + } + cols[i] = ColInfo{ + Idx: i, + SeqNum: i, + Type: types.T_any.ToType(), } } @@ -172,7 +197,7 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B colIdxs := make([]uint16, len(r.cols)) colTypes := make([]types.Type, len(r.cols)) for i := range r.cols { - colIdxs[i] = uint16(i) + colIdxs[i] = r.cols[i].SeqNum colTypes[i] = r.cols[i].Type } @@ -182,7 +207,7 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B return nil, nil, err } - release := func() { + releaseIOVector := func() { objectio.ReleaseIOVector(&ioVectors) } @@ -191,16 +216,65 @@ func (r *ObjectReader) ReadBlock(ctx context.Context, blockIdx uint32) (*batch.B for i := range colIdxs { obj, err := objectio.Decode(ioVectors.Entries[i].CachedData.Bytes()) if err != nil { - release() + releaseIOVector() return nil, nil, err } bat.Vecs[i] = obj.(*vector.Vector) bat.SetRowCount(bat.Vecs[i].Length()) } + release := func() { + bat.Clean(r.mp) + releaseIOVector() + } + return bat, release, nil } +// ReadBlockCommitTS reads the hidden commit timestamp column for a block. +// Objects that do not carry commit timestamps return a nil vector. +func (r *ObjectReader) ReadBlockCommitTS(ctx context.Context, blockIdx uint32) (*vector.Vector, func(), error) { + if blockIdx >= r.info.BlockCount { + return nil, nil, moerr.NewInternalErrorf(ctx, "block index %d out of range [0, %d)", blockIdx, r.info.BlockCount) + } + + ioVectors, err := r.objReader.ReadOneBlock( + ctx, + []uint16{objectio.SEQNUM_COMMITTS}, + []types.Type{types.T_TS.ToType()}, + uint16(blockIdx), + r.mp, + ) + if err != nil { + return nil, nil, err + } + + releaseIOVector := func() { + objectio.ReleaseIOVector(&ioVectors) + } + + obj, err := objectio.Decode(ioVectors.Entries[0].CachedData.Bytes()) + if err != nil { + releaseIOVector() + return nil, nil, err + } + vec := obj.(*vector.Vector) + if vec.GetType().Oid != types.T_TS { + releaseIOVector() + return nil, nil, moerr.NewInternalErrorf(ctx, "commit TS column type mismatch: expected TS, got %s", vec.GetType().String()) + } + if vec.GetNulls().GetCardinality() == vec.Length() { + vec.Free(r.mp) + releaseIOVector() + return nil, func() {}, nil + } + release := func() { + vec.Free(r.mp) + releaseIOVector() + } + return vec, release, nil +} + // BlockCount returns block count func (r *ObjectReader) BlockCount() uint32 { return r.info.BlockCount diff --git a/pkg/tools/toolfs/lazy_cache_fs.go b/pkg/tools/toolfs/lazy_cache_fs.go new file mode 100644 index 0000000000000..5601790a9053b --- /dev/null +++ b/pkg/tools/toolfs/lazy_cache_fs.go @@ -0,0 +1,415 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "io" + "iter" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const defaultLazyCacheMaxBytes int64 = 8 << 30 + +type lazyCacheEntry struct { + path string + size int64 + lastAccess uint64 +} + +type lazyCacheFS struct { + name string + remote fileservice.FileService + local *fileservice.LocalFS + root string + + mu sync.Mutex + caching map[string]*sync.Once + entries map[string]lazyCacheEntry + reservations map[string]int64 + usedBytes int64 + maxBytes int64 + accessCounter uint64 +} + +func newLazyCacheFS(ctx context.Context, remote fileservice.FileService) (fileservice.FileService, string, error) { + root, err := os.MkdirTemp("", "mo-tool-remote-cache-*") + if err != nil { + return nil, "", err + } + local, err := fileservice.NewLocalFS(ctx, remote.Name(), root, fileservice.DisabledCacheConfig, nil) + if err != nil { + _ = os.RemoveAll(root) + return nil, "", err + } + return &lazyCacheFS{ + name: remote.Name(), + remote: remote, + local: local, + root: root, + caching: make(map[string]*sync.Once), + entries: make(map[string]lazyCacheEntry), + reservations: make(map[string]int64), + maxBytes: lazyCacheMaxBytesFromEnv(), + }, root, nil +} + +func (l *lazyCacheFS) Name() string { return l.name } + +func (l *lazyCacheFS) Write(ctx context.Context, vector fileservice.IOVector) error { + if err := l.remote.Write(ctx, vector); err != nil { + return err + } + normalized := stripServiceName(vector.FilePath, l.name) + _ = os.Remove(filepath.Join(l.root, filepath.FromSlash(normalized))) + l.mu.Lock() + delete(l.caching, normalized) + l.removeCacheEntryLocked(normalized) + l.mu.Unlock() + return nil +} + +func (l *lazyCacheFS) Read(ctx context.Context, vector *fileservice.IOVector) error { + if err := l.ensureCached(ctx, vector.FilePath); err != nil { + return err + } + return l.readCachedFile(ctx, vector) +} + +func (l *lazyCacheFS) ReadCache(ctx context.Context, vector *fileservice.IOVector) error { + if err := l.ensureCached(ctx, vector.FilePath); err != nil { + return err + } + return l.readCachedFile(ctx, vector) +} + +func (l *lazyCacheFS) List(ctx context.Context, dirPath string) iter.Seq2[*fileservice.DirEntry, error] { + return l.remote.List(ctx, dirPath) +} + +func (l *lazyCacheFS) Delete(ctx context.Context, filePaths ...string) error { + _ = l.local.Delete(ctx, filePaths...) + l.mu.Lock() + for _, filePath := range filePaths { + normalized := stripServiceName(filePath, l.name) + delete(l.caching, normalized) + l.removeCacheEntryLocked(normalized) + } + l.mu.Unlock() + return l.remote.Delete(ctx, filePaths...) +} + +func (l *lazyCacheFS) StatFile(ctx context.Context, filePath string) (*fileservice.DirEntry, error) { + return l.remote.StatFile(ctx, filePath) +} + +func (l *lazyCacheFS) PrefetchFile(ctx context.Context, filePath string) error { + return l.ensureCached(ctx, filePath) +} + +func (l *lazyCacheFS) Cost() *fileservice.CostAttr { + return l.remote.Cost() +} + +func (l *lazyCacheFS) Close(ctx context.Context) { + l.local.Close(ctx) + l.remote.Close(ctx) + _ = os.RemoveAll(l.root) +} + +func (l *lazyCacheFS) ensureCached(ctx context.Context, filePath string) error { + normalized := stripServiceName(filePath, l.name) + localPath := filepath.Join(l.root, filepath.FromSlash(normalized)) + if stat, err := os.Stat(localPath); err == nil { + l.touchCacheEntry(normalized, localPath, stat.Size()) + return nil + } + + once := l.getOnce(normalized) + var cacheErr error + once.Do(func() { + cacheErr = l.cacheFile(ctx, normalized, localPath) + if cacheErr != nil { + l.mu.Lock() + delete(l.caching, normalized) + l.mu.Unlock() + } + }) + return cacheErr +} + +func (l *lazyCacheFS) getOnce(path string) *sync.Once { + l.mu.Lock() + defer l.mu.Unlock() + if once, ok := l.caching[path]; ok { + return once + } + once := &sync.Once{} + l.caching[path] = once + return once +} + +func (l *lazyCacheFS) cacheFile(ctx context.Context, normalized string, localPath string) error { + committed := false + defer func() { + if !committed { + l.releaseCacheReservation(normalized) + } + }() + if stat, err := l.remote.StatFile(ctx, normalized); err == nil && stat != nil && stat.Size > 0 { + l.reserveCacheSpace(stat.Size, normalized) + } + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(localPath), ".cache-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { + _ = tmp.Close() + _ = os.Remove(tmpName) + }() + + vec := &fileservice.IOVector{ + FilePath: normalized, + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + WriterForRead: tmp, + }}, + } + if err := l.remote.Read(ctx, vec); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + stat, err := os.Stat(tmpName) + if err != nil { + return err + } + l.reserveCacheSpace(stat.Size(), normalized) + if err := os.Rename(tmpName, localPath); err != nil { + return err + } + l.touchCacheEntry(normalized, localPath, stat.Size()) + committed = true + return nil +} + +func (l *lazyCacheFS) readCachedFile(ctx context.Context, vector *fileservice.IOVector) error { + normalized := stripServiceName(vector.FilePath, l.name) + localPath := filepath.Join(l.root, filepath.FromSlash(normalized)) + + file, err := os.Open(localPath) + if os.IsNotExist(err) { + if err := l.ensureCached(ctx, normalized); err != nil { + return err + } + file, err = os.Open(localPath) + if os.IsNotExist(err) { + return moerr.NewFileNotFoundNoCtx(normalized) + } + } + if err != nil { + return err + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return err + } + fileSize := stat.Size() + l.touchCacheEntry(normalized, localPath, fileSize) + + for i, entry := range vector.Entries { + if entry.Size == 0 { + return moerr.NewEmptyRangeNoCtx(normalized) + } + if entry.Offset < 0 { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + if entry.Size < 0 { + entry.Size = fileSize - entry.Offset + if entry.Size < 0 { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + } + if _, err := file.Seek(entry.Offset, io.SeekStart); err != nil { + return err + } + if err := entry.ReadFromOSFile(ctx, file, l.local); err != nil { + if err == io.ErrUnexpectedEOF { + return moerr.NewUnexpectedEOFNoCtx(normalized) + } + return err + } + vector.Entries[i] = entry + } + return nil +} + +func (l *lazyCacheFS) touchCacheEntry(normalized string, localPath string, size int64) { + l.mu.Lock() + defer l.mu.Unlock() + l.accessCounter++ + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } + if old, ok := l.entries[normalized]; ok { + l.usedBytes -= old.size + } + l.entries[normalized] = lazyCacheEntry{ + path: localPath, + size: size, + lastAccess: l.accessCounter, + } + l.usedBytes += size +} + +func (l *lazyCacheFS) reserveCacheSpace(needed int64, protected string) { + if l.maxBytes <= 0 || needed <= 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + reserved := l.reservations[protected] + if needed <= reserved { + return + } + delta := needed - reserved + l.evictCacheLocked(delta, protected) + l.reservations[protected] = needed + l.usedBytes += delta +} + +func (l *lazyCacheFS) evictCacheLocked(needed int64, protected string) { + if l.usedBytes+needed <= l.maxBytes { + return + } + entries := make([]lazyCacheEntry, 0, len(l.entries)) + for key, entry := range l.entries { + if key == protected { + continue + } + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].lastAccess < entries[j].lastAccess + }) + for _, entry := range entries { + if l.usedBytes+needed <= l.maxBytes { + return + } + _ = os.Remove(entry.path) + for key, cached := range l.entries { + if cached.path == entry.path { + delete(l.entries, key) + delete(l.caching, key) + l.usedBytes -= cached.size + break + } + } + } +} + +func (l *lazyCacheFS) removeCacheEntryLocked(normalized string) { + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } + if entry, ok := l.entries[normalized]; ok { + l.usedBytes -= entry.size + delete(l.entries, normalized) + } +} + +func (l *lazyCacheFS) releaseCacheReservation(normalized string) { + l.mu.Lock() + defer l.mu.Unlock() + if reservation, ok := l.reservations[normalized]; ok { + l.usedBytes -= reservation + delete(l.reservations, normalized) + } +} + +func lazyCacheMaxBytesFromEnv() int64 { + value := strings.TrimSpace(os.Getenv("MO_TOOL_REMOTE_CACHE_SIZE")) + if value == "" { + return defaultLazyCacheMaxBytes + } + if n, ok := parseLazyCacheSize(value); ok && n > 0 { + return n + } + return defaultLazyCacheMaxBytes +} + +func parseLazyCacheSize(value string) (int64, bool) { + value = strings.TrimSpace(strings.ToLower(value)) + if value == "" { + return 0, false + } + multiplier := int64(1) + for _, suffix := range []struct { + s string + m int64 + }{ + {s: "gib", m: 1 << 30}, + {s: "gb", m: 1 << 30}, + {s: "g", m: 1 << 30}, + {s: "mib", m: 1 << 20}, + {s: "mb", m: 1 << 20}, + {s: "m", m: 1 << 20}, + {s: "kib", m: 1 << 10}, + {s: "kb", m: 1 << 10}, + {s: "k", m: 1 << 10}, + } { + if strings.HasSuffix(value, suffix.s) { + multiplier = suffix.m + value = strings.TrimSpace(strings.TrimSuffix(value, suffix.s)) + break + } + } + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, false + } + return n * multiplier, true +} + +func stripServiceName(path string, service string) string { + prefix := strings.ToUpper(service) + ":" + if strings.HasPrefix(strings.ToUpper(path), prefix) { + return path[len(service)+1:] + } + if idx := strings.Index(path, ":"); idx >= 0 { + return path[idx+1:] + } + return path +} + +var _ fileservice.FileService = (*lazyCacheFS)(nil) diff --git a/pkg/tools/toolfs/lazy_cache_fs_test.go b/pkg/tools/toolfs/lazy_cache_fs_test.go new file mode 100644 index 0000000000000..ee13ccb29b949 --- /dev/null +++ b/pkg/tools/toolfs/lazy_cache_fs_test.go @@ -0,0 +1,269 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "bytes" + "context" + "encoding/binary" + "os" + "path/filepath" + "testing" + + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/stretchr/testify/require" +) + +const objectIOMagic = 0xFFFFFFFF + +func TestLazyCacheFSReadsMagicPrefixedRemoteObjectsWithoutLocalChecksum(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + payload := make([]byte, 4, 32) + binary.LittleEndian.PutUint32(payload, objectIOMagic) + payload = append(payload, []byte("checkpoint-object-bytes")...) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(payload)), + Data: payload, + }}, + })) + + fs, _, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + vec := &fileservice.IOVector{ + FilePath: "SHARED:ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + require.Equal(t, payload, vec.Entries[0].Data) + + var buf bytes.Buffer + cacheVec := &fileservice.IOVector{ + FilePath: "ckp/meta", + Entries: []fileservice.IOEntry{{ + Offset: 4, + Size: 10, + WriterForRead: &buf, + }}, + } + require.NoError(t, fs.ReadCache(ctx, cacheVec)) + require.Equal(t, payload[4:14], buf.Bytes()) +} + +func TestLazyCacheFSEvictsOldEntriesWhenOverLimit(t *testing.T) { + t.Setenv("MO_TOOL_REMOTE_CACHE_SIZE", "32") + + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + first := bytes.Repeat([]byte("a"), 24) + second := bytes.Repeat([]byte("b"), 24) + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/first", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(first)), + Data: first, + }}, + })) + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "ckp/second", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: int64(len(second)), + Data: second, + }}, + })) + + fs, root, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + readAll := func(path string) []byte { + vec := &fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + return vec.Entries[0].Data + } + + require.Equal(t, first, readAll("ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/first")) + + require.Equal(t, second, readAll("ckp/second")) + require.NoFileExists(t, filepath.Join(root, "ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/second")) + + require.Equal(t, first, readAll("ckp/first")) + require.FileExists(t, filepath.Join(root, "ckp/first")) + require.NoFileExists(t, filepath.Join(root, "ckp/second")) +} + +func TestLazyCacheFSDelegatesMetadataAndInvalidatesCachedWrites(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 3, + Data: []byte("old"), + }}, + })) + + fs, root, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + require.Equal(t, "SHARED", fs.Name()) + require.NotNil(t, fs.Cost()) + + require.NoError(t, fs.PrefetchFile(ctx, "SHARED:dir/file")) + require.FileExists(t, filepath.Join(root, "dir/file")) + + stat, err := fs.StatFile(ctx, "dir/file") + require.NoError(t, err) + require.Equal(t, int64(3), stat.Size) + + var listed []string + for entry, err := range fs.List(ctx, "dir") { + require.NoError(t, err) + listed = append(listed, entry.Name) + } + require.Contains(t, listed, "file") + + require.NoError(t, remote.Delete(ctx, "dir/file")) + require.NoError(t, fs.Write(ctx, fileservice.IOVector{ + FilePath: "SHARED:dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 3, + Data: []byte("new"), + }}, + })) + require.NoFileExists(t, filepath.Join(root, "dir/file")) + + vec := &fileservice.IOVector{ + FilePath: "dir/file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: -1, + }}, + } + require.NoError(t, fs.Read(ctx, vec)) + require.Equal(t, []byte("new"), vec.Entries[0].Data) + + require.NoError(t, fs.Delete(ctx, "SHARED:dir/file")) + require.NoFileExists(t, filepath.Join(root, "dir/file")) + _, err = fs.StatFile(ctx, "dir/file") + require.Error(t, err) +} + +func TestLazyCacheFSReadErrors(t *testing.T) { + ctx := context.Background() + remote, err := fileservice.NewMemoryFS("SHARED", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + + require.NoError(t, remote.Write(ctx, fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{ + Offset: 0, + Size: 4, + Data: []byte("data"), + }}, + })) + + fs, _, err := newLazyCacheFS(ctx, remote) + require.NoError(t, err) + defer fs.Close(ctx) + + require.Error(t, fs.PrefetchFile(ctx, "missing")) + + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: 0, Size: 0}}, + })) + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: -1, Size: 1}}, + })) + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{Offset: 10, Size: -1}}, + })) + + var buf bytes.Buffer + require.Error(t, fs.Read(ctx, &fileservice.IOVector{ + FilePath: "file", + Entries: []fileservice.IOEntry{{ + Offset: 3, + Size: 8, + WriterForRead: &buf, + }}, + })) +} + +func TestParseLazyCacheSize(t *testing.T) { + tests := []struct { + value string + want int64 + ok bool + }{ + {value: "42", want: 42, ok: true}, + {value: "2k", want: 2 << 10, ok: true}, + {value: "3MB", want: 3 << 20, ok: true}, + {value: "4GiB", want: 4 << 30, ok: true}, + {value: "", ok: false}, + {value: "bad", ok: false}, + } + + for _, tt := range tests { + got, ok := parseLazyCacheSize(tt.value) + require.Equal(t, tt.ok, ok) + if tt.ok { + require.Equal(t, tt.want, got) + } + } +} + +func TestLazyCacheMaxBytesFromEnvDefault(t *testing.T) { + old := os.Getenv("MO_TOOL_REMOTE_CACHE_SIZE") + t.Cleanup(func() { + if old == "" { + _ = os.Unsetenv("MO_TOOL_REMOTE_CACHE_SIZE") + } else { + _ = os.Setenv("MO_TOOL_REMOTE_CACHE_SIZE", old) + } + }) + _ = os.Unsetenv("MO_TOOL_REMOTE_CACHE_SIZE") + require.Equal(t, defaultLazyCacheMaxBytes, lazyCacheMaxBytesFromEnv()) +} diff --git a/pkg/tools/toolfs/storage.go b/pkg/tools/toolfs/storage.go new file mode 100644 index 0000000000000..406ca1d0e448e --- /dev/null +++ b/pkg/tools/toolfs/storage.go @@ -0,0 +1,166 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "fmt" + "strings" + + "github.com/BurntSushi/toml" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const defaultFSName = "SHARED" + +// StorageOptions describes a fileservice backend requested by a tool command. +type StorageOptions struct { + FSConfig string + FSName string + S3 string + Backend string +} + +// IsRemote reports whether the command should build an explicit fileservice. +func (o StorageOptions) IsRemote() bool { + return o.FSConfig != "" || o.S3 != "" || o.Backend != "" +} + +// Open creates a FileService from tool options. +func Open(ctx context.Context, opts StorageOptions) (fileservice.FileService, string, error) { + if opts.FSName == "" { + opts.FSName = defaultFSName + } + + if opts.FSConfig != "" { + return openFromConfig(ctx, opts.FSConfig, opts.FSName) + } + + backend := strings.ToUpper(opts.Backend) + if backend == "" { + backend = "S3" + } + if backend != "S3" && backend != "MINIO" { + return nil, "", moerr.NewInvalidInputNoCtxf("unsupported backend %q, use S3 or MINIO", opts.Backend) + } + if opts.S3 == "" { + return nil, "", moerr.NewInvalidInputNoCtx("missing --s3 arguments") + } + + args, err := ParseS3Arguments(opts.S3, opts.FSName) + if err != nil { + return nil, "", err + } + cfg := fileservice.Config{ + Name: opts.FSName, + Backend: backend, + S3: args, + Cache: fileservice.DisabledCacheConfig, + } + fs, err := fileservice.NewFileService(ctx, cfg, nil) + if err != nil { + return nil, "", err + } + if backend == "S3" || backend == "MINIO" { + fs, _, err = newLazyCacheFS(ctx, fs) + if err != nil { + return nil, "", err + } + } + return fs, fmt.Sprintf("%s:%s", strings.ToLower(backend), args.KeyPrefix), nil +} + +type launchConfig struct { + DataDir string `toml:"data-dir"` + FileService []fileservice.Config `toml:"fileservice"` + TN struct { + Txn struct { + Storage struct { + FileService string `toml:"fileservice"` + } `toml:"Storage"` + } `toml:"Txn"` + } `toml:"tn"` +} + +func openFromConfig(ctx context.Context, path string, fsName string) (fileservice.FileService, string, error) { + var cfg launchConfig + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return nil, "", err + } + + if cfg.TN.Txn.Storage.FileService != "" && fsName == defaultFSName { + fsName = cfg.TN.Txn.Storage.FileService + } + for _, fsCfg := range cfg.FileService { + if strings.EqualFold(fsCfg.Name, fsName) { + fs, err := fileservice.NewFileService(ctx, fsCfg, nil) + if err != nil { + return nil, "", err + } + backend := strings.ToUpper(fsCfg.Backend) + if backend == "S3" || backend == "MINIO" { + fs, _, err = newLazyCacheFS(ctx, fs) + if err != nil { + return nil, "", err + } + } + return fs, fmt.Sprintf("%s:%s", path, fsCfg.Name), nil + } + } + + if len(cfg.FileService) == 0 && cfg.DataDir != "" { + fsCfg := fileservice.Config{ + Name: fsName, + Backend: "DISK", + DataDir: cfg.DataDir, + Cache: fileservice.DisabledCacheConfig, + } + fs, err := fileservice.NewFileService(ctx, fsCfg, nil) + if err != nil { + return nil, "", err + } + return fs, fmt.Sprintf("%s:%s", path, cfg.DataDir), nil + } + + names := make([]string, 0, len(cfg.FileService)) + for _, fsCfg := range cfg.FileService { + names = append(names, fsCfg.Name) + } + return nil, "", moerr.NewInvalidInputNoCtxf( + "fileservice %q not found in %s; available: %s", + fsName, path, strings.Join(names, ","), + ) +} + +func ParseS3Arguments(s string, fsName string) (fileservice.ObjectStorageArguments, error) { + args := fileservice.ObjectStorageArguments{Name: fsName} + if err := args.SetFromString(splitArgs(s)); err != nil { + return fileservice.ObjectStorageArguments{}, err + } + return args, nil +} + +func splitArgs(s string) []string { + parts := strings.Split(s, ",") + ret := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + ret = append(ret, part) + } + } + return ret +} diff --git a/pkg/tools/toolfs/storage_test.go b/pkg/tools/toolfs/storage_test.go new file mode 100644 index 0000000000000..a7c6d194e0a26 --- /dev/null +++ b/pkg/tools/toolfs/storage_test.go @@ -0,0 +1,134 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package toolfs + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenFromConfigSelectsRequestedFileService(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "LOCAL" +`) + + fs, display, err := Open(context.Background(), StorageOptions{ + FSConfig: cfgPath, + FSName: "LOCAL", + }) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Contains(t, display, "LOCAL") + assert.Equal(t, "LOCAL", fs.Name()) +} + +func TestOpenFromConfigUsesTNStorageFileServiceByDefault(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "SHARED" + +[tn.Txn.Storage] +fileservice = "SHARED" +`) + + fs, _, err := Open(context.Background(), StorageOptions{FSConfig: cfgPath}) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Equal(t, "SHARED", fs.Name()) +} + +func TestOpenFromConfigFallsBackToDataDir(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, `data-dir = "`+dir+`"`) + + fs, display, err := Open(context.Background(), StorageOptions{FSConfig: cfgPath}) + require.NoError(t, err) + defer fs.Close(context.Background()) + + assert.Equal(t, "SHARED", fs.Name()) + assert.Contains(t, display, dir) +} + +func TestStorageOptionsIsRemote(t *testing.T) { + assert.False(t, (StorageOptions{}).IsRemote()) + assert.True(t, StorageOptions{FSConfig: "tn.toml"}.IsRemote()) + assert.True(t, StorageOptions{S3: "bucket=b"}.IsRemote()) + assert.True(t, StorageOptions{Backend: "minio"}.IsRemote()) +} + +func TestOpenRejectsInvalidRemoteOptions(t *testing.T) { + ctx := context.Background() + + _, _, err := Open(ctx, StorageOptions{Backend: "disk", S3: "bucket=b"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported backend") + + _, _, err = Open(ctx, StorageOptions{Backend: "s3"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing --s3") + + _, _, err = Open(ctx, StorageOptions{S3: "not-an-argument"}) + require.Error(t, err) +} + +func TestOpenFromConfigReportsMissingFileService(t *testing.T) { + dir := t.TempDir() + cfgPath := writeConfig(t, ` +[[fileservice]] +backend = "DISK" +data-dir = "`+dir+`" +name = "LOCAL" +`) + + _, _, err := Open(context.Background(), StorageOptions{ + FSConfig: cfgPath, + FSName: "MISSING", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "available: LOCAL") +} + +func TestParseS3Arguments(t *testing.T) { + args, err := ParseS3Arguments("bucket=b,endpoint=http://minio:9000,prefix=/dump/,key-id=k,key-secret=s", "OUT") + require.NoError(t, err) + + assert.Equal(t, "OUT", args.Name) + assert.Equal(t, "b", args.Bucket) + assert.Equal(t, "http://minio:9000", args.Endpoint) + assert.Equal(t, "/dump/", args.KeyPrefix) + assert.Equal(t, "k", args.KeyID) + assert.Equal(t, "s", args.KeySecret) +} + +func writeConfig(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "tn.toml") + require.NoError(t, os.WriteFile(path, []byte(content), 0644)) + return path +}