diff --git a/src/.vuepress/sidebar/V2.0.x/en-Table.ts b/src/.vuepress/sidebar/V2.0.x/en-Table.ts index 2093e570e..9a84def91 100644 --- a/src/.vuepress/sidebar/V2.0.x/en-Table.ts +++ b/src/.vuepress/sidebar/V2.0.x/en-Table.ts @@ -112,6 +112,8 @@ export const enSidebar = { { text: 'Monitor Tool', link: 'Monitor-Tool_apache' }, { text: 'Benchmark Tool', link: 'Benchmark' }, { text: 'Cluster Management Tool', link: 'Maintenance-Tool_apache' }, + { text: 'Data Import', link: 'Data-Import-Tool' }, + { text: 'Data Export', link: 'Data-Export-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar/V2.0.x/zh-Table.ts b/src/.vuepress/sidebar/V2.0.x/zh-Table.ts index aad81b998..1e50caa8d 100644 --- a/src/.vuepress/sidebar/V2.0.x/zh-Table.ts +++ b/src/.vuepress/sidebar/V2.0.x/zh-Table.ts @@ -104,6 +104,8 @@ export const zhSidebar = { { text: '命令行工具', link: 'CLI' }, { text: '监控工具', link: 'Monitor-Tool_apache' }, { text: '集群管理工具', link: 'Maintenance-Tool_apache' }, + { text: '数据导入', link: 'Data-Import-Tool' }, + { text: '数据导出', link: 'Data-Export-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V2.0.x/en-Table.ts b/src/.vuepress/sidebar_timecho/V2.0.x/en-Table.ts index e34adcdf3..7e1b5d4c9 100644 --- a/src/.vuepress/sidebar_timecho/V2.0.x/en-Table.ts +++ b/src/.vuepress/sidebar_timecho/V2.0.x/en-Table.ts @@ -117,6 +117,8 @@ export const enSidebar = { { text: 'Monitor Tool', link: 'Monitor-Tool_timecho' }, { text: 'Benchmark Tool', link: 'Benchmark' }, { text: 'Cluster Management Tool', link: 'Maintenance-Tool_timecho' }, + { text: 'Data Import', link: 'Data-Import-Tool' }, + { text: 'Data Export', link: 'Data-Export-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V2.0.x/zh-Table.ts b/src/.vuepress/sidebar_timecho/V2.0.x/zh-Table.ts index 058d1ba58..3e2345030 100644 --- a/src/.vuepress/sidebar_timecho/V2.0.x/zh-Table.ts +++ b/src/.vuepress/sidebar_timecho/V2.0.x/zh-Table.ts @@ -108,6 +108,8 @@ export const zhSidebar = { { text: '监控工具', link: 'Monitor-Tool_timecho' }, { text: '测试工具', link: 'Benchmark' }, { text: '集群管理工具', link: 'Maintenance-Tool_timecho' }, + { text: '数据导入', link: 'Data-Import-Tool' }, + { text: '数据导出', link: 'Data-Export-Tool' }, ], }, { diff --git a/src/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md b/src/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md new file mode 100644 index 000000000..1c564820c --- /dev/null +++ b/src/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md @@ -0,0 +1,146 @@ +# Data Export + +## 1. Function Overview +The data export tool `export-data.sh/bat` is located in the `tools` directory and can export query results from specified SQL statements into CSV, SQL, or TsFile (open-source time-series file format) formats. Its specific functionalities are as follows: + + + + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVexport-data.sh/batPlain text format for storing structured data. Must follow the CSV format specified below.
SQLFile containing custom SQL statements.
TsFileOpen-source time-series file format.
+ + +## 2. Detailed Features +### 2.1 Common Parameters +| Short | Full Parameter | Description | Required | Default | +| ---------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | Export file type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | Hostname of the IoTDB server. | No | `127.0.0.1` | +| `-p` | `--port` | Port number of the IoTDB server. | No | `6667` | +| `-u` | `--username` | Username for authentication. | No | `root` | +| `-pw` | `--password` | Password for authentication. | No | `root` | +| `-sql_dialect` | `--sql_dialect` | Select server model : tree or table | No | tree | +| `-db ` | `--database` | The target database to be exported only takes effect when `-sql_dialect` is of the table type.| Yes when `-sql_dialect = table`| -| +| `-table`|`--table` | The target table to be exported only takes effect when `-sql_dialect` is of the table type. If the `-q` parameter is specified, this parameter will not take effect. If the export type is tsfile/sql, this parameter is mandatory.| ​ No | - | +| `-start_time` | `--start_time` |The start time of the data to be exported only takes effect when `-sql_dialect` is of the table type. If `-q` is specified, this parameter will not take effect. The supported time formats are the same as those for the `-tf` parameter.|No | - | +|`-end_time` |`--end_time` | The end time of the data to be exported only takes effect when `-sql_dialect` is set to the table type. If `-q` is specified, this parameter will not take effect.| No | - | +| `-t` | `--target` | Target directory for the output files. If the path does not exist, it will be created. | ​**Yes** | - | +| `-pfn` | `--prefix_file_name` | Prefix for the exported file names. For example, `abc` will generate files like `abc_0.tsfile`, `abc_1.tsfile`. | No | `dump_0.tsfile` | +| `-q` | `--query` | SQL query command to execute. | No | - | +| `-timeout` | `--query_timeout` | Query timeout in milliseconds (ms). | No | `-1` (Range: -1~Long max=9223372036854775807) | +| `-help` | `--help` | Display help information. | No | - | + +### 2.2 CSV Format +#### 2.2.1 Command + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table + [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +``` +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------ | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |------------------------------------------| +| `-dt` | `--datatype` | Whether to include data types in the CSV file header (`true` or `false`). | No | `false` | +| `-lpf` | `--lines_per_file` | Number of rows per exported file. | No | `10000` (Range：0～Integer.Max=2147483647) | +| `-tf` | `--time_format` | Time format for the CSV file. Options: 1) Timestamp (numeric, long), 2) ISO8601 (default), 3) Custom pattern (e.g., `yyyy-MM-dd HH:mm:ss`). SQL file timestamps are unaffected by this setting. | No | `ISO8601` | +| `-tz` | `--timezone` | Timezone setting (e.g., `+08:00`, `-01:00`). | No | System default | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -db database1 -q "select * from table1" + +# Error Example +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -q "select * from table1" +Parse error: Missing required option: db +``` +## 2.3 SQL Format +#### 2.3.1 Command +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-aligned ] + -lpf - [-tf ] [-tz ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h -p -u -pw ] + -t [-pfn -aligned + -lpf -tf -tz -q -timeout ] +``` +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------------- | +| `-aligned` | `--use_aligned` | Whether to export as aligned SQL format (`true` or `false`). | No | `true` | +| `-lpf` | `--lines_per_file` | Number of rows per exported file. | No | `10000` (Range：0～Integer.Max=2147483647) | +| `-tf` | `--time_format` | Time format for the CSV file. Options: 1) Timestamp (numeric, long), 2) ISO8601 (default), 3) Custom pattern (e.g., `yyyy-MM-dd HH:mm:ss`). SQL file timestamps are unaffected by this setting. | No | `ISO8601` | +| `-tz` | `--timezone` | Timezone setting (e.g., `+08:00`, `-01:00`). | No | System default | + +#### 2.3.3 Examples +```Shell +# Valid Example +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -db database1 -start_time 1 + +# Error Example +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -start_time 1 +Parse error: Missing required option: db +``` + +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] +``` + +#### 2.4.2 TsFile-Specific Parameters + +* None + +#### 2.4.3 Examples + +```Shell +# Valid Example +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -db database1 -start_time 0 + +# Error Example +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -start_time 0 +Parse error: Missing required option: db +``` diff --git a/src/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md b/src/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..b56e4a4a0 --- /dev/null +++ b/src/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md @@ -0,0 +1,185 @@ +# Data Import + +## 1. Functional Overview +The data import tool `import-data.sh/bat` is located in the `tools` directory and can import data in ​CSV, ​SQL, and ​TsFile (an open-source time-series file format) into ​IoTDB. Its specific functionalities are as follows: + +
+ + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVimport-data.sh/batCan be used for single or batch import of CSV files into IoTDB
SQLCan be used for single or batch import of SQL files into IoTDB
TsFileCan be used for single or batch import of TsFile files into IoTDB
+ +## 2. Detailed Features +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| +|`-sql_dialect`|`--sql_dialect`|Select server model : tree or table | No | `tree` | +|` -db `|`--database` |​Target database , applies only to `-sql_dialect=table` |Yes when `-sql_dialect = table` | - | +|`-table` |`--table `|Target table , required for CSV imports in table model | No | - | +| +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table + [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_0.csv -db database1 -table table1 + +# Error Example +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -table table1 +Parse error: Missing required option: db + +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -db database1 -table table5 +There are no tables or the target table table5 does not exist +``` + +#### 2.2.4 Import Notes + +1. CSV Import Specifications + +- Special Character Escaping Rules: If a text-type field contains special characters (e.g., commas `,`), they must be escaped using a backslash (`\`). +- Supported Time Formats: `yyyy-MM-dd'T'HH:mm:ss`, `yyyy-MM-dd HH:mm:ss`, or `yyyy-MM-dd'T'HH:mm:ss.SSSZ`. +- Timestamp Column Requirement: The timestamp column must be the first column in the data file. + +2. CSV File Example + +```sql +time,region,device,model,temperature,humidity +1970-01-01T08:00:00.001+08:00,"SH","101","F",90.0,35.2 +1970-01-01T08:00:00.002+08:00,"SH","101","F",90.0,34.8 +``` + + +### 2.3 SQL Format + +#### 2.3.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | + +#### 2.3.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump0_0.sql -db database1 + +# Error Example +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump1_1.sql -db database1 +Source file or directory ./sql/dump1_1.sql does not exist + +# When the ​target table exists but metadata is incompatible or ​data is malformed, the system will generate a .failed file and log error details. +# Log Example +Fail to insert measurements '[column.name]' caused by [data type is not consistent, input '[column.value]', registered '[column.DataType]'] +``` +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 -os none -of none + +# Error Example +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 +Parse error: Missing required options: os, of +``` diff --git a/src/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md b/src/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md new file mode 100644 index 000000000..1c564820c --- /dev/null +++ b/src/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md @@ -0,0 +1,146 @@ +# Data Export + +## 1. Function Overview +The data export tool `export-data.sh/bat` is located in the `tools` directory and can export query results from specified SQL statements into CSV, SQL, or TsFile (open-source time-series file format) formats. Its specific functionalities are as follows: + +
+ + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVexport-data.sh/batPlain text format for storing structured data. Must follow the CSV format specified below.
SQLFile containing custom SQL statements.
TsFileOpen-source time-series file format.
+ + +## 2. Detailed Features +### 2.1 Common Parameters +| Short | Full Parameter | Description | Required | Default | +| ---------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | Export file type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | Hostname of the IoTDB server. | No | `127.0.0.1` | +| `-p` | `--port` | Port number of the IoTDB server. | No | `6667` | +| `-u` | `--username` | Username for authentication. | No | `root` | +| `-pw` | `--password` | Password for authentication. | No | `root` | +| `-sql_dialect` | `--sql_dialect` | Select server model : tree or table | No | tree | +| `-db ` | `--database` | The target database to be exported only takes effect when `-sql_dialect` is of the table type.| Yes when `-sql_dialect = table`| -| +| `-table`|`--table` | The target table to be exported only takes effect when `-sql_dialect` is of the table type. If the `-q` parameter is specified, this parameter will not take effect. If the export type is tsfile/sql, this parameter is mandatory.| ​ No | - | +| `-start_time` | `--start_time` |The start time of the data to be exported only takes effect when `-sql_dialect` is of the table type. If `-q` is specified, this parameter will not take effect. The supported time formats are the same as those for the `-tf` parameter.|No | - | +|`-end_time` |`--end_time` | The end time of the data to be exported only takes effect when `-sql_dialect` is set to the table type. If `-q` is specified, this parameter will not take effect.| No | - | +| `-t` | `--target` | Target directory for the output files. If the path does not exist, it will be created. | ​**Yes** | - | +| `-pfn` | `--prefix_file_name` | Prefix for the exported file names. For example, `abc` will generate files like `abc_0.tsfile`, `abc_1.tsfile`. | No | `dump_0.tsfile` | +| `-q` | `--query` | SQL query command to execute. | No | - | +| `-timeout` | `--query_timeout` | Query timeout in milliseconds (ms). | No | `-1` (Range: -1~Long max=9223372036854775807) | +| `-help` | `--help` | Display help information. | No | - | + +### 2.2 CSV Format +#### 2.2.1 Command + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table + [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +``` +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------ | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- |------------------------------------------| +| `-dt` | `--datatype` | Whether to include data types in the CSV file header (`true` or `false`). | No | `false` | +| `-lpf` | `--lines_per_file` | Number of rows per exported file. | No | `10000` (Range：0～Integer.Max=2147483647) | +| `-tf` | `--time_format` | Time format for the CSV file. Options: 1) Timestamp (numeric, long), 2) ISO8601 (default), 3) Custom pattern (e.g., `yyyy-MM-dd HH:mm:ss`). SQL file timestamps are unaffected by this setting. | No | `ISO8601` | +| `-tz` | `--timezone` | Timezone setting (e.g., `+08:00`, `-01:00`). | No | System default | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -db database1 -q "select * from table1" + +# Error Example +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -q "select * from table1" +Parse error: Missing required option: db +``` +## 2.3 SQL Format +#### 2.3.1 Command +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-aligned ] + -lpf - [-tf ] [-tz ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h -p -u -pw ] + -t [-pfn -aligned + -lpf -tf -tz -q -timeout ] +``` +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------------- | +| `-aligned` | `--use_aligned` | Whether to export as aligned SQL format (`true` or `false`). | No | `true` | +| `-lpf` | `--lines_per_file` | Number of rows per exported file. | No | `10000` (Range：0～Integer.Max=2147483647) | +| `-tf` | `--time_format` | Time format for the CSV file. Options: 1) Timestamp (numeric, long), 2) ISO8601 (default), 3) Custom pattern (e.g., `yyyy-MM-dd HH:mm:ss`). SQL file timestamps are unaffected by this setting. | No | `ISO8601` | +| `-tz` | `--timezone` | Timezone setting (e.g., `+08:00`, `-01:00`). | No | System default | + +#### 2.3.3 Examples +```Shell +# Valid Example +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -db database1 -start_time 1 + +# Error Example +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -start_time 1 +Parse error: Missing required option: db +``` + +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] +``` + +#### 2.4.2 TsFile-Specific Parameters + +* None + +#### 2.4.3 Examples + +```Shell +# Valid Example +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -db database1 -start_time 0 + +# Error Example +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -start_time 0 +Parse error: Missing required option: db +``` diff --git a/src/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md b/src/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..b56e4a4a0 --- /dev/null +++ b/src/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md @@ -0,0 +1,185 @@ +# Data Import + +## 1. Functional Overview +The data import tool `import-data.sh/bat` is located in the `tools` directory and can import data in ​CSV, ​SQL, and ​TsFile (an open-source time-series file format) into ​IoTDB. Its specific functionalities are as follows: + +
+ + + + + + + + + + + + + + + + + + + +
File FormatIoTDB ToolDescription
CSVimport-data.sh/batCan be used for single or batch import of CSV files into IoTDB
SQLCan be used for single or batch import of SQL files into IoTDB
TsFileCan be used for single or batch import of TsFile files into IoTDB
+ +## 2. Detailed Features +### 2.1 Common Parameters + +| Short | Full Parameter | Description | Required | Default | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |-----------------------------------------------| +| `-ft` | `--file_type` | File type: `csv`, `sql`, `tsfile`. | ​**Yes** | - | +| `-h` | `--host` | IoTDB server hostname. | No | `127.0.0.1` | +| `-p` | `--port` | IoTDB server port. | No | `6667` | +| `-u` | `--username` | Username. | No | `root` | +| `-pw` | `--password` | Password. | No | `root` | +| +|`-sql_dialect`|`--sql_dialect`|Select server model : tree or table | No | `tree` | +|` -db `|`--database` |​Target database , applies only to `-sql_dialect=table` |Yes when `-sql_dialect = table` | - | +|`-table` |`--table `|Target table , required for CSV imports in table model | No | - | +| +| `-s` | `--source` | Local path to the file/directory to import. ​​**Supported formats**​: CSV, SQL, TsFile. Unsupported formats trigger error: `The file name must end with "csv", "sql", or "tsfile"!` | ​**Yes** | - | +| `-tn` | `--thread_num` | Maximum parallel threads | No | `8`
Range: 0 to Integer.Max(2147483647). | +| `-tz` | `--timezone` | Timezone (e.g., `+08:00`, `-01:00`). | No | System default | +| `-help` | `--help` | Display help (general or format-specific: `-help csv`). | No | - | + +### 2.2 CSV Format + +#### 2.2.1 Command +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table + [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 CSV-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ---------------- | ------------------------------- |----------------------------------------------------------| ---------- |-----------------| +| `-fd` | `--fail_dir` | Directory to save failed files. | No | YOUR_CSV_FILE_PATH | +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-aligned` | `--use_aligned` | Import as aligned time series. | No | `false` | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-ti` | `--type_infer` | Type mapping (e.g., `BOOLEAN=text,INT=long`). | No | - | +| `-tp` | `--timestamp_precision` | Timestamp precision: `ms`, `us`, `ns`. | No | `ms` | + +#### 2.2.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_0.csv -db database1 -table table1 + +# Error Example +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -table table1 +Parse error: Missing required option: db + +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -db database1 -table table5 +There are no tables or the target table table5 does not exist +``` + +#### 2.2.4 Import Notes + +1. CSV Import Specifications + +- Special Character Escaping Rules: If a text-type field contains special characters (e.g., commas `,`), they must be escaped using a backslash (`\`). +- Supported Time Formats: `yyyy-MM-dd'T'HH:mm:ss`, `yyyy-MM-dd HH:mm:ss`, or `yyyy-MM-dd'T'HH:mm:ss.SSSZ`. +- Timestamp Column Requirement: The timestamp column must be the first column in the data file. + +2. CSV File Example + +```sql +time,region,device,model,temperature,humidity +1970-01-01T08:00:00.001+08:00,"SH","101","F",90.0,35.2 +1970-01-01T08:00:00.002+08:00,"SH","101","F",90.0,34.8 +``` + + +### 2.3 SQL Format + +#### 2.3.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 SQL-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| -------------- | ------------------------------- | -------------------------------------------------------------------- | ---------- | ------------------ | +| `-fd` | `--fail_dir` | Directory to save failed files. | No |YOUR_CSV_FILE_PATH| +| `-lpf` | `--lines_per_failed_file` | Max lines per failed file. | No | `100000`
Range: 0 to Integer.Max(2147483647). | +| `-batch` | `--batch_size` | Rows processed per API call. | No | `100000`
Range: 0 to Integer.Max(2147483647). | + +#### 2.3.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump0_0.sql -db database1 + +# Error Example +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump1_1.sql -db database1 +Source file or directory ./sql/dump1_1.sql does not exist + +# When the ​target table exists but metadata is incompatible or ​data is malformed, the system will generate a .failed file and log error details. +# Log Example +Fail to insert measurements '[column.name]' caused by [data type is not consistent, input '[column.value]', registered '[column.DataType]'] +``` +### 2.4 TsFile Format + +#### 2.4.1 Command + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` +#### 2.4.2 TsFile-Specific Parameters + +| Short | Full Parameter | Description | Required | Default | +| ----------- | ----------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | --------------------------- | +| `-os` | `--on_success` | Action for successful files:
`none`: Do not delete the file.
`mv`: Move the successful file to the target directory.
`cp`:Create a hard link (copy) of the successful file to the target directory.
`delete`:Delete the file. | ​**Yes** | - | +| `-sd` | `--success_dir` | Target directory for `mv`/`cp` actions on success. Required if `-os` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/success` | +| `-of` | `--on_fail` | Action for failed files:
`none`:Skip the file.
`mv`:Move the failed file to the target directory.
`cp`:Create a hard link (copy) of the failed file to the target directory.
`delete`:Delete the file.. | ​**Yes** | - | +| `-fd` | `--fail_dir` | Target directory for `mv`/`cp` actions on failure. Required if `-of` is `mv`/`cp`. The file name will be flattened and concatenated with the original file name. | Conditional | `${EXEC_DIR}/fail` | +| `-tp` | `--timestamp_precision` | TsFile timestamp precision: `ms`, `us`, `ns`.
For non-remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The system will manually verify if the timestamp precision matches the server. If it does not match, an error will be returned.
​For remote TsFile imports: Use -tp to specify the timestamp precision of the TsFile. The Pipe system will automatically verify if the timestamp precision matches. If it does not match, a Pipe error will be returned. | No | `ms` | + +#### 2.4.3 Examples + +```Shell +# Valid Example +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 -os none -of none + +# Error Example +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 +Parse error: Missing required options: os, of +``` diff --git a/src/zh/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md b/src/zh/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md new file mode 100644 index 000000000..185020c60 --- /dev/null +++ b/src/zh/UserGuide/Master/Table/Tools-System/Data-Export-Tool.md @@ -0,0 +1,156 @@ +# 数据导出 + +## 1. 功能概述 + +数据导出工具 `export-data.sh/bat` 位于 `tools` 目录下，能够将指定 SQL 的查询结果导出为 CSV、SQL 及 TsFile（开源时间序列文件格式）格式。具体功能如下： + +
+ + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVexport-data.sh/bat纯文本格式，存储格式化数据，需按照下文指定 CSV 格式进行构造
SQL包含自定义 SQL 语句的文件
TsFile开源时序数据文件格式
+ + +## 2. 功能详解 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ------------ | ---------------------- | ---------------------------------------------------------------------- | -------------- | ------------------------------------------ | +| -ft | --file\_type | 导出文件的类型，可以选择：csv、sql、tsfile | √ | | +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -sql_dialect | --sql_dialect | 选择 server 是树模型还是表模型，当前支持 tree 和 table 类型 | 否 | tree | +| -db | --database | ​将要导出的目标数据库，只在`-sql_dialect`为 table 类型下生效。| `-sql_dialect`为 table 时必填| -| +| -table|--table | 将要导出的目标表，只在`-sql_dialect`为 table 类型下生效。如果指定了`-q`参数则此参数不生效，如果导出类型为 tsfile/sql 则此参数必填。| ​ 否 | - | +| -start_time | --start_time |将要导出的数据起始时间，只有`-sql_dialect`为 table 类型时生效。如果填写了`-q`，则此参数不生效。支持的时间类型同`-tf`参数。|否 | - | +|-end_time |--end_time | 将要导出的数据的终止时间，只有`-sql_dialect`为 table 类型时生效。如果填写了`-q`，则此参数不生效。| 否 | - | +| -t | --target | 指定输出文件的目标文件夹，如果路径不存在新建文件夹 | √ | | +| -pfn | --prefix\_file\_name | 指定导出文件的名称。例如：abc,生成的文件是abc\_0.tsfile、abc\_1.tsfile | 否 | dump\_0.tsfile | +| -q | --query | 要执行的查询命令 | 否 | 无 | +| -timeout | --query\_timeout | 会话查询的超时时间(ms) | 否 | -1
范围：-1～Long max=9223372036854775807 | +| -help | --help | 显示帮助信息 | 否 | | + +### 2.2 CSV 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table + [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- |--------------------------------------| +| -dt | --datatype | 是否在 CSV 文件的表头输出时间序列的数据类型，可以选择`true`或`false` | 否 | false | +| -lpf | --lines\_per\_file | 每个转储文件的行数 | 否 | 10000
范围：0～Integer.Max=2147483647 | +| -tf | --time\_format | 指定 CSV 文件中的时间格式。可以选择：1) 时间戳（数字、长整型）；2) ISO8601（默认）；3) 用户自定义模式，如`yyyy-MM-dd HH:mm:ss`（默认为ISO8601）。SQL 文件中的时间戳输出不受时间格式设置影响 | 否| ISO8601 | +| -tz | --timezone | 设置时区，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | + +#### 2.2.3 运行示例： + +```Shell +# 正确示例 +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -db database1 -q "select * from table1" + +# 异常示例 +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -q "select * from table1" +Parse error: Missing required option: db +``` + +### 2.3 SQL 格式 + +#### 2.3.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-aligned ] + -lpf - [-tf ] [-tz ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h -p -u -pw ] + -t [-pfn -aligned + -lpf -tf -tz -q -timeout ] +``` + +#### 2.3.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | -------------------------------------- | +| -aligned | --use\_aligned | 是否导出为对齐的 SQL 格式 | 否 | true | +| -lpf | --lines\_per\_file | 每个转储文件的行数 | 否 | 10000
范围：0～Integer.Max=2147483647 | +| -tf | --time\_format | 指定 CSV 文件中的时间格式。可以选择：1) 时间戳（数字、长整型）；2) ISO8601（默认）；3) 用户自定义模式，如`yyyy-MM-dd HH:mm:ss`（默认为ISO8601）。SQL 文件中的时间戳输出不受时间格式设置影响 | 否| ISO8601| +| -tz | --timezone | 设置时区，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | + +#### 2.3.3 运行示例： + +```Shell +# 正确示例 +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -db database1 -start_time 1 + +# 异常示例 +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -start_time 1 +Parse error: Missing required option: db +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] +``` + +#### 2.4.2 私有参数 + +* 无 + +#### 2.4.3 运行示例： + +```Shell +# 正确示例 +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -db database1 -start_time 0 + +# 异常示例 +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -start_time 0 +Parse error: Missing required option: db +``` diff --git a/src/zh/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..b77509a3f --- /dev/null +++ b/src/zh/UserGuide/Master/Table/Tools-System/Data-Import-Tool.md @@ -0,0 +1,191 @@ +# 数据导入 + +## 1. 功能概述 + +数据导出工具 `import-data.sh/bat` 位于 `tools` 目录下，可以将 CSV、SQL、及 TsFile（开源时序文件格式）的数据导入 IoTDB。具体功能如下： + +
+ + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVimport-data.sh/bat可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQL可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFile可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
+ +- **表模型 TsFile 导入暂时只支持本地导入。** + +## 2. 功能详解 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为 csv sql tsfile 这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +|-sql_dialect|--sql_dialect|选择 server 是树模型还是表模型，当前支持 tree 和 table 类型| 否 | tree | +| -db |--database |数据将要导入的目标库，只在 `-sql_dialect` 为 table 类型下生效。|-sql_dialect 为 table 时必填 | - | +|-table |--table |数据将要导入的目标表，只在 `-sql_dialect` 为 table 类型且文件类型为 csv 条件下生效且必填。 | 否 | - | +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | + + +### 2.2 CSV 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table + [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_0.csv -db database1 -table table1 + +# 异常示例 +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -table table1 +Parse error: Missing required option: db + +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -db database1 -table table5 +There are no tables or the target table table5 does not exist +``` + +#### 2.2.4 导入说明 + +1. CSV 导入规范 + + - 特殊字符转义规则：若Text类型的字段中包含特殊字符（例如逗号`,`），需使用反斜杠（`\`）​进行转义处理。 + - 支持的时间格式：`yyyy-MM-dd'T'HH:mm:ss`， `yyy-MM-dd HH:mm:ss`， 或者 `yyyy-MM-dd'T'HH:mm:ss.SSSZ` 。 + - 时间戳列​必须作为数据文件的首列存在。 + +2. CSV 文件示例 + +```sql +time,region,device,model,temperature,humidity +1970-01-01T08:00:00.001+08:00,"上海","101","F",90.0,35.2 +1970-01-01T08:00:00.002+08:00,"上海","101","F",90.0,34.8 +``` + +### 2.3 SQL 格式 + +#### 2.3.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | + +#### 2.3.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump0_0.sql -db database1 + +# 异常示例 +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump1_1.sql -db database1 +Source file or directory ./sql/dump1_1.sql does not exist + +# 目标表存在但是元数据不适配/数据异常：生成.failed异常文件记录该条信息，日志打印错误信息如下 +Fail to insert measurements '[column.name]' caused by [data type is not consistent, input '[column.value]', registered '[column.DataType]'] +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` + +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为 mv 或 cp 时，mv 或 cp 的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为 mv 或 cp 时，mv 或 cp 的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为 mv 或 cp 时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile 非远程导入：-tp 指定 tsfile 文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定 tsfile 文件的时间精度 pipe 自动校验时间戳精度是否一致 不一致返回 pipe 报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 -os none -of none + +# 异常示例 +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 +Parse error: Missing required options: os, of +``` diff --git a/src/zh/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md b/src/zh/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md new file mode 100644 index 000000000..185020c60 --- /dev/null +++ b/src/zh/UserGuide/latest-Table/Tools-System/Data-Export-Tool.md @@ -0,0 +1,156 @@ +# 数据导出 + +## 1. 功能概述 + +数据导出工具 `export-data.sh/bat` 位于 `tools` 目录下，能够将指定 SQL 的查询结果导出为 CSV、SQL 及 TsFile（开源时间序列文件格式）格式。具体功能如下： + +
+ + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVexport-data.sh/bat纯文本格式，存储格式化数据，需按照下文指定 CSV 格式进行构造
SQL包含自定义 SQL 语句的文件
TsFile开源时序数据文件格式
+ + +## 2. 功能详解 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ------------ | ---------------------- | ---------------------------------------------------------------------- | -------------- | ------------------------------------------ | +| -ft | --file\_type | 导出文件的类型，可以选择：csv、sql、tsfile | √ | | +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -sql_dialect | --sql_dialect | 选择 server 是树模型还是表模型，当前支持 tree 和 table 类型 | 否 | tree | +| -db | --database | ​将要导出的目标数据库，只在`-sql_dialect`为 table 类型下生效。| `-sql_dialect`为 table 时必填| -| +| -table|--table | 将要导出的目标表，只在`-sql_dialect`为 table 类型下生效。如果指定了`-q`参数则此参数不生效，如果导出类型为 tsfile/sql 则此参数必填。| ​ 否 | - | +| -start_time | --start_time |将要导出的数据起始时间，只有`-sql_dialect`为 table 类型时生效。如果填写了`-q`，则此参数不生效。支持的时间类型同`-tf`参数。|否 | - | +|-end_time |--end_time | 将要导出的数据的终止时间，只有`-sql_dialect`为 table 类型时生效。如果填写了`-q`，则此参数不生效。| 否 | - | +| -t | --target | 指定输出文件的目标文件夹，如果路径不存在新建文件夹 | √ | | +| -pfn | --prefix\_file\_name | 指定导出文件的名称。例如：abc,生成的文件是abc\_0.tsfile、abc\_1.tsfile | 否 | dump\_0.tsfile | +| -q | --query | 要执行的查询命令 | 否 | 无 | +| -timeout | --query\_timeout | 会话查询的超时时间(ms) | 否 | -1
范围：-1～Long max=9223372036854775807 | +| -help | --help | 显示帮助信息 | 否 | | + +### 2.2 CSV 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table + [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-dt ] [-lpf ] [-tf ] + [-tz ] [-q ] [-timeout ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- |--------------------------------------| +| -dt | --datatype | 是否在 CSV 文件的表头输出时间序列的数据类型，可以选择`true`或`false` | 否 | false | +| -lpf | --lines\_per\_file | 每个转储文件的行数 | 否 | 10000
范围：0～Integer.Max=2147483647 | +| -tf | --time\_format | 指定 CSV 文件中的时间格式。可以选择：1) 时间戳（数字、长整型）；2) ISO8601（默认）；3) 用户自定义模式，如`yyyy-MM-dd HH:mm:ss`（默认为ISO8601）。SQL 文件中的时间戳输出不受时间格式设置影响 | 否| ISO8601 | +| -tz | --timezone | 设置时区，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | + +#### 2.2.3 运行示例： + +```Shell +# 正确示例 +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -db database1 -q "select * from table1" + +# 异常示例 +> export-data.sh -ft csv -sql_dialect table -t /path/export/dir -q "select * from table1" +Parse error: Missing required option: db +``` + +### 2.3 SQL 格式 + +#### 2.3.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-aligned ] + -lpf - [-tf ] [-tz ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h -p -u -pw ] + -t [-pfn -aligned + -lpf -tf -tz -q -timeout ] +``` + +#### 2.3.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | -------------------------------------- | +| -aligned | --use\_aligned | 是否导出为对齐的 SQL 格式 | 否 | true | +| -lpf | --lines\_per\_file | 每个转储文件的行数 | 否 | 10000
范围：0～Integer.Max=2147483647 | +| -tf | --time\_format | 指定 CSV 文件中的时间格式。可以选择：1) 时间戳（数字、长整型）；2) ISO8601（默认）；3) 用户自定义模式，如`yyyy-MM-dd HH:mm:ss`（默认为ISO8601）。SQL 文件中的时间戳输出不受时间格式设置影响 | 否| ISO8601| +| -tz | --timezone | 设置时区，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | + +#### 2.3.3 运行示例： + +```Shell +# 正确示例 +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -db database1 -start_time 1 + +# 异常示例 +> export-data.sh -ft sql -sql_dialect table -t /path/export/dir -start_time 1 +Parse error: Missing required option: db +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/export-data.sh -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] + +# Windows +> tools\export-data.bat -ft [-sql_dialect] -db -table
+ [-start_time] [-end_time] [-h ] [-p ] [-u ] [-pw ] + -t [-pfn ] [-q ] [-timeout ] +``` + +#### 2.4.2 私有参数 + +* 无 + +#### 2.4.3 运行示例： + +```Shell +# 正确示例 +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -db database1 -start_time 0 + +# 异常示例 +> /tools/export-data.sh -ft tsfile -sql_dialect table -t /path/export/dir -start_time 0 +Parse error: Missing required option: db +``` diff --git a/src/zh/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..aec8b0017 --- /dev/null +++ b/src/zh/UserGuide/latest-Table/Tools-System/Data-Import-Tool.md @@ -0,0 +1,192 @@ +# 数据导入 + +## 1. 功能概述 + +数据导出工具 `import-data.sh/bat` 位于 `tools` 目录下，可以将 CSV、SQL、及 TsFile（开源时序文件格式）的数据导入 IoTDB。具体功能如下： + +
+ + + + + + + + + + + + + + + + + + + +
文件格式IoTDB工具具体介绍
CSVimport-data.sh/bat可用于单个或一个目录的 CSV 文件批量导入 IoTDB
SQL可用于单个或一个目录的 SQL 文件批量导入 IoTDB
TsFile可用于单个或一个目录的 TsFile 文件批量导入 IoTDB
+ +- **表模型 TsFile 导入暂时只支持本地导入。** + +## 2. 功能详解 + +### 2.1 公共参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | --------------- |-----------------------------------------------------------------------------------------------------------------------------------------| -------------- |----------------------------------| +| -ft | --file\_type | 导入文件的类型，可以选择：csv、sql、tsfile | √ | +| -h | -- host | 主机名 | 否 | 127.0.0.1 | +| -p | --port | 端口号 | 否 | 6667 | +| -u | --username | 用户名 | 否 | root | +| -pw | --password | 密码 | 否 | root | +| -s | --source | 待加载的脚本文件(夹)的本地目录路径
如果为 csv sql tsfile 这三个支持的格式，直接导入
不支持的格式，报错提示`The file name must end with "csv" or "sql"or "tsfile"!` | √ | +|-sql_dialect|--sql_dialect|选择 server 是树模型还是表模型，当前支持 tree 和 table 类型| 否 | tree | +| -db |--database |数据将要导入的目标库，只在 `-sql_dialect` 为 table 类型下生效。|-sql_dialect 为 table 时必填 | - | +|-table |--table |数据将要导入的目标表，只在 `-sql_dialect` 为 table 类型且文件类型为 csv 条件下生效且必填。 | 否 | - | +| -tn | --thread\_num | 最大并行线程数 | 否 | 8
范围：0～Integer.Max=2147483647 | +| -tz | --timezone | 时区设置，例如`+08:00`或`-01:00` | 否 | 本机系统时间 | +| -help | --help | 显示帮助信息,支持分开展示和全部展示`-help`或`-help csv` | 否 | + + +### 2.2 CSV 格式 + +#### 2.2.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table + [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-aligned ] + [-ti ] [-tp ] [-tz ] [-batch ] + [-tn ] +``` + +#### 2.2.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- |-------------------------------------------|---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -aligned | --use\_aligned | 是否导入为对齐序列 | 否 | false | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -ti | --type\_infer | 通过选项定义类型信息，例如`"boolean=text,int=long, ..."` | 否 | 无 | +| -tp | --timestamp\_precision | 时间戳精度 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms +| + +#### 2.2.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_0.csv -db database1 -table table1 + +# 异常示例 +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -table table1 +Parse error: Missing required option: db + +> tools/import-data.sh -ft csv -sql_dialect table -s ./csv/dump0_1.csv -db database1 -table table5 +There are no tables or the target table table5 does not exist +``` + +#### 2.2.4 导入说明 + +1. CSV 导入规范 + + - 特殊字符转义规则：若Text类型的字段中包含特殊字符（例如逗号`,`），需使用反斜杠（`\`）​进行转义处理。 + - 支持的时间格式：`yyyy-MM-dd'T'HH:mm:ss`， `yyy-MM-dd HH:mm:ss`， 或者 `yyyy-MM-dd'T'HH:mm:ss.SSSZ` 。 + - 时间戳列​必须作为数据文件的首列存在。 + +2. CSV 文件示例 + +```sql +time,region,device,model,temperature,humidity +1970-01-01T08:00:00.001+08:00,"上海","101","F",90.0,35.2 +1970-01-01T08:00:00.002+08:00,"上海","101","F",90.0,34.8 +``` + + +### 2.3 SQL 格式 + +#### 2.3.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s [-fd ] [-lpf ] [-tz ] + [-batch ] [-tn ] +``` + +#### 2.3.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ---------------------------- | ----------------------------------------------------------------------------------- | -------------- |---------------------------------------| +| -fd | --fail\_dir | 指定保存失败文件的目录 | 否 | YOUR\_CSV\_FILE\_PATH | +| -lpf | --lines\_per\_failed\_file | 指定失败文件最大写入数据的行数 | 否 | 100000
范围：0～Integer.Max=2147483647 | +| -batch | --batch\_size | 指定每调用一次接口处理的数据行数（最小值为1，最大值为Integer.​*MAX\_VALUE*​） | 否 | 100000
范围：0～Integer.Max=2147483647 | + +#### 2.3.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump0_0.sql -db database1 + +# 异常示例 +> tools/import-data.sh -ft sql -sql_dialect table -s ./sql/dump1_1.sql -db database1 +Source file or directory ./sql/dump1_1.sql does not exist + +# 目标表存在但是元数据不适配/数据异常：生成.failed异常文件记录该条信息，日志打印错误信息如下 +Fail to insert measurements '[column.name]' caused by [data type is not consistent, input '[column.value]', registered '[column.DataType]'] +``` + +### 2.4 TsFile 格式 + +#### 2.4.1 运行命令 + +```Shell +# Unix/OS X +> tools/import-data.sh -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] + +# Windows +> tools\import-data.bat -ft [-sql_dialect] -db -table
+ [-h ] [-p ] [-u ] [-pw ] + -s -os [-sd ] -of [-fd ] + [-tn ] [-tz ] [-tp ] +``` + +#### 2.4.2 私有参数 + +| 参数缩写 | 参数全称 | 参数含义 | 是否为必填项 | 默认值 | +| ---------- | ------------------------ |----------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -------------------- | +| -os| --on\_succcess| 1. none：不删除
2. mv：移动成功的文件到目标文件夹
3. cp：硬连接（拷贝）成功的文件到目标文件夹
4. delete：删除 | √ || +| -sd | --success\_dir | 当`--on_succcess`为 mv 或 cp 时，mv 或 cp 的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_succcess`为mv或cp时需要填写 | `${EXEC_DIR}/success`| +| -of| --on\_fail| 1. none：跳过
2. mv：移动失败的文件到目标文件夹
3. cp：硬连接（拷贝）失败的文件到目标文件夹
4. delete：删除 | √ || +| -fd | --fail\_dir | 当`--on_fail`指定为 mv 或 cp 时，mv 或 cp 的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当`--on_fail`指定为 mv 或 cp 时需要填写 | `${EXEC_DIR}/fail` | +| -tp | --timestamp\_precision | 时间戳精度
tsfile 非远程导入：-tp 指定 tsfile 文件的时间精度 手动校验和服务器的时间戳是否一致 不一致返回报错信息
远程导入：-tp 指定 tsfile 文件的时间精度 pipe 自动校验时间戳精度是否一致 不一致返回 pipe 报错信息 | 否：
1. ms（毫秒）
2. us（纳秒）
3. ns（微秒） | ms| + + +#### 2.4.3 运行示例 + +```Shell +# 正确示例 +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 -os none -of none + +# 异常示例 +> tools/import-data.sh -ft tsfile -sql_dialect table -s ./tsfile -db database1 +Parse error: Missing required options: os, of +```