Skip to content

Commit 6c2cc9e

Browse files
jizezhangjizezhang
committed
update doc
1 parent 8f20060 commit 6c2cc9e

2 files changed

Lines changed: 48 additions & 14 deletions

File tree

‎datafusion-cli/src/functions.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,23 @@ impl TableFunctionImpl for StatisticsCacheFunc {
703703
}
704704
}
705705

706+
// Implementation of the `list_files_cache` table function in datafusion-cli.
707+
///
708+
/// This function returns the cached results of running a LIST command on a particular object store path for a table. The object metadata is returned as a List of Structs, with one Struct for each object.
709+
/// DataFusion uses these cached results to plan queries against external tables.
710+
/// # Schema
711+
/// ```sql
712+
/// > describe select * from list_files_cache();
713+
/// +---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+
714+
/// | column_name | data_type | is_nullable |
715+
/// +---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+
716+
/// | table | Utf8 | NO |
717+
/// | path | Utf8 | NO |
718+
/// | metadata_size_bytes | UInt64 | NO |
719+
/// | expires_in | Duration(ms) | YES |
720+
/// | metadata_list | List(Struct("file_path": non-null Utf8, "file_modified": non-null Timestamp(ms), "file_size_bytes": non-null UInt64, "e_tag": Utf8, "version": Utf8), field: 'metadata') | YES |
721+
/// +---------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------+
722+
/// ```
706723
#[derive(Debug)]
707724
struct ListFilesCacheTable {
708725
schema: SchemaRef,

‎docs/source/user-guide/cli/functions.md‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -177,21 +177,26 @@ The `list_files_cache` function shows information about the `ListFilesCache` tha
177177
You can inspect the cache by querying the `list_files_cache` function. For example,
178178

179179
```sql
180+
> set datafusion.runtime.list_files_cache_ttl = "30s";
181+
> create external table overturemaps
182+
stored as parquet
183+
location 's3://overturemaps-us-west-2/release/2025-12-17.0/theme=base/type=infrastructure';
184+
0 row(s) fetched.
180185
> select table, path, metadata_size_bytes, expires_in, unnest(metadata_list)['file_size_bytes'] as file_size_bytes, unnest(metadata_list)['e_tag'] as e_tag from list_files_cache() limit 10;
181-
+-------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
182-
| table | path | metadata_size_bytes | expires_in | file_size_bytes | e_tag |
183-
+-------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
184-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 999055952 | "35fc8fbe8400960b54c66fbb408c48e8-60" |
185-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 975592768 | "8a16e10b722681cdc00242564b502965-59" |
186-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1082925747 | "24cd13ddb5e0e438952d2499f5dabe06-65" |
187-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1008425557 | "37663e31c7c64d4ef355882bcd47e361-61" |
188-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1065561905 | "4e7c50d2d1b3c5ed7b82b4898f5ac332-64" |
189-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1045655427 | "8fff7e6a72d375eba668727c55d4f103-63" |
190-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1086822683 | "b67167d8022d778936c330a52a5f1922-65" |
191-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1016732378 | "6d70857a0473ed9ed3fc6e149814168b-61" |
192-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 991363784 | "c9cafb42fcbb413f851691c895dd7c2b-60" |
193-
| test | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 15.466 secs | 1032469715 | "7540252d0d67158297a67038a3365e0f-62" |
194-
+-------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
186+
+--------------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
187+
| table | path | metadata_size_bytes | expires_in | file_size_bytes | e_tag |
188+
+--------------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
189+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 999055952 | "35fc8fbe8400960b54c66fbb408c48e8-60" |
190+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 975592768 | "8a16e10b722681cdc00242564b502965-59" |
191+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1082925747 | "24cd13ddb5e0e438952d2499f5dabe06-65" |
192+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1008425557 | "37663e31c7c64d4ef355882bcd47e361-61" |
193+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1065561905 | "4e7c50d2d1b3c5ed7b82b4898f5ac332-64" |
194+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1045655427 | "8fff7e6a72d375eba668727c55d4f103-63" |
195+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1086822683 | "b67167d8022d778936c330a52a5f1922-65" |
196+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1016732378 | "6d70857a0473ed9ed3fc6e149814168b-61" |
197+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 991363784 | "c9cafb42fcbb413f851691c895dd7c2b-60" |
198+
| overturemaps | release/2025-12-17.0/theme=base/type=infrastructure | 2750 | 0 days 0 hours 0 mins 25.264 secs | 1032469715 | "7540252d0d67158297a67038a3365e0f-62" |
199+
+--------------+-----------------------------------------------------+---------------------+-----------------------------------+-----------------+---------------------------------------+
195200
```
196201

197202
The columns of the returned table are:
@@ -203,5 +208,17 @@ The columns of the returned table are:
203208
| expires_in | Duration(ms) | Last modified time of the file |
204209
| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
205210

211+
A metadata struct in the metadata_list contains the following fields:
212+
213+
```text
214+
{
215+
"file_path": "release/2025-12-17.0/theme=base/type=infrastructure/part-00000-d556e455-e0c5-4940-b367-daff3287a952-c000.zstd.parquet",
216+
"file_modified": "2025-12-17T22:20:29",
217+
"file_size_bytes": 999055952,
218+
"e_tag": "35fc8fbe8400960b54c66fbb408c48e8-60",
219+
"version": null
220+
}
221+
```
222+
206223
[`listingtable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html
207224
[entity tag]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag

0 commit comments

Comments
 (0)