-
Notifications
You must be signed in to change notification settings - Fork 8
Expand file tree
/
Copy path__main__.py
More file actions
36 lines (27 loc) · 778 Bytes
/
__main__.py
File metadata and controls
36 lines (27 loc) · 778 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
from pathlib import Path
from typing import Annotated
import typer
from rcs.envs.storage_wrapper import StorageWrapper
app = typer.Typer()
@app.command()
def consolidate(
path: Annotated[
Path,
typer.Argument(
exists=True,
file_okay=False,
dir_okay=True,
help="The root directory of the parquet dataset to consolidate.",
),
]
):
"""
Consolidates a fragmented Parquet dataset into larger files.
This is useful if the recording process crashed or was interrupted,
leaving many small files behind.
"""
typer.echo(f"Starting consolidation for: {path}")
StorageWrapper.consolidate(str(path), schema=None)
typer.echo("Done.")
if __name__ == "__main__":
app()