adm0-generator/sync.py at main · fieldmaps/adm0-generator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import subprocess
from pathlib import Path

cwd = Path(__file__).parent
exts = ["json", "csv", "xlsx"]


def sync(src, dest):
    subprocess.run(
        [
            "rclone",
            "sync",
            "--exclude=.*",
            "--progress",
            "--s3-no-check-bucket",
            "--s3-chunk-size=256M",
            src,
            dest,
        ],
        check=False,
    )


def copy(src, dest):
    subprocess.run(
        [
            "rclone",
            "copyto",
            "--s3-no-check-bucket",
            "--s3-chunk-size=256M",
            src,
            dest,
        ],
        check=False,
    )


if __name__ == "__main__":
    sync(cwd / "outputs/adm0", "r2://fieldmaps-data/adm0")
    for ext in exts:
        copy(cwd / f"outputs/adm0.{ext}", f"r2://fieldmaps-data/adm0.{ext}")