benchmarks/render.py at main · LiberTEM/benchmarks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/usr/bin/env python

import pathlib
import logging
import subprocess

import click

logger = logging.getLogger(__name__)


def index_for_all(dest_path: pathlib.Path):
    links = []
    for index_path in dest_path.glob("*/*/index.html"):
        human_name = "/".join(index_path.parent.parts[-2:])
        links.append(f"<li><a href='{index_path.relative_to(dest_path).as_posix()}'>{human_name}</a></li>")
    links = "\n".join(links)

    doc = f"""<!doctype html><html><head><title>Benchmark index</title></head>
    <body><ul>{links}</ul></body></html>"""

    with open(dest_path / "index.html", "w") as f:
        f.write(doc)


def index_for_repo(
    dest_path: pathlib.Path,
    indices: list[pathlib.Path],
):
    links = []

    for index_path in indices:
        human_name = "/".join(index_path.parent.parts[-3:])
        links.append(f"<li><a href='{index_path.relative_to(dest_path).as_posix()}'>{human_name}</a></li>")

    links = "\n".join(links)

    doc = f"""<!doctype html><html><head><title>Benchmark index</title></head>
    <body><ul>{links}</ul></body></html>"""

    with open(dest_path / "index.html", "w") as f:
        f.write(doc)


def make_html(folder: pathlib.Path, repo: str, benchmark_name: str):
    svgs = list(sorted(folder.glob("*.svg")))
    imgs = "\n".join(
        f"<p>Benchmark: {img.name}<br/><img src='{img.name}'/></p>"
        for img in svgs
    )

    doc = f"""<!doctype html><html><head><title>{repo}: {benchmark_name}</title></head>
    <body>{imgs}</body></html>"""

    dest_path = folder / "index.html"
    with open(dest_path, "w") as f:
        f.write(doc)
    return dest_path


@click.command()
@click.option("--src-folder", type=pathlib.Path, required=True)
@click.option("--dest-folder", type=pathlib.Path, required=True)
@click.option("--repo", type=str, required=True)
def main(
    src_folder: pathlib.Path,
    dest_folder: pathlib.Path,
    repo: str,
):
    """
    Render all benchmark results from one repo.

    src_folder should point to the top level of directories that contain the
    json benchmark results for one repository, for example
    ./collected/LiberTEM/cb-testing/, there should be a sub directory for every
    named "benchmark group" (i.e. juwels_cpu, juwels_gpu, ...)
    """
    logging.basicConfig(level=logging.INFO)
    indices = []
    for bench_path in (src_folder / repo).glob("*"):
        if not bench_path.is_dir():
            logging.warning(f"spurious file {bench_path}, ignoring")
            continue

        svgs_folder = dest_folder / repo / bench_path.name

        args = [
            "pytest-benchmark",
            "compare",
        ]
        args += [p.as_posix() for p in bench_path.glob("*.json")]
        args += [
            f"--histogram={svgs_folder.as_posix()}/{bench_path.name}",
            "--group-by=fullname",
            "--sort=name",
        ]
        logger.info(f"running {' '.join(args)}")
        subprocess.run(args, check=True)
        indices.append(make_html(svgs_folder, benchmark_name=bench_path.name, repo=repo))

    index_for_repo(dest_path=dest_folder / repo, indices=indices)
    index_for_all(dest_path=dest_folder)


if __name__ == "__main__":
    main()