Skip to content

Commit decc3aa

Browse files
CyanoHaoCyanoHao
committed
add overlayfs-based build dependency management
1 parent 079f0dc commit decc3aa

27 files changed

+1751
-1406
lines changed

‎.github/workflows/build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
podman run \
3333
--platform linux/amd64 \
3434
--rm \
35-
--security-opt seccomp=unconfined \
35+
--cap-add=sys_admin \
3636
--volume $PWD:/mnt \
3737
--workdir /mnt \
3838
docker.io/amd64/ubuntu:24.04 \

‎.gitignore‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
/assets
2-
/build
3-
/container/*/qt.tar
4-
/pkg
5-
6-
__pycache__/
1+
__pycache__
2+
assets
3+
build
4+
container/*/qt.tar
5+
layer
6+
pkg

‎README.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Red Panda C++ AppImage Build Environment
2+
3+
## Use (Build Red Panda C++)
4+
5+
Please refer to [Red Panda C++’s document](https://github.com/royqh1979/RedPanda-CPP/blob/master/BUILD.md).
6+
7+
## Build (Build This Project)
8+
9+
### Build the Tools and Libraries
10+
11+
1. Launch container:
12+
```bash
13+
podman run -it --rm \
14+
--cap-add=sys_admin \
15+
-v $PWD:/mnt -w /mnt \
16+
docker.io/amd64/ubuntu:24.04
17+
```
18+
To expose build directories for debugging:
19+
```bash
20+
podman run -it --rm \
21+
--cap-add=sys_admin \
22+
-v $PWD:/mnt -w /mnt \
23+
-v $PWD/build:/tmp/build \
24+
-v $PWD/layer:/tmp/layer \
25+
docker.io/amd64/ubuntu:24.04
26+
```
27+
2. Install dependencies:
28+
```bash
29+
./support/dep.sh
30+
```
31+
3. Build the project:
32+
```bash
33+
./main.sh -a <arch> -b <branch>
34+
```
35+
36+
### Build the Container Image
37+
38+
```
39+
podman build -t redpanda-cpp/appimage-builder-<arch> container/<arch>
40+
```

‎container/aarch64/Dockerfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM docker.io/amd64/ubuntu:24.04
22

33
RUN export DEBIAN_FRONTEND=noninteractive && \
44
apt update && \
5-
apt upgrade -y && \
65
apt install --no-install-recommends -y \
76
# Red Panda C++
87
git make \
@@ -11,8 +10,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1110
apt clean && \
1211
rm -rf /var/lib/apt/lists/*
1312

14-
ADD qt.tar /
13+
ADD qt.tar /usr/local
1514

1615
ENV ARCH=aarch64
17-
ENV PATH=/opt/qt-aarch64/aarch64-linux-musl/qt/bin:/opt/qt-aarch64/bin:${PATH}
18-
ENV APPIMAGE_RUNTIME=/opt/qt-aarch64/aarch64-linux-musl/bin/appimage-runtime
16+
ENV QMAKE=/usr/local/aarch64-linux-musl/bin/qmake
17+
ENV APPIMAGE_RUNTIME=/usr/local/aarch64-linux-musl/bin/appimage-runtime

‎container/i686/Dockerfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM docker.io/amd64/ubuntu:24.04
22

33
RUN export DEBIAN_FRONTEND=noninteractive && \
44
apt update && \
5-
apt upgrade -y && \
65
apt install --no-install-recommends -y \
76
# Red Panda C++
87
git make \
@@ -11,8 +10,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1110
apt clean && \
1211
rm -rf /var/lib/apt/lists/*
1312

14-
ADD qt.tar /
13+
ADD qt.tar /usr/local
1514

1615
ENV ARCH=i686
17-
ENV PATH=/opt/qt-i686/i686-linux-musl/qt/bin:/opt/qt-i686/bin:${PATH}
18-
ENV APPIMAGE_RUNTIME=/opt/qt-i686/i686-linux-musl/bin/appimage-runtime
16+
ENV QMAKE=/usr/local/i686-linux-musl/bin/qmake
17+
ENV APPIMAGE_RUNTIME=/usr/local/i686-linux-musl/bin/appimage-runtime

‎container/loong64/Dockerfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM docker.io/amd64/ubuntu:24.04
22

33
RUN export DEBIAN_FRONTEND=noninteractive && \
44
apt update && \
5-
apt upgrade -y && \
65
apt install --no-install-recommends -y \
76
# Red Panda C++
87
git make \
@@ -11,8 +10,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1110
apt clean && \
1211
rm -rf /var/lib/apt/lists/*
1312

14-
ADD qt.tar /
13+
ADD qt.tar /usr/local
1514

1615
ENV ARCH=loong64
17-
ENV PATH=/opt/qt-loong64/loongarch64-linux-musl/qt/bin:/opt/qt-loong64/bin:${PATH}
18-
ENV APPIMAGE_RUNTIME=/opt/qt-loong64/loongarch64-linux-musl/bin/appimage-runtime
16+
ENV QMAKE=/usr/local/loongarch64-linux-musl/bin/qmake
17+
ENV APPIMAGE_RUNTIME=/usr/local/loongarch64-linux-musl/bin/appimage-runtime

‎container/riscv64/Dockerfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM docker.io/amd64/ubuntu:24.04
22

33
RUN export DEBIAN_FRONTEND=noninteractive && \
44
apt update && \
5-
apt upgrade -y && \
65
apt install --no-install-recommends -y \
76
# Red Panda C++
87
git make \
@@ -11,8 +10,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1110
apt clean && \
1211
rm -rf /var/lib/apt/lists/*
1312

14-
ADD qt.tar /
13+
ADD qt.tar /usr/local
1514

1615
ENV ARCH=riscv64
17-
ENV PATH=/opt/qt-riscv64/riscv64-linux-musl/qt/bin:/opt/qt-riscv64/bin:${PATH}
18-
ENV APPIMAGE_RUNTIME=/opt/qt-riscv64/riscv64-linux-musl/bin/appimage-runtime
16+
ENV QMAKE=/usr/local/riscv64-linux-musl/bin/qmake
17+
ENV APPIMAGE_RUNTIME=/usr/local/riscv64-linux-musl/bin/appimage-runtime

‎container/x86_64/Dockerfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM docker.io/amd64/ubuntu:24.04
22

33
RUN export DEBIAN_FRONTEND=noninteractive && \
44
apt update && \
5-
apt upgrade -y && \
65
apt install --no-install-recommends -y \
76
# Red Panda C++
87
git make \
@@ -11,8 +10,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
1110
apt clean && \
1211
rm -rf /var/lib/apt/lists/*
1312

14-
ADD qt.tar /
13+
ADD qt.tar /usr/local
1514

1615
ENV ARCH=x86_64
17-
ENV PATH=/opt/qt-x86_64/x86_64-linux-musl/qt/bin:/opt/qt-x86_64/bin:${PATH}
18-
ENV APPIMAGE_RUNTIME=/opt/qt-x86_64/x86_64-linux-musl/bin/appimage-runtime
16+
ENV QMAKE=/usr/local/x86_64-linux-musl/bin/qmake
17+
ENV APPIMAGE_RUNTIME=/usr/local/x86_64-linux-musl/bin/appimage-runtime

‎main.py‎

Lines changed: 51 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,18 @@
33
import argparse
44
import logging
55
import os
6+
from pathlib import Path
67
import shutil
78
import subprocess
9+
from typing import Dict, List
810

9-
from module.cross_toolchain import build_cross_toolchain
10-
from module.host_lib import build_host_lib
1111
from module.path import ProjectPaths
1212
from module.prepare_source import download_and_patch
1313
from module.profile import get_full_profile
14+
from module.util import ensure, overlayfs_ro
15+
16+
from module.host_lib import build_host_lib
17+
from module.cross_toolchain import build_cross_toolchain
1418
from module.target_lib import build_target_lib
1519

1620
def parse_args() -> argparse.Namespace:
@@ -53,25 +57,55 @@ def parse_args() -> argparse.Namespace:
5357
return result
5458

5559
def clean(config: argparse.Namespace, paths: ProjectPaths):
56-
if paths.build.exists():
57-
shutil.rmtree(paths.build)
58-
if paths.h_prefix.exists():
59-
shutil.rmtree(paths.h_prefix)
60+
if paths.build_dir.exists():
61+
shutil.rmtree(paths.build_dir)
62+
if paths.layer_dir.exists():
63+
shutil.rmtree(paths.layer_dir)
6064

6165
def prepare_dirs(paths: ProjectPaths):
62-
paths.assets.mkdir(parents = True, exist_ok = True)
63-
paths.build.mkdir(parents = True, exist_ok = True)
64-
paths.dist.mkdir(parents = True, exist_ok = True)
66+
ensure(paths.assets_dir)
67+
ensure(paths.build_dir)
68+
ensure(paths.dist_dir)
69+
70+
def check_file_collision(layers: list[Path]):
71+
file_to_package_map: Dict[str, List[str]] = {}
72+
for layer in layers:
73+
for file in layer.glob('**/*'):
74+
if not file.is_dir():
75+
file_path = str(file.relative_to(layer))
76+
if file_path in file_to_package_map:
77+
file_to_package_map[file_path].append(str(layer))
78+
else:
79+
file_to_package_map[file_path] = [str(layer)]
80+
81+
ok = True
82+
for file, packages in file_to_package_map.items():
83+
if len(packages) > 1:
84+
ok = False
85+
print(f'file collision: {file} in {packages}')
86+
87+
if not ok:
88+
raise Exception('file collision')
6589

6690
def package(paths: ProjectPaths):
67-
ret = subprocess.run([
68-
'tar',
69-
'-cf',
70-
paths.container / 'qt.tar',
71-
paths.h_prefix,
72-
])
73-
if ret.returncode != 0:
74-
raise Exception('Failed to package')
91+
layers = []
92+
for layer_group in (paths.layer_host, paths.layer_x, paths.layer_target):
93+
for k, v in layer_group._asdict().items():
94+
if k in ('prefix', 'freetype_decycle'):
95+
continue
96+
layers.append(v)
97+
98+
check_file_collision(layers)
99+
100+
with overlayfs_ro('/usr/local', [
101+
*map(lambda layer: layer / 'usr/local', layers),
102+
]):
103+
subprocess.run([
104+
'tar',
105+
'-C', '/usr/local',
106+
'-c', '.',
107+
'-f', paths.container_dir / 'qt.tar',
108+
], check = True)
75109

76110
def main():
77111
config = parse_args()
@@ -93,17 +127,11 @@ def main():
93127

94128
download_and_patch(profile.ver, paths, profile.info)
95129

96-
os.environ['PATH'] = f'{paths.h_prefix}/bin:{os.environ['PATH']}'
97-
98-
os.environ['PKG_CONFIG_LIBDIR'] = f'{paths.h_prefix}/lib/pkgconfig:{paths.h_prefix}/share/pkgconfig'
99130
build_host_lib(profile.ver, paths, profile.info, config)
100-
del os.environ['PKG_CONFIG_LIBDIR']
101131

102132
build_cross_toolchain(profile.ver, paths, profile.info, config)
103133

104-
os.environ['PKG_CONFIG_LIBDIR'] = f'{paths.prefix}/lib/pkgconfig:{paths.prefix}/share/pkgconfig'
105134
build_target_lib(profile.ver, paths, profile.info, config)
106-
del os.environ['PKG_CONFIG_LIBDIR']
107135

108136
package(paths)
109137

‎module/checksum.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
'musl-1.1.24.tar.gz': '1370c9a812b2cf2a7d92802510cca0058cc37e66a7bedd70051f0a34015022a3',
4444
'musl-1.2.5.tar.gz': 'a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4',
4545

46+
'pkgconf-pkgconf-2.5.1.tar.gz': '79721badcad1987dead9c3609eb4877ab9b58821c06bdacb824f2c8897c11f2a',
47+
4648
'qtbase-everywhere-src-6.8.1.tar.xz': '40b14562ef3bd779bc0e0418ea2ae08fa28235f8ea6e8c0cb3bce1d6ad58dcaf',
4749

4850
'qtsvg-everywhere-src-6.8.1.tar.xz': '3d0de73596e36b2daa7c48d77c4426bb091752856912fba720215f756c560dd0',

0 commit comments

Comments
 (0)