From 3414a69b26233552394a4fa7bc0b865addf543cd Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Tue, 17 Dec 2024 19:06:04 +0100 Subject: [PATCH 1/7] Backend building through nix --- .gitignore | 1 + backend/Dockerfile | 65 +++++++++++++++++++++++++--------------- backend/Dockerfile.build | 38 ----------------------- backend/Dockerfile.dev | 28 ----------------- docker-compose.yml | 14 ++++----- 5 files changed, 47 insertions(+), 99 deletions(-) delete mode 100644 backend/Dockerfile.build delete mode 100644 backend/Dockerfile.dev diff --git a/.gitignore b/.gitignore index 729fcca..6c1928c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ git-crypt-key +result .idea .cache .vscode diff --git a/backend/Dockerfile b/backend/Dockerfile index 4287e2f..dc500a3 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,32 +1,49 @@ # syntax=docker/dockerfile:1 -FROM phusion/baseimage:jammy-1.0.4 +FROM nixos/nix:2.23.0 AS base + +# Image to capture store closure +FROM base AS builder # We take the battery option as an argument ARG BATTERY_SUPPORT=OFF -# Copy the binaries and libs from the builder -COPY --from=typicalam/yarilo-build:latest /app/deps /app/deps - -# Prepare deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential cmake ninja-build libpcap-dev libssl-dev libspdlog-dev libnl-3-dev libnl-nf-3-dev libnl-route-3-dev libnl-genl-3-dev golang-go doxygen \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && GOPATH=/app/deps go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v1.5.1 - -# Build yarilo -RUN git clone https://github.com/TypicalAM/Yarilo /app/src \ - && /app/deps/bin/protoc -I /app/src/protos --doc_opt=markdown,proto.md --doc_out=/app/src/backend/docs --plugin=protoc-gen-doc=/app/deps/bin/protoc-gen-doc /app/src/protos/service.proto \ - && cmake -DCMAKE_PREFIX_PATH=/app/deps -DYARILO_BUILD_DOCS=ON -DYARILO_BATTERY_SUPPORT=$BATTERY_SUPPORT -B /app/src/backend/build -G Ninja /app/src/backend \ - && ninja -C /app/src/backend/build && mv /app/src/backend/build/yarilo /yarilo - -# Set up the run script -COPY </dev/null 2>&1 & -/yarilo \$* +# Capture deps into /nix/store, even when building dev image +COPY . /src +WORKDIR /src +RUN if [ "$BATTERY_SUPPORT" = "ON" ] ; then \ + nix --extra-experimental-features "nix-command flakes" build .#YariloBattery; \ + else \ + nix --extra-experimental-features "nix-command flakes" build .; \ + fi; + +# Development image +FROM builder AS development + +COPY </dev/null 2>&1 & +/yarilo/bin/yarilo \$* +EOF + +RUN chmod +x /run.sh +WORKDIR /src +EXPOSE 9090 +ENTRYPOINT ["sh", "/run.sh"] + +FROM builder AS production + +COPY </dev/null 2>&1 & +/src/result/bin/yarilo \$* EOF -# Set the script as executable -RUN chmod +x run.sh +RUN chmod +x /run.sh + +EXPOSE 9090 +ENTRYPOINT ["sh", "/run.sh"] diff --git a/backend/Dockerfile.build b/backend/Dockerfile.build deleted file mode 100644 index f826260..0000000 --- a/backend/Dockerfile.build +++ /dev/null @@ -1,38 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM phusion/baseimage:jammy-1.0.4 - -# We take the target platform and build it conditionally -ARG TARGETPLATFORM - -# Prepare deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential cmake ninja-build curl git libpcap-dev libssl-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app/deps - -# Download and install grpc -RUN git clone --recurse-submodules -b v1.62.1 --depth 1 --shallow-submodules https://github.com/grpc/grpc /opt/grpc \ - && mkdir /opt/grpc/cmake/build \ - && cmake -B /opt/grpc/cmake/build -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=/app/deps /opt/grpc \ - && make -C /opt/grpc/cmake/build -j 4 \ - && make -C /opt/grpc/cmake/build install \ - && rm -rf /opt/grpc - -# Download and install libtins -RUN git clone --recurse-submodules -b v4.5 --depth 1 --shallow-submodules https://github.com/mfontanini/libtins /opt/libtins \ - && mkdir /opt/libtins/build \ - && cmake -B /opt/libtins/build -DLIBTINS_ENABLE_CXX11=1 -DLIBTINS_ENABLE_ACK_TRACKER=0 -DCMAKE_INSTALL_PREFIX=/app/deps /opt/libtins \ - && make -C /opt/libtins/build -j 4 \ - && make -C /opt/libtins/build install \ - && rm -rf /opt/libtins - -# Download the envoy proxy -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \ - curl -L https://github.com/envoyproxy/envoy/releases/download/v1.30.1/envoy-1.30.1-linux-aarch_64 -o /app/deps/bin/envoy; \ - else \ - curl -L https://github.com/envoyproxy/envoy/releases/download/v1.30.1/envoy-1.30.1-linux-x86_64 -o /app/deps/bin/envoy; \ - fi; -RUN chmod +x /app/deps/bin/envoy diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev deleted file mode 100644 index c64bce4..0000000 --- a/backend/Dockerfile.dev +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM phusion/baseimage:jammy-1.0.4 - -# Copy the binaries and libs from the builder -COPY --from=typicalam/yarilo-build:latest /app/deps /app/deps - -# Prepare runtime deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential cmake ninja-build libpcap-dev libssl-dev libspdlog-dev libnl-3-dev libnl-nf-3-dev libnl-route-3-dev libnl-genl-3-dev doxygen golang-go \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@latest - -# Create the run script -COPY </dev/null 2>&1 & -protoc -I /app/src/protos --cpp_out=/app/src/backend/src/proto --doc_opt=markdown,proto.md --doc_out=/app/src/backend/docs --grpc_out=/app/src/backend/src/proto --plugin=protoc-gen-grpc=$(which grpc_cpp_plugin) /app/src/protos/service.proto -cmake -DCMAKE_PREFIX_PATH=/app/deps -DYARILO_BATTERY_SUPPORT=ON -G Ninja -B /app/build /app/src/backend -ninja -C /app/build -/app/build/yarilo \$* -EOF - -# Set the script as executable -RUN chmod +x run.sh diff --git a/docker-compose.yml b/docker-compose.yml index 1beb1e7..08a6dd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,18 +2,14 @@ services: yarilo-dev: build: context: . - dockerfile: ./backend/Dockerfile.dev - command: ./run.sh --sniff_file=/app/src/pcap/wireshark_sample.pcap --save_path=/app/saves + dockerfile: ./backend/Dockerfile + args: + BATTERY_SUPPORT: ON + command: --sniff_file=/src/pcap/wireshark_sample.pcap --save_path=/app/saves ports: - 8080:8080 # Envoy (for grpc-web) - 9090:9090 # Yarilo volumes: - - ./backend:/app/src/backend - - ./pcap:/app/src/pcap - - ./protos:/app/src/protos + - .:/src - ./dev-save-data:/app/saves - - ./dev-build-data:/app/build - /tmp/battery_level:/tmp/battery_level - -volumes: - build_data: From 05ef59b5370136134745770899c5599b0940d994 Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Tue, 17 Dec 2024 19:33:27 +0100 Subject: [PATCH 2/7] Working docker for nix --- backend/Dockerfile | 6 +++--- docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index dc500a3..0109f95 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -23,12 +23,12 @@ FROM builder AS development COPY </dev/null 2>&1 & -/yarilo/bin/yarilo \$* +/src/result/bin/yarilo \$* EOF RUN chmod +x /run.sh diff --git a/docker-compose.yml b/docker-compose.yml index 08a6dd0..d39357b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: build: context: . dockerfile: ./backend/Dockerfile + target: development args: BATTERY_SUPPORT: ON command: --sniff_file=/src/pcap/wireshark_sample.pcap --save_path=/app/saves From 33e54cc84864afc1f71daeaa54a1a8bede871b7a Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Wed, 18 Dec 2024 01:39:23 +0100 Subject: [PATCH 3/7] Working frontend for nix --- frontend/Dockerfile | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2e3b24e..3ce5a96 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,38 +1,14 @@ # syntax=docker/dockerfile:1 -FROM node:23.4-bookworm-slim AS builder +FROM nixos/nix:2.23.0 AS base -# Prepare deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends git nodejs npm \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +# Image to capture store closure +FROM base AS production -# Set up workdir -WORKDIR /app +# Building the project +COPY . /src +WORKDIR /src +RUN nix --extra-experimental-features "nix-command flakes" build .#YariloFrontend -# Copy package deps -COPY package.json package-lock.json . - -# Install npm deps -RUN npm ci - -# Copy source -COPY . . - -# Build sveltekit -RUN npm run build - -FROM node:23.4-bookworm-slim - -# Copy deps from builder -COPY --from=builder /app/build /app/build -COPY --from=builder /app/node_modules /app/node_modules - -# Install runtime deps -RUN apt-get update \ - && apt-get install -y --no-install-recommends git nodejs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -ENTRYPOINT ["node", "/app/build"] +EXPOSE 3000 +ENTRYPOINT ["/src/result/bin/yarilo-frontend"] From 542879d91551c838ee345a570fb69830e67bbe11 Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Wed, 18 Dec 2024 21:33:52 +0100 Subject: [PATCH 4/7] ARM8 nix corrections --- backend/Dockerfile | 32 ++++++++++++++++++++++++++------ frontend/Dockerfile | 5 ++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 0109f95..d80cc88 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -12,9 +12,14 @@ ARG BATTERY_SUPPORT=OFF COPY . /src WORKDIR /src RUN if [ "$BATTERY_SUPPORT" = "ON" ] ; then \ - nix --extra-experimental-features "nix-command flakes" build .#YariloBattery; \ + nix \ + --extra-experimental-features "nix-command flakes" \ + --option filter-syscalls false \ + build .#YariloBattery; \ else \ - nix --extra-experimental-features "nix-command flakes" build .; \ + nix --extra-experimental-features "nix-command flakes" \ + --option filter-syscalls false \ + build .; \ fi; # Development image @@ -23,11 +28,21 @@ FROM builder AS development COPY </dev/null 2>&1 & +nix \ + --extra-experimental-features "nix-command flakes" \ + --option filter-syscalls false \ + shell nixpkgs#envoy --command envoy \ + -c /src/backend/envoy.yaml \ + --log-path /app/envoy.log >/dev/null 2>&1 & /src/result/bin/yarilo \$* EOF @@ -39,7 +54,12 @@ ENTRYPOINT ["sh", "/run.sh"] FROM builder AS production COPY </dev/null 2>&1 & +nix \ + --extra-experimental-features "nix-command flakes" \ + --option filter-syscalls false \ + shell nixpkgs#envoy --command envoy \ + -c /src/backend/envoy.yaml \ + --log-path /app/envoy.log >/dev/null 2>&1 & /src/result/bin/yarilo \$* EOF diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 3ce5a96..9025dd5 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,7 +8,10 @@ FROM base AS production # Building the project COPY . /src WORKDIR /src -RUN nix --extra-experimental-features "nix-command flakes" build .#YariloFrontend +RUN nix \ + --extra-experimental-features "nix-command flakes" \ + --option filter-syscalls false \ + build .#YariloFrontend EXPOSE 3000 ENTRYPOINT ["/src/result/bin/yarilo-frontend"] From 7f47267221c86943cf70ab93a1cbdefd0468591a Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Wed, 18 Dec 2024 21:34:08 +0100 Subject: [PATCH 5/7] Dockerignore --- .dockerignore | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..562604a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +.vscode +.idea +.git +.vscode +compile_commands.json +dev-save-data +dev-build-data +cmake-build-debug +*Dockerfile* +*docker-compose* + +backend/cmake-build-debug +backend/build +backend/.idea +backend/.cache +backend/.vscode +backend/result + +frontend/.idea +frontend/.cache +frontend/.svelte-kit +frontend/.DS_Store +frontend/.env +frontend/.env.* +frontend/build +frontend/node_modules +frontend/vite.config.js.timestamp-* +frontend/vite.config.ts.timestamp-* +frontend/.vscode +frontend/result From 24e4f03026aa3ac2ebe36c0b73b13815534d5ef5 Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Wed, 18 Dec 2024 22:19:51 +0100 Subject: [PATCH 6/7] Github CI action on tag push --- .github/workflows/docker.yml | 77 ++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..34a6973 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,77 @@ +name: Build Docker Image + +on: + push: + tags: + - '*' + +jobs: + backend: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + username: typicalam + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: "typicalam/yarilo:latest,typicalam/yarilo:${{ github.ref_name }}" + platforms: linux/amd64,linux/arm64 + file: ./backend/Dockerfile + + backend-battery: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + username: typicalam + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v4 + context: . + push: true + tags: "typicalam/yarilo:latest-hardware,typicalam/yarilo:${{ github.ref_name }}-hardware" + platforms: linux/amd64,linux/arm64 + file: ./backend/Dockerfile + build-args: + BATTERY_SUPPORT=ON + + frontend: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + username: typicalam + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: "typicalam/yarilo-front:latest,typicalam/yarilo-front:${{ github.ref_name }}" + platforms: linux/amd64,linux/arm64 + file: ./frontend/Dockerfile From 3c1a55342fee7c6b3067e98a2cdcaa3d69953282 Mon Sep 17 00:00:00 2001 From: TypicalAM Date: Wed, 18 Dec 2024 23:01:02 +0100 Subject: [PATCH 7/7] Ignore encrypted files in docker images --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 562604a..3513373 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ dev-build-data cmake-build-debug *Dockerfile* *docker-compose* +pcap backend/cmake-build-debug backend/build