diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f3d2d9272c87..72dfc6bae494 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/dfinity/ic-dev@sha256:16f8614341dee3f04e528262a6e23b94a524cd33e543908eccf8fb8b9cda8f27", + "image": "ghcr.io/dfinity/ic-dev@sha256:b001f2fa9ed28fc86b055879303a9dbcf3d20d0cc0625504941b855f5a688ed6", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ diff --git a/.github/workflows/api-bn-recovery-test.yml b/.github/workflows/api-bn-recovery-test.yml index 255d3d544471..2d8ef0bff450 100644 --- a/.github/workflows/api-bn-recovery-test.yml +++ b/.github/workflows/api-bn-recovery-test.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 683542a25019..3e8723ecb488 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -33,7 +33,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 90 diff --git a/.github/workflows/ci-pr-only.yml b/.github/workflows/ci-pr-only.yml index bd529821902d..a9d9e7a11e7b 100644 --- a/.github/workflows/ci-pr-only.yml +++ b/.github/workflows/ci-pr-only.yml @@ -37,7 +37,7 @@ jobs: runs-on: &dind-small-setup labels: dind-small container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --mount type=tmpfs,target="/tmp/containers" steps: diff --git a/.github/workflows/container-api-bn-recovery.yml b/.github/workflows/container-api-bn-recovery.yml index 068129629f61..8b22d14fa233 100644 --- a/.github/workflows/container-api-bn-recovery.yml +++ b/.github/workflows/container-api-bn-recovery.yml @@ -28,7 +28,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/container-scan-nightly.yml b/.github/workflows/container-scan-nightly.yml index d27b87801142..2ddfdb80da9c 100644 --- a/.github/workflows/container-scan-nightly.yml +++ b/.github/workflows/container-scan-nightly.yml @@ -12,7 +12,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 60 diff --git a/.github/workflows/pocket-ic-tests-windows.yml b/.github/workflows/pocket-ic-tests-windows.yml index 29fc3d263d2c..0743d04eb2c0 100644 --- a/.github/workflows/pocket-ic-tests-windows.yml +++ b/.github/workflows/pocket-ic-tests-windows.yml @@ -45,7 +45,7 @@ jobs: bazel-build-pocket-ic: name: Bazel Build PocketIC container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 90 diff --git a/.github/workflows/rate-limits-backend-release.yml b/.github/workflows/rate-limits-backend-release.yml index 31816c9d1e29..094800e65752 100644 --- a/.github/workflows/rate-limits-backend-release.yml +++ b/.github/workflows/rate-limits-backend-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" diff --git a/.github/workflows/release-testing.yml b/.github/workflows/release-testing.yml index ee2f14588834..f338eb29e8f4 100644 --- a/.github/workflows/release-testing.yml +++ b/.github/workflows/release-testing.yml @@ -34,7 +34,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 180 diff --git a/.github/workflows/rosetta-release.yml b/.github/workflows/rosetta-release.yml index b24379567eb8..22b0f011dae8 100644 --- a/.github/workflows/rosetta-release.yml +++ b/.github/workflows/rosetta-release.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" environment: DockerHub diff --git a/.github/workflows/salt-sharing-canister-release.yml b/.github/workflows/salt-sharing-canister-release.yml index afc56f0144b3..b72809297aca 100644 --- a/.github/workflows/salt-sharing-canister-release.yml +++ b/.github/workflows/salt-sharing-canister-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml index 2a42f6c86477..5c145ead667d 100644 --- a/.github/workflows/schedule-daily.yml +++ b/.github/workflows/schedule-daily.yml @@ -14,7 +14,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml index f86f400e667f..92aaa4c16cb2 100644 --- a/.github/workflows/schedule-rust-bench.yml +++ b/.github/workflows/schedule-rust-bench.yml @@ -24,7 +24,7 @@ jobs: # see linux-x86-64 runner group labels: rust-benchmarks container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 # running on bare metal machine using ubuntu user options: --user ubuntu --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/system-tests-benchmarks-nightly.yml b/.github/workflows/system-tests-benchmarks-nightly.yml index 9ddaa14afb9d..5d77c5d6c2f8 100644 --- a/.github/workflows/system-tests-benchmarks-nightly.yml +++ b/.github/workflows/system-tests-benchmarks-nightly.yml @@ -16,7 +16,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 480 diff --git a/.github/workflows/update-mainnet-canister-revisions.yaml b/.github/workflows/update-mainnet-canister-revisions.yaml index a675662e5fb5..c2bbd0ad66ba 100644 --- a/.github/workflows/update-mainnet-canister-revisions.yaml +++ b/.github/workflows/update-mainnet-canister-revisions.yaml @@ -25,7 +25,7 @@ jobs: labels: dind-small environment: CREATE_PR container: - image: ghcr.io/dfinity/ic-build@sha256:ac24f205995a1c36921bb8606e0ee16b8b26d73be7dd0cac786c6882f3569680 + image: ghcr.io/dfinity/ic-build@sha256:d7f562394dc9f2b801d7d9154f3a276a156d01d03a39fb37da5848bc116d1b36 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" env: diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index 49ff0f69961a..ce10921e4e2a 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -5,18 +5,29 @@ FROM ghcr.io/dfinity/library/ubuntu@sha256:985be7c735afdf6f18aaa122c23f87d989c30 ENV TZ=UTC RUN export DEBIAN_FRONTEND=noninteractive && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -# copy/write config files -ARG PACKAGE_FILE=ci/container/files/packages.common -COPY ${PACKAGE_FILE} /tmp/ -COPY ./ci/container/files/gitconfig /etc/gitconfig -COPY ./ci/container/files/containers.conf /etc/containers/containers.conf -RUN echo "[storage]\nrootless_storage_path=\"/tmp/containers\"" > /etc/containers/storage.conf -# marker for scripts to check if they are running in this container -RUN touch /home/ubuntu/.ic-build-container - RUN apt -yq update && \ - apt -yqq install $(sed -e "s/#.*//" "/tmp/$(basename $PACKAGE_FILE)") && \ - rm "/tmp/$(basename $PACKAGE_FILE)" + apt -yqq install --no-install-recommends ca-certificates curl sudo gnupg git build-essential zlib1g-dev + +# AFLplusplus build image +FROM base as afl + +# Install AFLplusplus for fuzzing +# LLVM is only a build time dependency now since we link the fuzzer lib from the hermetic toolchain directly +ARG AFLPLUSPLUS_RELEASE_VERSION=v4.35c +ARG LLVM_VERSION=21 +RUN curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | apt-key add - && \ + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-${LLVM_VERSION} main" | tee -a /etc/apt/sources.list.d/llvm.list && \ + apt -yq update && \ + apt -yqq install --no-install-recommends lld-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev clang-${LLVM_VERSION} libclang-rt-${LLVM_VERSION}-dev && \ + mkdir -p /afl && \ + chown -R ubuntu:ubuntu /afl && \ + cd /afl && \ + git clone --depth=1 --branch=${AFLPLUSPLUS_RELEASE_VERSION} https://github.com/AFLplusplus/AFLplusplus.git && \ + cd AFLplusplus && \ + STATIC=1 LLVM_CONFIG=/usr/bin/llvm-config-${LLVM_VERSION} CC=/usr/bin/clang-${LLVM_VERSION} CXX=/usr/bin/clang++-${LLVM_VERSION} make all && \ + STATIC=1 LLVM_CONFIG=/usr/bin/llvm-config-${LLVM_VERSION} CC=/usr/bin/clang-${LLVM_VERSION} CXX=/usr/bin/clang++-${LLVM_VERSION} make install + +FROM base as build ## Because the container is used for both CI and development, we need to have a user that matches the UID of the runner (1001) and a user that matches the UID of ubuntu (1000) RUN groupadd -g 1001 buildifier && useradd -ms /bin/bash -u 1001 -g 1001 -G ubuntu buildifier && \ @@ -59,24 +70,6 @@ RUN curl -sSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION} ARG motoko_version=0.16.3 RUN curl -fsSL https://github.com/dfinity/motoko/releases/download/${motoko_version}/motoko-linux-x86_64-${motoko_version}.tar.gz | tar -xz -C /usr/local/bin && chmod +x /usr/local/bin/moc -# Install AFLplusplus for fuzzing -# LLVM is only a build time dependency now since we link the fuzzer lib from the hermetic toolchain directly -ARG AFLPLUSPLUS_RELEASE_VERSION=v4.35c -ARG LLVM_VERSION=21 -RUN curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | apt-key add - && \ - echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-${LLVM_VERSION} main" | tee -a /etc/apt/sources.list.d/llvm.list && \ - apt -yq update && \ - apt -yqq install --no-install-recommends lld-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev clang-${LLVM_VERSION} libclang-rt-${LLVM_VERSION}-dev && \ - mkdir -p /afl && \ - chown -R ubuntu:ubuntu /afl && \ - cd /afl && \ - git clone --depth=1 --branch=${AFLPLUSPLUS_RELEASE_VERSION} https://github.com/AFLplusplus/AFLplusplus.git && \ - cd AFLplusplus && \ - STATIC=1 LLVM_CONFIG=/usr/bin/llvm-config-${LLVM_VERSION} CC=/usr/bin/clang-${LLVM_VERSION} CXX=/usr/bin/clang++-${LLVM_VERSION} make all && \ - STATIC=1 LLVM_CONFIG=/usr/bin/llvm-config-${LLVM_VERSION} CC=/usr/bin/clang-${LLVM_VERSION} CXX=/usr/bin/clang++-${LLVM_VERSION} make install && \ - mv afl-fuzz afl-showmap /afl && \ - cd .. && rm -rf AFLplusplus - # install cargo with the ubuntu user USER ubuntu ENV PATH=/ic/bin:/home/ubuntu/.cargo/bin:/home/ubuntu/.local/bin:$PATH @@ -99,11 +92,29 @@ USER root WORKDIR / CMD ["/bin/bash"] -FROM base as build + +# copy/write config files + +ARG PACKAGE_FILE=ci/container/files/packages.common +COPY ${PACKAGE_FILE} /tmp/ +RUN apt -yqq install $(sed -e "s/#.*//" "/tmp/$(basename $PACKAGE_FILE)") && \ + rm "/tmp/$(basename $PACKAGE_FILE)" +COPY ./ci/container/files/gitconfig /etc/gitconfig +COPY ./ci/container/files/containers.conf /etc/containers/containers.conf +RUN echo hello +RUN echo "[storage]\nrootless_storage_path=\"/tmp/containers\"" > /etc/containers/storage.conf + +# copy executables built elsewhere +COPY --from=afl /usr/local/bin/afl-fuzz /usr/local/bin/afl-fuzz + +# marker for scripts to check if they are running in this container +RUN touch /home/ubuntu/.ic-build-container USER buildifier -FROM base as dev +FROM build as dev + +USER root # Add zshrc generated from zsh-newuser-install (option 2) COPY --chown=ubuntu:ubuntu ./ci/container/files/zshrc /home/ubuntu/.zshrc diff --git a/ci/container/TAG b/ci/container/TAG index 54317033020b..348126a11dc5 100644 --- a/ci/container/TAG +++ b/ci/container/TAG @@ -1 +1 @@ -d0dbd29b1eb7ca6666e77fde4ca0c785293974ac02bdab3c0dfe14e91f7e657a +1c7befc831d88679bfe678cf9cea6a2f447bab0c2e85e98690ad5e69cf3b54cf