diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml index 74ee7f2f3de8..acd6fe17e56b 100644 --- a/.github/workflows/build-src.yml +++ b/.github/workflows/build-src.yml @@ -113,14 +113,36 @@ jobs: /cache/ccache key: ccache-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'depends/packages/*') }}-${{ inputs.build-target }}-${{ github.sha }} + - name: Restore ctcache cache + if: inputs.build-target == 'linux64_multiprocess' + uses: actions/cache/restore@v4 + with: + path: | + /cache/ctcache + key: ctcache-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'depends/packages/*') }}-${{ inputs.build-target }}-${{ github.sha }} + restore-keys: | + ctcache-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'depends/packages/*') }}-${{ inputs.build-target }}- + - name: Run linters if: inputs.build-target == 'linux64_multiprocess' run: | + CACHE_DIR="/cache" export BUILD_TARGET="${{ inputs.build-target }}" source ./ci/dash/matrix.sh ./ci/dash/lint-tidy.sh shell: bash + - name: Save ctcache cache + if: | + inputs.build-target == 'linux64_multiprocess' && + github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch + uses: actions/cache/save@v4 + with: + path: | + /cache/ctcache + key: ctcache-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'depends/packages/*') }}-${{ inputs.build-target }}-${{ github.sha }} + - name: Run unit tests run: | BUILD_TARGET="${{ inputs.build-target }}" diff --git a/ci/dash/lint-tidy.sh b/ci/dash/lint-tidy.sh index bd78e85ef22b..e8153a62d844 100755 --- a/ci/dash/lint-tidy.sh +++ b/ci/dash/lint-tidy.sh @@ -11,13 +11,76 @@ set -eo pipefail # only on nor do they set the requisite build parameters. Make sure you do # that *before* running this script. +# Cleanup old ctcache entries to keep cache size under limit +cleanup_ctcache() { + local cache_dir=$1 + local max_size_mb=$2 + local cache_size_mb + + cache_size_mb=$(du -sm "${cache_dir}" 2>/dev/null | cut -f1) + if [ "${cache_size_mb}" -gt "${max_size_mb}" ]; then + echo "Cache size ${cache_size_mb}MB exceeds limit ${max_size_mb}MB, cleaning old entries..." + # Remove files older than 7 days, or if still too large, oldest 20% of files + find "${cache_dir}" -type f -mtime +7 -delete 2>/dev/null || true + cache_size_mb=$(du -sm "${cache_dir}" 2>/dev/null | cut -f1) + if [ "${cache_size_mb}" -gt "${max_size_mb}" ]; then + local file_count remove_count + file_count=$(find "${cache_dir}" -type f | wc -l) + remove_count=$((file_count / 5)) + find "${cache_dir}" -type f -printf '%T+ %p\0' | sort -z | head -z -n "${remove_count}" | cut -z -d' ' -f2- | xargs -0 rm -f 2>/dev/null || true + echo "Attempted to remove ${remove_count} oldest cache entries" + fi + echo "Cache size after cleanup: $(du -sh "${cache_dir}" 2>/dev/null | cut -f1)" + fi +} + +# Verify clang-tidy is available +CLANG_TIDY_BIN=$(command -v clang-tidy) +if [ -z "${CLANG_TIDY_BIN}" ]; then + echo "Error: clang-tidy not found in PATH" + exit 1 +fi + +# Setup ctcache for clang-tidy result caching +export CTCACHE_SAVE_OUTPUT=1 +export CTCACHE_CLANG_TIDY="${CLANG_TIDY_BIN}" +mkdir -p "${CTCACHE_DIR}" + +CLANG_TIDY_CACHE=$(command -v clang-tidy-cache) + +# Verify ctcache is installed +if [ -z "${CLANG_TIDY_CACHE}" ]; then + echo "Error: clang-tidy-cache not found in PATH" + exit 1 +fi + +# Derive Python script path from wrapper location (matches wrapper's own logic) +CLANG_TIDY_CACHE_PY="$(dirname "${CLANG_TIDY_CACHE}")/src/ctcache/clang_tidy_cache.py" +if [ ! -f "${CLANG_TIDY_CACHE_PY}" ]; then + echo "Error: ctcache Python script not found at ${CLANG_TIDY_CACHE_PY}" + exit 1 +fi + +# Zero stats before run to get accurate statistics for this run only +python3 "${CLANG_TIDY_CACHE_PY}" --zero-stats 2>&1 || true + cd "${BASE_ROOT_DIR}/build-ci/dashcore-${BUILD_TARGET}/src" -if ! ( run-clang-tidy -quiet "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then + +if ! ( run-clang-tidy -clang-tidy-binary="${CLANG_TIDY_CACHE}" -quiet "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then grep -C5 "error: " tmp.tidy-out.txt echo "^^^ ⚠️ Failure generated from clang-tidy" false fi +# Show ctcache statistics and manage cache size +echo "=== ctcache statistics ===" +du -sh "${CTCACHE_DIR}" 2>/dev/null || echo "Cache directory not found" +python3 "${CLANG_TIDY_CACHE_PY}" --show-stats 2>&1 || true + +# Limit cache size (ctcache has no built-in size management) +cleanup_ctcache "${CTCACHE_DIR}" "${CTCACHE_MAXSIZE_MB}" +echo "==========================" + cd "${BASE_ROOT_DIR}/build-ci/dashcore-${BUILD_TARGET}" iwyu_tool.py \ "src/compat" \ diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 9ff3c333754e..fc4944ae1d35 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -61,6 +61,9 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. # This folder exists on the ci host and ci guest. Changes are propagated back and forth. export CCACHE_DIR=${CCACHE_DIR:-$CACHE_DIR/ccache} +# ctcache (clang-tidy cache) configuration +export CTCACHE_DIR=${CTCACHE_DIR:-$CACHE_DIR/ctcache} +export CTCACHE_MAXSIZE_MB=${CTCACHE_MAXSIZE_MB:-50} # The depends dir. # This folder exists on the ci host and ci guest. Changes are propagated back and forth. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} diff --git a/contrib/containers/ci/ci.Dockerfile b/contrib/containers/ci/ci.Dockerfile index 4265a0bc83c1..2f0659cbf01f 100644 --- a/contrib/containers/ci/ci.Dockerfile +++ b/contrib/containers/ci/ci.Dockerfile @@ -65,8 +65,20 @@ RUN set -ex; \ make install -j "$(( $(nproc) - 1 ))"; \ cd /opt && rm -rf /opt/iwyu; +# Install ctcache for clang-tidy result caching +# Pin to specific commit to ensure patch applies correctly +ARG CTCACHE_COMMIT=e393144d5c49b060a1dbc7ae15b9c6973efb967d +RUN set -ex; \ + mkdir -p /usr/local/bin/src/ctcache; \ + curl -fsSL "https://raw.githubusercontent.com/matus-chochlik/ctcache/${CTCACHE_COMMIT}/src/ctcache/clang_tidy_cache.py" \ + -o /usr/local/bin/src/ctcache/clang_tidy_cache.py; \ + curl -fsSL "https://raw.githubusercontent.com/matus-chochlik/ctcache/${CTCACHE_COMMIT}/clang-tidy" \ + -o /usr/local/bin/clang-tidy-cache; \ + chmod +x /usr/local/bin/clang-tidy-cache; + RUN \ mkdir -p /cache/ccache && \ + mkdir /cache/ctcache && \ mkdir /cache/depends && \ mkdir /cache/sdk-sources && \ chown ${USER_ID}:${GROUP_ID} /cache && \