From 6f3297a4a08ece0f47d5a10ab38b7df82c1bff6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 4 Nov 2026 13:52:05 +0100 Subject: [PATCH] [CI] Size compiler caches for one configuration and split the Fedora leg The bounds in #7667 were set below what one build configuration writes, so ccache filled to its limit and evicted the entries the next build was about to ask for. The probe measured what that costs: a 6G cache on one leg served 1059 of 1059 calls and the 700M bound on the same leg served 14%. The quota also went to legs that finish under the Windows leg, which has no compiler cache. Debug information is most of an object's size. A measured Arch Debug configuration is 1.2 GB with the built-in flags and 660 MB at -g1; Debian 13 Release is 1.1 GB and 614 MB. CI therefore builds every non-tag leg at level 1 through a new COCKATRICE_DEBUG_INFO_LEVEL option, which stays empty by default so local, distro and release builds are unchanged. Level 1 keeps file and line, which is what a failing test or a sanitizer backtrace reports. Tags and the nightly legs keep full DWARF for the Sentry symbols. With one configuration at half the size, the quota goes to fewer legs at a size that holds one whole configuration, and the legs that duplicate another leg's coverage give their cache up: Servatrice, Ubuntu 26.04 behind the sanitize leg of the same distro, macOS 15 and macOS 26 Debug behind macOS 14. Their keys leave LIVE_CCACHE_KEYS, so prune-caches reclaims the existing entries. Fedora 44 built Debug for the suite and Release for the package in one job, so the package compile queued behind the debug one and the leg took 30.8 min - the only leg above Windows - while one 1.6 GB cache held two configurations. It is split into two legs with one cache each, running in parallel. The option reaches compile.sh inside the container through docker.sh, and compile.sh turns it into the CMake flag. Bounds: Arch 850M, Debian 13 800M, Fedora 44 debug 850M and release 800M, Ubuntu 26.04 sanitize 1200M, macOS 14 800M. That is 5.3 GB of compiler cache on top of roughly 4.5 GB of buildkit, vcpkg, Qt and CodeQL entries. Windows still sets the pull-request wall clock: 29.5 min, 23.7 of it compiling with no compiler cache, because ccache and sccache cannot store the 711 of 1054 translation units that use PCH under MSVC (#7381). That is a separate trial. --- .ci/compile.sh | 5 ++ .ci/docker.sh | 5 ++ .github/workflows/desktop-build.yml | 132 +++++++++++++++++++--------- CMakeLists.txt | 21 +++++ 4 files changed, 122 insertions(+), 41 deletions(-) diff --git a/.ci/compile.sh b/.ci/compile.sh index 569ae9f6b..7414b2dc3 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -203,6 +203,11 @@ if [[ ${WITH_CRASH_REPORTING:-} == "ON" ]]; then # developer builds never reach Sentry. flags+=("-DWITH_CRASH_REPORTING=ON") fi +if [[ ${COCKATRICE_DEBUG_INFO_LEVEL:-} ]]; then + # CI lowers the DWARF level so one build configuration fits its compiler-cache bound; + # see the option in CMakeLists.txt. Empty keeps the built-in flags. + flags+=("-DCOCKATRICE_DEBUG_INFO_LEVEL=$COCKATRICE_DEBUG_INFO_LEVEL") +fi if [[ ${BUILDTYPE:-Release} == "Release" && -n ${COCKATRICE_SENTRY_DSN:-} ]]; then # Release-only: Debug builds stay DSN-free so crashes from developer builds # never reach Sentry. Empty (local builds, forks without the secret, and non-tag diff --git a/.ci/docker.sh b/.ci/docker.sh index 64ea6baa7..c78ba3c31 100644 --- a/.ci/docker.sh +++ b/.ci/docker.sh @@ -181,6 +181,11 @@ function RUN () if [[ ${WITH_CRASH_REPORTING:-} ]]; then args+=(--env "WITH_CRASH_REPORTING=${WITH_CRASH_REPORTING}") fi + # Same reason: the DWARF level reaches compile.sh only if it is handed to the container, + # and without it CI would cache objects twice the size its bounds were measured for. + if [[ ${COCKATRICE_DEBUG_INFO_LEVEL:-} ]]; then + args+=(--env "COCKATRICE_DEBUG_INFO_LEVEL=${COCKATRICE_DEBUG_INFO_LEVEL}") + fi # shellcheck disable=2086 docker run "${args[@]}" $RUN_ARGS "$IMAGE_NAME" bash "$BUILD_SCRIPT" $RUN_OPTS "$@" return $? diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index be11c3748..70becd740 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -150,6 +150,7 @@ jobs: allow-failure: yes package: skip # We are packaged in Arch already debug_test: yes # see the note on Fedora 44 below + ccache_size: 850M # Debug plus the suite: 660 MB of objects at -g1 - distro: Servatrice_Debian version: 12 @@ -157,30 +158,46 @@ jobs: package: DEB server_only: yes test: skip - ccache_size: 400M # server only, no client to compile + # 4.7 min and the server alone: nothing here for a compiler cache to save. + persist_ccache: 'false' - distro: Debian version: 13 package: DEB - ccache_size: 850M # the Release build plus the suite it runs + ccache_size: 800M # Release plus the suite: 614 MB of objects at -g1 + + # Fedora 44 used to build Debug for the suite and Release for the package in one + # job, so the package compile waited behind the Debug one and the leg took 30.8 min + # - the only leg slower than Windows. Two legs run in parallel and each keeps a + # cache that holds exactly one configuration. + # + # Arch and Fedora 44 are the two newest toolchains, and the -Wno-error= exceptions + # in CMakeLists.txt exist for exactly their warnings. This leg compiles Debug + # because only Debug passes -Werror, so it is where a new warning still fails. + - distro: Fedora + version: 44 + variant: debug # separate image and ccache from the release leg below + + package: skip + debug_test: yes + ccache_size: 850M # Debug plus the suite: 660 MB of objects at -g1 - distro: Fedora version: 44 + variant: release # separate image and ccache from the debug leg above package: RPM - # Arch and Fedora 44 are the two newest toolchains, and the -Wno-error= exceptions - # in CMakeLists.txt exist for exactly their warnings. They keep compiling Debug: - # only the Debug configuration passes -Werror, so these two are the legs where a - # new warning in either compiler still fails the build. - debug_test: yes - ccache_size: 900M # Debug for the suite and Release for the package + test: skip # The debug leg above runs this distro's suite + ccache_size: 800M # Release plus the package: 614 MB of objects at -g1 - distro: Ubuntu version: 26.04 package: DEB test: skip # The sanitizer leg below runs this distro's suite + # 18.1 min, and the same distro as the sanitize leg below, which runs this suite. + persist_ccache: 'false' # The one leg that runs the suite under AddressSanitizer and # UndefinedBehaviorSanitizer. Debian 13 and Fedora 44 keep covering the @@ -196,7 +213,7 @@ jobs: package: skip sanitize: yes - ccache_size: 800M # instrumented objects are larger + ccache_size: 1200M # instrumented objects are larger than the 614 MB measured; estimated name: ${{ matrix.distro }} ${{ matrix.version }}${{ matrix.variant && format(' {0}', matrix.variant) || '' }} needs: configure @@ -211,25 +228,47 @@ jobs: CCACHE_EVICTION_AGE: 7d # The bound is sized against the repository's 10 GB cache quota # (https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy), - # not against what one leg would like. The quota currently holds about 1.1 GB of - # CodeQL trap, 1.1 GB of vcpkg, 0.7 GB of Qt and 2.1 GB of docker buildkit layers - # (mode=min in docker-release.yml, added here, brings the last one down to roughly - # 0.3 GB - the one input here that is an expectation rather than a measurement, since - # mode=min still writes one layer per pushed binary) plus the compiler caches of the nine legs a pull request runs. The five - # nightly-only legs keep no compiler cache (see PERSIST_CCACHE below), which is what - # makes these per-leg bounds affordable. Accounting for all of it: 6.45G of ccache - # bounds + 3.2G of the fixed entries = 9.65G against the 10G quota. + # not against what one leg would like: the quota also holds about 0.9 GB of buildkit + # layers (mode=min in docker-release.yml), 0.8 GB of vcpkg, 0.7 GB of Qt and the CodeQL + # trap caches (about 1 GB each, which prune-caches below does not manage). # - # A configuration needs roughly 500-850 MB on Linux, so the old uniform 600M bound sat - # below what a leg writes: ccache spent each build deleting the entries the next run - # asked for (measured: 2624-3720 cleanup passes per build, 8% of calls served). Each - # leg states the size it needs instead: one configuration for most, two for Fedora 44 - # (Debug for the suite, Release for the package), the server alone for Servatrice. - # Raising these further means caching fewer legs, since GitHub evicts a whole entry - # when the total is exceeded. + # Measured 2026-11-04: one configuration costs 1.2 GB with full DWARF and 660 MB with + # -g1 (COCKATRICE_DEBUG_INFO_LEVEL below), and the size of the cache decides the hit + # rate far more sharply than its fill suggests. A 6G cache served 1059 of 1059 calls; + # the 700M bound on the same leg served 14%, because ccache fills to the limit and then + # evicts the entries the next build is about to ask for. A cache that cannot hold a + # whole configuration therefore buys almost nothing, and the quota goes to fewer legs + # at a size that holds one: + # + # Arch Debug + suite 660 MB working set 850M + # Debian 13 Release + suite 614 MB 800M + # Fedora 44 debug Debug + suite 660 MB 850M + # Fedora 44 release Release + package 614 MB 800M + # Ubuntu 26.04 sanitize ~1.0 GB 1200M + # + # That is about 5.3 GB of bounds, and with the fixed entries above it puts the quota at + # roughly 9.8 GB of 10 - so only legs worth their share get one. A leg keeps no cache + # when it duplicates another leg's coverage (Ubuntu 26.04 behind the sanitize leg of + # the same distro; macOS 15 and macOS 26 Debug behind macOS 14; Servatrice covers the + # server alone and finishes in 4.7 min). Those four run cold: 4.7, 18.1, 16.5 and + # 15.8 min. The five that stay compile the client on the toolchains a change can break + # and are the ones a cache shortens by the most. Measured job totals on master + # 2026-11-04 also show the ceiling this leaves: Windows 10 is 29.5 min and compiles for + # 23.7 of it with no compiler cache at all, so Windows still sets the pull-request + # wall clock no matter how warm these legs are (see build-vcpkg). + # + # Re-add a leg only with a bound that holds its whole configuration: a smaller one is + # worse than none. # # Every key listed in LIVE_CCACHE_KEYS in prune-caches below must match this matrix. CCACHE_SIZE: ${{ matrix.ccache_size || '700M' }} + # Debug information dominates a cached object: with the built-in flags one + # configuration is 1.2 GB and no per-leg bound can hold it, so CI lowers the level to + # 1 (file and line, which is what a failing test or a sanitizer backtrace reports) and + # one configuration halves. Empty on tags and on the nightly legs, which ship binaries + # and keep full DWARF for the Sentry symbols; a tag restores a cache it cannot use and + # is slower, which is accepted because a tag is rare and only pull requests gate. + COCKATRICE_DEBUG_INFO_LEVEL: ${{ github.ref_type != 'tag' && '1' || '' }} CMAKE_GENERATOR: 'Ninja' NAME: ${{ matrix.distro }}${{ matrix.version }} IMAGE_SUFFIX: ${{ matrix.variant }} @@ -242,11 +281,13 @@ jobs: # also package, so testing and packaging cost one compile instead of two. DEBUG_TEST: ${{ matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes' }} RUN_TESTS: ${{ matrix.test != 'skip' && !(matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes') }} - # Whether this leg restores and saves its compiler cache. Every leg a pull request - # runs keeps one; the nightly-only legs do not, so that the quota is spent on the - # builds that review a change. A tag build runs both sets, and the nightly legs are - # cold there - accepted, because a tag is rare and only the pull-request legs gate. - PERSIST_CCACHE: 'true' + # Whether this leg restores and saves its compiler cache. Not every pull-request leg + # keeps one: a cache only pays for itself when its bound holds a whole configuration + # (see the sizing comment above), and a leg that duplicates another leg's coverage + # spends quota without adding any. The nightly-only legs keep none either. Fedora 44 + # states it per leg: its two configurations have their own job and key, so neither + # bound is shared with a build whose objects it would evict. + PERSIST_CCACHE: ${{ matrix.persist_ccache || 'true' }} # Shared with build-linux-nightly so the two leg sets run the same steps. steps: &linux-steps @@ -534,6 +575,7 @@ jobs: make_package: 1 override_target: 14 package_suffix: "-macOS14" + ccache_size: 800M # one Release configuration plus the suite at -g1 qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets soc: Apple @@ -553,6 +595,7 @@ jobs: make_package: 1 override_target: 15 package_suffix: "-macOS15" + persist_ccache: 'false' # 16.5 min: shorter than the Windows leg qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets soc: Apple @@ -568,6 +611,7 @@ jobs: ccache_eviction_age: 7d cmake_generator: Ninja + persist_ccache: 'false' # 15.8 min: shorter than the Windows leg qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets soc: Apple @@ -606,13 +650,17 @@ jobs: timeout-minutes: 100 env: CCACHE_DIR: ${{ github.workspace }}/.cache/ - # macOS configurations compile about twice the TUs of a Linux one, so each of these - # three legs gets a bound above what it has been measured to write. The four macOS - # legs used to be bounded at 600M from the same quota as the Linux ones; the three - # pull-request legs now get 700M each while the nightly Intel leg keeps none, and - # that is what fits (see CCACHE_SIZE in build-linux for the full arithmetic). - CCACHE_SIZE: 700M - PERSIST_CCACHE: 'true' + # One macOS configuration is about 600 MB of objects at -g1, so its bound needs to be + # 800M to hold one whole; a smaller cache serves a few percent and is worthless (see + # the sizing comment in build-linux). macOS 14 keeps the one macOS cache: it runs the + # suite against the shipping Release build and is the longest of the three at 24.2 min. + # macOS 15 (16.5) and macOS 26 Debug (15.8) duplicate its coverage and are shorter, so + # they run cold. + CCACHE_SIZE: ${{ matrix.ccache_size || '700M' }} + PERSIST_CCACHE: ${{ matrix.persist_ccache || 'true' }} + # MSVC has no -g level, so the Windows leg does not ask for one; it has no compiler + # cache to shrink anyway (see the sizing comment in build-linux). + COCKATRICE_DEBUG_INFO_LEVEL: ${{ matrix.os != 'Windows' && github.ref_type != 'tag' && '1' || '' }} # Shared with build-vcpkg-nightly so the two leg sets run the same steps. steps: &vcpkg-steps @@ -1164,9 +1212,10 @@ jobs: LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' # Must list the ccache key of every leg that keeps a compiler cache # (PERSIST_CCACHE in build-linux and build-vcpkg), or a live entry is deleted. - # The nightly legs keep none, so their leftover entries are reclaimed here - # instead of waiting out the age rule. - LIVE_CCACHE_KEYS: 'ccache-Arch ccache-Servatrice_Debian12 ccache-Debian13 ccache-Fedora44 ccache-Ubuntu26.04 ccache-Ubuntu26.04sanitize ccache-macos-26_14-Xcode26.6 ccache-macos-26_15-Xcode26.6 ccache-macos-26_-Xcode26.6' + # The nightly legs and the pull-request legs that finish under the Windows leg + # keep none, so their leftover entries are reclaimed here instead of waiting out + # the age rule. + LIVE_CCACHE_KEYS: 'ccache-Arch ccache-Debian13 ccache-Fedora44debug ccache-Fedora44release ccache-Ubuntu26.04sanitize ccache-macos-26_14-Xcode26.6' run: | gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" @@ -1190,8 +1239,9 @@ jobs: if [[ $ref != refs/heads/master ]]; then reason="scoped to $ref" elif [[ $key == ccache-* ]]; then - # An exact token compare: ccache-Ubuntu26.04 is a prefix of - # ccache-Ubuntu26.04sanitize, and both keys are live + # An exact token compare, because one key can be a prefix of another: + # ccache-Ubuntu26.04 is a prefix of ccache-Ubuntu26.04sanitize, and only + # the sanitize entry is live. if [[ $live_ccache != *" $key "* ]]; then reason="no pull-request leg keeps it" fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 9044bc005..53508fcd1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,6 +238,27 @@ include(Sanitizers) include(Coverage) include(CrashReporting) +# Debug information is what makes a cached object large, and a build configuration needs more +# room than a per-leg cache bound can spare when it carries full DWARF: ccache then fills to +# its limit and evicts the entries the next build is about to ask for, so it serves a few +# percent of calls while holding most of a build. CI sets a lower level so one whole +# configuration fits; level 1 keeps file and line, which is what a failing test or a sanitizer +# backtrace reports. Empty (the default) leaves local, distro and release builds exactly as +# they were. This comes after CrashReporting, so the -g it adds for Sentry symbols cannot +# raise the level again. +set(COCKATRICE_DEBUG_INFO_LEVEL + "" + CACHE STRING "DWARF level for Debug and Release objects (empty keeps the built-in flags)" +) +if(NOT COCKATRICE_DEBUG_INFO_LEVEL STREQUAL "") + if(MSVC) + message(WARNING "COCKATRICE_DEBUG_INFO_LEVEL is ignored on MSVC") + else() + string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g${COCKATRICE_DEBUG_INFO_LEVEL}") + string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g${COCKATRICE_DEBUG_INFO_LEVEL}") + endif() +endif() + # GNU systems need to define the Mersenne exponent for the RNG to compile w/o warning if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") add_definitions("-DSFMT_MEXP=19937")