Merge 6f3297a4a0 into 7ca697348b

.ci/compile.sh

@@ -203,6 +203,11 @@ if [[ ${WITH_CRASH_REPORTING:-} == "ON" ]]; then
   # developer builds never reach Sentry.
   flags+=("-DWITH_CRASH_REPORTING=ON")
 fi
+if [[ ${COCKATRICE_DEBUG_INFO_LEVEL:-} ]]; then
+  # CI lowers the DWARF level so one build configuration fits its compiler-cache bound;
+  # see the option in CMakeLists.txt. Empty keeps the built-in flags.
+  flags+=("-DCOCKATRICE_DEBUG_INFO_LEVEL=$COCKATRICE_DEBUG_INFO_LEVEL")
+fi
 if [[ ${BUILDTYPE:-Release} == "Release" && -n ${COCKATRICE_SENTRY_DSN:-} ]]; then
   # Release-only: Debug builds stay DSN-free so crashes from developer builds
   # never reach Sentry. Empty (local builds, forks without the secret, and non-tag

.ci/docker.sh

@@ -181,6 +181,11 @@ function RUN ()
     if [[ ${WITH_CRASH_REPORTING:-} ]]; then
       args+=(--env "WITH_CRASH_REPORTING=${WITH_CRASH_REPORTING}")
     fi
+    # Same reason: the DWARF level reaches compile.sh only if it is handed to the container,
+    # and without it CI would cache objects twice the size its bounds were measured for.
+    if [[ ${COCKATRICE_DEBUG_INFO_LEVEL:-} ]]; then
+      args+=(--env "COCKATRICE_DEBUG_INFO_LEVEL=${COCKATRICE_DEBUG_INFO_LEVEL}")
+    fi
     # shellcheck disable=2086
     docker run "${args[@]}" $RUN_ARGS "$IMAGE_NAME" bash "$BUILD_SCRIPT" $RUN_OPTS "$@"
     return $?

.github/workflows/desktop-build.yml

@@ -150,6 +150,7 @@ jobs:
             allow-failure: yes
             package: skip    # We are packaged in Arch already
             debug_test: yes    # see the note on Fedora 44 below
+            ccache_size: 850M    # Debug plus the suite: 660 MB of objects at -g1
 
           - distro: Servatrice_Debian
             version: 12
@@ -157,30 +158,46 @@ jobs:
             package: DEB
             server_only: yes
             test: skip
-            ccache_size: 400M    # server only, no client to compile
+            # 4.7 min and the server alone: nothing here for a compiler cache to save.
+            persist_ccache: 'false'
 
           - distro: Debian
             version: 13
 
             package: DEB
-            ccache_size: 850M    # the Release build plus the suite it runs
+            ccache_size: 800M    # Release plus the suite: 614 MB of objects at -g1
+
+          # Fedora 44 used to build Debug for the suite and Release for the package in one
+          # job, so the package compile waited behind the Debug one and the leg took 30.8 min
+          # - the only leg slower than Windows. Two legs run in parallel and each keeps a
+          # cache that holds exactly one configuration.
+          #
+          # Arch and Fedora 44 are the two newest toolchains, and the -Wno-error= exceptions
+          # in CMakeLists.txt exist for exactly their warnings. This leg compiles Debug
+          # because only Debug passes -Werror, so it is where a new warning still fails.
+          - distro: Fedora
+            version: 44
+            variant: debug    # separate image and ccache from the release leg below
+
+            package: skip
+            debug_test: yes
+            ccache_size: 850M    # Debug plus the suite: 660 MB of objects at -g1
 
           - distro: Fedora
             version: 44
+            variant: release    # separate image and ccache from the debug leg above
 
             package: RPM
-            # Arch and Fedora 44 are the two newest toolchains, and the -Wno-error= exceptions
+            test: skip    # The debug leg above runs this distro's suite
-            # in CMakeLists.txt exist for exactly their warnings. They keep compiling Debug:
+            ccache_size: 800M    # Release plus the package: 614 MB of objects at -g1
-            # only the Debug configuration passes -Werror, so these two are the legs where a
-            # new warning in either compiler still fails the build.
-            debug_test: yes
-            ccache_size: 900M    # Debug for the suite and Release for the package
 
           - distro: Ubuntu
             version: 26.04
 
             package: DEB
             test: skip    # The sanitizer leg below runs this distro's suite
+            # 18.1 min, and the same distro as the sanitize leg below, which runs this suite.
+            persist_ccache: 'false'
 
           # The one leg that runs the suite under AddressSanitizer and
           # UndefinedBehaviorSanitizer. Debian 13 and Fedora 44 keep covering the
@@ -196,7 +213,7 @@ jobs:
 
             package: skip
             sanitize: yes
-            ccache_size: 800M    # instrumented objects are larger
+            ccache_size: 1200M    # instrumented objects are larger than the 614 MB measured; estimated
 
     name: ${{ matrix.distro }} ${{ matrix.version }}${{ matrix.variant && format(' {0}', matrix.variant) || '' }}
     needs: configure
@@ -211,25 +228,47 @@ jobs:
       CCACHE_EVICTION_AGE: 7d
       # The bound is sized against the repository's 10 GB cache quota
       # (https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy),
-      # not against what one leg would like. The quota currently holds about 1.1 GB of
+      # not against what one leg would like: the quota also holds about 0.9 GB of buildkit
-      # CodeQL trap, 1.1 GB of vcpkg, 0.7 GB of Qt and 2.1 GB of docker buildkit layers
+      # layers (mode=min in docker-release.yml), 0.8 GB of vcpkg, 0.7 GB of Qt and the CodeQL
-      # (mode=min in docker-release.yml, added here, brings the last one down to roughly
+      # trap caches (about 1 GB each, which prune-caches below does not manage).
-      # 0.3 GB - the one input here that is an expectation rather than a measurement, since
-      # mode=min still writes one layer per pushed binary) plus the compiler caches of the nine legs a pull request runs. The five
-      # nightly-only legs keep no compiler cache (see PERSIST_CCACHE below), which is what
-      # makes these per-leg bounds affordable. Accounting for all of it: 6.45G of ccache
-      # bounds + 3.2G of the fixed entries = 9.65G against the 10G quota.
       #
-      # A configuration needs roughly 500-850 MB on Linux, so the old uniform 600M bound sat
+      # Measured 2026-11-04: one configuration costs 1.2 GB with full DWARF and 660 MB with
-      # below what a leg writes: ccache spent each build deleting the entries the next run
+      # -g1 (COCKATRICE_DEBUG_INFO_LEVEL below), and the size of the cache decides the hit
-      # asked for (measured: 2624-3720 cleanup passes per build, 8% of calls served). Each
+      # rate far more sharply than its fill suggests. A 6G cache served 1059 of 1059 calls;
-      # leg states the size it needs instead: one configuration for most, two for Fedora 44
+      # the 700M bound on the same leg served 14%, because ccache fills to the limit and then
-      # (Debug for the suite, Release for the package), the server alone for Servatrice.
+      # evicts the entries the next build is about to ask for. A cache that cannot hold a
-      # Raising these further means caching fewer legs, since GitHub evicts a whole entry
+      # whole configuration therefore buys almost nothing, and the quota goes to fewer legs
-      # when the total is exceeded.
+      # at a size that holds one:
+      #
+      #   Arch                 Debug + suite       660 MB working set   850M
+      #   Debian 13            Release + suite     614 MB               800M
+      #   Fedora 44 debug      Debug + suite       660 MB               850M
+      #   Fedora 44 release    Release + package   614 MB               800M
+      #   Ubuntu 26.04 sanitize                     ~1.0 GB             1200M
+      #
+      # That is about 5.3 GB of bounds, and with the fixed entries above it puts the quota at
+      # roughly 9.8 GB of 10 - so only legs worth their share get one. A leg keeps no cache
+      # when it duplicates another leg's coverage (Ubuntu 26.04 behind the sanitize leg of
+      # the same distro; macOS 15 and macOS 26 Debug behind macOS 14; Servatrice covers the
+      # server alone and finishes in 4.7 min). Those four run cold: 4.7, 18.1, 16.5 and
+      # 15.8 min. The five that stay compile the client on the toolchains a change can break
+      # and are the ones a cache shortens by the most. Measured job totals on master
+      # 2026-11-04 also show the ceiling this leaves: Windows 10 is 29.5 min and compiles for
+      # 23.7 of it with no compiler cache at all, so Windows still sets the pull-request
+      # wall clock no matter how warm these legs are (see build-vcpkg).
+      #
+      # Re-add a leg only with a bound that holds its whole configuration: a smaller one is
+      # worse than none.
       #
       # Every key listed in LIVE_CCACHE_KEYS in prune-caches below must match this matrix.
       CCACHE_SIZE: ${{ matrix.ccache_size || '700M' }}
+      # Debug information dominates a cached object: with the built-in flags one
+      # configuration is 1.2 GB and no per-leg bound can hold it, so CI lowers the level to
+      # 1 (file and line, which is what a failing test or a sanitizer backtrace reports) and
+      # one configuration halves. Empty on tags and on the nightly legs, which ship binaries
+      # and keep full DWARF for the Sentry symbols; a tag restores a cache it cannot use and
+      # is slower, which is accepted because a tag is rare and only pull requests gate.
+      COCKATRICE_DEBUG_INFO_LEVEL: ${{ github.ref_type != 'tag' && '1' || '' }}
       CMAKE_GENERATOR: 'Ninja'
       NAME: ${{ matrix.distro }}${{ matrix.version }}
       IMAGE_SUFFIX: ${{ matrix.variant }}
@@ -242,11 +281,13 @@ jobs:
       #               also package, so testing and packaging cost one compile instead of two.
       DEBUG_TEST: ${{ matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes' }}
       RUN_TESTS: ${{ matrix.test != 'skip' && !(matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes') }}
-      # Whether this leg restores and saves its compiler cache. Every leg a pull request
+      # Whether this leg restores and saves its compiler cache. Not every pull-request leg
-      # runs keeps one; the nightly-only legs do not, so that the quota is spent on the
+      # keeps one: a cache only pays for itself when its bound holds a whole configuration
-      # builds that review a change. A tag build runs both sets, and the nightly legs are
+      # (see the sizing comment above), and a leg that duplicates another leg's coverage
-      # cold there - accepted, because a tag is rare and only the pull-request legs gate.
+      # spends quota without adding any. The nightly-only legs keep none either. Fedora 44
-      PERSIST_CCACHE: 'true'
+      # states it per leg: its two configurations have their own job and key, so neither
+      # bound is shared with a build whose objects it would evict.
+      PERSIST_CCACHE: ${{ matrix.persist_ccache || 'true' }}
 
     # Shared with build-linux-nightly so the two leg sets run the same steps.
     steps: &linux-steps
@@ -534,6 +575,7 @@ jobs:
             make_package: 1
             override_target: 14
             package_suffix: "-macOS14"
+            ccache_size: 800M    # one Release configuration plus the suite at -g1
             qt_version: 6.11.*
             qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets
             soc: Apple
@@ -553,6 +595,7 @@ jobs:
             make_package: 1
             override_target: 15
             package_suffix: "-macOS15"
+            persist_ccache: 'false'    # 16.5 min: shorter than the Windows leg
             qt_version: 6.11.*
             qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets
             soc: Apple
@@ -568,6 +611,7 @@ jobs:
 
             ccache_eviction_age: 7d
             cmake_generator: Ninja
+            persist_ccache: 'false'    # 15.8 min: shorter than the Windows leg
             qt_version: 6.11.*
             qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets
             soc: Apple
@@ -606,13 +650,17 @@ jobs:
     timeout-minutes: 100
     env:
       CCACHE_DIR: ${{ github.workspace }}/.cache/
-      # macOS configurations compile about twice the TUs of a Linux one, so each of these
+      # One macOS configuration is about 600 MB of objects at -g1, so its bound needs to be
-      # three legs gets a bound above what it has been measured to write. The four macOS
+      # 800M to hold one whole; a smaller cache serves a few percent and is worthless (see
-      # legs used to be bounded at 600M from the same quota as the Linux ones; the three
+      # the sizing comment in build-linux). macOS 14 keeps the one macOS cache: it runs the
-      # pull-request legs now get 700M each while the nightly Intel leg keeps none, and
+      # suite against the shipping Release build and is the longest of the three at 24.2 min.
-      # that is what fits (see CCACHE_SIZE in build-linux for the full arithmetic).
+      # macOS 15 (16.5) and macOS 26 Debug (15.8) duplicate its coverage and are shorter, so
-      CCACHE_SIZE: 700M
+      # they run cold.
-      PERSIST_CCACHE: 'true'
+      CCACHE_SIZE: ${{ matrix.ccache_size || '700M' }}
+      PERSIST_CCACHE: ${{ matrix.persist_ccache || 'true' }}
+      # MSVC has no -g level, so the Windows leg does not ask for one; it has no compiler
+      # cache to shrink anyway (see the sizing comment in build-linux).
+      COCKATRICE_DEBUG_INFO_LEVEL: ${{ matrix.os != 'Windows' && github.ref_type != 'tag' && '1' || '' }}
 
     # Shared with build-vcpkg-nightly so the two leg sets run the same steps.
     steps: &vcpkg-steps
@@ -1164,9 +1212,10 @@ jobs:
           LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/'
           # Must list the ccache key of every leg that keeps a compiler cache
           # (PERSIST_CCACHE in build-linux and build-vcpkg), or a live entry is deleted.
-          # The nightly legs keep none, so their leftover entries are reclaimed here
+          # The nightly legs and the pull-request legs that finish under the Windows leg
-          # instead of waiting out the age rule.
+          # keep none, so their leftover entries are reclaimed here instead of waiting out
-          LIVE_CCACHE_KEYS: 'ccache-Arch ccache-Servatrice_Debian12 ccache-Debian13 ccache-Fedora44 ccache-Ubuntu26.04 ccache-Ubuntu26.04sanitize ccache-macos-26_14-Xcode26.6 ccache-macos-26_15-Xcode26.6 ccache-macos-26_-Xcode26.6'
+          # the age rule.
+          LIVE_CCACHE_KEYS: 'ccache-Arch ccache-Debian13 ccache-Fedora44debug ccache-Fedora44release ccache-Ubuntu26.04sanitize ccache-macos-26_14-Xcode26.6'
         run: |
           gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \
             --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv"
@@ -1190,8 +1239,9 @@ jobs:
             if [[ $ref != refs/heads/master ]]; then
               reason="scoped to $ref"
             elif [[ $key == ccache-* ]]; then
-              # An exact token compare: ccache-Ubuntu26.04 is a prefix of
+              # An exact token compare, because one key can be a prefix of another:
-              # ccache-Ubuntu26.04sanitize, and both keys are live
+              # ccache-Ubuntu26.04 is a prefix of ccache-Ubuntu26.04sanitize, and only
+              # the sanitize entry is live.
               if [[ $live_ccache != *" $key "* ]]; then
                 reason="no pull-request leg keeps it"
               fi

CMakeLists.txt

@@ -238,6 +238,27 @@ include(Sanitizers)
 include(Coverage)
 include(CrashReporting)
 
+# Debug information is what makes a cached object large, and a build configuration needs more
+# room than a per-leg cache bound can spare when it carries full DWARF: ccache then fills to
+# its limit and evicts the entries the next build is about to ask for, so it serves a few
+# percent of calls while holding most of a build. CI sets a lower level so one whole
+# configuration fits; level 1 keeps file and line, which is what a failing test or a sanitizer
+# backtrace reports. Empty (the default) leaves local, distro and release builds exactly as
+# they were. This comes after CrashReporting, so the -g it adds for Sentry symbols cannot
+# raise the level again.
+set(COCKATRICE_DEBUG_INFO_LEVEL
+    ""
+    CACHE STRING "DWARF level for Debug and Release objects (empty keeps the built-in flags)"
+)
+if(NOT COCKATRICE_DEBUG_INFO_LEVEL STREQUAL "")
+  if(MSVC)
+    message(WARNING "COCKATRICE_DEBUG_INFO_LEVEL is ignored on MSVC")
+  else()
+    string(APPEND CMAKE_CXX_FLAGS_DEBUG " -g${COCKATRICE_DEBUG_INFO_LEVEL}")
+    string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g${COCKATRICE_DEBUG_INFO_LEVEL}")
+  endif()
+endif()
+
 # GNU systems need to define the Mersenne exponent for the RNG to compile w/o warning
 if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
   add_definitions("-DSFMT_MEXP=19937")