diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index d72cb881e..be11c3748 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -157,11 +157,13 @@ jobs: package: DEB server_only: yes test: skip + ccache_size: 400M # server only, no client to compile - distro: Debian version: 13 package: DEB + ccache_size: 850M # the Release build plus the suite it runs - distro: Fedora version: 44 @@ -172,6 +174,7 @@ jobs: # only the Debug configuration passes -Werror, so these two are the legs where a # new warning in either compiler still fails the build. debug_test: yes + ccache_size: 900M # Debug for the suite and Release for the package - distro: Ubuntu version: 26.04 @@ -193,6 +196,7 @@ jobs: package: skip sanitize: yes + ccache_size: 800M # instrumented objects are larger name: ${{ matrix.distro }} ${{ matrix.version }}${{ matrix.variant && format(' {0}', matrix.variant) || '' }} needs: configure @@ -205,7 +209,27 @@ jobs: env: CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }}${{ matrix.variant }} # directory for caching docker image and ccache CCACHE_EVICTION_AGE: 7d - CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # The bound is sized against the repository's 10 GB cache quota + # (https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy), + # not against what one leg would like. The quota currently holds about 1.1 GB of + # CodeQL trap, 1.1 GB of vcpkg, 0.7 GB of Qt and 2.1 GB of docker buildkit layers + # (mode=min in docker-release.yml, added here, brings the last one down to roughly + # 0.3 GB - the one input here that is an expectation rather than a measurement, since + # mode=min still writes one layer per pushed binary) plus the compiler caches of the nine legs a pull request runs. The five + # nightly-only legs keep no compiler cache (see PERSIST_CCACHE below), which is what + # makes these per-leg bounds affordable. Accounting for all of it: 6.45G of ccache + # bounds + 3.2G of the fixed entries = 9.65G against the 10G quota. + # + # A configuration needs roughly 500-850 MB on Linux, so the old uniform 600M bound sat + # below what a leg writes: ccache spent each build deleting the entries the next run + # asked for (measured: 2624-3720 cleanup passes per build, 8% of calls served). Each + # leg states the size it needs instead: one configuration for most, two for Fedora 44 + # (Debug for the suite, Release for the package), the server alone for Servatrice. + # Raising these further means caching fewer legs, since GitHub evicts a whole entry + # when the total is exceeded. + # + # Every key listed in LIVE_CCACHE_KEYS in prune-caches below must match this matrix. + CCACHE_SIZE: ${{ matrix.ccache_size || '700M' }} CMAKE_GENERATOR: 'Ninja' NAME: ${{ matrix.distro }}${{ matrix.version }} IMAGE_SUFFIX: ${{ matrix.variant }} @@ -218,6 +242,11 @@ jobs: # also package, so testing and packaging cost one compile instead of two. DEBUG_TEST: ${{ matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes' }} RUN_TESTS: ${{ matrix.test != 'skip' && !(matrix.sanitize == 'yes' || matrix.coverage == 'yes' || matrix.debug_test == 'yes') }} + # Whether this leg restores and saves its compiler cache. Every leg a pull request + # runs keeps one; the nightly-only legs do not, so that the quota is spent on the + # builds that review a change. A tag build runs both sets, and the nightly legs are + # cold there - accepted, because a tag is rare and only the pull-request legs gate. + PERSIST_CCACHE: 'true' # Shared with build-linux-nightly so the two leg sets run the same steps. steps: &linux-steps @@ -226,6 +255,7 @@ jobs: - name: "Restore compiler cache (ccache)" id: restore_ccache + if: env.PERSIST_CCACHE == 'true' uses: actions/cache/restore@v6 with: key: ccache-${{ matrix.distro }}${{ matrix.version }}${{ matrix.variant }} @@ -371,7 +401,7 @@ jobs: # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "Delete remote compiler cache (ccache)" - if: github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit + if: github.ref == 'refs/heads/master' && env.PERSIST_CCACHE == 'true' && steps.restore_ccache.outputs.cache-hit continue-on-error: true env: CACHE_PRIMARY_KEY: ${{ steps.restore_ccache.outputs.cache-primary-key }} @@ -382,7 +412,7 @@ jobs: fi - name: "Save updated compiler cache (ccache)" - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' && env.PERSIST_CCACHE == 'true' uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} @@ -479,7 +509,12 @@ jobs: env: CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }}${{ matrix.variant }} # directory for caching docker image and ccache CCACHE_EVICTION_AGE: 7d - CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # These legs do not keep a compiler cache: they run once a day (and on tags) and gate + # nothing, so their share of the 10 GB quota goes to the pull-request legs instead. + # A cold nightly build costs minutes on a runner nobody waits for; a cold pull request + # build costs a review. CCACHE_SIZE is still passed so ccache trims its scratch dir. + PERSIST_CCACHE: 'false' + CCACHE_SIZE: 600M CMAKE_GENERATOR: 'Ninja' NAME: ${{ matrix.distro }}${{ matrix.version }} IMAGE_SUFFIX: ${{ matrix.variant }} @@ -571,7 +606,13 @@ jobs: timeout-minutes: 100 env: CCACHE_DIR: ${{ github.workspace }}/.cache/ - CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # macOS configurations compile about twice the TUs of a Linux one, so each of these + # three legs gets a bound above what it has been measured to write. The four macOS + # legs used to be bounded at 600M from the same quota as the Linux ones; the three + # pull-request legs now get 700M each while the nightly Intel leg keeps none, and + # that is what fits (see CCACHE_SIZE in build-linux for the full arithmetic). + CCACHE_SIZE: 700M + PERSIST_CCACHE: 'true' # Shared with build-vcpkg-nightly so the two leg sets run the same steps. steps: &vcpkg-steps @@ -591,7 +632,7 @@ jobs: run: brew install ccache - name: "[macOS] Restore compiler cache (ccache)" - if: matrix.os == 'macOS' && matrix.use_ccache == 1 + if: matrix.os == 'macOS' && matrix.use_ccache == 1 && env.PERSIST_CCACHE == 'true' id: restore_ccache uses: actions/cache/restore@v6 with: @@ -922,7 +963,7 @@ jobs: # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "[macOS] Delete remote compiler cache (ccache)" - if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit + if: matrix.os == 'macOS' && matrix.use_ccache == 1 && env.PERSIST_CCACHE == 'true' && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit continue-on-error: true env: CACHE_PRIMARY_KEY: ${{ steps.restore_ccache.outputs.cache-primary-key }} @@ -933,7 +974,7 @@ jobs: fi - name: "[macOS] Cache updated compiler cache (ccache)" - if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' + if: matrix.os == 'macOS' && matrix.use_ccache == 1 && env.PERSIST_CCACHE == 'true' && github.ref == 'refs/heads/master' uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} @@ -1082,7 +1123,10 @@ jobs: timeout-minutes: 100 env: CCACHE_DIR: ${{ github.workspace }}/.cache/ - CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # Like build-linux-nightly: this leg keeps no compiler cache, so the quota stays with + # the pull-request legs (see PERSIST_CCACHE there). + PERSIST_CCACHE: 'false' + CCACHE_SIZE: 600M steps: *vcpkg-steps prune-caches: @@ -1093,12 +1137,23 @@ jobs: continue-on-error: true # freeing space must never fail a build steps: - # The repository holds 10Gi of cache against entries that want more, and GitHub - # evicts the least recently used entry whenever that is exceeded. The vcpkg - # entries are the reason that never happens: the restore above touches every - # entry under the prefix on every run, so a dead ABI generation keeps its access - # date fresh forever and age based eviction can never reach it. Deleting by age - # alone cannot fix that, only knowing which scope an entry belongs to can. + # The repository holds 10 GB of cache against entries that want more, and GitHub evicts + # the least recently used entry whenever that is exceeded. The vcpkg entries are the + # reason that never happens: the restore above touches every entry under the prefix on + # every run, so a dead ABI generation keeps its access date fresh forever and age based + # eviction can never reach it. Deleting by age alone cannot fix that, only knowing which + # scope an entry belongs to can. The same is true of a compiler cache of a leg that no + # longer keeps one: its restore would keep touching it, so the live key list below is + # what retires it. + # + # docker-release's image builds keep their layer cache in the same quota: one blob per + # layer, keyed buildkit-blob-1-, plus one index--1- entry per image + # scope that lists them. They are written with mode=min, so only the final image stage + # is kept and the set stays small. A layer a build actually needs is downloaded, which + # refreshes its access date, so an old date means the builds have stopped using it and + # age is the right test. Dropping a blob an index still lists costs the next image build + # one rebuilt layer; docker-release gates nothing, so that is cheaper than keeping the + # entries forever. - name: "Delete unusable cache entries" shell: bash env: @@ -1107,25 +1162,39 @@ jobs: # matrices above, or the entries of a scope that is still in use get deleted # on the next run. LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' + # Must list the ccache key of every leg that keeps a compiler cache + # (PERSIST_CCACHE in build-linux and build-vcpkg), or a live entry is deleted. + # The nightly legs keep none, so their leftover entries are reclaimed here + # instead of waiting out the age rule. + LIVE_CCACHE_KEYS: 'ccache-Arch ccache-Servatrice_Debian12 ccache-Debian13 ccache-Fedora44 ccache-Ubuntu26.04 ccache-Ubuntu26.04sanitize ccache-macos-26_14-Xcode26.6 ccache-macos-26_15-Xcode26.6 ccache-macos-26_-Xcode26.6' run: | gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" cutoff="$(( $(date -u +%s) - 7 * 24 * 3600 ))" live=" ${LIVE_VCPKG_SCOPES} " + live_ccache=" ${LIVE_CCACHE_KEYS} " + deleted=0 while IFS=$'\t' read -r id ref key accessed; do [[ -n $id ]] || continue - # Only ever touch the key shapes this workflow creates, so that an entry of - # some other workflow can never be deleted by accident + # Only ever touch the key shapes our workflows create (buildkit-* and index-* + # come from docker-release.yml), so that an entry of some other workflow can + # never be deleted by accident case "$key" in - vcpkg/* | ccache-* | Qt*) ;; + vcpkg/* | ccache-* | Qt* | buildkit-* | index-*) ;; *) continue ;; esac reason="" if [[ $ref != refs/heads/master ]]; then reason="scoped to $ref" + elif [[ $key == ccache-* ]]; then + # An exact token compare: ccache-Ubuntu26.04 is a prefix of + # ccache-Ubuntu26.04sanitize, and both keys are live + if [[ $live_ccache != *" $key "* ]]; then + reason="no pull-request leg keeps it" + fi elif [[ $key == vcpkg/* ]]; then # "/", cut the hash off the end to get the scope scope="vcpkg/${key#vcpkg/}" diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 937386940..1537d6d4d 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -84,12 +84,18 @@ jobs: password: ${{ github.token }} # Don't push for non-publishing triggers + # + # The layer cache shares the repository's 10 GB Actions cache quota with the compiler + # caches (see CCACHE_SIZE in desktop-build.yml). mode=min keeps only the final image + # stage: the build stage sits behind `COPY . .`, so its compile layer is invalidated by + # every source change and was never read back, yet it was the bulk of the ~2 GB this + # cache held. The final runtime stage is the part that still pays off. - name: "Build image" if: steps.login.outcome != 'success' uses: docker/build-push-action@v7 with: cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} - cache-to: ${{ case(github.ref == 'refs/heads/master', format('type=gha,mode=max,scope={0}', env.CACHE_SCOPE), '') }} + cache-to: ${{ case(github.ref == 'refs/heads/master', format('type=gha,mode=min,scope={0}', env.CACHE_SCOPE), '') }} context: . platforms: ${{ matrix.platform }} push: false @@ -101,7 +107,7 @@ jobs: uses: docker/build-push-action@v7 with: cache-from: type=gha,scope=${{ env.CACHE_SCOPE }} - cache-to: type=gha,mode=max,scope=${{ env.CACHE_SCOPE }} + cache-to: type=gha,mode=min,scope=${{ env.CACHE_SCOPE }} context: . labels: | org.opencontainers.image.description=${{ env.OCI_DESCRIPTION }}