From ccf92e2ea24a4823484b25d0078dec353a5361b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sat, 3 Oct 2026 17:55:51 +0200 Subject: [PATCH 1/6] [CI] Key the vcpkg binary cache per ABI and let pull requests heal it The cache had no prefix, and TAServers/vcpkg-cache restores every entry it finds under the prefix rather than only the hashes vcpkg asks for. With all legs sharing the single vcpkg/ namespace, each of the five of them downloads all 66 archives of all five on every run. Each restore also refreshes the entry's last access date, so a dead ABI generation stays fresh forever and GitHub's least recently used eviction can never reach it. Name the scope in the key after the platform, target and toolchain it belongs to, so a leg restores only its own archives. macOS 14, 15 and 26 on the same Xcode build identical binaries when they share the lowest deployment target, so point all three at 14 and let them share one scope instead of three. A pull request cannot write the shared cache either. Its entries are scoped to refs/pull//merge, so nothing it saves is ever readable again, and letting it write anyway only burns the repository's quota, which is what the previous attempt at this did. Keep the build read-only and hand the archives it had to build itself to a new workflow running in the default branch scope, which saves them where master can restore them. Forks stay read-only, they are the one case whose archives must not be trusted. Because the prefix changes every key, add a job that deletes the entries of retired scopes, which is the one-time clean up of the current flat namespace, and the per-run ones pull requests leave behind, which age based eviction also never reaches. Also pin the Linux ccache cache to its own subdirectory, since the directory is shared with the docker build context, and drop the clang-format install from the build images, which format nothing at runtime. --- .ci/Debian12/Dockerfile | 1 - .ci/Debian13/Dockerfile | 1 - .ci/Servatrice_Debian12/Dockerfile | 1 - .ci/Ubuntu24.04/Dockerfile | 1 - .ci/Ubuntu26.04/Dockerfile | 1 - .ci/compile.sh | 12 +- .github/workflows/desktop-build.yml | 144 ++++++++++++++++++++++- .github/workflows/vcpkg-cache-import.yml | 112 ++++++++++++++++++ 8 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/vcpkg-cache-import.yml diff --git a/.ci/Debian12/Dockerfile b/.ci/Debian12/Dockerfile index fc756aac2..14c6f018c 100644 --- a/.ci/Debian12/Dockerfile +++ b/.ci/Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Debian13/Dockerfile b/.ci/Debian13/Dockerfile index bdecb56df..b597307c5 100644 --- a/.ci/Debian13/Dockerfile +++ b/.ci/Debian13/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Servatrice_Debian12/Dockerfile b/.ci/Servatrice_Debian12/Dockerfile index 321aa7c0f..f1a16843a 100644 --- a/.ci/Servatrice_Debian12/Dockerfile +++ b/.ci/Servatrice_Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu24.04/Dockerfile b/.ci/Ubuntu24.04/Dockerfile index 715997474..b9fb2b701 100644 --- a/.ci/Ubuntu24.04/Dockerfile +++ b/.ci/Ubuntu24.04/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu26.04/Dockerfile b/.ci/Ubuntu26.04/Dockerfile index 96dd10763..a7e010ff5 100644 --- a/.ci/Ubuntu26.04/Dockerfile +++ b/.ci/Ubuntu26.04/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/compile.sh b/.ci/compile.sh index 72a017f33..b4f9e7782 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -14,7 +14,7 @@ # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION VCPKG_TARGET_MACOS_VERSION # (correspond to args: --debug/--release --install --package --suffix --server --test --ccache --dir ) # exitcode: 1 for failure, 3 for invalid arguments @@ -212,8 +212,14 @@ if [[ $RUNNER_OS == macOS ]]; then triplet_source="../vcpkg/triplets/community/$arch-osx.cmake" fi cp "$triplet_source" "$triplet_file" - echo "set(VCPKG_CMAKE_SYSTEM_VERSION $TARGET_MACOS_VERSION)" >>"$triplet_file" - echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $TARGET_MACOS_VERSION)" >>"$triplet_file" + # The triplet is what vcpkg hashes into the ABI hash of every port, so two legs that + # agree here reuse each other's binary cache. Targets 14, 15 and 26 on the same Xcode + # produce identical binaries when they share the lowest deployment target, so they all + # build against 14 to keep one vcpkg cache scope instead of three. + # Keep in sync with "vcpkg_scope" and "vcpkg_deployment_target" in desktop-build.yml. + vcpkg_target="${VCPKG_TARGET_MACOS_VERSION:-$TARGET_MACOS_VERSION}" + echo "set(VCPKG_CMAKE_SYSTEM_VERSION $vcpkg_target)" >>"$triplet_file" + echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $vcpkg_target)" >>"$triplet_file" flags+=("-DVCPKG_OVERLAY_TRIPLETS=$triplets_dir") flags+=("-DVCPKG_HOST_TRIPLET=$triplet_version") flags+=("-DVCPKG_TARGET_TRIPLET=$triplet_version") diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 17c42b07d..1bcc044f2 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -150,7 +150,7 @@ jobs: continue-on-error: ${{ matrix.allow-failure == 'yes' }} timeout-minutes: 70 env: - CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }} # directory for caching docker image and ccache + CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }} # directory for caching the compiler cache CCACHE_EVICTION_AGE: 7d CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy CMAKE_GENERATOR: 'Ninja' @@ -165,7 +165,7 @@ jobs: uses: actions/cache/restore@v6 with: key: ccache-${{ matrix.distro }}${{ matrix.version }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Build ${{ matrix.distro }} ${{ matrix.version }} Docker image" shell: bash @@ -219,7 +219,7 @@ jobs: uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Upload artifact" id: upload_artifact @@ -277,6 +277,7 @@ jobs: soc: Intel type: Release use_ccache: 1 + vcpkg_scope: 'vcpkg/macos13-x64-xcode26.3/' xcode: "26.3" - os: macOS @@ -293,6 +294,8 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: macOS @@ -309,6 +312,8 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: macOS @@ -322,6 +327,8 @@ jobs: soc: Apple type: Debug use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: Windows @@ -335,6 +342,7 @@ jobs: qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets type: Release + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} needs: configure @@ -432,6 +440,15 @@ jobs: id: vcpkg-cache uses: TAServers/vcpkg-cache@v3 with: + # Every entry is keyed by vcpkg ABI hash, and one hash set exists per platform, + # target and toolchain. The action restores every entry it finds under the + # prefix, not just the hashes vcpkg asks for, so with one shared vcpkg/ + # namespace every leg downloads all 66 archives of all 5 legs on every run. + # Each of those restores also refreshes the entry's last access date, which + # is why a dead ABI generation is never evicted by age. Naming the scope is + # what makes those generations identifiable, and prune-caches below able to + # delete them. + prefix: ${{ matrix.vcpkg_scope }} token: ${{ secrets.GITHUB_TOKEN }} # Uses environment variables, see compile.sh for more details @@ -453,9 +470,15 @@ jobs: PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' TARGET_MACOS_VERSION: ${{ matrix.override_target }} USE_CCACHE: ${{ matrix.use_ccache }} + # Only master may write the cache itself. A pull request run has its own + # merge ref scope, which nothing else can read, so anything it saved there + # would be unreachable for every later run. Handing the new archives over + # to vcpkg-cache-import.yml is what lets a pull request heal the shared + # cache after a runner image update changes every ABI hash. VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master', 'readwrite', 'read') }}" VCPKG_DISABLE_METRICS: 1 VCPKG_FEATURE_FLAGS: dependencygraph + VCPKG_TARGET_MACOS_VERSION: ${{ matrix.vcpkg_deployment_target }} run: .ci/compile.sh --server --test --vcpkg # CPack reports a failed makensis run as "Problem running NSIS command" @@ -469,6 +492,57 @@ jobs: shell: bash run: find . -name NSISOutput.log -exec cat {} + + # A pull request cannot write the shared cache itself: its cache entries are scoped + # to refs/pull//merge and stay unreadable for every other run, so writing them + # from a pull request only burns the repository's quota. So hand over only the + # archives this leg had to build itself and let vcpkg-cache-import.yml, which runs + # in the default branch scope, save them where later runs can restore them. + # Forks are skipped, they are the one case where the archives must not be trusted. + - name: "Collect new vcpkg entries" + id: vcpkg_new + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + GH_TOKEN: ${{ github.token }} + VCPKG_SCOPE: '${{ matrix.vcpkg_scope }}' + run: | + # The archive name is the ABI hash, so a missing key is exactly what vcpkg had + # to compile from source in this job. + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/caches?ref=refs/heads/master&key=$VCPKG_SCOPE&per_page=100" \ + --jq '.actions_caches[].key' | sort -u >"$RUNNER_TEMP/master-keys.txt" + find "$CACHE_DIR" -type f -name '*.zip' -exec basename {} .zip \; | + sed "s|^|$VCPKG_SCOPE|" | sort -u >"$RUNNER_TEMP/local-keys.txt" + comm -23 "$RUNNER_TEMP/local-keys.txt" "$RUNNER_TEMP/master-keys.txt" >"$RUNNER_TEMP/new-keys.txt" + + mkdir -p "$RUNNER_TEMP/vcpkg-new" + : >"$RUNNER_TEMP/vcpkg-new/keys.txt" + while read -r key; do + hash="${key##*/}" + archive="$(find "$CACHE_DIR" -type f -name "$hash.zip" -print -quit)" + dir="$RUNNER_TEMP/vcpkg-new/${hash:0:2}" # vcpkg's own two character fanout + mkdir -p "$dir" + cp "$archive" "$dir/" + echo "$key ${hash:0:2}/$hash.zip" >>"$RUNNER_TEMP/vcpkg-new/keys.txt" + done <"$RUNNER_TEMP/new-keys.txt" + + count="$(wc -l <"$RUNNER_TEMP/new-keys.txt")" + echo "vcpkg scope $VCPKG_SCOPE: $count entries to import" + echo "count=$count" >>"$GITHUB_OUTPUT" + echo "slug=$(sed -e 's|^vcpkg/||' -e 's|/$||' -e 's|/|-|g' <<<"$VCPKG_SCOPE")" >>"$GITHUB_OUTPUT" + + - name: "Upload new vcpkg entries" + # An empty count means the step above did not run at all, on master and on forks + if: steps.vcpkg_new.outputs.count != '' && steps.vcpkg_new.outputs.count != '0' + uses: actions/upload-artifact@v7 + with: + archive: false + if-no-files-found: error # a count above zero with nothing staged is a bug + name: vcpkg-new-${{ steps.vcpkg_new.outputs.slug }} + path: ${{ runner.temp }}/vcpkg-new + retention-days: 1 + # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "[macOS] Delete remote compiler cache (ccache)" if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit @@ -581,3 +655,67 @@ jobs: BUILD_PATH: ${{ steps.build.outputs.path }} GH_TOKEN: ${{ github.token }} run: gh attestation verify "$BUILD_PATH" --repo Cockatrice/Cockatrice + + prune-caches: + name: Prune caches + # Only master writes caches at all, so only master may clean them up too + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + continue-on-error: true # freeing space must never fail a build + + steps: + # The repository holds 10Gi of cache against entries that want more, and GitHub + # evicts the least recently used entry whenever that is exceeded. The vcpkg + # entries are the reason that never happens: the restore above touches every + # entry under the prefix on every run, so a dead ABI generation keeps its access + # date fresh forever and age based eviction can never reach it. Deleting by age + # alone cannot fix that, only knowing which scope an entry belongs to can. + - name: "Delete unusable cache entries" + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Must list every vcpkg_scope of the build-vcpkg matrix above, or the entries of + # a scope that is still in use get deleted on the next run. + LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ + --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" + + cutoff="$(( $(date -u +%s) - 7 * 24 * 3600 ))" + live=" ${LIVE_VCPKG_SCOPES} " + deleted=0 + while IFS=$'\t' read -r id ref key accessed; do + [[ -n $id ]] || continue + # Only ever touch the key shapes this workflow creates, so that an entry of + # some other workflow can never be deleted by accident + case "$key" in + vcpkg/* | ccache-* | Qt*) ;; + *) continue ;; + esac + + reason="" + if [[ $ref != refs/heads/master ]]; then + reason="scoped to $ref" + elif [[ $key == vcpkg/* ]]; then + # "/", cut the hash off the end to get the scope + scope="vcpkg/${key#vcpkg/}" + scope="${scope%/*}/" + if [[ $live != *" $scope "* ]]; then + reason="no matrix leg uses $scope" + fi + fi + if [[ -z $reason ]]; then + stamp="$(date -u -d "$accessed" +%s 2>/dev/null || echo 0)" + if [[ $stamp -ne 0 && $stamp -lt $cutoff ]]; then + reason="unused since $accessed" + fi + fi + + if [[ -n $reason ]]; then + echo "::notice::deleting $key, $reason" + gh api -X DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" + deleted=$((deleted + 1)) + fi + done <"$RUNNER_TEMP/caches.tsv" + + echo "::notice::deleted $deleted of $(wc -l <"$RUNNER_TEMP/caches.tsv") cache entries" diff --git a/.github/workflows/vcpkg-cache-import.yml b/.github/workflows/vcpkg-cache-import.yml new file mode 100644 index 000000000..01bfb7f7a --- /dev/null +++ b/.github/workflows/vcpkg-cache-import.yml @@ -0,0 +1,112 @@ +name: Import vcpkg cache + +# A pull request run can only write caches into its own refs/pull//merge scope, which +# nothing else is allowed to restore. So after a runner image update changes every port's +# ABI hash, the first pull request rebuilds protobuf, openssl and abseil from source and +# the next one does the same, until somebody merges and a master run finally repopulates +# the shared cache. This workflow takes the archives such a run produced and saves them +# in the default branch scope instead, where every later run can restore them. +# +# repository_dispatch is not used on purpose: it would also resolve to the default branch, +# but it counts as a low-trust trigger and may only write there when it declares a +# write-capable cache-mode, which puts it in the same position as workflow_run. + +permissions: + actions: write # needed to save entries into the default branch's cache scope + contents: read + +on: + workflow_run: + workflows: ["Build Desktop"] + types: [completed] + +# Every import writes the same scopes, so let them queue instead of racing each other +concurrency: + group: vcpkg-cache-import + cancel-in-progress: false + +jobs: + list: + name: List new vcpkg entries + # Only a successful same-repository pull request has archives worth importing. Forks + # build untrusted code and must never reach a cache that master runs restore and link. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.head_repository.full_name == github.repository + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + # workflow_run is a low-trust trigger, so cache access is read-only by default even + # though this run resolves to the default branch. Assert below that this took effect, + # because a save under read-only access only warns and would import nothing silently. + cache-mode: write + outputs: + entries: ${{ steps.scan.outputs.entries }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: vcpkg-new-* + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Assert write access" + shell: bash + run: | + if [[ $ACTIONS_CACHE_MODE != *write* ]]; then + echo "::error::expected a write-capable cache, got '$ACTIONS_CACHE_MODE'" + exit 1 + fi + + - name: "List cache entries" + id: scan + shell: bash + working-directory: incoming + run: | + entries="$(find . -name keys.txt -type f -exec sh -c ' + for manifest; do + artifact="$(basename "$(dirname "$manifest")")" + while read -r key path; do + printf "{\"artifact\": \"%s\", \"key\": \"%s\", \"path\": \"%s\"}\n" \ + "$artifact" "$key" "$path" + done <"$manifest" + done + ' _ {} +)" + [[ -n $entries ]] || exit 0 + # Compact, because a multi line value cannot be written to GITHUB_OUTPUT + entries="$(jq -cs '.' <<<"$entries")" + echo "entries=$entries" >>"$GITHUB_OUTPUT" + echo "::notice::importing $(jq length <<<"$entries") vcpkg entries" + + save: + name: "Save ${{ matrix.entry.key }}" + needs: list + if: needs.list.outputs.entries + runs-on: ubuntu-slim + cache-mode: write + strategy: + # One job per archive, they run in parallel and each only uploads a single file + fail-fast: false + max-parallel: 20 + matrix: + entry: ${{ fromJSON(needs.list.outputs.entries) }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: ${{ matrix.entry.artifact }} + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Save vcpkg entry" + uses: actions/cache/save@v6 + with: + # The payload is a Windows or macOS archive while the upload runs on Linux, and + # without this the entry would be stamped as a Linux cache that neither of them + # is allowed to restore. + enableCrossOsArchive: true + key: ${{ matrix.entry.key }} + path: incoming/${{ matrix.entry.path }} \ No newline at end of file From 4b6833584eecc6060d2df0530b48f0b691dff634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Mon, 5 Oct 2026 15:26:57 +0200 Subject: [PATCH 2/6] [CI] Create the vcpkg cache directory when a scope is still empty --- .github/workflows/desktop-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 1bcc044f2..afa694ffc 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -451,6 +451,17 @@ jobs: prefix: ${{ matrix.vcpkg_scope }} token: ${{ secrets.GITHUB_TOKEN }} + # The action only creates its directory when the prefix matched at least one + # entry, and it reads that directory unguarded when it saves at the end of the + # job. A scope nobody has written to yet therefore leaves the directory missing + # and fails the job, which is exactly the state every scope starts in. Make the + # directory up front so an empty scope is a cache miss instead of an error. + - name: "Create vcpkg cache directory" + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + run: mkdir -p "$CACHE_DIR" + # Uses environment variables, see compile.sh for more details - name: "Build Cockatrice" id: build From 070efcab11135a63ee0ac449021cfedb4347334c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Mon, 5 Oct 2026 18:22:28 +0200 Subject: [PATCH 3/6] [CI] Let a pull request write the vcpkg archives it builds --- .github/workflows/desktop-build.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index afa694ffc..111947ef8 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -481,12 +481,15 @@ jobs: PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' TARGET_MACOS_VERSION: ${{ matrix.override_target }} USE_CCACHE: ${{ matrix.use_ccache }} - # Only master may write the cache itself. A pull request run has its own - # merge ref scope, which nothing else can read, so anything it saved there - # would be unreachable for every later run. Handing the new archives over - # to vcpkg-cache-import.yml is what lets a pull request heal the shared - # cache after a runner image update changes every ABI hash. - VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master', 'readwrite', 'read') }}" + # vcpkg has to be allowed to write into the cache directory even on a pull + # request, because that is where "Collect new vcpkg entries" looks for the + # archives this run had to compile itself. With the source left at read those + # archives are never written, the collect step finds nothing, and vcpkg-cache-import.yml + # gets no artifact to heal the shared cache with. The importer is what publishes + # to master's scope; a pull request's own merge ref stays unreadable by every + # later run, so the action's own save there costs quota but nothing else. + # Forks keep the read-only mode, they are the case that must not be trusted. + VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository), 'readwrite', 'read') }}" VCPKG_DISABLE_METRICS: 1 VCPKG_FEATURE_FLAGS: dependencygraph VCPKG_TARGET_MACOS_VERSION: ${{ matrix.vcpkg_deployment_target }} From 5ddeaf2afb62ec289c10ef1b8a8b127a38fd37d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Mon, 5 Oct 2026 19:47:05 +0200 Subject: [PATCH 4/6] [CI] Upload the vcpkg handover as one archive instead of loose files --- .github/workflows/desktop-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 111947ef8..fd2463487 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -546,12 +546,15 @@ jobs: echo "count=$count" >>"$GITHUB_OUTPUT" echo "slug=$(sed -e 's|^vcpkg/||' -e 's|/$||' -e 's|/|-|g' <<<"$VCPKG_SCOPE")" >>"$GITHUB_OUTPUT" + # archive: false accepts a single file and nothing else, and a scope hands over one + # archive per ABI hash vcpkg had to rebuild, which is 23 files on a cold runner. + # Zipping the directory keeps the fanout at the artifact root, which is where + # vcpkg-cache-import.yml looks for keys.txt once download-artifact unpacks it. - name: "Upload new vcpkg entries" # An empty count means the step above did not run at all, on master and on forks if: steps.vcpkg_new.outputs.count != '' && steps.vcpkg_new.outputs.count != '0' uses: actions/upload-artifact@v7 with: - archive: false if-no-files-found: error # a count above zero with nothing staged is a bug name: vcpkg-new-${{ steps.vcpkg_new.outputs.slug }} path: ${{ runner.temp }}/vcpkg-new From 54a0993d02e28219104899cad0960dfe0e4a955f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Tue, 6 Oct 2026 05:49:58 +0200 Subject: [PATCH 5/6] [CI] Hand the vcpkg cache over from one leg per scope --- .github/workflows/desktop-build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index fd2463487..46c8b7ed0 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -278,6 +278,7 @@ jobs: type: Release use_ccache: 1 vcpkg_scope: 'vcpkg/macos13-x64-xcode26.3/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.3" - os: macOS @@ -296,6 +297,7 @@ jobs: use_ccache: 1 vcpkg_deployment_target: 14 vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.6" - os: macOS @@ -343,6 +345,7 @@ jobs: qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets type: Release vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} needs: configure @@ -512,9 +515,13 @@ jobs: # archives this leg had to build itself and let vcpkg-cache-import.yml, which runs # in the default branch scope, save them where later runs can restore them. # Forks are skipped, they are the one case where the archives must not be trusted. + # vcpkg_upload keeps this to one leg per scope: macOS 14, 15 and 26 share a scope + # by design, and all three rebuild the same archives, so uploading from each of them + # would publish three identical 269 MB artifacts under one artifact name and make + # the importer spawn a save job per copy. - name: "Collect new vcpkg entries" id: vcpkg_new - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && matrix.vcpkg_upload == 1 shell: bash env: CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' From f8ce0dd0e5ce555be9489ef5bd858774b2d6abf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Fri, 2 Oct 2026 13:49:50 +0200 Subject: [PATCH 6/6] [CI] Collect an MSBuild performance summary and binary log on request The Windows job spends about 28 minutes inside one "Build project" group, with nothing in it that says where the time goes: MSBuild runs single-node because no --parallel is passed, so projects build one after another and it is not visible which projects, tasks or phases are responsible. Add a --profile flag that makes the Windows build write -flp:PerformanceSummary and -bl, repeat the summary in the job log so the numbers need no download, and report the core count, which decides how much parallelism is worth asking for. The binary log is uploaded as an artifact only on the runs that asked for it. The workflow drives it from a build_profile matrix key that is empty by default, so no build changes behaviour until someone sets it. --- .ci/compile.sh | 41 +++++++++++++++++++++++++++-- .github/workflows/desktop-build.yml | 16 +++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.ci/compile.sh b/.ci/compile.sh index b4f9e7782..45f8b2e98 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -14,8 +14,9 @@ # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION VCPKG_TARGET_MACOS_VERSION -# (correspond to args: --debug/--release --install --package --suffix --server --test --ccache --dir ) +# --profile writes an MSBuild performance summary and binary log, Windows only +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION VCPKG_TARGET_MACOS_VERSION BUILD_PROFILE +# (correspond to args: --debug/--release --install --package --suffix --server --test --ccache --dir --profile) # exitcode: 1 for failure, 3 for invalid arguments # Read arguments @@ -104,6 +105,10 @@ while [[ $# != 0 ]]; do export CMAKE_GENERATOR=$1 shift ;; + '--profile') + BUILD_PROFILE=1 + shift + ;; '--target-macos-version') shift if [[ $# == 0 ]]; then @@ -178,6 +183,22 @@ function ccachestatsverbose() { fi } +function cpuinfo() { + # The number of cores decides how much build parallelism is useful; the hosted + # runners are not all sized alike, so read it off the machine instead of assuming. + case "$RUNNER_OS" in + Windows) + echo "cores: ${NUMBER_OF_PROCESSORS:-unknown}" + ;; + macOS) + echo "cores: $(sysctl -n hw.ncpu)" + ;; + *) + echo "cores: $(nproc 2>/dev/null || echo unknown)" + ;; + esac +} + # Compile if [[ $RUNNER_OS == macOS ]]; then # QTDIR is needed for macOS since we actually only use the cached thin Qt binaries instead of the install-qt-action, @@ -265,6 +286,13 @@ elif [[ $RUNNER_OS == Windows ]]; then # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true) + if [[ $BUILD_PROFILE ]]; then + # A performance summary says which projects and tasks cost what, the binary log + # keeps the full timeline; both are far too noisy to always produce. The files + # land next to the solution, i.e. in the build dir the workflow uploads. These + # join the MTT options above, which already carried the "--" separator. + buildflags+=("-flp:PerformanceSummary;v=q;LogFile=msbuild-perf.log" -bl:msbuild.binlog) + fi fi if [[ $USE_CCACHE ]]; then @@ -280,10 +308,19 @@ cmake .. "${flags[@]}" echo "::endgroup::" echo "::group::Build project" +cpuinfo echo "Running cmake --build with flags: ${buildflags[*]}" cmake --build . "${buildflags[@]}" echo "::endgroup::" +if [[ $BUILD_PROFILE && $RUNNER_OS == Windows && -f msbuild-perf.log ]]; then + # Repeat the summary in the job log so the numbers are readable without + # downloading the binary log, which is hundreds of megabytes. + echo "::group::MSBuild performance summary" + cat msbuild-perf.log + echo "::endgroup::" +fi + if [[ $USE_CCACHE ]]; then if [[ $CCACHE_EVICTION_AGE ]]; then echo "::group::evict ccache files older than $CCACHE_EVICTION_AGE" diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 46c8b7ed0..2d6044d5f 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -337,6 +337,10 @@ jobs: target: 10 runner: windows-2025 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2025-VS2026-Readme.md + # Set build_profile: 1 to have the build write an MSBuild performance + # summary and a binary log; empty on every entry that runs by default. + build_profile: '' + cmake_generator: "Visual Studio 18 2026" cmake_generator_platform: x64 make_package: 1 @@ -352,6 +356,7 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 100 env: + BUILD_PROFILE: '${{ matrix.build_profile }}' CCACHE_DIR: ${{ github.workspace }}/.cache/ CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy @@ -653,6 +658,17 @@ jobs: build/oracle/Release/*.pdb build/servatrice/Release/*.pdb + # Written only when the matrix entry sets build_profile, so the artifact + # exists exactly on the runs that were asked to produce one. + - name: "[Windows] Upload MSBuild binary log" + if: always() && matrix.os == 'Windows' && env.BUILD_PROFILE == '1' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error + name: ${{ steps.build.outputs.name }}-msbuild + path: build/msbuild.binlog + retention-days: 7 + - name: "Upload to release" if: needs.configure.outputs.tag != null && matrix.make_package == '1' id: upload_release