diff --git a/.ci/Debian12/Dockerfile b/.ci/Debian12/Dockerfile index fc756aac2..14c6f018c 100644 --- a/.ci/Debian12/Dockerfile +++ b/.ci/Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Debian13/Dockerfile b/.ci/Debian13/Dockerfile index bdecb56df..b597307c5 100644 --- a/.ci/Debian13/Dockerfile +++ b/.ci/Debian13/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Servatrice_Debian12/Dockerfile b/.ci/Servatrice_Debian12/Dockerfile index 321aa7c0f..f1a16843a 100644 --- a/.ci/Servatrice_Debian12/Dockerfile +++ b/.ci/Servatrice_Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu24.04/Dockerfile b/.ci/Ubuntu24.04/Dockerfile index 715997474..b9fb2b701 100644 --- a/.ci/Ubuntu24.04/Dockerfile +++ b/.ci/Ubuntu24.04/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu26.04/Dockerfile b/.ci/Ubuntu26.04/Dockerfile index 96dd10763..a7e010ff5 100644 --- a/.ci/Ubuntu26.04/Dockerfile +++ b/.ci/Ubuntu26.04/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/compile.sh b/.ci/compile.sh index 72a017f33..39df165a5 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -11,11 +11,13 @@ # --debug or --release sets the build type ie CMAKE_BUILD_TYPE # --ccache [] uses ccache and shows stats, optionally provide size # --evict-ccache runs ccache eviction based on given age after build +# --sccache uses sccache instead, needs a generator that runs the compiler itself # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION -# (correspond to args: --debug/--release --install --package --suffix --server --test --ccache --dir ) +# --profile writes an MSBuild performance summary and binary log, Windows only +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION VCPKG_TARGET_MACOS_VERSION BUILD_PARALLEL_LEVEL BUILD_PROFILE USE_SCCACHE +# (correspond to args: --debug/--release --install --package --suffix --server --test --ccache --dir --profile --sccache) # exitcode: 1 for failure, 3 for invalid arguments # Read arguments @@ -82,6 +84,10 @@ while [[ $# != 0 ]]; do CCACHE_EVICTION_AGE=$1 shift ;; + '--sccache') + USE_SCCACHE=1 + shift + ;; '--vcpkg') USE_VCPKG=1 shift @@ -104,6 +110,10 @@ while [[ $# != 0 ]]; do export CMAKE_GENERATOR=$1 shift ;; + '--profile') + BUILD_PROFILE=1 + shift + ;; '--target-macos-version') shift if [[ $# == 0 ]]; then @@ -122,6 +132,12 @@ done set -e +# cmake reads the generator platform from the environment, so a leftover platform +# follows the generator around; Ninja has no platform to select. +if [[ $CMAKE_GENERATOR != "Visual Studio"* ]]; then + unset CMAKE_GENERATOR_PLATFORM +fi + # Setup ./servatrice/check_schema_version.sh if [[ ! $BUILDTYPE ]]; then @@ -138,6 +154,16 @@ export CMAKE_POLICY_VERSION_MINIMUM=3.10 # Add cmake flags flags=("-DCMAKE_BUILD_TYPE=$BUILDTYPE") +if [[ $USE_CCACHE && $USE_SCCACHE ]]; then + echo "::error file=$0::ccache and sccache at the same time is not a supported combination" + exit 1 +fi +if [[ $USE_SCCACHE && $CMAKE_GENERATOR != "Ninja"* ]]; then + # A compiler cache wraps the compiler, and only generators that run the compiler + # themselves are given a launcher to wrap: cmake does not pass one to MSBuild. + echo "::error file=$0::sccache needs a Ninja generator, not '$CMAKE_GENERATOR', which would ignore it" + exit 1 +fi if [[ $MAKE_SERVER ]]; then flags+=("-DWITH_SERVER=1") fi @@ -157,6 +183,10 @@ if [[ $USE_CCACHE ]]; then ccache --max-size "$CCACHE_SIZE" fi fi +if [[ $USE_SCCACHE ]]; then + flags+=("-DUSE_SCCACHE=1") + flags+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache" "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache") +fi if [[ $PACKAGE_TYPE ]]; then flags+=("-DCPACK_GENERATOR=$PACKAGE_TYPE") fi @@ -178,6 +208,22 @@ function ccachestatsverbose() { fi } +function cpuinfo() { + # The number of cores decides how much build parallelism is useful; the hosted + # runners are not all sized alike, so read it off the machine instead of assuming. + case "$RUNNER_OS" in + Windows) + echo "cores: ${NUMBER_OF_PROCESSORS:-unknown}" + ;; + macOS) + echo "cores: $(sysctl -n hw.ncpu)" + ;; + *) + echo "cores: $(nproc 2>/dev/null || echo unknown)" + ;; + esac +} + # Compile if [[ $RUNNER_OS == macOS ]]; then # QTDIR is needed for macOS since we actually only use the cached thin Qt binaries instead of the install-qt-action, @@ -212,8 +258,14 @@ if [[ $RUNNER_OS == macOS ]]; then triplet_source="../vcpkg/triplets/community/$arch-osx.cmake" fi cp "$triplet_source" "$triplet_file" - echo "set(VCPKG_CMAKE_SYSTEM_VERSION $TARGET_MACOS_VERSION)" >>"$triplet_file" - echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $TARGET_MACOS_VERSION)" >>"$triplet_file" + # The triplet is what vcpkg hashes into the ABI hash of every port, so two legs that + # agree here reuse each other's binary cache. Targets 14, 15 and 26 on the same Xcode + # produce identical binaries when they share the lowest deployment target, so they all + # build against 14 to keep one vcpkg cache scope instead of three. + # Keep in sync with "vcpkg_scope" and "vcpkg_deployment_target" in desktop-build.yml. + vcpkg_target="${VCPKG_TARGET_MACOS_VERSION:-$TARGET_MACOS_VERSION}" + echo "set(VCPKG_CMAKE_SYSTEM_VERSION $vcpkg_target)" >>"$triplet_file" + echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $vcpkg_target)" >>"$triplet_file" flags+=("-DVCPKG_OVERLAY_TRIPLETS=$triplets_dir") flags+=("-DVCPKG_HOST_TRIPLET=$triplet_version") flags+=("-DVCPKG_TARGET_TRIPLET=$triplet_version") @@ -256,9 +308,33 @@ if [[ $RUNNER_OS == macOS ]]; then fi elif [[ $RUNNER_OS == Windows ]]; then - # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ - # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt - buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true) + if [[ $CMAKE_GENERATOR == "Visual Studio"* ]]; then + # cmake only hands MSBuild a job count when it is asked for, and without it + # MSBuild builds a single node: one project at a time, whatever MTT does inside + # it. Ask for a node per core. + if [[ ! $BUILD_PARALLEL_LEVEL ]]; then + BUILD_PARALLEL_LEVEL="${NUMBER_OF_PROCESSORS:-1}" + fi + buildflags+=(--parallel "$BUILD_PARALLEL_LEVEL") + + # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ + # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt + # EnforceProcessCountAcrossBuilds makes the CL_MPCount ceiling apply across all + # the projects the nodes are building, instead of each node running its own + # cl.exe processes and /m and MTT multiplying into an oversubscribed machine. + buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true -p:EnforceProcessCountAcrossBuilds=true -p:CL_MPCount="$BUILD_PARALLEL_LEVEL") + if [[ $BUILD_PROFILE ]]; then + # A performance summary says which projects and tasks cost what, the binary log + # keeps the full timeline; both are far too noisy to always produce. The files + # land next to the solution, i.e. in the build dir the workflow uploads. These + # join the MTT options above, which already carried the "--" separator. + buildflags+=("-flp:PerformanceSummary;v=q;LogFile=msbuild-perf.log" -bl:msbuild.binlog) + fi + else + # Ninja runs the compiler itself, so it neither needs the MSVC switches nor + # wants a job count: it sizes its own pool for the cores it finds. + echo "ninja $(ninja --version)" + fi fi if [[ $USE_CCACHE ]]; then @@ -274,10 +350,19 @@ cmake .. "${flags[@]}" echo "::endgroup::" echo "::group::Build project" +cpuinfo echo "Running cmake --build with flags: ${buildflags[*]}" cmake --build . "${buildflags[@]}" echo "::endgroup::" +if [[ $BUILD_PROFILE && $RUNNER_OS == Windows && -f msbuild-perf.log ]]; then + # Repeat the summary in the job log so the numbers are readable without + # downloading the binary log, which is hundreds of megabytes. + echo "::group::MSBuild performance summary" + cat msbuild-perf.log + echo "::endgroup::" +fi + if [[ $USE_CCACHE ]]; then if [[ $CCACHE_EVICTION_AGE ]]; then echo "::group::evict ccache files older than $CCACHE_EVICTION_AGE" @@ -287,6 +372,10 @@ if [[ $USE_CCACHE ]]; then echo "::group::Show ccache stats again" ccachestatsverbose echo "::endgroup::" +elif [[ $USE_SCCACHE ]]; then + echo "::group::Show sccache stats" + sccache --show-stats + echo "::endgroup::" elif [[ $CCACHE_EVICTION_AGE ]]; then echo "::error file=$0::ccache eviction is enabled while ccache is disabled!" fi diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 17c42b07d..5bc5e2cbc 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -150,7 +150,7 @@ jobs: continue-on-error: ${{ matrix.allow-failure == 'yes' }} timeout-minutes: 70 env: - CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }} # directory for caching docker image and ccache + CACHE: ${{ github.workspace }}/.cache/${{ matrix.distro }}${{ matrix.version }} # directory for caching the compiler cache CCACHE_EVICTION_AGE: 7d CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy CMAKE_GENERATOR: 'Ninja' @@ -165,7 +165,7 @@ jobs: uses: actions/cache/restore@v6 with: key: ccache-${{ matrix.distro }}${{ matrix.version }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Build ${{ matrix.distro }} ${{ matrix.version }} Docker image" shell: bash @@ -219,7 +219,7 @@ jobs: uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Upload artifact" id: upload_artifact @@ -277,6 +277,8 @@ jobs: soc: Intel type: Release use_ccache: 1 + vcpkg_scope: 'vcpkg/macos13-x64-xcode26.3/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.3" - os: macOS @@ -293,6 +295,9 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.6" - os: macOS @@ -309,6 +314,8 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: macOS @@ -322,12 +329,20 @@ jobs: soc: Apple type: Debug use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: Windows target: 10 runner: windows-2025 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2025-VS2026-Readme.md + # Set build_profile: 1 to have the build write an MSBuild performance + # summary and a binary log; empty on every entry that runs by default. + # Enabled here on purpose: this pull request measures MSBuild + # parallelism, so its run is the one worth profiling. + build_profile: 1 + cmake_generator: "Visual Studio 18 2026" cmake_generator_platform: x64 make_package: 1 @@ -335,12 +350,40 @@ jobs: qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets type: Release + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over - name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} + # A/B run: is Ninja faster than the Visual Studio generator on this runner? + # It is allowed to fail and is never published, see the steps that build + # a release below. Delete this entry once the answer is known. + - os: Windows + target: 10 + runner: windows-2025 + + cmake_generator: Ninja + experiment: yes + make_package: 1 + package_suffix: "-Win10-ninja" + qt_version: 6.11.* + qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets + type: Release + use_sccache: 1 + variant: ' (Ninja)' + # The same scope as the Visual Studio leg, so both halves of the A/B + # start from the same vcpkg cache and the comparison is about the + # generator alone. Without it this leg's prefix is empty, which makes + # the cache action restore the whole vcpkg/ namespace and write entries + # that no scope claims. vcpkg_upload is deliberately left unset, so only + # the Visual Studio leg hands its archives over. + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + + name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }}${{ matrix.variant }} needs: configure runs-on: ${{ matrix.runner }} + continue-on-error: ${{ matrix.experiment == 'yes' }} timeout-minutes: 100 env: + BUILD_PROFILE: '${{ matrix.build_profile }}' CCACHE_DIR: ${{ github.workspace }}/.cache/ CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy @@ -423,6 +466,28 @@ jobs: modules: ${{ matrix.qt_modules }} version: ${{ matrix.qt_version }} + # Ninja invokes cl.exe directly and needs the MSVC environment on PATH; the + # Visual Studio generator locates the compiler on its own and skips this. + - name: "[Windows/Ninja] Set up MSVC environment" + if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja' + uses: TheMrMilchmann/setup-msvc-dev@v4.1.0 + with: + arch: x64 + + # sccache is only useful for the Ninja job: a compiler cache has to wrap the + # compiler, and MSBuild is driven by cmake instead of running it, so a + # launcher handed to MSBuild is ignored without a word. + - name: "[Windows/Ninja] Set up sccache" + if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja' + uses: mozilla-actions/sccache-action@v0.0.11 + env: + SCCACHE_GHA_ENABLED: "true" + # A fork pull request can read the cache, but the token it runs with + # cannot write to the base branch's scope. + SCCACHE_GHA_RW_MODE: ${{ github.event.pull_request.head.repo.fork && 'READ_ONLY' || 'READ_WRITE' }} + # A cache that cannot be reached must not fail the build. + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + - name: "[Windows] Install NSIS" if: matrix.os == 'Windows' shell: bash @@ -432,8 +497,28 @@ jobs: id: vcpkg-cache uses: TAServers/vcpkg-cache@v3 with: + # Every entry is keyed by vcpkg ABI hash, and one hash set exists per platform, + # target and toolchain. The action restores every entry it finds under the + # prefix, not just the hashes vcpkg asks for, so with one shared vcpkg/ + # namespace every leg downloads all 66 archives of all 5 legs on every run. + # Each of those restores also refreshes the entry's last access date, which + # is why a dead ABI generation is never evicted by age. Naming the scope is + # what makes those generations identifiable, and prune-caches below able to + # delete them. + prefix: ${{ matrix.vcpkg_scope }} token: ${{ secrets.GITHUB_TOKEN }} + # The action only creates its directory when the prefix matched at least one + # entry, and it reads that directory unguarded when it saves at the end of the + # job. A scope nobody has written to yet therefore leaves the directory missing + # and fails the job, which is exactly the state every scope starts in. Make the + # directory up front so an empty scope is a cache miss instead of an error. + - name: "Create vcpkg cache directory" + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + run: mkdir -p "$CACHE_DIR" + # Uses environment variables, see compile.sh for more details - name: "Build Cockatrice" id: build @@ -451,11 +536,21 @@ jobs: MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} MAKE_PACKAGE: '${{ matrix.make_package }}' PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' - TARGET_MACOS_VERSION: ${{ matrix.override_target }} + TARGET_MACOS_VERSION: '${{ matrix.override_target }}' USE_CCACHE: ${{ matrix.use_ccache }} - VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master', 'readwrite', 'read') }}" + USE_SCCACHE: ${{ matrix.use_sccache }} + # vcpkg has to be allowed to write into the cache directory even on a pull + # request, because that is where "Collect new vcpkg entries" looks for the + # archives this run had to compile itself. With the source left at read those + # archives are never written, the collect step finds nothing, and vcpkg-cache-import.yml + # gets no artifact to heal the shared cache with. The importer is what publishes + # to master's scope; a pull request's own merge ref stays unreadable by every + # later run, so the action's own save there costs quota but nothing else. + # Forks keep the read-only mode, they are the case that must not be trusted. + VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository), 'readwrite', 'read') }}" VCPKG_DISABLE_METRICS: 1 VCPKG_FEATURE_FLAGS: dependencygraph + VCPKG_TARGET_MACOS_VERSION: ${{ matrix.vcpkg_deployment_target }} run: .ci/compile.sh --server --test --vcpkg # CPack reports a failed makensis run as "Problem running NSIS command" @@ -469,6 +564,64 @@ jobs: shell: bash run: find . -name NSISOutput.log -exec cat {} + + # A pull request cannot write the shared cache itself: its cache entries are scoped + # to refs/pull//merge and stay unreadable for every other run, so writing them + # from a pull request only burns the repository's quota. So hand over only the + # archives this leg had to build itself and let vcpkg-cache-import.yml, which runs + # in the default branch scope, save them where later runs can restore them. + # Forks are skipped, they are the one case where the archives must not be trusted. + # vcpkg_upload keeps this to one leg per scope: macOS 14, 15 and 26 share a scope + # by design, and all three rebuild the same archives, so uploading from each of them + # would publish three identical 269 MB artifacts under one artifact name and make + # the importer spawn a save job per copy. + - name: "Collect new vcpkg entries" + id: vcpkg_new + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && matrix.vcpkg_upload == 1 + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + GH_TOKEN: ${{ github.token }} + VCPKG_SCOPE: '${{ matrix.vcpkg_scope }}' + run: | + # The archive name is the ABI hash, so a missing key is exactly what vcpkg had + # to compile from source in this job. + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/caches?ref=refs/heads/master&key=$VCPKG_SCOPE&per_page=100" \ + --jq '.actions_caches[].key' | sort -u >"$RUNNER_TEMP/master-keys.txt" + find "$CACHE_DIR" -type f -name '*.zip' -exec basename {} .zip \; | + sed "s|^|$VCPKG_SCOPE|" | sort -u >"$RUNNER_TEMP/local-keys.txt" + comm -23 "$RUNNER_TEMP/local-keys.txt" "$RUNNER_TEMP/master-keys.txt" >"$RUNNER_TEMP/new-keys.txt" + + mkdir -p "$RUNNER_TEMP/vcpkg-new" + : >"$RUNNER_TEMP/vcpkg-new/keys.txt" + while read -r key; do + hash="${key##*/}" + archive="$(find "$CACHE_DIR" -type f -name "$hash.zip" -print -quit)" + dir="$RUNNER_TEMP/vcpkg-new/${hash:0:2}" # vcpkg's own two character fanout + mkdir -p "$dir" + cp "$archive" "$dir/" + echo "$key ${hash:0:2}/$hash.zip" >>"$RUNNER_TEMP/vcpkg-new/keys.txt" + done <"$RUNNER_TEMP/new-keys.txt" + + count="$(wc -l <"$RUNNER_TEMP/new-keys.txt")" + echo "vcpkg scope $VCPKG_SCOPE: $count entries to import" + echo "count=$count" >>"$GITHUB_OUTPUT" + echo "slug=$(sed -e 's|^vcpkg/||' -e 's|/$||' -e 's|/|-|g' <<<"$VCPKG_SCOPE")" >>"$GITHUB_OUTPUT" + + # archive: false accepts a single file and nothing else, and a scope hands over one + # archive per ABI hash vcpkg had to rebuild, which is 23 files on a cold runner. + # Zipping the directory keeps the fanout at the artifact root, which is where + # vcpkg-cache-import.yml looks for keys.txt once download-artifact unpacks it. + - name: "Upload new vcpkg entries" + # An empty count means the step above did not run at all, on master and on forks + if: steps.vcpkg_new.outputs.count != '' && steps.vcpkg_new.outputs.count != '0' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error # a count above zero with nothing staged is a bug + name: vcpkg-new-${{ steps.vcpkg_new.outputs.slug }} + path: ${{ runner.temp }}/vcpkg-new + retention-days: 1 + # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "[macOS] Delete remote compiler cache (ccache)" if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit @@ -544,8 +697,11 @@ jobs: if-no-files-found: error path: ${{ steps.build.outputs.path }} + # Ninja writes its PDBs next to the binaries instead of into a Release + # subdirectory, and the experimental job is not a source of release debug + # symbols, so it keeps out of both. - name: "[Windows] Upload PDBs (Program Databases)" - if: matrix.os == 'Windows' && github.ref_type != 'tag' + if: matrix.os == 'Windows' && matrix.experiment != 'yes' && github.ref_type != 'tag' uses: actions/upload-artifact@v7 with: if-no-files-found: error @@ -555,8 +711,19 @@ jobs: build/oracle/Release/*.pdb build/servatrice/Release/*.pdb + # Written only when the matrix entry sets build_profile, so the artifact + # exists exactly on the runs that were asked to produce one. + - name: "[Windows] Upload MSBuild binary log" + if: always() && matrix.os == 'Windows' && env.BUILD_PROFILE == '1' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error + name: ${{ steps.build.outputs.name }}-msbuild + path: build/msbuild.binlog + retention-days: 7 + - name: "Upload to release" - if: needs.configure.outputs.tag != null && matrix.make_package == '1' + if: needs.configure.outputs.tag != null && matrix.make_package == '1' && matrix.experiment != 'yes' id: upload_release shell: bash env: @@ -581,3 +748,67 @@ jobs: BUILD_PATH: ${{ steps.build.outputs.path }} GH_TOKEN: ${{ github.token }} run: gh attestation verify "$BUILD_PATH" --repo Cockatrice/Cockatrice + + prune-caches: + name: Prune caches + # Only master writes caches at all, so only master may clean them up too + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + continue-on-error: true # freeing space must never fail a build + + steps: + # The repository holds 10Gi of cache against entries that want more, and GitHub + # evicts the least recently used entry whenever that is exceeded. The vcpkg + # entries are the reason that never happens: the restore above touches every + # entry under the prefix on every run, so a dead ABI generation keeps its access + # date fresh forever and age based eviction can never reach it. Deleting by age + # alone cannot fix that, only knowing which scope an entry belongs to can. + - name: "Delete unusable cache entries" + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Must list every vcpkg_scope of the build-vcpkg matrix above, or the entries of + # a scope that is still in use get deleted on the next run. + LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ + --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" + + cutoff="$(( $(date -u +%s) - 7 * 24 * 3600 ))" + live=" ${LIVE_VCPKG_SCOPES} " + deleted=0 + while IFS=$'\t' read -r id ref key accessed; do + [[ -n $id ]] || continue + # Only ever touch the key shapes this workflow creates, so that an entry of + # some other workflow can never be deleted by accident + case "$key" in + vcpkg/* | ccache-* | Qt*) ;; + *) continue ;; + esac + + reason="" + if [[ $ref != refs/heads/master ]]; then + reason="scoped to $ref" + elif [[ $key == vcpkg/* ]]; then + # "/", cut the hash off the end to get the scope + scope="vcpkg/${key#vcpkg/}" + scope="${scope%/*}/" + if [[ $live != *" $scope "* ]]; then + reason="no matrix leg uses $scope" + fi + fi + if [[ -z $reason ]]; then + stamp="$(date -u -d "$accessed" +%s 2>/dev/null || echo 0)" + if [[ $stamp -ne 0 && $stamp -lt $cutoff ]]; then + reason="unused since $accessed" + fi + fi + + if [[ -n $reason ]]; then + echo "::notice::deleting $key, $reason" + gh api -X DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" + deleted=$((deleted + 1)) + fi + done <"$RUNNER_TEMP/caches.tsv" + + echo "::notice::deleted $deleted of $(wc -l <"$RUNNER_TEMP/caches.tsv") cache entries" diff --git a/.github/workflows/vcpkg-cache-import.yml b/.github/workflows/vcpkg-cache-import.yml new file mode 100644 index 000000000..01bfb7f7a --- /dev/null +++ b/.github/workflows/vcpkg-cache-import.yml @@ -0,0 +1,112 @@ +name: Import vcpkg cache + +# A pull request run can only write caches into its own refs/pull//merge scope, which +# nothing else is allowed to restore. So after a runner image update changes every port's +# ABI hash, the first pull request rebuilds protobuf, openssl and abseil from source and +# the next one does the same, until somebody merges and a master run finally repopulates +# the shared cache. This workflow takes the archives such a run produced and saves them +# in the default branch scope instead, where every later run can restore them. +# +# repository_dispatch is not used on purpose: it would also resolve to the default branch, +# but it counts as a low-trust trigger and may only write there when it declares a +# write-capable cache-mode, which puts it in the same position as workflow_run. + +permissions: + actions: write # needed to save entries into the default branch's cache scope + contents: read + +on: + workflow_run: + workflows: ["Build Desktop"] + types: [completed] + +# Every import writes the same scopes, so let them queue instead of racing each other +concurrency: + group: vcpkg-cache-import + cancel-in-progress: false + +jobs: + list: + name: List new vcpkg entries + # Only a successful same-repository pull request has archives worth importing. Forks + # build untrusted code and must never reach a cache that master runs restore and link. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.head_repository.full_name == github.repository + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + # workflow_run is a low-trust trigger, so cache access is read-only by default even + # though this run resolves to the default branch. Assert below that this took effect, + # because a save under read-only access only warns and would import nothing silently. + cache-mode: write + outputs: + entries: ${{ steps.scan.outputs.entries }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: vcpkg-new-* + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Assert write access" + shell: bash + run: | + if [[ $ACTIONS_CACHE_MODE != *write* ]]; then + echo "::error::expected a write-capable cache, got '$ACTIONS_CACHE_MODE'" + exit 1 + fi + + - name: "List cache entries" + id: scan + shell: bash + working-directory: incoming + run: | + entries="$(find . -name keys.txt -type f -exec sh -c ' + for manifest; do + artifact="$(basename "$(dirname "$manifest")")" + while read -r key path; do + printf "{\"artifact\": \"%s\", \"key\": \"%s\", \"path\": \"%s\"}\n" \ + "$artifact" "$key" "$path" + done <"$manifest" + done + ' _ {} +)" + [[ -n $entries ]] || exit 0 + # Compact, because a multi line value cannot be written to GITHUB_OUTPUT + entries="$(jq -cs '.' <<<"$entries")" + echo "entries=$entries" >>"$GITHUB_OUTPUT" + echo "::notice::importing $(jq length <<<"$entries") vcpkg entries" + + save: + name: "Save ${{ matrix.entry.key }}" + needs: list + if: needs.list.outputs.entries + runs-on: ubuntu-slim + cache-mode: write + strategy: + # One job per archive, they run in parallel and each only uploads a single file + fail-fast: false + max-parallel: 20 + matrix: + entry: ${{ fromJSON(needs.list.outputs.entries) }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: ${{ matrix.entry.artifact }} + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Save vcpkg entry" + uses: actions/cache/save@v6 + with: + # The payload is a Windows or macOS archive while the upload runs on Linux, and + # without this the entry would be stamped as a Linux cache that neither of them + # is allowed to restore. + enableCrossOsArchive: true + key: ${{ matrix.entry.key }} + path: incoming/${{ matrix.entry.path }} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ea17b664..1ebe84a62 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,19 @@ # 3.16 required for Qt6 and target_precompile_headers() cmake_minimum_required(VERSION 3.16) +# Let CMake pick the MSVC debug information format through +# CMAKE_MSVC_DEBUG_INFORMATION_FORMAT instead of writing it into the +# CMAKE__FLAGS_ defaults. The policy has to be settled before the +# first project() enables a language, and it does not exist before CMake 3.25, +# where the old defaults are the only thing available. +if(POLICY CMP0141) + cmake_policy(SET CMP0141 NEW) +endif() + # Use compiler cache (ccache) option(USE_CCACHE "Cache the build results with ccache" ON) +# Use compiler cache (sccache), for generators that invoke the compiler themselves +option(USE_SCCACHE "Cache the build results with sccache" OFF) # Treat warnings as errors (Debug builds only) option(WARNING_AS_ERROR "Treat warnings as errors in debug builds" ON) # Check for translation updates @@ -152,8 +163,17 @@ endif() # Define proper compilation flags if(MSVC) - # Disable Warning C4251, C++20 compatibility, Multi-threaded Builds, Warn Detection, Unwind Semantics, Debug Symbols - set(CMAKE_CXX_FLAGS "/wd4251 /Zc:__cplusplus /std:c++20 /permissive- /W4 /MP /EHsc /Zi") + # Debug information: a cache can only store and restore objects that carry it + # inside, so a cached build asks for /Z7 (Embedded); otherwise /Zi (ProgramDatabase). + # Either way the linker is told to write the PDB, in every configuration. + if(USE_SCCACHE) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT Embedded) + else() + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT ProgramDatabase) + endif() + + # Disable Warning C4251, C++20 compatibility, Multi-threaded Builds, Warn Detection, Unwind Semantics + set(CMAKE_CXX_FLAGS "/wd4251 /Zc:__cplusplus /std:c++20 /permissive- /W4 /MP /EHsc") # Visual Studio: Maximum Optimization, Multi-threaded DLL set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD") # Visual Studio: No Optimization, Multi-threaded Debug DLL