diff --git a/.ci/compile.sh b/.ci/compile.sh index 302972f30..a9386400f 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -14,11 +14,13 @@ # --debug or --release sets the build type ie CMAKE_BUILD_TYPE # --ccache [] uses ccache and shows stats, optionally provide size # --evict-ccache runs ccache eviction based on given age after build +# --sccache uses sccache instead, needs a generator that runs the compiler itself # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE VCPKG_TARGET_MACOS_VERSION -# (correspond to args: --debug/--release --install --package --suffix --server --test --sanitize --coverage --ccache --dir ) +# --profile writes an MSBuild performance summary and binary log, Windows only +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE VCPKG_TARGET_MACOS_VERSION BUILD_PARALLEL_LEVEL BUILD_PROFILE USE_SCCACHE +# (correspond to args: --debug/--release --install --package --suffix --server --test --sanitize --coverage --ccache --dir --profile --sccache) # exitcode: 1 for failure, 3 for invalid arguments # Read arguments @@ -93,6 +95,10 @@ while [[ $# != 0 ]]; do CCACHE_EVICTION_AGE=$1 shift ;; + '--sccache') + USE_SCCACHE=1 + shift + ;; '--vcpkg') USE_VCPKG=1 shift @@ -115,6 +121,10 @@ while [[ $# != 0 ]]; do export CMAKE_GENERATOR=$1 shift ;; + '--profile') + BUILD_PROFILE=1 + shift + ;; '--target-macos-version') shift if [[ $# == 0 ]]; then @@ -133,6 +143,12 @@ done set -e +# cmake reads the generator platform from the environment, so a leftover platform +# follows the generator around; Ninja has no platform to select. +if [[ $CMAKE_GENERATOR != "Visual Studio"* ]]; then + unset CMAKE_GENERATOR_PLATFORM +fi + # Setup ./servatrice/check_schema_version.sh if [[ ! $BUILDTYPE ]]; then @@ -158,6 +174,16 @@ export CMAKE_POLICY_VERSION_MINIMUM=3.10 # Add cmake flags flags=("-DCMAKE_BUILD_TYPE=$BUILDTYPE") +if [[ $USE_CCACHE && $USE_SCCACHE ]]; then + echo "::error file=$0::ccache and sccache at the same time is not a supported combination" + exit 1 +fi +if [[ $USE_SCCACHE && $CMAKE_GENERATOR != "Ninja"* ]]; then + # A compiler cache wraps the compiler, and only generators that run the compiler + # themselves are given a launcher to wrap: cmake does not pass one to MSBuild. + echo "::error file=$0::sccache needs a Ninja generator, not '$CMAKE_GENERATOR', which would ignore it" + exit 1 +fi if [[ $MAKE_SERVER ]]; then flags+=("-DWITH_SERVER=1") fi @@ -188,6 +214,10 @@ if [[ $USE_CCACHE ]]; then ccache --max-size "$CCACHE_SIZE" fi fi +if [[ $USE_SCCACHE ]]; then + flags+=("-DUSE_SCCACHE=1") + flags+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache" "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache") +fi if [[ $PACKAGE_TYPE ]]; then flags+=("-DCPACK_GENERATOR=$PACKAGE_TYPE") fi @@ -209,6 +239,22 @@ function ccachestatsverbose() { fi } +function cpuinfo() { + # The number of cores decides how much build parallelism is useful; the hosted + # runners are not all sized alike, so read it off the machine instead of assuming. + case "$RUNNER_OS" in + Windows) + echo "cores: ${NUMBER_OF_PROCESSORS:-unknown}" + ;; + macOS) + echo "cores: $(sysctl -n hw.ncpu)" + ;; + *) + echo "cores: $(nproc 2>/dev/null || echo unknown)" + ;; + esac +} + # Compile if [[ $RUNNER_OS == macOS ]]; then # QTDIR is needed for macOS since we actually only use the cached thin Qt binaries instead of the install-qt-action, @@ -293,9 +339,33 @@ if [[ $RUNNER_OS == macOS ]]; then fi elif [[ $RUNNER_OS == Windows ]]; then - # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ - # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt - buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true) + if [[ $CMAKE_GENERATOR == "Visual Studio"* ]]; then + # cmake only hands MSBuild a job count when it is asked for, and without it + # MSBuild builds a single node: one project at a time, whatever MTT does inside + # it. Ask for a node per core. + if [[ ! $BUILD_PARALLEL_LEVEL ]]; then + BUILD_PARALLEL_LEVEL="${NUMBER_OF_PROCESSORS:-1}" + fi + buildflags+=(--parallel "$BUILD_PARALLEL_LEVEL") + + # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ + # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt + # EnforceProcessCountAcrossBuilds makes the CL_MPCount ceiling apply across all + # the projects the nodes are building, instead of each node running its own + # cl.exe processes and /m and MTT multiplying into an oversubscribed machine. + buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true -p:EnforceProcessCountAcrossBuilds=true -p:CL_MPCount="$BUILD_PARALLEL_LEVEL") + if [[ $BUILD_PROFILE ]]; then + # A performance summary says which projects and tasks cost what, the binary log + # keeps the full timeline; both are far too noisy to always produce. The files + # land next to the solution, i.e. in the build dir the workflow uploads. These + # join the MTT options above, which already carried the "--" separator. + buildflags+=("-flp:PerformanceSummary;v=q;LogFile=msbuild-perf.log" -bl:msbuild.binlog) + fi + else + # Ninja runs the compiler itself, so it neither needs the MSVC switches nor + # wants a job count: it sizes its own pool for the cores it finds. + echo "ninja $(ninja --version)" + fi fi if [[ $USE_CCACHE ]]; then @@ -311,10 +381,19 @@ cmake .. "${flags[@]}" echo "::endgroup::" echo "::group::Build project" +cpuinfo echo "Running cmake --build with flags: ${buildflags[*]}" cmake --build . "${buildflags[@]}" echo "::endgroup::" +if [[ $BUILD_PROFILE && $RUNNER_OS == Windows && -f msbuild-perf.log ]]; then + # Repeat the summary in the job log so the numbers are readable without + # downloading the binary log, which is hundreds of megabytes. + echo "::group::MSBuild performance summary" + cat msbuild-perf.log + echo "::endgroup::" +fi + if [[ $USE_CCACHE ]]; then if [[ $CCACHE_EVICTION_AGE ]]; then echo "::group::evict ccache files older than $CCACHE_EVICTION_AGE" @@ -324,6 +403,10 @@ if [[ $USE_CCACHE ]]; then echo "::group::Show ccache stats again" ccachestatsverbose echo "::endgroup::" +elif [[ $USE_SCCACHE ]]; then + echo "::group::Show sccache stats" + sccache --show-stats + echo "::endgroup::" elif [[ $CCACHE_EVICTION_AGE ]]; then echo "::error file=$0::ccache eviction is enabled while ccache is disabled!" fi diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 1ce51afaa..8a491483e 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -385,6 +385,12 @@ jobs: target: 10 runner: windows-2025 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2025-VS2026-Readme.md + # Set build_profile: 1 to have the build write an MSBuild performance + # summary and a binary log; empty on every entry that runs by default. + # Enabled here on purpose: this pull request measures MSBuild + # parallelism, so its run is the one worth profiling. + build_profile: 1 + cmake_generator: "Visual Studio 18 2026" cmake_generator_platform: x64 make_package: 1 @@ -395,11 +401,37 @@ jobs: vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over - name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} + # A/B run: is Ninja faster than the Visual Studio generator on this runner? + # It is allowed to fail and is never published, see the steps that build + # a release below. Delete this entry once the answer is known. + - os: Windows + target: 10 + runner: windows-2025 + + cmake_generator: Ninja + experiment: yes + make_package: 1 + package_suffix: "-Win10-ninja" + qt_version: 6.11.* + qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets + type: Release + use_sccache: 1 + variant: ' (Ninja)' + # The same scope as the Visual Studio leg, so both halves of the A/B + # start from the same vcpkg cache and the comparison is about the + # generator alone. Without it this leg's prefix is empty, which makes + # the cache action restore the whole vcpkg/ namespace and write entries + # that no scope claims. vcpkg_upload is deliberately left unset, so only + # the Visual Studio leg hands its archives over. + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + + name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }}${{ matrix.variant }} needs: configure runs-on: ${{ matrix.runner }} + continue-on-error: ${{ matrix.experiment == 'yes' }} timeout-minutes: 100 env: + BUILD_PROFILE: '${{ matrix.build_profile }}' CCACHE_DIR: ${{ github.workspace }}/.cache/ CCACHE_SIZE: 600M # space of all repo is 10Gi: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy @@ -482,6 +514,41 @@ jobs: modules: ${{ matrix.qt_modules }} version: ${{ matrix.qt_version }} + # Ninja invokes cl.exe directly and needs the MSVC environment on PATH; the + # Visual Studio generator locates the compiler on its own and skips this. + - name: "[Windows/Ninja] Set up MSVC environment" + if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja' + uses: TheMrMilchmann/setup-msvc-dev@v4.1.0 + with: + arch: x64 + + # sccache is only useful for the Ninja job: a compiler cache has to wrap the + # compiler, and MSBuild is driven by cmake instead of running it, so a + # launcher handed to MSBuild is ignored without a word. + # Experiment: sccache 0.18 may read ACTIONS_CACHE_URL (v1) while the runner + # only provides ACTIONS_RESULTS_URL (v2, see ACTIONS_CACHE_SERVICE_V2), which + # would leave the GHA backend disengaged and everything local-only with zero + # cross-run hits. Export both spellings and watch for sccache-* keys. + - name: "[Windows/Ninja] Export cache service URLs for sccache" + if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja' + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + - name: "[Windows/Ninja] Set up sccache" + if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja' + uses: mozilla-actions/sccache-action@v0.0.11 + env: + SCCACHE_GHA_ENABLED: "true" + # A fork pull request can read the cache, but the token it runs with + # cannot write to the base branch's scope. + SCCACHE_GHA_RW_MODE: ${{ github.event.pull_request.head.repo.fork && 'READ_ONLY' || 'READ_WRITE' }} + # Experiment: surface backend errors for one run instead of swallowing + # them. Three runs stored 343 entries locally with zero cross-run hits + # and no sccache-* keys; if writes are refused (quota) or the backend + # never engages (URL skew), the error string names it. Revert after. + - name: "[Windows] Install NSIS" if: matrix.os == 'Windows' shell: bash @@ -530,8 +597,9 @@ jobs: MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }} MAKE_PACKAGE: '${{ matrix.make_package }}' PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' - TARGET_MACOS_VERSION: ${{ matrix.override_target }} + TARGET_MACOS_VERSION: '${{ matrix.override_target }}' USE_CCACHE: ${{ matrix.use_ccache }} + USE_SCCACHE: ${{ matrix.use_sccache }} # Only pushes to the default branch may write the shared binary cache: the # archives a run compiles are only as trustworthy as the code they were # built from, and only merged code may populate the scope that master and @@ -690,8 +758,11 @@ jobs: if-no-files-found: error path: ${{ steps.build.outputs.path }} + # Ninja writes its PDBs next to the binaries instead of into a Release + # subdirectory, and the experimental job is not a source of release debug + # symbols, so it keeps out of both. - name: "[Windows] Upload PDBs (Program Databases)" - if: matrix.os == 'Windows' && github.ref_type != 'tag' + if: matrix.os == 'Windows' && matrix.experiment != 'yes' && github.ref_type != 'tag' uses: actions/upload-artifact@v7 with: if-no-files-found: error @@ -701,8 +772,19 @@ jobs: build/oracle/Release/*.pdb build/servatrice/Release/*.pdb + # Written only when the matrix entry sets build_profile, so the artifact + # exists exactly on the runs that were asked to produce one. + - name: "[Windows] Upload MSBuild binary log" + if: always() && matrix.os == 'Windows' && env.BUILD_PROFILE == '1' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error + name: ${{ steps.build.outputs.name }}-msbuild + path: build/msbuild.binlog + retention-days: 7 + - name: "Upload to release" - if: needs.configure.outputs.tag != null && matrix.make_package == '1' + if: needs.configure.outputs.tag != null && matrix.make_package == '1' && matrix.experiment != 'yes' id: upload_release shell: bash env: diff --git a/CMakeLists.txt b/CMakeLists.txt index bee47972c..2ae7b94a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,19 @@ # 3.16 required for Qt6 and target_precompile_headers() cmake_minimum_required(VERSION 3.16) +# Let CMake pick the MSVC debug information format through +# CMAKE_MSVC_DEBUG_INFORMATION_FORMAT instead of writing it into the +# CMAKE__FLAGS_ defaults. The policy has to be settled before the +# first project() enables a language, and it does not exist before CMake 3.25, +# where the old defaults are the only thing available. +if(POLICY CMP0141) + cmake_policy(SET CMP0141 NEW) +endif() + # Use compiler cache (ccache) option(USE_CCACHE "Cache the build results with ccache" ON) +# Use compiler cache (sccache), for generators that invoke the compiler themselves +option(USE_SCCACHE "Cache the build results with sccache" OFF) # Treat warnings as errors (Debug builds only) option(WARNING_AS_ERROR "Treat warnings as errors in debug builds" ON) # Check for translation updates @@ -156,8 +167,17 @@ endif() # Define proper compilation flags if(MSVC) - # Disable Warning C4251, C++20 compatibility, Multi-threaded Builds, Warn Detection, Unwind Semantics, Debug Symbols - set(CMAKE_CXX_FLAGS "/wd4251 /Zc:__cplusplus /std:c++20 /permissive- /W4 /MP /EHsc /Zi") + # Debug information: a cache can only store and restore objects that carry it + # inside, so a cached build asks for /Z7 (Embedded); otherwise /Zi (ProgramDatabase). + # Either way the linker is told to write the PDB, in every configuration. + if(USE_SCCACHE) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT Embedded) + else() + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT ProgramDatabase) + endif() + + # Disable Warning C4251, C++20 compatibility, Multi-threaded Builds, Warn Detection, Unwind Semantics + set(CMAKE_CXX_FLAGS "/wd4251 /Zc:__cplusplus /std:c++20 /permissive- /W4 /MP /EHsc") # Visual Studio: Maximum Optimization, Multi-threaded DLL set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD") # Visual Studio: No Optimization, Multi-threaded Debug DLL