[CI] Measure Ninja against the Visual Studio generator on Windows

Ninja scales its job pool to the machine it runs on, and it lets a
compiler launcher through, which MSBuild does not; those two are the
reasons to expect it to beat the Visual Studio generator on a 4-core
runner. That is an expectation, not a measurement, so run it as an A/B
next to the Visual Studio job for a week: the extra matrix entry is
allowed to fail, keeps its own " (Ninja)" suffix in the job name, is
never uploaded to a release and never contributes PDBs, so it can
neither block a pull request nor ship anything.

Ninja needs cl.exe and the MSVC environment on PATH, which the Visual
Studio generator gets for free by locating the compiler itself, hence
the setup-msvc-dev step for the experimental job alone. Its build gets
neither the MSBuild switches nor a job count: Ninja rejects -p: options
and sizes its own pool, while the Visual Studio job keeps both. cmake
takes the generator platform from the environment, so it is unset for
everything that is not a Visual Studio generator, where it would
otherwise follow the generator around.

Delete this entry once the numbers are in: if Ninja wins, switch the
Windows matrix to it and fix the PDB paths with it.

.ci/compile.sh

@@ -138,6 +138,12 @@ done
 
 set -e
 
+# cmake reads the generator platform from the environment, so a leftover platform
+# follows the generator around; Ninja has no platform to select.
+if [[ $CMAKE_GENERATOR != "Visual Studio"* ]]; then
+  unset CMAKE_GENERATOR_PLATFORM
+fi
+
 # Setup
 ./servatrice/check_schema_version.sh
 if [[ ! $BUILDTYPE ]]; then
@@ -314,26 +320,32 @@ if [[ $RUNNER_OS == macOS ]]; then
   fi
 
 elif [[ $RUNNER_OS == Windows ]]; then
-  # cmake only hands MSBuild a job count when one is asked for, and without it
-  # MSBuild builds a single node: one project at a time, whatever MTT does inside
-  # it. Ask for a node per core.
-  if [[ ! $BUILD_PARALLEL_LEVEL ]]; then
-    BUILD_PARALLEL_LEVEL="${NUMBER_OF_PROCESSORS:-1}"
-  fi
-  buildflags+=(--parallel "$BUILD_PARALLEL_LEVEL")
+  if [[ $CMAKE_GENERATOR == "Visual Studio"* ]]; then
+    # cmake only hands MSBuild a job count when it is asked for, and without it
+    # MSBuild builds a single node: one project at a time, whatever MTT does inside
+    # it. Ask for a node per core.
+    if [[ ! $BUILD_PARALLEL_LEVEL ]]; then
+      BUILD_PARALLEL_LEVEL="${NUMBER_OF_PROCESSORS:-1}"
+    fi
+    buildflags+=(--parallel "$BUILD_PARALLEL_LEVEL")
 
-  # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/
-  # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt
-  # EnforceProcessCountAcrossBuilds makes the CL_MPCount ceiling apply across all
-  # the projects the nodes are building, instead of each node running its own
-  # cl.exe processes and /m and MTT multiplying into an oversubscribed machine.
-  buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true -p:EnforceProcessCountAcrossBuilds=true -p:CL_MPCount="$BUILD_PARALLEL_LEVEL")
-  if [[ $BUILD_PROFILE ]]; then
-    # A performance summary says which projects and tasks cost what, the binary log
-    # keeps the full timeline; both are far too noisy to always produce. The files
-    # land next to the solution, i.e. in the build dir the workflow uploads. These
-    # join the MTT options above, which already carried the "--" separator.
-    buildflags+=("-flp:PerformanceSummary;v=q;LogFile=msbuild-perf.log" -bl:msbuild.binlog)
+    # Enable MTT, see https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/
+    # and https://devblogs.microsoft.com/cppblog/cpp-build-throughput-investigation-and-tune-up/#multitooltask-mtt
+    # EnforceProcessCountAcrossBuilds makes the CL_MPCount ceiling apply across all
+    # the projects the nodes are building, instead of each node running its own
+    # cl.exe processes and /m and MTT multiplying into an oversubscribed machine.
+    buildflags+=(-- -p:UseMultiToolTask=true -p:EnableClServerMode=true -p:EnforceProcessCountAcrossBuilds=true -p:CL_MPCount="$BUILD_PARALLEL_LEVEL")
+    if [[ $BUILD_PROFILE ]]; then
+      # A performance summary says which projects and tasks cost what, the binary log
+      # keeps the full timeline; both are far too noisy to always produce. The files
+      # land next to the solution, i.e. in the build dir the workflow uploads. These
+      # join the MTT options above, which already carried the "--" separator.
+      buildflags+=("-flp:PerformanceSummary;v=q;LogFile=msbuild-perf.log" -bl:msbuild.binlog)
+    fi
+  else
+    # Ninja runs the compiler itself, so it neither needs the MSVC switches nor
+    # wants a job count: it sizes its own pool for the cores it finds.
+    echo "ninja $(ninja --version)"
   fi
 fi
 

.github/workflows/desktop-build.yml

@@ -401,9 +401,26 @@ jobs:
             vcpkg_scope: 'vcpkg/windows10-x64-vs2026/'
             vcpkg_upload: 1    # only one leg per vcpkg_scope hands its archives over
 
-    name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }}
+          # A/B run: is Ninja faster than the Visual Studio generator on this runner?
+          # It is allowed to fail and is never published, see the steps that build
+          # a release below. Delete this entry once the answer is known.
+          - os: Windows
+            target: 10
+            runner: windows-2025
+
+            cmake_generator: Ninja
+            experiment: yes
+            make_package: 1
+            package_suffix: "-Win10-ninja"
+            qt_version: 6.11.*
+            qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets
+            type: Release
+            variant: ' (Ninja)'
+
+    name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }}${{ matrix.variant }}
     needs: configure
     runs-on: ${{ matrix.runner }}
+    continue-on-error: ${{ matrix.experiment == 'yes' }}
     timeout-minutes: 100
     env:
       BUILD_PROFILE: '${{ matrix.build_profile }}'
@@ -489,6 +506,14 @@ jobs:
           modules: ${{ matrix.qt_modules }}
           version: ${{ matrix.qt_version }}
 
+      # Ninja invokes cl.exe directly and needs the MSVC environment on PATH; the
+      # Visual Studio generator locates the compiler on its own and skips this.
+      - name: "[Windows/Ninja] Set up MSVC environment"
+        if: matrix.os == 'Windows' && matrix.cmake_generator == 'Ninja'
+        uses: TheMrMilchmann/setup-msvc-dev@v4.1.0
+        with:
+          arch: x64
+
       - name: "[Windows] Install NSIS"
         if: matrix.os == 'Windows'
         shell: bash
@@ -697,8 +722,11 @@ jobs:
           if-no-files-found: error
           path: ${{ steps.build.outputs.path }}
 
+      # Ninja writes its PDBs next to the binaries instead of into a Release
+      # subdirectory, and the experimental job is not a source of release debug
+      # symbols, so it keeps out of both.
       - name: "[Windows] Upload PDBs (Program Databases)"
-        if: matrix.os == 'Windows' && github.ref_type != 'tag'
+        if: matrix.os == 'Windows' && matrix.experiment != 'yes' && github.ref_type != 'tag'
         uses: actions/upload-artifact@v7
         with:
           if-no-files-found: error
@@ -720,7 +748,7 @@ jobs:
           retention-days: 7
 
       - name: "Upload to release"
-        if: needs.configure.outputs.tag != null && matrix.make_package == '1'
+        if: needs.configure.outputs.tag != null && matrix.make_package == '1' && matrix.experiment != 'yes'
         id: upload_release
         shell: bash
         env: