[Tests] Replay XML test files through a headless runner

Testing/README.md

@@ -0,0 +1,52 @@
+# GUI tests
+
+Widget-level playback tests for the Cockatrice client, driven by
+[QtTesting](https://github.com/Kitware/QtTesting) (`external/qttesting`, a submodule).
+Each `*_test.xml` file is a list of events that QtTesting replays against the running
+application; a test passes when every event is handled without error.
+
+## Running
+
+```bash
+git submodule update --init external/qttesting      # once
+cmake -S . -B build/debug -DCMAKE_BUILD_TYPE=Debug -DWITH_SERVER=0 -DWITH_GUI_TEST=ON
+cmake --build build/debug -j 24
+
+./Testing/run_test.sh --all                        # headless, offscreen
+./Testing/run_test.sh Testing/smoke_test.xml       # one file
+./Testing/run_test.sh --visual Testing/smoke_test.xml   # shows the window
+```
+
+`WITH_GUI_TEST` is on by default; a checkout without the submodule configures with a
+warning and no harness. `COCKATRICE_BIN` points the runner at another build, and
+`TEST_TIMEOUT` (seconds, default 180) bounds a single test — a failing event inside a
+modal dialog can otherwise leave the process alive. The client also hard-exits a few
+seconds after playback returns, whatever the result.
+
+The tests are **not** part of `ctest`: they need a card database and a developer's
+settings, so they stay a local loop for now.
+
+## Caveats
+
+- **Not hermetic.** The client reads the local settings and card database, so a test can
+  pass on one machine and fail on another. Avoid asserting things that restored
+  workspace state or user preferences decide (for example the number of open tabs).
+- **English labels only.** Menu entries are addressed by their (translated) text, so the
+  tests expect the source language. Widgets are addressed by objectName paths, which are
+  stable across translations.
+- Actions that open a dialog are followed by a `pause` event: playback continues in the
+  dialog's own event loop, and the pause gives the UI time to build itself.
+- Widget paths include Qt-internal names where a QTabWidget is involved
+  (`qt_tabwidget_stackedwidget`), and the deck views in a game are indexed
+  (`tabbedDeckView_0`, `tabbedDeckView_1`).
+
+## Writing a test
+
+A test is a `<events>` list. Widgets are named by their path from the main window, e.g.
+`MainWindow/tabSupervisor/stackedWidget/TabGame/phasesMenu`. Useful commands:
+
+| Command | Arguments | Effect |
+|---------|-----------|--------|
+| `activate` | action or menu name | triggers a QAction (menus are opened first) |
+| `key` | Qt key code | presses that key on the focused widget |
+| `pause` | milliseconds | waits, letting the UI settle |

Testing/run_test.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+# Run QtTesting XML test files against the Cockatrice build.
+#
+# Usage:
+#   ./Testing/run_test.sh [--visual] [--all | <test-file.xml> ...]
+#
+# Options:
+#   --visual    Run with native display (shows the GUI window).
+#               Default is headless (QT_QPA_PLATFORM=offscreen).
+#   --all       Run every *.xml file in Testing/.
+#
+# Environment:
+#   COCKATRICE_BIN   Path to the client binary (default: build/debug/cockatrice/cockatrice).
+#   TEST_TIMEOUT     Seconds a single test may run before it is killed (default: 180).
+#
+# Requires the debug build to exist at build/debug/cockatrice/cockatrice.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+COCKATRICE="${COCKATRICE_BIN:-$REPO_ROOT/build/debug/cockatrice/cockatrice}"
+TEST_DIR="$REPO_ROOT/Testing"
+TEST_TIMEOUT="${TEST_TIMEOUT:-180}"
+
+if [ ! -f "$COCKATRICE" ]; then
+    echo "Error: $COCKATRICE not found. Build first:" >&2
+    echo "  cmake --build build/debug" >&2
+    exit 1
+fi
+
+VISUAL=0
+if [ "${1:-}" = "--visual" ]; then
+    VISUAL=1
+    shift
+fi
+
+if [ "${1:-}" = "--all" ]; then
+    shift
+    TEST_FILES=("$TEST_DIR"/*.xml)
+    if [ ${#TEST_FILES[@]} -eq 0 ]; then
+        echo "No *.xml files found in $TEST_DIR" >&2
+        exit 1
+    fi
+elif [ $# -ge 1 ]; then
+    TEST_FILES=("$@")
+else
+    echo "Usage: $0 [--visual] [--all | <test-file.xml> ...]" >&2
+    echo "  Run a single test:  $0 Testing/smoke_test.xml" >&2
+    echo "  Run all tests:      $0 --all" >&2
+    echo "  Run visually:       $0 --visual Testing/smoke_test.xml" >&2
+    echo "  Run all visually:   $0 --visual --all" >&2
+    exit 1
+fi
+
+PASS=0
+FAIL=0
+FAILED_TESTS=()
+
+run_one() {
+    local file="$1"
+    local name
+    name="$(basename "$file")"
+
+    if [ "$VISUAL" -eq 1 ]; then
+        if timeout --foreground "$TEST_TIMEOUT" "$COCKATRICE" --no-connect --test "$file"; then
+            return 0
+        else
+            return 1
+        fi
+    else
+        if timeout --foreground "$TEST_TIMEOUT" env QT_QPA_PLATFORM=offscreen "$COCKATRICE" --no-connect --test "$file"; then
+            return 0
+        else
+            return 1
+        fi
+    fi
+}
+
+for f in "${TEST_FILES[@]}"; do
+    name="$(basename "$f")"
+    printf "%-40s " "$name"
+    if run_one "$f"; then
+        echo "PASS"
+        PASS=$((PASS + 1))
+    else
+        echo "FAIL"
+        FAIL=$((FAIL + 1))
+        FAILED_TESTS+=("$name")
+    fi
+done
+
+TOTAL=$((PASS + FAIL))
+echo ""
+echo "Results: $PASS/$TOTAL passed, $FAIL failed"
+if [ "$FAIL" -gt 0 ]; then
+    echo "Failed:"
+    for t in "${FAILED_TESTS[@]}"; do
+        echo "  $t"
+    done
+    exit 1
+fi

Testing/smoke_test.xml

@@ -0,0 +1,8 @@
+<events>
+  <!-- Open Help menu by activating it on the menu bar -->
+  <event widget="MainWindow/menubar" command="activate" arguments="helpMenu" />
+  <!-- Click About Cockatrice (opens modal QMessageBox) -->
+  <event widget="MainWindow/menubar/helpMenu" command="activate" arguments="&amp;About Cockatrice" />
+  <!-- Close the modal dialog by pressing Escape (Qt::Key_Escape = 16777216) -->
+  <event widget="MainWindow/aboutDialog" command="key" arguments="16777216" />
+</events>

cockatrice/CMakeLists.txt

@@ -563,6 +563,9 @@ target_link_libraries(
 cockatrice_target_enable_crash_reporting(cockatrice)
 
 if(WITH_GUI_TEST)
+  # recording.cpp translates between XML test files and the QtTesting event pipeline;
+  # it needs the QtTesting headers, so it only exists with the harness.
+  target_sources(cockatrice PRIVATE src/interface/recording.cpp)
   target_link_libraries(cockatrice PRIVATE QtTesting)
   target_compile_definitions(cockatrice PRIVATE WITH_GUI_TEST)
 endif()

cockatrice/src/interface/recording.cpp

@@ -0,0 +1,102 @@
+#include "recording.h"
+
+#include "pqEventTypes.h"
+
+#include <QFile>
+#include <QTextStream>
+#include <QXmlStreamReader>
+#include <QXmlStreamWriter>
+
+XMLEventObserver::XMLEventObserver(QObject *parent) : pqEventObserver(parent)
+{
+}
+
+XMLEventObserver::~XMLEventObserver() = default;
+
+void XMLEventObserver::setStream(QTextStream *stream)
+{
+    if (xmlStream) {
+        xmlStream->writeEndElement();
+        xmlStream->writeEndDocument();
+        xmlStream.reset();
+    }
+    if (Stream) {
+        *Stream << xmlString;
+    }
+    xmlString.clear();
+    pqEventObserver::setStream(stream);
+    if (Stream) {
+        xmlStream.reset(new QXmlStreamWriter(&xmlString));
+        xmlStream->setAutoFormatting(true);
+        xmlStream->writeStartDocument();
+        xmlStream->writeStartElement(QStringLiteral("events"));
+    }
+}
+
+void XMLEventObserver::onRecordEvent(const QString &widget,
+                                     const QString &command,
+                                     const QString &arguments,
+                                     const int &eventType)
+{
+    if (!xmlStream) {
+        return;
+    }
+
+    xmlStream->writeStartElement(QStringLiteral("event"));
+    xmlStream->writeAttribute(QStringLiteral("widget"), widget);
+    if (eventType == pqEventTypes::ACTION_EVENT) {
+        xmlStream->writeAttribute(QStringLiteral("command"), command);
+    } else { // pqEventTypes::CHECK_EVENT
+        xmlStream->writeAttribute(QStringLiteral("property"), command);
+    }
+    xmlStream->writeAttribute(QStringLiteral("arguments"), arguments);
+    xmlStream->writeEndElement();
+}
+
+XMLEventSource::XMLEventSource(QObject *parent) : pqEventSource(parent)
+{
+}
+
+XMLEventSource::~XMLEventSource() = default;
+
+void XMLEventSource::setContent(const QString &xmlFileName)
+{
+    xmlStream.reset();
+
+    QFile xml(xmlFileName);
+    if (!xml.open(QIODevice::ReadOnly)) {
+        qCWarning(RecordingLog) << "Could not read test file" << xmlFileName;
+        return;
+    }
+
+    xmlStream.reset(new QXmlStreamReader(xml.readAll()));
+}
+
+int XMLEventSource::getNextEvent(QString &widget, QString &command, QString &arguments, int &eventType)
+{
+    if (!xmlStream) {
+        return DONE;
+    }
+
+    while (!xmlStream->atEnd()) {
+        if (xmlStream->readNext() != QXmlStreamReader::StartElement || xmlStream->name() != QStringLiteral("event")) {
+            continue;
+        }
+
+        const QXmlStreamAttributes attributes = xmlStream->attributes();
+        eventType = pqEventTypes::ACTION_EVENT;
+        widget = attributes.value(QStringLiteral("widget")).toString();
+        command = attributes.value(QStringLiteral("command")).toString();
+        if (command.isEmpty()) {
+            // A recorded check event carries a property instead of a command.
+            command = attributes.value(QStringLiteral("property")).toString();
+        }
+        arguments = attributes.value(QStringLiteral("arguments")).toString();
+        return SUCCESS;
+    }
+
+    if (xmlStream->hasError()) {
+        qCWarning(RecordingLog) << "Malformed test file:" << xmlStream->errorString();
+    }
+    return DONE;
+}

cockatrice/src/interface/recording.h

@@ -0,0 +1,54 @@
+#ifndef COCKATRICE_RECORDING_H
+#define COCKATRICE_RECORDING_H
+
+#include "pqEventObserver.h"
+#include "pqEventSource.h"
+
+#include <QLoggingCategory>
+#include <QScopedPointer>
+
+inline Q_LOGGING_CATEGORY(RecordingLog, "recording");
+
+class QXmlStreamReader;
+class QXmlStreamWriter;
+
+/**
+ * @brief Writes a session recording as an XML test file.
+ *
+ * The events are buffered in memory and flushed to the stream when recording
+ * ends, because the stream is handed over once the file name is known.
+ */
+class XMLEventObserver : public pqEventObserver
+{
+    QScopedPointer<QXmlStreamWriter> xmlStream;
+    QString xmlString;
+
+public:
+    explicit XMLEventObserver(QObject *parent);
+    ~XMLEventObserver() override;
+
+protected:
+    void setStream(QTextStream *stream) override;
+    void onRecordEvent(const QString &widget,
+                       const QString &command,
+                       const QString &arguments,
+                       const int &eventType) override;
+};
+
+/**
+ * @brief Plays the events of an XML test file back, one at a time.
+ */
+class XMLEventSource : public pqEventSource
+{
+    QScopedPointer<QXmlStreamReader> xmlStream;
+
+public:
+    explicit XMLEventSource(QObject *parent);
+    ~XMLEventSource() override;
+
+protected:
+    void setContent(const QString &xmlFileName) override;
+    int getNextEvent(QString &widget, QString &command, QString &arguments, int &eventType) override;
+};
+
+#endif // COCKATRICE_RECORDING_H

cockatrice/src/interface/window_main.cpp

@@ -49,6 +49,13 @@
 #include "intents/url_parser.h"
 #include "latency_tracker.h"
 #include "logger.h"
+#ifdef WITH_GUI_TEST
+#include "pqEventObserver.h"
+#include "pqEventSource.h"
+#include "pqEventTypes.h"
+#include "pqTestUtility.h"
+#include "recording.h"
+#endif
 #include "pixel_map_generator.h"
 #include "version_string.h"
 #include "widgets/dialogs/dlg_connect.h"
@@ -306,6 +313,7 @@ void MainWindow::actAbout()
     mb.setIconPixmap(
         themePixmap(QStringLiteral("cockatrice")).scaled(64, 64, Qt::KeepAspectRatio, Qt::SmoothTransformation));
     mb.setTextInteractionFlags(Qt::TextBrowserInteraction);
+    mb.setObjectName("aboutDialog");
     mb.exec();
 }
 
@@ -344,6 +352,24 @@ void MainWindow::actOpenSettingsFolder()
     QtUtils::openPathInFileManager(dir);
 }
 
+#ifdef WITH_GUI_TEST
+void MainWindow::actRecord()
+{
+    QString filename = QFileDialog::getSaveFileName(this, tr("Test File Name"), QString(), tr("XML Files (*.xml)"));
+    if (!filename.isEmpty()) {
+        testUtility->recordTests(filename);
+    }
+}
+
+void MainWindow::actPlayRecording()
+{
+    QString filename = QFileDialog::getOpenFileName(this, tr("Test File Name"), QString(), tr("XML Files (*.xml)"));
+    if (!filename.isEmpty()) {
+        testUtility->playTests(filename);
+    }
+}
+#endif
+
 void MainWindow::retranslateUi()
 {
     connectionController->refreshWindowTitle();
@@ -388,6 +414,10 @@ void MainWindow::retranslateUi()
     aViewLog->setText(tr("View &Debug Log"));
     aOpenSettingsFolder->setText(tr("Open Settings Folder"));
     aFirstRunWizard->setText(tr("Re-run Onboarding Wizard..."));
+#ifdef WITH_GUI_TEST
+    aRecord->setText(tr("&Record Test..."));
+    aPlayRecording->setText(tr("&Play Test..."));
+#endif
 
     aShow->setText(tr("Show/Hide"));
 
@@ -451,6 +481,12 @@ void MainWindow::createActions()
     connect(aOpenSettingsFolder, &QAction::triggered, this, &MainWindow::actOpenSettingsFolder);
     aFirstRunWizard = new QAction(this);
     connect(aFirstRunWizard, &QAction::triggered, this, [this] { runFirstRunWizard(); });
+#ifdef WITH_GUI_TEST
+    aRecord = new QAction(this);
+    connect(aRecord, &QAction::triggered, this, &MainWindow::actRecord);
+    aPlayRecording = new QAction(this);
+    connect(aPlayRecording, &QAction::triggered, this, &MainWindow::actPlayRecording);
+#endif
 
     aShow = new QAction(this);
     connect(aShow, &QAction::triggered, this, &MainWindow::actShow);
@@ -491,6 +527,7 @@ void MainWindow::createActions()
 
 void MainWindow::createMenus()
 {
+    menuBar()->setObjectName("menubar");
     cockatriceMenu = menuBar()->addMenu(QString());
     cockatriceMenu->addAction(aConnect);
     cockatriceMenu->addAction(aDisconnect);
@@ -519,6 +556,7 @@ void MainWindow::createMenus()
     tabsMenu = menuBar()->addMenu(QString());
 
     helpMenu = menuBar()->addMenu(QString());
+    helpMenu->setObjectName("helpMenu");
     helpMenu->addAction(aAbout);
     helpMenu->addAction(aTips);
     helpMenu->addSeparator();
@@ -531,12 +569,19 @@ void MainWindow::createMenus()
     helpMenu->addAction(aOpenSettingsFolder);
     helpMenu->addSeparator();
     helpMenu->addAction(aFirstRunWizard);
+#ifdef WITH_GUI_TEST
+    helpMenu->addSeparator();
+    helpMenu->addAction(aRecord);
+    helpMenu->addAction(aPlayRecording);
+#endif
 }
 
 MainWindow::MainWindow(QWidget *parent)
-    : QMainWindow(parent), localServer(nullptr), bHasActivated(false), askedForDbUpdater(false),
+    : QMainWindow(parent), localServer(nullptr), bHasActivated(false), askedForDbUpdater(false), noConnect(false),
       cardUpdateProcess(nullptr), logviewDialog(nullptr)
 {
+    setObjectName("MainWindow");
+
     connect(&SettingsCache::instance().cacheStorage(), &CacheStorageSettings::pixmapCacheSizeChanged, this,
             &MainWindow::pixmapCacheSizeChanged);
     pixmapCacheSizeChanged(SettingsCache::instance().cacheStorage().getPixmapCacheSize());
@@ -622,6 +667,13 @@ MainWindow::MainWindow(QWidget *parent)
     // run startup check async
     QTimer::singleShot(0, this, &MainWindow::startupConfigCheck);
     QTimer::singleShot(0, this, &MainWindow::applyStartupDestination);
+
+#ifdef WITH_GUI_TEST
+    testUtility = new pqTestUtility(this);
+    testUtility->setRecordWithDialog(true);
+    testUtility->addEventObserver("xml", new XMLEventObserver(this));
+    testUtility->addEventSource("xml", new XMLEventSource(this));
+#endif
 }
 
 void MainWindow::startupConfigCheck()
@@ -776,6 +828,11 @@ void MainWindow::runFirstRunWizard(bool firstRun)
  */
 void MainWindow::applyStartupDestination()
 {
+    if (noConnect) {
+        qCInfo(WindowMainStartupAutoconnectLog) << "Startup destination suppressed (--no-connect)";
+        return;
+    }
+
     // An explicit command-line connect takes precedence over the startup destination.
     if (!connectTo.isEmpty()) {
         return;
@@ -1045,6 +1102,10 @@ void MainWindow::handleCockatriceLink(const QString &url)
 
 void MainWindow::attemptStartupAutoConnect()
 {
+    if (noConnect) {
+        qCInfo(WindowMainStartupAutoconnectLog) << "Auto-connect suppressed (--no-connect)";
+        return;
+    }
     if (startupAutoConnectAttempted || skipStartupAutoConnect) {
         return;
     }

cockatrice/src/interface/window_main.h

@@ -48,6 +48,7 @@
 class QAction;
 class QToolButton;
 class QWidget;
+class pqTestUtility;
 struct LocalGameOptions;
 
 inline Q_LOGGING_CATEGORY(WindowMainLog, "window_main");
@@ -104,6 +105,11 @@ private slots:
     void actUpdate();
     void actViewLog();
     void actOpenSettingsFolder();
+#ifdef WITH_GUI_TEST
+    void actRecord();
+    void actPlayRecording();
+#endif
+
     void actShow();
     void showWindowIfHidden();
     void onUrlChainFinished(bool connected);
@@ -185,6 +191,9 @@ private:
         *aReloadCardDatabase;
     QAction *aTips, *aUpdate, *aCheckCardUpdates, *aCheckCardUpdatesBackground, *aFirstRunWizard, *aStatusBar,
         *aViewLog, *aOpenSettingsFolder;
+#ifdef WITH_GUI_TEST
+    QAction *aRecord, *aPlayRecording;
+#endif
 
     TabSupervisor *tabSupervisor;
     IntentUrlParser *urlParser;
@@ -208,6 +217,7 @@ private:
     bool skipStartupAutoConnect = false;
     bool startupAutoConnectAttempted = false;
     bool firstRunWizardActive = false;
+    bool noConnect = false; ///< --no-connect: skip every startup connection
     QProcess *cardUpdateProcess;
     QByteArray cardUpdateOutputBuffer;
     QByteArray cardUpdateErrorOutput;  ///< diagnostic output collected from the updater's stderr
@@ -219,6 +229,9 @@ private:
     GameReplay *replay;
     DlgTipOfTheDay *tip;
     QUrl connectTo;
+#ifdef WITH_GUI_TEST
+    pqTestUtility *testUtility;
+#endif
 
 public:
     explicit MainWindow(QWidget *parent = nullptr);
@@ -226,6 +239,16 @@ public:
     {
         connectTo = QUrl(QString("cockatrice://%1").arg(url));
     }
+    /**
+     * @brief Suppresses every startup connection: --connect, auto-connect and the
+     *        configured startup destination. Used by the GUI test harness so a
+     *        playback run never dials out.
+     */
+    void setNoConnect(bool value)
+    {
+        noConnect = value;
+    }
+
     // When set, the window's own startup connection (--connect or auto-connect
     // on first activation) is skipped. Used for activation launches: the intent
     // chain triggered by a cockatrice:// URL owns the connection, and letting
@@ -266,6 +289,13 @@ public:
      */
     void requestInstallerClose();
 
+#ifdef WITH_GUI_TEST
+    pqTestUtility *getTestUtility() const
+    {
+        return testUtility;
+    }
+#endif
+
 protected:
     void closeEvent(QCloseEvent *event) override;
     void changeEvent(QEvent *event) override;

cockatrice/src/main.cpp

@@ -39,6 +39,10 @@
 #include "single_instance_manager.h"
 #include "version_string.h"
 
+#ifdef WITH_GUI_TEST
+#include "pqTestUtility.h"
+#endif
+
 #include <QApplication>
 #include <QByteArray>
 #include <QByteArrayView>
@@ -62,10 +66,12 @@
 #include <QString>
 #include <QStringList>
 #include <QStringLiteral>
+#include <QThread>
 #include <QTranslator>
 #include <QUrl>
 #include <QtGlobal>
 #include <algorithm>
+#include <cstdlib>
 #include <libcockatrice/card/database/card_database_manager.h>
 #include <libcockatrice/rng/rng_sfmt.h>
 #include <libcockatrice/settings/card_database_settings.h>
@@ -343,9 +349,16 @@ int main(int argc, char *argv[])
     parser.addOptions(
         {{{"c", "connect"}, QCoreApplication::translate("main", "Connect on startup"), "user:pass@host:port"},
          {{"d", "debug-output"}, QCoreApplication::translate("main", "Debug to file")},
-         {"simulate-crash",
-          QCoreApplication::translate("main",
-                                      "Show the crash-report dialog (saves a mock file when no DSN is configured)")}});
+         {"simulate-crash", QCoreApplication::translate(
+                                "main", "Show the crash-report dialog (saves a mock file when no DSN is configured)")},
+         {"no-connect", QCoreApplication::translate("main", "Suppress auto-connect on startup")}
+#ifdef WITH_GUI_TEST
+         ,
+         {{"t", "test"},
+          QCoreApplication::translate("main", "Play back a QtTesting XML test file and exit"),
+          "test-file.xml"}
+#endif
+        });
 
     parser.process(app);
 
@@ -483,6 +496,9 @@ int main(int argc, char *argv[])
     if (parser.isSet("connect")) {
         ui.setConnectTo(parser.value("connect"));
     }
+    if (parser.isSet("no-connect")) {
+        ui.setNoConnect(true);
+    }
     qCInfo(MainLog) << "MainWindow constructor finished";
 
     ui.setWindowIcon(themePixmap(QStringLiteral("cockatrice")));
@@ -523,6 +539,27 @@ int main(int argc, char *argv[])
         handleActivation(file);
     }
 
+#ifdef WITH_GUI_TEST
+    if (parser.isSet("test")) {
+        QString testFile = parser.value("test");
+        qCInfo(MainLog) << "Playing test file:" << testFile;
+        QTimer::singleShot(0, &ui, [&ui, testFile]() {
+            bool success = ui.getTestUtility()->playTests(testFile);
+            qCInfo(MainLog) << "Test playback" << (success ? "succeeded" : "FAILED");
+            qApp->exit(success ? 0 : 1);
+
+            // An event that fails while a modal dialog is open aborts the dispatcher,
+            // but the dialog's nested event loop can then keep the application alive
+            // forever. Hard-exit after a grace period so a red test always terminates.
+            auto *watchdog = QThread::create([success] {
+                QThread::sleep(10);
+                ::_Exit(success ? 0 : 1);
+            });
+            watchdog->start();
+        });
+    }
+#endif
+
     // Connect to future file/URL events from other instances
     QObject::connect(&instance, &SingleInstanceManager::filesReceived, [&handleActivation](const QStringList &files) {
         for (const QString &file : files) {