[Oracle/Client] Report card database download progress (#7253)

* [Oracle/Client] Report card database download progress

Card database updates ran invisibly: MTGJSON parsing spun an indeterminate
bar on the UI thread and the set import blocked the window, while the
onboarding wizard spawned `oracle -b` with no progress to show at all.

- Add byte-level scan progress to `RawJson::scanSetRanges` via an optional
  callback, throttled to ~100 reports per scan.
- Emit `OracleImporter::dataReadProgress` during the scan and import sets on a
  worker thread, driving the wizard's progress bar per set.
- With `-b`, write machine-readable `PROGRESS <stage> <done> <total>` lines to
  stdout for the download/scan/import stages; stderr keeps the log output.
- Parse the oracle stdout in `MainWindow` and forward it to the onboarding
  wizard, giving the card database step a determinate bar with stage-specific
  status text.
- Guard the async workers against the wizard being closed mid-run.
- Add Google Test coverage for scan progress reporting.

* [Oracle/Client] Harden oracle progress workers and quit prompt

Address review feedback on the download-progress change: decompress and read
sets files off the UI thread, cancel the load/import workers before the
wizard can tear down the importer, and show an 'Extracting file...' status
plus a clean 100% tail so the poll never looks stuck. Quitting Cockatrice
while a card database update runs now asks for confirmation.

* Show 100% for 500ms on complete.

* Disable buttons on set import until done.

* Clean up progress bar.

* Drop wrapper around lambda

---------

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>

oracle/src/oracleimporter.cpp

@@ -47,8 +47,16 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s
 
 bool OracleImporter::readSetsFromByteArray(QByteArray data)
 {
+    const RawJson::ScanProgressCallback progress =
+        progressReporting
+            ? [this](
+                  qsizetype bytesRead,
+                  qsizetype
+                      totalBytes) { emit dataReadProgress(static_cast<int>(bytesRead), static_cast<int>(totalBytes)); }
+            : RawJson::ScanProgressCallback{};
+
     RawJson::ScanError error;
-    const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error);
+    const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error, progress);
     if (error.isError()) {
         qDebug() << "error: RawJson::scanSetRanges():" << error.message;
         return false;
@@ -572,6 +580,8 @@ int OracleImporter::startImport()
 {
     static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController();
 
+    importCancelled.storeRelease(0);
+
     // Pre-allocate the cards hash to avoid rehashing during import. Keys are
     // distinct card names while raw ranges only count printings (AllPrintings
     // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact
@@ -591,6 +601,12 @@ int OracleImporter::startImport()
     int setIndex = 0;
 
     for (const SetToDownload &curSetToParse : allSets) {
+        if (importCancelled.loadAcquire()) {
+            // The wizard was closed mid-import: stop at the next set boundary so
+            // the caller can wait for this future without processing every set.
+            break;
+        }
+
         CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(),
                                                  curSetToParse.getLongName(), curSetToParse.getSetType(),
                                                  curSetToParse.getReleaseDate(), curSetToParse.getPriority());

oracle/src/oracleimporter.h

@@ -3,6 +3,7 @@
 
 #include "raw_json_scanner.h"
 
+#include <QAtomicInt>
 #include <QByteArray>
 #include <QJsonArray>
 #include <QJsonObject>
@@ -155,6 +156,21 @@ private:
      */
     QByteArray rawSetsData;
 
+    /**
+     * Whether readSetsFromByteArray() should report scan progress via
+     * dataReadProgress. A background run routes that signal to stdout (for the
+     * hosting Cockatrice client to parse); the flag exists to skip the scanner
+     * instrumentation entirely when no consumer needs it.
+     */
+    bool progressReporting = true;
+
+    /**
+     * Atomic "please stop importing" flag. startImport() checks it between sets
+     * so a wizard being closed mid-import can be torn down without waiting for
+     * the whole import (or racing it).
+     */
+    QAtomicInt importCancelled;
+
     CardInfoPtr addCard(QString name,
                         const QString &text,
                         bool isToken,
@@ -167,12 +183,35 @@ signals:
 
 public:
     explicit OracleImporter(QObject *parent = nullptr);
+    /**
+     * @brief Controls whether readSetsFromByteArray() instruments the raw scan.
+     *
+     * When enabled (the default) the raw scanner reports progress via
+     * dataReadProgress(), which an interactive wizard shows on its progress bar
+     * and a background run routes to stdout for the hosting client. Switch it
+     * off only when nothing will consume scan progress.
+     */
+    void setProgressReporting(bool enabled)
+    {
+        progressReporting = enabled;
+    }
     /**
      * Scans the given JSON document for set metadata. Takes the data by value so
      * the wizard can hand over its decompressed buffer without copying it.
      */
     bool readSetsFromByteArray(QByteArray data);
     int startImport();
+    /**
+     * @brief Requests an in-flight startImport() to stop at the next set boundary.
+     *
+     * Works by setting an atomic flag that startImport() polls between sets, so
+     * cancelImport() followed by a short waitForFinished() on the running future is
+     * safe the moment the wizard is about to be destroyed.
+     */
+    void cancelImport()
+    {
+        importCancelled.storeRelease(1);
+    }
     bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion);
     int importCardsFromSet(const CardSetPtr &currentSet, const QJsonArray &cardsList);
     /**

oracle/src/oraclewizard.cpp

@@ -110,6 +110,24 @@ void OracleWizard::accept()
     QDialog::accept();
 }
 
+void OracleWizard::reject()
+{
+    // The wizard is being closed while a page may still run a worker on the
+    // importer. Ask it to stop before the wizard (and the importer child) is
+    // destroyed, so the worker thread never touches freed memory.
+    if (auto *active = dynamic_cast<OracleWizardPage *>(currentPage())) {
+        active->cancelWork();
+    }
+    QWizard::reject();
+}
+
+void OracleWizard::runInBackground()
+{
+    backgroundMode = true;
+    hide();
+    currentPage()->initializePage();
+}
+
 void OracleWizard::enableButtons()
 {
     button(QWizard::NextButton)->setDisabled(false);

oracle/src/oraclewizard.h

@@ -23,6 +23,7 @@ class OracleWizard : public QWizard
 public:
     explicit OracleWizard(QWidget *parent = nullptr);
     void accept() override;
+    void reject() override;
     void enableButtons();
     void disableButtons();
     void retranslateUi();
@@ -52,12 +53,7 @@ public:
     }
     bool saveTokensToFile(const QString &fileName);
 
-    void runInBackground()
-    {
-        backgroundMode = true;
-        hide();
-        currentPage()->initializePage();
-    }
+    void runInBackground();
 
 public:
     OracleImporter *importer;

oracle/src/pages.cpp

@@ -12,6 +12,7 @@
 #include <QBuffer>
 #include <QCheckBox>
 #include <QComboBox>
+#include <QDebug>
 #include <QDir>
 #include <QFileDialog>
 #include <QGridLayout>
@@ -20,14 +21,17 @@
 #include <QMessageBox>
 #include <QNetworkAccessManager>
 #include <QNetworkReply>
+#include <QPointer>
 #include <QProgressBar>
 #include <QPushButton>
 #include <QRadioButton>
 #include <QScrollBar>
 #include <QStandardPaths>
 #include <QTextEdit>
+#include <QTextStream>
 #include <QtConcurrent>
 #include <QtGui>
+#include <cstdio>
 #include <libcockatrice/settings/personal_settings.h>
 
 #ifdef HAS_LZMA
@@ -53,6 +57,115 @@
 #define ALLSETS_URL "https://www.mtgjson.com/api/v5/AllPrintings.json"
 #endif
 
+/**
+ * @brief Emits one machine-readable background-run progress line to stdout.
+ *
+ * Used only in background mode, so the hosting Cockatrice client can parse these
+ * lines to drive a determinate progress bar. stderr stays reserved for
+ * human-readable log output.
+ */
+static void emitBackgroundProgress(const char *stage, qint64 done, qint64 total)
+{
+    QTextStream out(stdout);
+    out << "PROGRESS " << stage << ' ' << done << ' ' << total << '\n';
+    out.flush();
+}
+
+namespace
+{
+
+/**
+ * @brief Decompresses and dispatches a sets-file payload on a worker thread.
+ *
+ * Iteratively unwraps xz/zip compression, then either hands the JSON to the
+ * importer (which reports scan progress via dataReadProgress) or returns the raw
+ * XML for the plain-XML path. Must never touch the wizard or the page: the caller
+ * consumes the returned LoadSetsResult on the UI thread in importFinished().
+ */
+LoadSetsResult loadSetsData(const QPointer<OracleImporter> &importer, QByteArray data)
+{
+    LoadSetsResult result;
+
+    while (true) {
+        if (data.startsWith(XZ_SIGNATURE)) {
+#ifdef HAS_LZMA
+            QBuffer inBuffer(&data);
+            QByteArray decompressed;
+            QBuffer outBuffer(&decompressed);
+            inBuffer.open(QBuffer::ReadOnly);
+            outBuffer.open(QBuffer::WriteOnly);
+            XzDecompressor xz;
+            if (!xz.decompress(&inBuffer, &outBuffer)) {
+                result.errorMessage = LoadSetsPage::tr("Xz extraction failed.");
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            data = decompressed;
+            continue;
+#else
+            result.errorMessage =
+                LoadSetsPage::tr("Sorry, this version of Oracle does not support xz compressed files.");
+            result.offerUncompressedFallback = true;
+            return result;
+#endif
+        }
+
+        if (data.startsWith(ZIP_SIGNATURE)) {
+#ifdef HAS_ZLIB
+            QBuffer inBuffer(&data);
+            UnZip uz;
+            const UnZip::ErrorCode openEc = uz.openArchive(&inBuffer);
+            if (openEc != UnZip::Ok) {
+                result.errorMessage = LoadSetsPage::tr("Failed to open Zip archive: %1.").arg(uz.formatError(openEc));
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            if (uz.fileList().size() != 1) {
+                result.errorMessage =
+                    LoadSetsPage::tr("Zip extraction failed: the Zip archive doesn't contain exactly one file.");
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            const QString fileName = uz.fileList().at(0);
+            QByteArray decompressed;
+            QBuffer outBuffer(&decompressed);
+            outBuffer.open(QBuffer::ReadWrite);
+            const UnZip::ErrorCode ec = uz.extractFile(fileName, &outBuffer);
+            uz.closeArchive();
+            if (ec != UnZip::Ok) {
+                result.errorMessage = LoadSetsPage::tr("Zip extraction failed: %1.").arg(uz.formatError(ec));
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            data = decompressed;
+            continue;
+#else
+            result.errorMessage = LoadSetsPage::tr("Sorry, this version of Oracle does not support zipped files.");
+            result.offerUncompressedFallback = true;
+            return result;
+#endif
+        }
+        break;
+    }
+
+    if (data.startsWith("<")) {
+        result.ok = true;
+        result.plainXml = true;
+        result.xmlData = std::move(data);
+        return result;
+    }
+
+    if (data.startsWith("{")) {
+        result.ok = importer && importer->readSetsFromByteArray(std::move(data));
+        return result;
+    }
+
+    result.errorMessage = LoadSetsPage::tr("Failed to interpret downloaded data.");
+    return result;
+}
+
+} // namespace
+
 #define TOKENS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Token/master/tokens.xml"
 #define SPOILERS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Spoiler/files/spoiler.xml"
 
@@ -182,6 +295,11 @@ LoadSetsPage::LoadSetsPage(QWidget *parent) : OracleWizardPage(parent)
     setLayout(layout);
 }
 
+bool LoadSetsPage::isComplete() const
+{
+    return !loadActive;
+}
+
 void LoadSetsPage::initializePage()
 {
     urlLineEdit->setText(wizard()->settings->value("allsetsurl", ALLSETS_URL).toString());
@@ -279,18 +397,13 @@ bool LoadSetsPage::validatePage()
             return false;
         }
 
-        if (!setsFile.open(QIODevice::ReadOnly)) {
-            QMessageBox::critical(nullptr, tr("Error"), tr("Cannot open file '%1'.").arg(fileLineEdit->text()));
-            return false;
-        }
-
         wizard()->disableButtons();
         setEnabled(false);
 
         wizard()->setCardSourceUrl(setsFile.fileName());
         wizard()->setCardSourceVersion("unknown");
 
-        readSetsFromByteArray(setsFile.readAll());
+        readSetsFromFile(setsFile.fileName());
     }
 
     return false;
@@ -339,6 +452,9 @@ void LoadSetsPage::downloadSetsFile(const QUrl &url)
 
 void LoadSetsPage::actDownloadProgressSetsFile(qint64 received, qint64 total)
 {
+    if (wizard()->backgroundMode) {
+        emitBackgroundProgress("download", received, total);
+    }
     if (total > 0) {
         progressBar->setMaximum(static_cast<int>(total));
         progressBar->setValue(static_cast<int>(received));
@@ -384,106 +500,97 @@ void LoadSetsPage::actDownloadFinishedSetsFile()
     reply->deleteLater();
 }
 
-void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
+void LoadSetsPage::updateParsingProgress(int bytesRead, int totalBytes)
 {
-    // show an infinite progressbar
+    if (totalBytes <= 0) {
+        return;
+    }
+    progressBar->setRange(0, totalBytes);
+    progressBar->setValue(bytesRead);
+    const int percent = static_cast<int>((100.0 * bytesRead) / totalBytes);
+    progressLabel->setText(tr("Parsing file (%1%)").arg(percent));
+}
+
+void LoadSetsPage::scanProgressToStdout(int bytesRead, int totalBytes)
+{
+    emitBackgroundProgress("scan", bytesRead, totalBytes);
+}
+
+void LoadSetsPage::beginLoadSets(bool compressedFile)
+{
+    // Show an infinite progressbar while the worker decompresses; the scan
+    // steals the label via dataReadProgress as soon as it starts.
     progressBar->setMaximum(0);
     progressBar->setMinimum(0);
     progressBar->setValue(0);
-    progressLabel->setText(tr("Parsing file"));
+    progressLabel->setText(compressedFile ? tr("Extracting file...") : tr("Parsing file"));
     progressLabel->show();
     progressBar->show();
 
+    // Keep Next disabled (via completeChanged) until the worker reports in;
+    // updateButtonStates() re-evaluates button state whenever we re-enable.
+    loadActive = true;
+    emit completeChanged();
+
     wizard()->downloadedPlainXml = false;
     wizard()->xmlData.clear();
-    readSetsFromByteArrayRef(_data);
+
+    if (wizard()->backgroundMode) {
+        connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::scanProgressToStdout,
+                Qt::UniqueConnection);
+    } else {
+        connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::updateParsingProgress,
+                Qt::UniqueConnection);
+    }
 }
 
-void LoadSetsPage::readSetsFromByteArrayRef(QByteArray &_data)
+void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
 {
-    // unzip the file if needed
-    if (_data.startsWith(XZ_SIGNATURE)) {
-#ifdef HAS_LZMA
-        // zipped file
-        auto *inBuffer = new QBuffer(&_data);
-        auto newData = QByteArray();
-        auto *outBuffer = new QBuffer(&newData);
-        inBuffer->open(QBuffer::ReadOnly);
-        outBuffer->open(QBuffer::WriteOnly);
-        XzDecompressor xz;
-        if (!xz.decompress(inBuffer, outBuffer)) {
-            zipDownloadFailed(tr("Xz extraction failed."));
-            return;
+    const bool compressed = _data.startsWith(XZ_SIGNATURE) || _data.startsWith(ZIP_SIGNATURE);
+    beginLoadSets(compressed);
+
+    // Decompress and scan off the UI thread so a large download can't freeze the window.
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    future = QtConcurrent::run(
+        [importer, data = std::move(_data)]() mutable { return loadSetsData(importer, std::move(data)); });
+    watcher.setFuture(future);
+}
+
+void LoadSetsPage::readSetsFromFile(const QString &fileName)
+{
+    // Peek at the header on the UI thread so the status text can distinguish
+    // "Extracting file..." from a plain JSON parse; the full read happens in the worker.
+    QFile headerFile(fileName);
+    bool compressed = false;
+    if (headerFile.open(QIODevice::ReadOnly)) {
+        const QByteArray header = headerFile.read(6);
+        compressed = header.startsWith(XZ_SIGNATURE) || header.startsWith(ZIP_SIGNATURE);
+    }
+    beginLoadSets(compressed);
+
+    // Read, decompress and scan off the UI thread (a plain JSON can be hundreds
+    // of MB, so even the read itself must not block the window).
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    future = QtConcurrent::run([importer, fileName]() mutable -> LoadSetsResult {
+        QFile file(fileName);
+        if (!file.open(QIODevice::ReadOnly)) {
+            LoadSetsResult readError;
+            readError.errorMessage = LoadSetsPage::tr("Cannot open file '%1'.").arg(fileName);
+            return readError;
         }
-        _data.clear();
-        readSetsFromByteArrayRef(newData);
-        return;
-#else
-        zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files."));
+        return loadSetsData(importer, file.readAll());
+    });
+    watcher.setFuture(future);
+}
 
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        return;
-#endif
-    } else if (_data.startsWith(ZIP_SIGNATURE)) {
-#ifdef HAS_ZLIB
-        // zipped file
-        auto *inBuffer = new QBuffer(&_data);
-        auto newData = QByteArray();
-        auto *outBuffer = new QBuffer(&newData);
-        QString fileName;
-        UnZip::ErrorCode ec;
-        UnZip uz;
-
-        ec = uz.openArchive(inBuffer);
-        if (ec != UnZip::Ok) {
-            zipDownloadFailed(tr("Failed to open Zip archive: %1.").arg(uz.formatError(ec)));
-            return;
-        }
-
-        if (uz.fileList().size() != 1) {
-            zipDownloadFailed(tr("Zip extraction failed: the Zip archive doesn't contain exactly one file."));
-            return;
-        }
-        fileName = uz.fileList().at(0);
-
-        outBuffer->open(QBuffer::ReadWrite);
-        ec = uz.extractFile(fileName, outBuffer);
-        if (ec != UnZip::Ok) {
-            zipDownloadFailed(tr("Zip extraction failed: %1.").arg(uz.formatError(ec)));
-            uz.closeArchive();
-            return;
-        }
-        _data.clear();
-        readSetsFromByteArrayRef(newData);
-        return;
-#else
-        zipDownloadFailed(tr("Sorry, this version of Oracle does not support zipped files."));
-
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        return;
-#endif
-    } else if (_data.startsWith("{")) {
-        // Start the computation.
-        jsonData = std::move(_data);
-        future = QtConcurrent::run([this] { return wizard()->importer->readSetsFromByteArray(std::move(jsonData)); });
-        watcher.setFuture(future);
-    } else if (_data.startsWith("<")) {
-        // save xml file and don't do any processing
-        wizard()->downloadedPlainXml = true;
-        wizard()->xmlData = std::move(_data);
-        importFinished();
-    } else {
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        QMessageBox::critical(this, tr("Error"), tr("Failed to interpret downloaded data."));
+void LoadSetsPage::cancelWork()
+{
+    // The scan is short-lived; just wait it out before the wizard (and its
+    // importer) can be torn down underneath the worker thread.
+    if (future.isRunning()) {
+        future.cancel();
+        watcher.cancel();
+        future.waitForFinished();
     }
 }
 
@@ -509,17 +616,65 @@ void LoadSetsPage::zipDownloadFailed(const QString &message)
 
 void LoadSetsPage::importFinished()
 {
+    loadActive = false;
+    emit completeChanged();
     wizard()->enableButtons();
     setEnabled(true);
-    progressLabel->hide();
-    progressBar->hide();
 
-    if (wizard()->downloadedPlainXml || watcher.future().result()) {
-        wizard()->next();
-    } else {
-        QMessageBox::critical(this, tr("Error"),
-                              tr("The file was retrieved successfully, but it does not contain any sets data."));
+    const LoadSetsResult result = watcher.result();
+
+    if (result.plainXml) {
+        wizard()->downloadedPlainXml = true;
+        wizard()->xmlData = result.xmlData;
     }
+
+    if (wizard()->backgroundMode) {
+        progressLabel->hide();
+        progressBar->hide();
+        if (!result.errorMessage.isEmpty()) {
+            qWarning() << result.errorMessage;
+        } else if (!result.ok && !result.plainXml) {
+            qWarning() << tr("The file was retrieved successfully, but it does not contain any sets data.");
+        }
+        emit readyToContinue();
+        return;
+    }
+
+    const auto fail = [this](const QString &message) {
+        progressLabel->hide();
+        progressBar->hide();
+        QMessageBox::critical(this, tr("Error"), message);
+    };
+
+    if (!result.errorMessage.isEmpty()) {
+        if (result.offerUncompressedFallback) {
+            zipDownloadFailed(result.errorMessage);
+            return;
+        }
+        fail(result.errorMessage);
+        return;
+    }
+
+    if (!result.ok && !result.plainXml) {
+        fail(tr("The file was retrieved successfully, but it does not contain any sets data."));
+        return;
+    }
+
+    // Snap the bar to 100% and hold it there for a moment so the completed state
+    // is actually visible before the next page's own import progress takes over
+    // (a zero-length deferral can still fire before the repaint is delivered).
+    progressBar->setMaximum(1);
+    progressBar->setValue(1);
+    progressLabel->setText(tr("Parsing file (100%)"));
+    QTimer::singleShot(500, this, [this] {
+        if (wizard()->currentPage() == this) {
+            // Leave the page pristine: hide the completed load bar so a later
+            // Back from the save page doesn't show stale progress.
+            progressLabel->hide();
+            progressBar->hide();
+            wizard()->next();
+        }
+    });
 }
 
 SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
@@ -527,49 +682,113 @@ SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
     pathLabel = new QLabel(this);
     saveLabel = new QLabel(this);
 
+    progressBar = new QProgressBar(this);
+    progressBar->hide();
+
     defaultPathCheckBox = new QCheckBox(this);
 
     messageLog = new QTextEdit(this);
     messageLog->setReadOnly(true);
 
     auto *layout = new QGridLayout(this);
-    layout->addWidget(messageLog, 0, 0);
-    layout->addWidget(saveLabel, 1, 0);
-    layout->addWidget(pathLabel, 2, 0);
-    layout->addWidget(defaultPathCheckBox, 3, 0);
+    layout->addWidget(progressBar, 0, 0);
+    layout->addWidget(messageLog, 1, 0);
+    layout->addWidget(saveLabel, 2, 0);
+    layout->addWidget(pathLabel, 3, 0);
+    layout->addWidget(defaultPathCheckBox, 4, 0);
 
     setLayout(layout);
 }
 
+bool SaveSetsPage::isComplete() const
+{
+    return !importActive;
+}
+
 void SaveSetsPage::cleanupPage()
 {
+    cancelWork();
+    disconnect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
+    disconnect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished);
     wizard()->importer->clear();
-    disconnect(wizard()->importer, &OracleImporter::setIndexChanged, nullptr, nullptr);
 }
 
 void SaveSetsPage::initializePage()
 {
-    messageLog->clear();
-
     retranslateUi();
     if (wizard()->downloadedPlainXml) {
         messageLog->hide();
-    } else {
-        messageLog->show();
-        connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
-
-        int setsImported = wizard()->importer->startImport();
-
-        // JSON data no longer needed after CardInfo objects are built
-        wizard()->importer->releaseSetData();
-
-        if (setsImported == 0) {
-            QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
+        progressBar->hide();
+        if (wizard()->backgroundMode) {
+            emit readyToContinue();
         }
+        return;
+    }
+
+    messageLog->clear();
+    messageLog->show();
+    progressBar->show();
+
+    totalSets = wizard()->importer->getSets().size();
+    progressBar->setRange(0, totalSets);
+    progressBar->setValue(0);
+
+    connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress,
+            Qt::UniqueConnection);
+    connect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished, Qt::UniqueConnection);
+
+    wizard()->disableButtons();
+    importActive = true;
+    emit completeChanged();
+
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    importFuture = QtConcurrent::run([importer] { return importer ? importer->startImport() : 0; });
+    importWatcher.setFuture(importFuture);
+}
+
+void SaveSetsPage::cancelWork()
+{
+    if (!importActive) {
+        return;
+    }
+    // Ask the worker to stop at the next set boundary, then wait it out so the
+    // wizard (and the importer it owns) is never torn down under a running thread.
+    importActive = false;
+    emit completeChanged();
+    wizard()->importer->cancelImport();
+    importFuture.cancel();
+    importWatcher.cancel();
+    importFuture.waitForFinished();
+}
+
+void SaveSetsPage::importFinished()
+{
+    if (!importActive) {
+        return;
+    }
+    importActive = false;
+    emit completeChanged();
+
+    wizard()->enableButtons();
+
+    const int setsImported = importWatcher.result();
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    if (importer) {
+        importer->releaseSetData();
     }
 
     if (wizard()->backgroundMode) {
+        if (setsImported == 0) {
+            qWarning() << tr("No set has been imported.");
+        }
         emit readyToContinue();
+        return;
+    }
+
+    progressBar->setValue(progressBar->maximum());
+
+    if (setsImported == 0) {
+        QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
     }
 }
 
@@ -591,13 +810,28 @@ void SaveSetsPage::retranslateUi()
     setButtonText(QWizard::NextButton, tr("&Save"));
 }
 
-void SaveSetsPage::updateTotalProgress(int cardsImported, int /* setIndex */, const QString &setName)
+void SaveSetsPage::updateTotalProgress(int cardsImported, int setIndex, const QString &setName)
 {
+    if (!importActive) {
+        return;
+    }
     if (setName.isEmpty()) {
-        messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()) +
-                           "</b>");
+        progressBar->setValue(progressBar->maximum());
+        const int cardCount = wizard()->importer->getCardList().size();
+        if (wizard()->backgroundMode) {
+            qInfo() << tr("Import finished: %1 cards.").arg(cardCount);
+            emitBackgroundProgress("import", totalSets, totalSets);
+        } else {
+            messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(cardCount) + "</b>");
+        }
     } else {
-        messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
+        progressBar->setValue(setIndex);
+        if (wizard()->backgroundMode) {
+            qInfo() << tr("%1: %2 cards imported").arg(setName).arg(cardsImported);
+            emitBackgroundProgress("import", setIndex, totalSets);
+        } else {
+            messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
+        }
     }
 
     messageLog->verticalScrollBar()->setValue(messageLog->verticalScrollBar()->maximum());

oracle/src/pages.h

@@ -3,8 +3,10 @@
 
 #include "pagetemplates.h"
 
+#include <QByteArray>
 #include <QFuture>
 #include <QFutureWatcher>
+#include <QString>
 #include <QTimer>
 #include <QWizard>
 #include <utility>
@@ -57,19 +59,31 @@ protected:
     void initializePage() override;
 };
 
+/** @brief Result of a worker-thread sets-file load (read + decompress + dispatch). */
+struct LoadSetsResult
+{
+    bool ok = false;                        ///< JSON scan produced set data (or plain XML was handled)
+    bool plainXml = false;                  ///< input was a plain Cockatrice XML database
+    QByteArray xmlData;                     ///< raw XML for the plain-XML path
+    QString errorMessage;                   ///< set when the input could not be processed
+    bool offerUncompressedFallback = false; ///< decompression-only failure: offer the uncompressed URL
+};
+
 class LoadSetsPage : public OracleWizardPage
 {
     Q_OBJECT
 public:
     explicit LoadSetsPage(QWidget *parent = nullptr);
     void retranslateUi() override;
+    bool isComplete() const override;
 
 protected:
     void initializePage() override;
     bool validatePage() override;
     void readSetsFromByteArray(QByteArray _data);
-    void readSetsFromByteArrayRef(QByteArray &_data);
+    void readSetsFromFile(const QString &fileName);
     void downloadSetsFile(const QUrl &url);
+    void cancelWork() override;
 
 private:
     QRadioButton *urlRadioButton;
@@ -81,15 +95,19 @@ private:
     QLabel *progressLabel;
     QProgressBar *progressBar;
 
-    QFutureWatcher<bool> watcher;
-    QFuture<bool> future;
-    QByteArray jsonData;
+    QFutureWatcher<LoadSetsResult> watcher;
+    QFuture<LoadSetsResult> future;
+    bool loadActive = false;
+
+    void beginLoadSets(bool compressedFile = false);
 
 private slots:
     void actLoadSetsFile();
     void actRestoreDefaultUrl();
     void actDownloadProgressSetsFile(qint64 received, qint64 total);
     void actDownloadFinishedSetsFile();
+    void updateParsingProgress(int bytesRead, int totalBytes);
+    void scanProgressToStdout(int bytesRead, int totalBytes);
     void importFinished();
     void zipDownloadFailed(const QString &message);
 };
@@ -100,19 +118,28 @@ class SaveSetsPage : public OracleWizardPage
 public:
     explicit SaveSetsPage(QWidget *parent = nullptr);
     void retranslateUi() override;
+    bool isComplete() const override;
 
 private:
     QTextEdit *messageLog;
+    QProgressBar *progressBar;
     QCheckBox *defaultPathCheckBox;
     QLabel *pathLabel;
     QLabel *saveLabel;
 
+    QFutureWatcher<int> importWatcher;
+    QFuture<int> importFuture;
+    int totalSets = 0;
+    bool importActive = false;
+
 protected:
     void initializePage() override;
     void cleanupPage() override;
     bool validatePage() override;
+    void cancelWork() override;
 
 private slots:
+    void importFinished();
     void updateTotalProgress(int cardsImported, int setIndex, const QString &setName);
 };
 

oracle/src/pagetemplates.h

@@ -20,6 +20,14 @@ public:
     }
     virtual void retranslateUi() = 0;
 
+    /**
+     * @brief Asks an active page to stop any background worker before the wizard
+     *        (and its importer) can be torn down underneath it. Default is a no-op.
+     */
+    virtual void cancelWork()
+    {
+    }
+
 signals:
     void readyToContinue();
 

oracle/src/raw_json_scanner.cpp

@@ -1,5 +1,6 @@
 #include "raw_json_scanner.h"
 
+#include <algorithm>
 #include <cstring>
 
 namespace
@@ -11,6 +12,46 @@ namespace
 // reason and reports DeepNesting).
 constexpr int kMaxNestingDepth = 1024;
 
+/**
+ * @brief Throttled byte-position reporting for scanSetRanges().
+ *
+ * Threaded through the skip walk so progress can be reported without materializing
+ * the whole document. Reports are rate-limited so a GUI showing progress isn't
+ * flooded with interrupts: a callback is invoked at most ~100 times per scan
+ * regardless of element count. The closing stretch (the last ~1%) is reported
+ * more finely so a large document doesn't stall the progress bar on the final
+ * percent before the scan wraps up.
+ */
+struct ScanProgress
+{
+    const char *begin = nullptr;
+    qsizetype size = 0;
+    RawJson::ScanProgressCallback callback;
+    qsizetype step = 1;
+    qsizetype lastReported = 0;
+
+    /**
+     * @brief Reports the scanner's absolute offset, unless within @p step bytes
+     * of the previous report and not yet at the end of the document.
+     */
+    void report(const char *p)
+    {
+        if (!callback) {
+            return;
+        }
+        const qsizetype offset = p - begin;
+        if (offset == lastReported) {
+            return; // the final element often already sits exactly at the end
+        }
+        const qsizetype reportingStep = offset >= size - step ? std::max<qsizetype>(1, step / 16) : step;
+        if (offset - lastReported < reportingStep && offset < size) {
+            return;
+        }
+        lastReported = offset;
+        callback(offset, size);
+    }
+};
+
 inline bool isWhitespace(char c)
 {
     return c == ' ' || c == '\t' || c == '\r' || c == '\n';
@@ -296,9 +337,9 @@ bool skipNumber(const char *&p, const char *end)
     return true;
 }
 
-bool skipValue(const char *&p, const char *end, int depth);
-bool skipObject(const char *&p, const char *end, int depth);
-bool skipArray(const char *&p, const char *end, int depth);
+bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan);
+bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan);
+bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan);
 
 bool skipPrimitive(const char *&p, const char *end)
 {
@@ -324,7 +365,7 @@ bool skipPrimitive(const char *&p, const char *end)
     return false;
 }
 
-bool skipObject(const char *&p, const char *end, int depth)
+bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan)
 {
     if (depth <= 0) {
         return false; // nest deeper than the cap
@@ -348,9 +389,10 @@ bool skipObject(const char *&p, const char *end, int depth)
             return false;
         }
         ++p;
-        if (!skipValue(p, end, depth - 1)) {
+        if (!skipValue(p, end, depth - 1, scan)) {
             return false;
         }
+        scan.report(p);
         p = skipWhitespace(p, end);
         if (p >= end) {
             return false;
@@ -367,7 +409,7 @@ bool skipObject(const char *&p, const char *end, int depth)
     }
 }
 
-bool skipArray(const char *&p, const char *end, int depth)
+bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan)
 {
     if (depth <= 0) {
         return false; // nest deeper than the cap
@@ -379,9 +421,10 @@ bool skipArray(const char *&p, const char *end, int depth)
         return true;
     }
     for (;;) {
-        if (!skipValue(p, end, depth - 1)) {
+        if (!skipValue(p, end, depth - 1, scan)) {
             return false;
         }
+        scan.report(p);
         p = skipWhitespace(p, end);
         if (p >= end) {
             return false;
@@ -398,7 +441,7 @@ bool skipArray(const char *&p, const char *end, int depth)
     }
 }
 
-bool skipValue(const char *&p, const char *end, int depth)
+bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan)
 {
     p = skipWhitespace(p, end);
     if (p >= end) {
@@ -407,10 +450,10 @@ bool skipValue(const char *&p, const char *end, int depth)
     const char c = *p;
     if (c == '{') {
         // pass depth through: skipObject consumes the single decrement for this level
-        return skipObject(p, end, depth);
+        return skipObject(p, end, depth, scan);
     }
     if (c == '[') {
-        return skipArray(p, end, depth);
+        return skipArray(p, end, depth, scan);
     }
     // a primitive is a leaf, so it never wastes a nesting level
     return skipPrimitive(p, end);
@@ -422,7 +465,8 @@ bool skipValue(const char *&p, const char *end, int depth)
  * For each member invokes @p memberCallback with the key and the byte range of
  * its value. Advancing @p p is unaffected by the callback.
  */
-template <typename F> bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback)
+template <typename F>
+bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback, ScanProgress &scan)
 {
     if (depth <= 0) {
         return false; // nest deeper than the cap
@@ -449,7 +493,7 @@ template <typename F> bool forEachObjectMember(const char *&p, const char *end,
         ++p;
         const char *valueStart = skipWhitespace(p, end);
         const char *valueEnd = valueStart;
-        if (!skipValue(valueEnd, end, depth - 1)) {
+        if (!skipValue(valueEnd, end, depth - 1, scan)) {
             return false;
         }
         if (!memberCallback(key, valueStart, valueEnd)) {
@@ -473,7 +517,7 @@ template <typename F> bool forEachObjectMember(const char *&p, const char *end,
 }
 
 // Counts the direct elements of an array value; returns -1 if the array is malformed.
-int countArrayElements(const char *p, const char *end, int depth)
+int countArrayElements(const char *p, const char *end, int depth, ScanProgress &scan)
 {
     if (depth <= 0) {
         return -1; // nest deeper than the cap
@@ -485,9 +529,10 @@ int countArrayElements(const char *p, const char *end, int depth)
         return 0;
     }
     for (;;) {
-        if (!skipValue(p, end, depth - 1)) {
+        if (!skipValue(p, end, depth - 1, scan)) {
             return -1;
         }
+        scan.report(p);
         ++count;
         p = skipWhitespace(p, end);
         if (p >= end) {
@@ -509,7 +554,7 @@ int countArrayElements(const char *p, const char *end, int depth)
 namespace RawJson
 {
 
-QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
+QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error, const ScanProgressCallback &progress)
 {
     QList<SetRange> ranges;
     if (error) {
@@ -529,6 +574,14 @@ QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
         return fail(QStringLiteral("empty JSON document"));
     }
 
+    // Throttle reports to ~100 per scan so a GUI thread unthrottling them never
+    // drowns under per-card interrupts, whatever the document size.
+    ScanProgress scan;
+    scan.begin = begin;
+    scan.size = end - begin;
+    scan.step = std::max<qsizetype>(1, scan.size / 100);
+    scan.callback = progress;
+
     const char *p = skipWhitespace(begin, end);
     if (p >= end || *p != '{') {
         return fail(QStringLiteral("top-level JSON must be an object"));
@@ -545,55 +598,57 @@ QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
                 return false;
             }
             const char *setP = valueStart;
-            const bool ok = forEachObjectMember(setP, valueEnd, kMaxNestingDepth - 1,
-                                                [&](const QString &setCode, const char *setStart, const char *setEnd) {
-                                                    if (setStart >= setEnd || *setStart != '{') {
-                                                        malformedSetData = true;
-                                                        return false;
-                                                    }
-                                                    SetRange range;
-                                                    range.dataRange.start = setStart - begin;
-                                                    range.dataRange.length = setEnd - setStart;
-                                                    range.code = setCode;
+            const bool ok = forEachObjectMember(
+                setP, valueEnd, kMaxNestingDepth - 1,
+                [&](const QString &setCode, const char *setStart, const char *setEnd) {
+                    if (setStart >= setEnd || *setStart != '{') {
+                        malformedSetData = true;
+                        return false;
+                    }
+                    SetRange range;
+                    range.dataRange.start = setStart - begin;
+                    range.dataRange.length = setEnd - setStart;
+                    range.code = setCode;
 
-                                                    const char *memberP = setStart;
-                                                    const bool metaOk = forEachObjectMember(
-                                                        memberP, setEnd, kMaxNestingDepth - 2,
-                                                        [&](const QString &field, const char *fs, const char *fe) {
-                                                            if (field == QStringLiteral("code")) {
-                                                                return decodeStringMember(fs, fe, range.code);
-                                                            }
-                                                            if (field == QStringLiteral("name")) {
-                                                                return decodeStringMember(fs, fe, range.name);
-                                                            }
-                                                            if (field == QStringLiteral("type")) {
-                                                                return decodeStringMember(fs, fe, range.type);
-                                                            }
-                                                            if (field == QStringLiteral("releaseDate")) {
-                                                                return decodeStringMember(fs, fe, range.releaseDate);
-                                                            }
-                                                            if (field == QStringLiteral("cards")) {
-                                                                if (fs >= fe) {
-                                                                    return false;
-                                                                }
-                                                                if (*fs != '[') {
-                                                                    // e.g. "cards": null — treat as an empty array,
-                                                                    // matching Qt's tolerance.
-                                                                    return true;
-                                                                }
-                                                                range.dataRange.cardCount =
-                                                                    countArrayElements(fs, fe, kMaxNestingDepth - 2);
-                                                                return range.dataRange.cardCount >= 0;
-                                                            }
-                                                            return true;
-                                                        });
-                                                    if (!metaOk) {
-                                                        malformedSetData = true;
-                                                        return false;
-                                                    }
-                                                    ranges.append(range);
-                                                    return true;
-                                                });
+                    const char *memberP = setStart;
+                    const bool metaOk = forEachObjectMember(
+                        memberP, setEnd, kMaxNestingDepth - 2,
+                        [&](const QString &field, const char *fs, const char *fe) {
+                            if (field == QStringLiteral("code")) {
+                                return decodeStringMember(fs, fe, range.code);
+                            }
+                            if (field == QStringLiteral("name")) {
+                                return decodeStringMember(fs, fe, range.name);
+                            }
+                            if (field == QStringLiteral("type")) {
+                                return decodeStringMember(fs, fe, range.type);
+                            }
+                            if (field == QStringLiteral("releaseDate")) {
+                                return decodeStringMember(fs, fe, range.releaseDate);
+                            }
+                            if (field == QStringLiteral("cards")) {
+                                if (fs >= fe) {
+                                    return false;
+                                }
+                                if (*fs != '[') {
+                                    // e.g. "cards": null — treat as an empty array,
+                                    // matching Qt's tolerance.
+                                    return true;
+                                }
+                                range.dataRange.cardCount = countArrayElements(fs, fe, kMaxNestingDepth - 2, scan);
+                                return range.dataRange.cardCount >= 0;
+                            }
+                            return true;
+                        },
+                        scan);
+                    if (!metaOk) {
+                        malformedSetData = true;
+                        return false;
+                    }
+                    ranges.append(range);
+                    return true;
+                },
+                scan);
             if (!ok) {
                 malformedSetData = true;
                 return false;
@@ -602,7 +657,7 @@ QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
         return true;
     };
 
-    if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback)) {
+    if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback, scan)) {
         return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON"));
     }
     p = skipWhitespace(p, end);
@@ -615,6 +670,7 @@ QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
     if (ranges.isEmpty()) {
         return fail(QStringLiteral("no sets found in \"data\""));
     }
+    scan.report(end);
     return ranges;
 }
 

oracle/src/raw_json_scanner.h

@@ -4,6 +4,7 @@
 #include <QByteArray>
 #include <QList>
 #include <QString>
+#include <functional>
 
 namespace RawJson
 {
@@ -42,6 +43,13 @@ struct ScanError
     QString message;
 };
 
+/**
+ * @brief Optional progress callback receiving @c (bytesRead, totalBytes) while
+ * the document is walked. Invoked from the scanning thread; the caller decides
+ * how the throttled offsets are relayed to a GUI event loop.
+ */
+using ScanProgressCallback = std::function<void(qsizetype bytesRead, qsizetype totalBytes)>;
+
 /**
  * @brief Scans a full MTGJSON document without materializing the JSON tree.
  *
@@ -67,9 +75,14 @@ struct ScanError
  * @param error Out parameter. Set to an error ScanError when the document
  *              cannot be parsed, otherwise left empty. Passing a null
  *              pointer disables error reporting.
+ * @param progress Optional progress callback. When non-empty it is invoked as
+ *                 the scanner advances through the document, throttled to a
+ *                 tiny fraction of the total size.
  * @return The detected per-set ranges, or an empty list on failure.
  */
-QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error = nullptr);
+QList<SetRange> scanSetRanges(const QByteArray &json,
+                              ScanError *error = nullptr,
+                              const ScanProgressCallback &progress = ScanProgressCallback());
 
 } // namespace RawJson