[Oracle/Client] Report card database download progress (#7253)

* [Oracle/Client] Report card database download progress

Card database updates ran invisibly: MTGJSON parsing spun an indeterminate
bar on the UI thread and the set import blocked the window, while the
onboarding wizard spawned `oracle -b` with no progress to show at all.

- Add byte-level scan progress to `RawJson::scanSetRanges` via an optional
  callback, throttled to ~100 reports per scan.
- Emit `OracleImporter::dataReadProgress` during the scan and import sets on a
  worker thread, driving the wizard's progress bar per set.
- With `-b`, write machine-readable `PROGRESS <stage> <done> <total>` lines to
  stdout for the download/scan/import stages; stderr keeps the log output.
- Parse the oracle stdout in `MainWindow` and forward it to the onboarding
  wizard, giving the card database step a determinate bar with stage-specific
  status text.
- Guard the async workers against the wizard being closed mid-run.
- Add Google Test coverage for scan progress reporting.

* [Oracle/Client] Harden oracle progress workers and quit prompt

Address review feedback on the download-progress change: decompress and read
sets files off the UI thread, cancel the load/import workers before the
wizard can tear down the importer, and show an 'Extracting file...' status
plus a clean 100% tail so the poll never looks stuck. Quitting Cockatrice
while a card database update runs now asks for confirmation.

* Show 100% for 500ms on complete.

* Disable buttons on set import until done.

* Clean up progress bar.

* Drop wrapper around lambda

---------

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>

oracle/src/pages.cpp

@@ -12,6 +12,7 @@
 #include <QBuffer>
 #include <QCheckBox>
 #include <QComboBox>
+#include <QDebug>
 #include <QDir>
 #include <QFileDialog>
 #include <QGridLayout>
@@ -20,14 +21,17 @@
 #include <QMessageBox>
 #include <QNetworkAccessManager>
 #include <QNetworkReply>
+#include <QPointer>
 #include <QProgressBar>
 #include <QPushButton>
 #include <QRadioButton>
 #include <QScrollBar>
 #include <QStandardPaths>
 #include <QTextEdit>
+#include <QTextStream>
 #include <QtConcurrent>
 #include <QtGui>
+#include <cstdio>
 #include <libcockatrice/settings/personal_settings.h>
 
 #ifdef HAS_LZMA
@@ -53,6 +57,115 @@
 #define ALLSETS_URL "https://www.mtgjson.com/api/v5/AllPrintings.json"
 #endif
 
+/**
+ * @brief Emits one machine-readable background-run progress line to stdout.
+ *
+ * Used only in background mode, so the hosting Cockatrice client can parse these
+ * lines to drive a determinate progress bar. stderr stays reserved for
+ * human-readable log output.
+ */
+static void emitBackgroundProgress(const char *stage, qint64 done, qint64 total)
+{
+    QTextStream out(stdout);
+    out << "PROGRESS " << stage << ' ' << done << ' ' << total << '\n';
+    out.flush();
+}
+
+namespace
+{
+
+/**
+ * @brief Decompresses and dispatches a sets-file payload on a worker thread.
+ *
+ * Iteratively unwraps xz/zip compression, then either hands the JSON to the
+ * importer (which reports scan progress via dataReadProgress) or returns the raw
+ * XML for the plain-XML path. Must never touch the wizard or the page: the caller
+ * consumes the returned LoadSetsResult on the UI thread in importFinished().
+ */
+LoadSetsResult loadSetsData(const QPointer<OracleImporter> &importer, QByteArray data)
+{
+    LoadSetsResult result;
+
+    while (true) {
+        if (data.startsWith(XZ_SIGNATURE)) {
+#ifdef HAS_LZMA
+            QBuffer inBuffer(&data);
+            QByteArray decompressed;
+            QBuffer outBuffer(&decompressed);
+            inBuffer.open(QBuffer::ReadOnly);
+            outBuffer.open(QBuffer::WriteOnly);
+            XzDecompressor xz;
+            if (!xz.decompress(&inBuffer, &outBuffer)) {
+                result.errorMessage = LoadSetsPage::tr("Xz extraction failed.");
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            data = decompressed;
+            continue;
+#else
+            result.errorMessage =
+                LoadSetsPage::tr("Sorry, this version of Oracle does not support xz compressed files.");
+            result.offerUncompressedFallback = true;
+            return result;
+#endif
+        }
+
+        if (data.startsWith(ZIP_SIGNATURE)) {
+#ifdef HAS_ZLIB
+            QBuffer inBuffer(&data);
+            UnZip uz;
+            const UnZip::ErrorCode openEc = uz.openArchive(&inBuffer);
+            if (openEc != UnZip::Ok) {
+                result.errorMessage = LoadSetsPage::tr("Failed to open Zip archive: %1.").arg(uz.formatError(openEc));
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            if (uz.fileList().size() != 1) {
+                result.errorMessage =
+                    LoadSetsPage::tr("Zip extraction failed: the Zip archive doesn't contain exactly one file.");
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            const QString fileName = uz.fileList().at(0);
+            QByteArray decompressed;
+            QBuffer outBuffer(&decompressed);
+            outBuffer.open(QBuffer::ReadWrite);
+            const UnZip::ErrorCode ec = uz.extractFile(fileName, &outBuffer);
+            uz.closeArchive();
+            if (ec != UnZip::Ok) {
+                result.errorMessage = LoadSetsPage::tr("Zip extraction failed: %1.").arg(uz.formatError(ec));
+                result.offerUncompressedFallback = true;
+                return result;
+            }
+            data = decompressed;
+            continue;
+#else
+            result.errorMessage = LoadSetsPage::tr("Sorry, this version of Oracle does not support zipped files.");
+            result.offerUncompressedFallback = true;
+            return result;
+#endif
+        }
+        break;
+    }
+
+    if (data.startsWith("<")) {
+        result.ok = true;
+        result.plainXml = true;
+        result.xmlData = std::move(data);
+        return result;
+    }
+
+    if (data.startsWith("{")) {
+        result.ok = importer && importer->readSetsFromByteArray(std::move(data));
+        return result;
+    }
+
+    result.errorMessage = LoadSetsPage::tr("Failed to interpret downloaded data.");
+    return result;
+}
+
+} // namespace
+
 #define TOKENS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Token/master/tokens.xml"
 #define SPOILERS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Spoiler/files/spoiler.xml"
 
@@ -182,6 +295,11 @@ LoadSetsPage::LoadSetsPage(QWidget *parent) : OracleWizardPage(parent)
     setLayout(layout);
 }
 
+bool LoadSetsPage::isComplete() const
+{
+    return !loadActive;
+}
+
 void LoadSetsPage::initializePage()
 {
     urlLineEdit->setText(wizard()->settings->value("allsetsurl", ALLSETS_URL).toString());
@@ -279,18 +397,13 @@ bool LoadSetsPage::validatePage()
             return false;
         }
 
-        if (!setsFile.open(QIODevice::ReadOnly)) {
-            QMessageBox::critical(nullptr, tr("Error"), tr("Cannot open file '%1'.").arg(fileLineEdit->text()));
-            return false;
-        }
-
         wizard()->disableButtons();
         setEnabled(false);
 
         wizard()->setCardSourceUrl(setsFile.fileName());
         wizard()->setCardSourceVersion("unknown");
 
-        readSetsFromByteArray(setsFile.readAll());
+        readSetsFromFile(setsFile.fileName());
     }
 
     return false;
@@ -339,6 +452,9 @@ void LoadSetsPage::downloadSetsFile(const QUrl &url)
 
 void LoadSetsPage::actDownloadProgressSetsFile(qint64 received, qint64 total)
 {
+    if (wizard()->backgroundMode) {
+        emitBackgroundProgress("download", received, total);
+    }
     if (total > 0) {
         progressBar->setMaximum(static_cast<int>(total));
         progressBar->setValue(static_cast<int>(received));
@@ -384,106 +500,97 @@ void LoadSetsPage::actDownloadFinishedSetsFile()
     reply->deleteLater();
 }
 
-void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
+void LoadSetsPage::updateParsingProgress(int bytesRead, int totalBytes)
 {
-    // show an infinite progressbar
+    if (totalBytes <= 0) {
+        return;
+    }
+    progressBar->setRange(0, totalBytes);
+    progressBar->setValue(bytesRead);
+    const int percent = static_cast<int>((100.0 * bytesRead) / totalBytes);
+    progressLabel->setText(tr("Parsing file (%1%)").arg(percent));
+}
+
+void LoadSetsPage::scanProgressToStdout(int bytesRead, int totalBytes)
+{
+    emitBackgroundProgress("scan", bytesRead, totalBytes);
+}
+
+void LoadSetsPage::beginLoadSets(bool compressedFile)
+{
+    // Show an infinite progressbar while the worker decompresses; the scan
+    // steals the label via dataReadProgress as soon as it starts.
     progressBar->setMaximum(0);
     progressBar->setMinimum(0);
     progressBar->setValue(0);
-    progressLabel->setText(tr("Parsing file"));
+    progressLabel->setText(compressedFile ? tr("Extracting file...") : tr("Parsing file"));
     progressLabel->show();
     progressBar->show();
 
+    // Keep Next disabled (via completeChanged) until the worker reports in;
+    // updateButtonStates() re-evaluates button state whenever we re-enable.
+    loadActive = true;
+    emit completeChanged();
+
     wizard()->downloadedPlainXml = false;
     wizard()->xmlData.clear();
-    readSetsFromByteArrayRef(_data);
+
+    if (wizard()->backgroundMode) {
+        connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::scanProgressToStdout,
+                Qt::UniqueConnection);
+    } else {
+        connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::updateParsingProgress,
+                Qt::UniqueConnection);
+    }
 }
 
-void LoadSetsPage::readSetsFromByteArrayRef(QByteArray &_data)
+void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
 {
-    // unzip the file if needed
-    if (_data.startsWith(XZ_SIGNATURE)) {
-#ifdef HAS_LZMA
-        // zipped file
-        auto *inBuffer = new QBuffer(&_data);
-        auto newData = QByteArray();
-        auto *outBuffer = new QBuffer(&newData);
-        inBuffer->open(QBuffer::ReadOnly);
-        outBuffer->open(QBuffer::WriteOnly);
-        XzDecompressor xz;
-        if (!xz.decompress(inBuffer, outBuffer)) {
-            zipDownloadFailed(tr("Xz extraction failed."));
-            return;
+    const bool compressed = _data.startsWith(XZ_SIGNATURE) || _data.startsWith(ZIP_SIGNATURE);
+    beginLoadSets(compressed);
+
+    // Decompress and scan off the UI thread so a large download can't freeze the window.
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    future = QtConcurrent::run(
+        [importer, data = std::move(_data)]() mutable { return loadSetsData(importer, std::move(data)); });
+    watcher.setFuture(future);
+}
+
+void LoadSetsPage::readSetsFromFile(const QString &fileName)
+{
+    // Peek at the header on the UI thread so the status text can distinguish
+    // "Extracting file..." from a plain JSON parse; the full read happens in the worker.
+    QFile headerFile(fileName);
+    bool compressed = false;
+    if (headerFile.open(QIODevice::ReadOnly)) {
+        const QByteArray header = headerFile.read(6);
+        compressed = header.startsWith(XZ_SIGNATURE) || header.startsWith(ZIP_SIGNATURE);
+    }
+    beginLoadSets(compressed);
+
+    // Read, decompress and scan off the UI thread (a plain JSON can be hundreds
+    // of MB, so even the read itself must not block the window).
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    future = QtConcurrent::run([importer, fileName]() mutable -> LoadSetsResult {
+        QFile file(fileName);
+        if (!file.open(QIODevice::ReadOnly)) {
+            LoadSetsResult readError;
+            readError.errorMessage = LoadSetsPage::tr("Cannot open file '%1'.").arg(fileName);
+            return readError;
         }
-        _data.clear();
-        readSetsFromByteArrayRef(newData);
-        return;
-#else
-        zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files."));
+        return loadSetsData(importer, file.readAll());
+    });
+    watcher.setFuture(future);
+}
 
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        return;
-#endif
-    } else if (_data.startsWith(ZIP_SIGNATURE)) {
-#ifdef HAS_ZLIB
-        // zipped file
-        auto *inBuffer = new QBuffer(&_data);
-        auto newData = QByteArray();
-        auto *outBuffer = new QBuffer(&newData);
-        QString fileName;
-        UnZip::ErrorCode ec;
-        UnZip uz;
-
-        ec = uz.openArchive(inBuffer);
-        if (ec != UnZip::Ok) {
-            zipDownloadFailed(tr("Failed to open Zip archive: %1.").arg(uz.formatError(ec)));
-            return;
-        }
-
-        if (uz.fileList().size() != 1) {
-            zipDownloadFailed(tr("Zip extraction failed: the Zip archive doesn't contain exactly one file."));
-            return;
-        }
-        fileName = uz.fileList().at(0);
-
-        outBuffer->open(QBuffer::ReadWrite);
-        ec = uz.extractFile(fileName, outBuffer);
-        if (ec != UnZip::Ok) {
-            zipDownloadFailed(tr("Zip extraction failed: %1.").arg(uz.formatError(ec)));
-            uz.closeArchive();
-            return;
-        }
-        _data.clear();
-        readSetsFromByteArrayRef(newData);
-        return;
-#else
-        zipDownloadFailed(tr("Sorry, this version of Oracle does not support zipped files."));
-
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        return;
-#endif
-    } else if (_data.startsWith("{")) {
-        // Start the computation.
-        jsonData = std::move(_data);
-        future = QtConcurrent::run([this] { return wizard()->importer->readSetsFromByteArray(std::move(jsonData)); });
-        watcher.setFuture(future);
-    } else if (_data.startsWith("<")) {
-        // save xml file and don't do any processing
-        wizard()->downloadedPlainXml = true;
-        wizard()->xmlData = std::move(_data);
-        importFinished();
-    } else {
-        wizard()->enableButtons();
-        setEnabled(true);
-        progressLabel->hide();
-        progressBar->hide();
-        QMessageBox::critical(this, tr("Error"), tr("Failed to interpret downloaded data."));
+void LoadSetsPage::cancelWork()
+{
+    // The scan is short-lived; just wait it out before the wizard (and its
+    // importer) can be torn down underneath the worker thread.
+    if (future.isRunning()) {
+        future.cancel();
+        watcher.cancel();
+        future.waitForFinished();
     }
 }
 
@@ -509,17 +616,65 @@ void LoadSetsPage::zipDownloadFailed(const QString &message)
 
 void LoadSetsPage::importFinished()
 {
+    loadActive = false;
+    emit completeChanged();
     wizard()->enableButtons();
     setEnabled(true);
-    progressLabel->hide();
-    progressBar->hide();
 
-    if (wizard()->downloadedPlainXml || watcher.future().result()) {
-        wizard()->next();
-    } else {
-        QMessageBox::critical(this, tr("Error"),
-                              tr("The file was retrieved successfully, but it does not contain any sets data."));
+    const LoadSetsResult result = watcher.result();
+
+    if (result.plainXml) {
+        wizard()->downloadedPlainXml = true;
+        wizard()->xmlData = result.xmlData;
     }
+
+    if (wizard()->backgroundMode) {
+        progressLabel->hide();
+        progressBar->hide();
+        if (!result.errorMessage.isEmpty()) {
+            qWarning() << result.errorMessage;
+        } else if (!result.ok && !result.plainXml) {
+            qWarning() << tr("The file was retrieved successfully, but it does not contain any sets data.");
+        }
+        emit readyToContinue();
+        return;
+    }
+
+    const auto fail = [this](const QString &message) {
+        progressLabel->hide();
+        progressBar->hide();
+        QMessageBox::critical(this, tr("Error"), message);
+    };
+
+    if (!result.errorMessage.isEmpty()) {
+        if (result.offerUncompressedFallback) {
+            zipDownloadFailed(result.errorMessage);
+            return;
+        }
+        fail(result.errorMessage);
+        return;
+    }
+
+    if (!result.ok && !result.plainXml) {
+        fail(tr("The file was retrieved successfully, but it does not contain any sets data."));
+        return;
+    }
+
+    // Snap the bar to 100% and hold it there for a moment so the completed state
+    // is actually visible before the next page's own import progress takes over
+    // (a zero-length deferral can still fire before the repaint is delivered).
+    progressBar->setMaximum(1);
+    progressBar->setValue(1);
+    progressLabel->setText(tr("Parsing file (100%)"));
+    QTimer::singleShot(500, this, [this] {
+        if (wizard()->currentPage() == this) {
+            // Leave the page pristine: hide the completed load bar so a later
+            // Back from the save page doesn't show stale progress.
+            progressLabel->hide();
+            progressBar->hide();
+            wizard()->next();
+        }
+    });
 }
 
 SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
@@ -527,49 +682,113 @@ SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
     pathLabel = new QLabel(this);
     saveLabel = new QLabel(this);
 
+    progressBar = new QProgressBar(this);
+    progressBar->hide();
+
     defaultPathCheckBox = new QCheckBox(this);
 
     messageLog = new QTextEdit(this);
     messageLog->setReadOnly(true);
 
     auto *layout = new QGridLayout(this);
-    layout->addWidget(messageLog, 0, 0);
-    layout->addWidget(saveLabel, 1, 0);
-    layout->addWidget(pathLabel, 2, 0);
-    layout->addWidget(defaultPathCheckBox, 3, 0);
+    layout->addWidget(progressBar, 0, 0);
+    layout->addWidget(messageLog, 1, 0);
+    layout->addWidget(saveLabel, 2, 0);
+    layout->addWidget(pathLabel, 3, 0);
+    layout->addWidget(defaultPathCheckBox, 4, 0);
 
     setLayout(layout);
 }
 
+bool SaveSetsPage::isComplete() const
+{
+    return !importActive;
+}
+
 void SaveSetsPage::cleanupPage()
 {
+    cancelWork();
+    disconnect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
+    disconnect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished);
     wizard()->importer->clear();
-    disconnect(wizard()->importer, &OracleImporter::setIndexChanged, nullptr, nullptr);
 }
 
 void SaveSetsPage::initializePage()
 {
-    messageLog->clear();
-
     retranslateUi();
     if (wizard()->downloadedPlainXml) {
         messageLog->hide();
-    } else {
-        messageLog->show();
-        connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
-
-        int setsImported = wizard()->importer->startImport();
-
-        // JSON data no longer needed after CardInfo objects are built
-        wizard()->importer->releaseSetData();
-
-        if (setsImported == 0) {
-            QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
+        progressBar->hide();
+        if (wizard()->backgroundMode) {
+            emit readyToContinue();
         }
+        return;
+    }
+
+    messageLog->clear();
+    messageLog->show();
+    progressBar->show();
+
+    totalSets = wizard()->importer->getSets().size();
+    progressBar->setRange(0, totalSets);
+    progressBar->setValue(0);
+
+    connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress,
+            Qt::UniqueConnection);
+    connect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished, Qt::UniqueConnection);
+
+    wizard()->disableButtons();
+    importActive = true;
+    emit completeChanged();
+
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    importFuture = QtConcurrent::run([importer] { return importer ? importer->startImport() : 0; });
+    importWatcher.setFuture(importFuture);
+}
+
+void SaveSetsPage::cancelWork()
+{
+    if (!importActive) {
+        return;
+    }
+    // Ask the worker to stop at the next set boundary, then wait it out so the
+    // wizard (and the importer it owns) is never torn down under a running thread.
+    importActive = false;
+    emit completeChanged();
+    wizard()->importer->cancelImport();
+    importFuture.cancel();
+    importWatcher.cancel();
+    importFuture.waitForFinished();
+}
+
+void SaveSetsPage::importFinished()
+{
+    if (!importActive) {
+        return;
+    }
+    importActive = false;
+    emit completeChanged();
+
+    wizard()->enableButtons();
+
+    const int setsImported = importWatcher.result();
+    const QPointer<OracleImporter> importer = wizard()->importer;
+    if (importer) {
+        importer->releaseSetData();
     }
 
     if (wizard()->backgroundMode) {
+        if (setsImported == 0) {
+            qWarning() << tr("No set has been imported.");
+        }
         emit readyToContinue();
+        return;
+    }
+
+    progressBar->setValue(progressBar->maximum());
+
+    if (setsImported == 0) {
+        QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
     }
 }
 
@@ -591,13 +810,28 @@ void SaveSetsPage::retranslateUi()
     setButtonText(QWizard::NextButton, tr("&Save"));
 }
 
-void SaveSetsPage::updateTotalProgress(int cardsImported, int /* setIndex */, const QString &setName)
+void SaveSetsPage::updateTotalProgress(int cardsImported, int setIndex, const QString &setName)
 {
+    if (!importActive) {
+        return;
+    }
     if (setName.isEmpty()) {
-        messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()) +
-                           "</b>");
+        progressBar->setValue(progressBar->maximum());
+        const int cardCount = wizard()->importer->getCardList().size();
+        if (wizard()->backgroundMode) {
+            qInfo() << tr("Import finished: %1 cards.").arg(cardCount);
+            emitBackgroundProgress("import", totalSets, totalSets);
+        } else {
+            messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(cardCount) + "</b>");
+        }
     } else {
-        messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
+        progressBar->setValue(setIndex);
+        if (wizard()->backgroundMode) {
+            qInfo() << tr("%1: %2 cards imported").arg(setName).arg(cardsImported);
+            emitBackgroundProgress("import", setIndex, totalSets);
+        } else {
+            messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
+        }
     }
 
     messageLog->verticalScrollBar()->setValue(messageLog->verticalScrollBar()->maximum());