diff --git a/cockatrice/src/interface/widgets/onboarding/first_run_wizard.cpp b/cockatrice/src/interface/widgets/onboarding/first_run_wizard.cpp index 618ac6f26..4af02fe4f 100644 --- a/cockatrice/src/interface/widgets/onboarding/first_run_wizard.cpp +++ b/cockatrice/src/interface/widgets/onboarding/first_run_wizard.cpp @@ -182,6 +182,13 @@ void FirstRunWizard::onCardDatabaseUpdateFinished(bool success) } } +void FirstRunWizard::onCardDatabaseUpdateProgress(const QString &stage, qint64 done, qint64 total) +{ + if (cardDatabasePage) { + cardDatabasePage->onUpdateProgress(stage, done, total); + } +} + void FirstRunWizard::finish() { accept(); diff --git a/cockatrice/src/interface/widgets/onboarding/first_run_wizard.h b/cockatrice/src/interface/widgets/onboarding/first_run_wizard.h index 2c186ef95..21d7b6e06 100644 --- a/cockatrice/src/interface/widgets/onboarding/first_run_wizard.h +++ b/cockatrice/src/interface/widgets/onboarding/first_run_wizard.h @@ -37,6 +37,9 @@ public slots: /** @brief Forwarded from MainWindow once the background card database update process exits. */ void onCardDatabaseUpdateFinished(bool success); + /** @brief Forwarded from MainWindow while the background card database update process runs. */ + void onCardDatabaseUpdateProgress(const QString &stage, qint64 done, qint64 total); + protected: void closeEvent(QCloseEvent *event) override; void changeEvent(QEvent *event) override; diff --git a/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.cpp b/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.cpp index 12116de7a..50e8ff63d 100644 --- a/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.cpp +++ b/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,25 @@ void CardDatabaseSetupPage::onUpdateFinished(bool success) } } +void CardDatabaseSetupPage::onUpdateProgress(const QString &stage, qint64 done, qint64 total) +{ + if (state != State::Running) { + return; + } + progressBar->setRange(0, total > 0 ? static_cast(qMin(total, INT_MAX)) : 0); + progressBar->setValue(static_cast(qMin(done, INT_MAX))); + if (total > 0) { + const int percent = static_cast((100.0 * done) / total); + if (stage == QLatin1String("download")) { + statusLabel->setText(tr("Downloading the card database (%1%)…").arg(percent)); + } else if (stage == QLatin1String("scan")) { + statusLabel->setText(tr("Parsing the card database (%1%)…").arg(percent)); + } else if (stage == QLatin1String("import")) { + statusLabel->setText(tr("Importing cards (%1%)…").arg(percent)); + } + } +} + QString CardDatabaseSetupPage::nextButtonText() const { return state == State::NotStarted ? tr("Download") : QString(); diff --git a/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.h b/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.h index 0461d11d5..870e759ea 100644 --- a/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.h +++ b/cockatrice/src/interface/widgets/onboarding/pages/card_database_setup_page.h @@ -30,6 +30,7 @@ public: void retranslateUi() override; void onUpdateFinished(bool success); + void onUpdateProgress(const QString &stage, qint64 done, qint64 total); signals: void updateRequested(); diff --git a/cockatrice/src/interface/window_main.cpp b/cockatrice/src/interface/window_main.cpp index 4567991c8..5afa5fbf0 100644 --- a/cockatrice/src/interface/window_main.cpp +++ b/cockatrice/src/interface/window_main.cpp @@ -682,6 +682,7 @@ void MainWindow::runFirstRunWizard() connect(wizard, &FirstRunWizard::cardDatabaseUpdateRequested, this, &MainWindow::actCheckCardUpdatesBackground); connect(wizard, &FirstRunWizard::manualCardDatabaseSetupRequested, this, &MainWindow::actCheckCardUpdates); connect(this, &MainWindow::cardDatabaseUpdateFinished, wizard, &FirstRunWizard::onCardDatabaseUpdateFinished); + connect(this, &MainWindow::cardDatabaseUpdateProgress, wizard, &FirstRunWizard::onCardDatabaseUpdateProgress); connect(wizard, &FirstRunWizard::registerRequested, connectionController, &ConnectionController::registerToServer); connect(wizard, &FirstRunWizard::connectRequested, connectionController, &ConnectionController::connectToServer); @@ -1057,11 +1058,45 @@ void MainWindow::createCardUpdateProcess(bool background) if (!background) { cardUpdateProcess->start(updaterCmd, QStringList()); } else { + cardUpdateOutputBuffer.clear(); + connect(cardUpdateProcess, &QProcess::readyReadStandardOutput, this, &MainWindow::cardUpdateProgressOutput); cardUpdateProcess->start(updaterCmd, QStringList("-b")); statusBar()->showMessage(tr("Card database update running.")); } } +void MainWindow::cardUpdateProgressOutput() +{ + if (!cardUpdateProcess) { + return; + } + cardUpdateOutputBuffer.append(cardUpdateProcess->readAllStandardOutput()); + while (true) { + const int newline = cardUpdateOutputBuffer.indexOf('\n'); + if (newline < 0) { + break; + } + const QByteArray line = cardUpdateOutputBuffer.left(newline).trimmed(); + cardUpdateOutputBuffer.remove(0, newline + 1); + // Protocol emitted by `oracle -b`: "PROGRESS " + if (!line.startsWith("PROGRESS ")) { + continue; + } + const QList parts = line.split(' '); + if (parts.size() != 4) { + continue; + } + bool doneOk = false; + bool totalOk = false; + const qint64 done = parts.at(2).toLongLong(&doneOk); + const qint64 total = parts.at(3).toLongLong(&totalOk); + if (!doneOk || !totalOk || done < 0 || total < 0) { + continue; + } + emit cardDatabaseUpdateProgress(QString::fromLatin1(parts.at(1)), done, total); + } +} + void MainWindow::exitCardDatabaseUpdate() { if (!cardUpdateProcess) { @@ -1109,6 +1144,8 @@ void MainWindow::cardUpdateError(QProcess::ProcessError err) void MainWindow::cardUpdateFinished(int exitCode, QProcess::ExitStatus exitStatus) { + cardUpdateProgressOutput(); // drain any progress lines not yet parsed + const bool success = (exitStatus == QProcess::NormalExit) && (exitCode == 0); if (exitStatus == QProcess::NormalExit) { SettingsCache::instance().updates().setLastCardUpdateCheck(QDateTime::currentDateTime().date()); diff --git a/cockatrice/src/interface/window_main.h b/cockatrice/src/interface/window_main.h index 920145552..08481fd36 100644 --- a/cockatrice/src/interface/window_main.h +++ b/cockatrice/src/interface/window_main.h @@ -68,6 +68,11 @@ signals: /** @brief Emitted after the background card-database update subprocess exits. */ void cardDatabaseUpdateFinished(bool success); + /** @brief Emitted while the background card-database update subprocess runs. + * @p stage is one of "download", "scan" or "import"; @p done/@p total + * are byte counts for the first two stages and set indices for "import". */ + void cardDatabaseUpdateProgress(const QString &stage, qint64 done, qint64 total); + public slots: void actCheckCardUpdates(); void actCheckCardUpdatesBackground(); @@ -96,6 +101,7 @@ private slots: void cardUpdateError(QProcess::ProcessError err); void cardUpdateFinished(int exitCode, QProcess::ExitStatus exitStatus); + void cardUpdateProgressOutput(); void refreshShortcuts(); void cardDatabaseLoadingFailed(); void cardDatabaseNewSetsFound(int numUnknownSets, QStringList unknownSetsNames); @@ -159,6 +165,7 @@ private: LatencyStatusWidget *latencyStatus = nullptr; ///< status bar widget with live round-trip stats and history graph bool bHasActivated, askedForDbUpdater; QProcess *cardUpdateProcess; + QByteArray cardUpdateOutputBuffer; DlgViewLog *logviewDialog; GameReplay *replay; DlgTipOfTheDay *tip; diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index d745b250c..6a83fd96c 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -47,8 +47,14 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s bool OracleImporter::readSetsFromByteArray(QByteArray data) { + const RawJson::ScanProgressCallback progress = + progressReporting ? RawJson::ScanProgressCallback([this](qsizetype bytesRead, qsizetype totalBytes) { + emit dataReadProgress(static_cast(bytesRead), static_cast(totalBytes)); + }) + : RawJson::ScanProgressCallback(); + RawJson::ScanError error; - const QList ranges = RawJson::scanSetRanges(data, &error); + const QList ranges = RawJson::scanSetRanges(data, &error, progress); if (error.isError()) { qDebug() << "error: RawJson::scanSetRanges():" << error.message; return false; diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 8cb30ca40..62927fefb 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -155,6 +155,14 @@ private: */ QByteArray rawSetsData; + /** + * Whether readSetsFromByteArray() should report scan progress via + * dataReadProgress. A background run routes that signal to stdout (for the + * hosting Cockatrice client to parse); the flag exists to skip the scanner + * instrumentation entirely when no consumer needs it. + */ + bool progressReporting = true; + CardInfoPtr addCard(QString name, const QString &text, bool isToken, @@ -167,6 +175,18 @@ signals: public: explicit OracleImporter(QObject *parent = nullptr); + /** + * @brief Controls whether readSetsFromByteArray() instruments the raw scan. + * + * When enabled (the default) the raw scanner reports progress via + * dataReadProgress(), which an interactive wizard shows on its progress bar + * and a background run routes to stdout for the hosting client. Switch it + * off only when nothing will consume scan progress. + */ + void setProgressReporting(bool enabled) + { + progressReporting = enabled; + } /** * Scans the given JSON document for set metadata. Takes the data by value so * the wizard can hand over its decompressed buffer without copying it. diff --git a/oracle/src/oraclewizard.cpp b/oracle/src/oraclewizard.cpp index 11200dfc5..ec9cb41bc 100644 --- a/oracle/src/oraclewizard.cpp +++ b/oracle/src/oraclewizard.cpp @@ -110,6 +110,13 @@ void OracleWizard::accept() QDialog::accept(); } +void OracleWizard::runInBackground() +{ + backgroundMode = true; + hide(); + currentPage()->initializePage(); +} + void OracleWizard::enableButtons() { button(QWizard::NextButton)->setDisabled(false); diff --git a/oracle/src/oraclewizard.h b/oracle/src/oraclewizard.h index 78427175c..e6c09fedf 100644 --- a/oracle/src/oraclewizard.h +++ b/oracle/src/oraclewizard.h @@ -52,12 +52,7 @@ public: } bool saveTokensToFile(const QString &fileName); - void runInBackground() - { - backgroundMode = true; - hide(); - currentPage()->initializePage(); - } + void runInBackground(); public: OracleImporter *importer; diff --git a/oracle/src/pages.cpp b/oracle/src/pages.cpp index 94e662ffe..263712e5f 100644 --- a/oracle/src/pages.cpp +++ b/oracle/src/pages.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -20,14 +21,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include #ifdef HAS_LZMA @@ -53,6 +57,20 @@ #define ALLSETS_URL "https://www.mtgjson.com/api/v5/AllPrintings.json" #endif +/** + * @brief Emits one machine-readable background-run progress line to stdout. + * + * Used only in background mode, so the hosting Cockatrice client can parse these + * lines to drive a determinate progress bar. stderr stays reserved for + * human-readable log output. + */ +static void emitBackgroundProgress(const char *stage, qint64 done, qint64 total) +{ + QTextStream out(stdout); + out << "PROGRESS " << stage << ' ' << done << ' ' << total << '\n'; + out.flush(); +} + #define TOKENS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Token/master/tokens.xml" #define SPOILERS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Spoiler/files/spoiler.xml" @@ -339,6 +357,9 @@ void LoadSetsPage::downloadSetsFile(const QUrl &url) void LoadSetsPage::actDownloadProgressSetsFile(qint64 received, qint64 total) { + if (wizard()->backgroundMode) { + emitBackgroundProgress("download", received, total); + } if (total > 0) { progressBar->setMaximum(static_cast(total)); progressBar->setValue(static_cast(received)); @@ -384,6 +405,21 @@ void LoadSetsPage::actDownloadFinishedSetsFile() reply->deleteLater(); } +void LoadSetsPage::updateParsingProgress(int bytesRead, int totalBytes) +{ + if (totalBytes <= 0) { + return; + } + progressBar->setValue(bytesRead); + const int percent = static_cast((100.0 * bytesRead) / totalBytes); + progressLabel->setText(tr("Parsing file (%1%)").arg(percent)); +} + +void LoadSetsPage::scanProgressToStdout(int bytesRead, int totalBytes) +{ + emitBackgroundProgress("scan", bytesRead, totalBytes); +} + void LoadSetsPage::readSetsFromByteArray(QByteArray _data) { // show an infinite progressbar @@ -469,9 +505,23 @@ void LoadSetsPage::readSetsFromByteArrayRef(QByteArray &_data) return; #endif } else if (_data.startsWith("{")) { - // Start the computation. - jsonData = std::move(_data); - future = QtConcurrent::run([this] { return wizard()->importer->readSetsFromByteArray(std::move(jsonData)); }); + if (wizard()->backgroundMode) { + qInfo() << tr("Parsing file"); + connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::scanProgressToStdout, + Qt::UniqueConnection); + } else { + // Start the computation. + progressBar->setRange(0, static_cast(_data.size())); + progressBar->setValue(0); + progressLabel->setText(tr("Parsing file (0%)")); + connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::updateParsingProgress, + Qt::UniqueConnection); + } + + const QPointer importer = wizard()->importer; + future = QtConcurrent::run([importer, data = std::move(_data)]() mutable { + return importer ? importer->readSetsFromByteArray(std::move(data)) : false; + }); watcher.setFuture(future); } else if (_data.startsWith("<")) { // save xml file and don't do any processing @@ -514,7 +564,16 @@ void LoadSetsPage::importFinished() progressLabel->hide(); progressBar->hide(); - if (wizard()->downloadedPlainXml || watcher.future().result()) { + const bool hasData = wizard()->downloadedPlainXml || watcher.future().result(); + if (wizard()->backgroundMode) { + if (!hasData) { + qWarning() << tr("The file was retrieved successfully, but it does not contain any sets data."); + } + emit readyToContinue(); + return; + } + + if (hasData) { wizard()->next(); } else { QMessageBox::critical(this, tr("Error"), @@ -527,16 +586,22 @@ SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent) pathLabel = new QLabel(this); saveLabel = new QLabel(this); + progressBar = new QProgressBar(this); + progressBar->hide(); + defaultPathCheckBox = new QCheckBox(this); messageLog = new QTextEdit(this); messageLog->setReadOnly(true); auto *layout = new QGridLayout(this); - layout->addWidget(messageLog, 0, 0); - layout->addWidget(saveLabel, 1, 0); - layout->addWidget(pathLabel, 2, 0); - layout->addWidget(defaultPathCheckBox, 3, 0); + layout->addWidget(progressBar, 0, 0); + layout->addWidget(messageLog, 1, 0); + layout->addWidget(saveLabel, 2, 0); + layout->addWidget(pathLabel, 3, 0); + layout->addWidget(defaultPathCheckBox, 4, 0); + + connect(&importWatcher, &QFutureWatcher::finished, this, &SaveSetsPage::importFinished); setLayout(layout); } @@ -549,27 +614,54 @@ void SaveSetsPage::cleanupPage() void SaveSetsPage::initializePage() { - messageLog->clear(); - retranslateUi(); if (wizard()->downloadedPlainXml) { messageLog->hide(); - } else { - messageLog->show(); - connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress); - - int setsImported = wizard()->importer->startImport(); - - // JSON data no longer needed after CardInfo objects are built - wizard()->importer->releaseSetData(); - - if (setsImported == 0) { - QMessageBox::critical(this, tr("Error"), tr("No set has been imported.")); + progressBar->hide(); + if (wizard()->backgroundMode) { + emit readyToContinue(); } + return; + } + + messageLog->clear(); + messageLog->show(); + progressBar->show(); + progressBar->setRange(0, wizard()->importer->getSets().size()); + progressBar->setValue(0); + + connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress, + Qt::UniqueConnection); + + wizard()->disableButtons(); + + const QPointer importer = wizard()->importer; + importFuture = QtConcurrent::run([importer] { return importer ? importer->startImport() : 0; }); + importWatcher.setFuture(importFuture); +} + +void SaveSetsPage::importFinished() +{ + wizard()->enableButtons(); + + const int setsImported = importWatcher.result(); + const QPointer importer = wizard()->importer; + if (importer) { + importer->releaseSetData(); } if (wizard()->backgroundMode) { + if (setsImported == 0) { + qWarning() << tr("No set has been imported."); + } emit readyToContinue(); + return; + } + + progressBar->setValue(progressBar->maximum()); + + if (setsImported == 0) { + QMessageBox::critical(this, tr("Error"), tr("No set has been imported.")); } } @@ -591,13 +683,27 @@ void SaveSetsPage::retranslateUi() setButtonText(QWizard::NextButton, tr("&Save")); } -void SaveSetsPage::updateTotalProgress(int cardsImported, int /* setIndex */, const QString &setName) +void SaveSetsPage::updateTotalProgress(int cardsImported, int setIndex, const QString &setName) { + const bool background = wizard()->backgroundMode; + const int totalSets = wizard()->importer->getSets().size(); if (setName.isEmpty()) { - messageLog->append("" + tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()) + - ""); + progressBar->setValue(progressBar->maximum()); + if (background) { + qInfo() << tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()); + emitBackgroundProgress("import", totalSets, totalSets); + } else { + messageLog->append("" + tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()) + + ""); + } } else { - messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported)); + progressBar->setValue(setIndex); + if (background) { + qInfo() << tr("%1: %2 cards imported").arg(setName).arg(cardsImported); + emitBackgroundProgress("import", setIndex, totalSets); + } else { + messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported)); + } } messageLog->verticalScrollBar()->setValue(messageLog->verticalScrollBar()->maximum()); diff --git a/oracle/src/pages.h b/oracle/src/pages.h index 066cc2e1b..b0543e98f 100644 --- a/oracle/src/pages.h +++ b/oracle/src/pages.h @@ -83,13 +83,14 @@ private: QFutureWatcher watcher; QFuture future; - QByteArray jsonData; private slots: void actLoadSetsFile(); void actRestoreDefaultUrl(); void actDownloadProgressSetsFile(qint64 received, qint64 total); void actDownloadFinishedSetsFile(); + void updateParsingProgress(int bytesRead, int totalBytes); + void scanProgressToStdout(int bytesRead, int totalBytes); void importFinished(); void zipDownloadFailed(const QString &message); }; @@ -103,16 +104,21 @@ public: private: QTextEdit *messageLog; + QProgressBar *progressBar; QCheckBox *defaultPathCheckBox; QLabel *pathLabel; QLabel *saveLabel; + QFutureWatcher importWatcher; + QFuture importFuture; + protected: void initializePage() override; void cleanupPage() override; bool validatePage() override; private slots: + void importFinished(); void updateTotalProgress(int cardsImported, int setIndex, const QString &setName); }; diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp index db972491c..747a34814 100644 --- a/oracle/src/raw_json_scanner.cpp +++ b/oracle/src/raw_json_scanner.cpp @@ -1,5 +1,6 @@ #include "raw_json_scanner.h" +#include #include namespace @@ -11,6 +12,43 @@ namespace // reason and reports DeepNesting). constexpr int kMaxNestingDepth = 1024; +/** + * @brief Throttled byte-position reporting for scanSetRanges(). + * + * Threaded through the skip walk so progress can be reported without materializing + * the whole document. Reports are rate-limited so a GUI showing progress isn't + * flooded with interrupts: a callback is invoked at most ~100 times per scan + * regardless of element count. + */ +struct ScanProgress +{ + const char *begin = nullptr; + qsizetype size = 0; + RawJson::ScanProgressCallback callback; + qsizetype step = 1; + qsizetype lastReported = 0; + + /** + * @brief Reports the scanner's absolute offset, unless within @p step bytes + * of the previous report and not yet at the end of the document. + */ + void report(const char *p) + { + if (!callback) { + return; + } + const qsizetype offset = p - begin; + if (offset == lastReported) { + return; // the final element often already sits exactly at the end + } + if (offset - lastReported < step && offset < size) { + return; + } + lastReported = offset; + callback(offset, size); + } +}; + inline bool isWhitespace(char c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; @@ -296,9 +334,9 @@ bool skipNumber(const char *&p, const char *end) return true; } -bool skipValue(const char *&p, const char *end, int depth); -bool skipObject(const char *&p, const char *end, int depth); -bool skipArray(const char *&p, const char *end, int depth); +bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan); +bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan); +bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan); bool skipPrimitive(const char *&p, const char *end) { @@ -324,7 +362,7 @@ bool skipPrimitive(const char *&p, const char *end) return false; } -bool skipObject(const char *&p, const char *end, int depth) +bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan) { if (depth <= 0) { return false; // nest deeper than the cap @@ -348,9 +386,10 @@ bool skipObject(const char *&p, const char *end, int depth) return false; } ++p; - if (!skipValue(p, end, depth - 1)) { + if (!skipValue(p, end, depth - 1, scan)) { return false; } + scan.report(p); p = skipWhitespace(p, end); if (p >= end) { return false; @@ -367,7 +406,7 @@ bool skipObject(const char *&p, const char *end, int depth) } } -bool skipArray(const char *&p, const char *end, int depth) +bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan) { if (depth <= 0) { return false; // nest deeper than the cap @@ -379,9 +418,10 @@ bool skipArray(const char *&p, const char *end, int depth) return true; } for (;;) { - if (!skipValue(p, end, depth - 1)) { + if (!skipValue(p, end, depth - 1, scan)) { return false; } + scan.report(p); p = skipWhitespace(p, end); if (p >= end) { return false; @@ -398,7 +438,7 @@ bool skipArray(const char *&p, const char *end, int depth) } } -bool skipValue(const char *&p, const char *end, int depth) +bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan) { p = skipWhitespace(p, end); if (p >= end) { @@ -407,10 +447,10 @@ bool skipValue(const char *&p, const char *end, int depth) const char c = *p; if (c == '{') { // pass depth through: skipObject consumes the single decrement for this level - return skipObject(p, end, depth); + return skipObject(p, end, depth, scan); } if (c == '[') { - return skipArray(p, end, depth); + return skipArray(p, end, depth, scan); } // a primitive is a leaf, so it never wastes a nesting level return skipPrimitive(p, end); @@ -422,7 +462,8 @@ bool skipValue(const char *&p, const char *end, int depth) * For each member invokes @p memberCallback with the key and the byte range of * its value. Advancing @p p is unaffected by the callback. */ -template bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback) +template +bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback, ScanProgress &scan) { if (depth <= 0) { return false; // nest deeper than the cap @@ -449,7 +490,7 @@ template bool forEachObjectMember(const char *&p, const char *end, ++p; const char *valueStart = skipWhitespace(p, end); const char *valueEnd = valueStart; - if (!skipValue(valueEnd, end, depth - 1)) { + if (!skipValue(valueEnd, end, depth - 1, scan)) { return false; } if (!memberCallback(key, valueStart, valueEnd)) { @@ -473,7 +514,7 @@ template bool forEachObjectMember(const char *&p, const char *end, } // Counts the direct elements of an array value; returns -1 if the array is malformed. -int countArrayElements(const char *p, const char *end, int depth) +int countArrayElements(const char *p, const char *end, int depth, ScanProgress &scan) { if (depth <= 0) { return -1; // nest deeper than the cap @@ -485,9 +526,10 @@ int countArrayElements(const char *p, const char *end, int depth) return 0; } for (;;) { - if (!skipValue(p, end, depth - 1)) { + if (!skipValue(p, end, depth - 1, scan)) { return -1; } + scan.report(p); ++count; p = skipWhitespace(p, end); if (p >= end) { @@ -509,7 +551,7 @@ int countArrayElements(const char *p, const char *end, int depth) namespace RawJson { -QList scanSetRanges(const QByteArray &json, ScanError *error) +QList scanSetRanges(const QByteArray &json, ScanError *error, const ScanProgressCallback &progress) { QList ranges; if (error) { @@ -529,6 +571,14 @@ QList scanSetRanges(const QByteArray &json, ScanError *error) return fail(QStringLiteral("empty JSON document")); } + // Throttle reports to ~100 per scan so a GUI thread unthrottling them never + // drowns under per-card interrupts, whatever the document size. + ScanProgress scan; + scan.begin = begin; + scan.size = end - begin; + scan.step = std::max(1, scan.size / 100); + scan.callback = progress; + const char *p = skipWhitespace(begin, end); if (p >= end || *p != '{') { return fail(QStringLiteral("top-level JSON must be an object")); @@ -545,55 +595,57 @@ QList scanSetRanges(const QByteArray &json, ScanError *error) return false; } const char *setP = valueStart; - const bool ok = forEachObjectMember(setP, valueEnd, kMaxNestingDepth - 1, - [&](const QString &setCode, const char *setStart, const char *setEnd) { - if (setStart >= setEnd || *setStart != '{') { - malformedSetData = true; - return false; - } - SetRange range; - range.dataRange.start = setStart - begin; - range.dataRange.length = setEnd - setStart; - range.code = setCode; + const bool ok = forEachObjectMember( + setP, valueEnd, kMaxNestingDepth - 1, + [&](const QString &setCode, const char *setStart, const char *setEnd) { + if (setStart >= setEnd || *setStart != '{') { + malformedSetData = true; + return false; + } + SetRange range; + range.dataRange.start = setStart - begin; + range.dataRange.length = setEnd - setStart; + range.code = setCode; - const char *memberP = setStart; - const bool metaOk = forEachObjectMember( - memberP, setEnd, kMaxNestingDepth - 2, - [&](const QString &field, const char *fs, const char *fe) { - if (field == QStringLiteral("code")) { - return decodeStringMember(fs, fe, range.code); - } - if (field == QStringLiteral("name")) { - return decodeStringMember(fs, fe, range.name); - } - if (field == QStringLiteral("type")) { - return decodeStringMember(fs, fe, range.type); - } - if (field == QStringLiteral("releaseDate")) { - return decodeStringMember(fs, fe, range.releaseDate); - } - if (field == QStringLiteral("cards")) { - if (fs >= fe) { - return false; - } - if (*fs != '[') { - // e.g. "cards": null — treat as an empty array, - // matching Qt's tolerance. - return true; - } - range.dataRange.cardCount = - countArrayElements(fs, fe, kMaxNestingDepth - 2); - return range.dataRange.cardCount >= 0; - } - return true; - }); - if (!metaOk) { - malformedSetData = true; - return false; - } - ranges.append(range); - return true; - }); + const char *memberP = setStart; + const bool metaOk = forEachObjectMember( + memberP, setEnd, kMaxNestingDepth - 2, + [&](const QString &field, const char *fs, const char *fe) { + if (field == QStringLiteral("code")) { + return decodeStringMember(fs, fe, range.code); + } + if (field == QStringLiteral("name")) { + return decodeStringMember(fs, fe, range.name); + } + if (field == QStringLiteral("type")) { + return decodeStringMember(fs, fe, range.type); + } + if (field == QStringLiteral("releaseDate")) { + return decodeStringMember(fs, fe, range.releaseDate); + } + if (field == QStringLiteral("cards")) { + if (fs >= fe) { + return false; + } + if (*fs != '[') { + // e.g. "cards": null — treat as an empty array, + // matching Qt's tolerance. + return true; + } + range.dataRange.cardCount = countArrayElements(fs, fe, kMaxNestingDepth - 2, scan); + return range.dataRange.cardCount >= 0; + } + return true; + }, + scan); + if (!metaOk) { + malformedSetData = true; + return false; + } + ranges.append(range); + return true; + }, + scan); if (!ok) { malformedSetData = true; return false; @@ -602,7 +654,7 @@ QList scanSetRanges(const QByteArray &json, ScanError *error) return true; }; - if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback)) { + if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback, scan)) { return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")); } p = skipWhitespace(p, end); @@ -615,6 +667,7 @@ QList scanSetRanges(const QByteArray &json, ScanError *error) if (ranges.isEmpty()) { return fail(QStringLiteral("no sets found in \"data\"")); } + scan.report(end); return ranges; } diff --git a/oracle/src/raw_json_scanner.h b/oracle/src/raw_json_scanner.h index f6e3a4647..086f2b3e3 100644 --- a/oracle/src/raw_json_scanner.h +++ b/oracle/src/raw_json_scanner.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace RawJson { @@ -42,6 +43,13 @@ struct ScanError QString message; }; +/** + * @brief Optional progress callback receiving @c (bytesRead, totalBytes) while + * the document is walked. Invoked from the scanning thread; the caller decides + * how the throttled offsets are relayed to a GUI event loop. + */ +using ScanProgressCallback = std::function; + /** * @brief Scans a full MTGJSON document without materializing the JSON tree. * @@ -67,9 +75,14 @@ struct ScanError * @param error Out parameter. Set to an error ScanError when the document * cannot be parsed, otherwise left empty. Passing a null * pointer disables error reporting. + * @param progress Optional progress callback. When non-empty it is invoked as + * the scanner advances through the document, throttled to a + * tiny fraction of the total size. * @return The detected per-set ranges, or an empty list on failure. */ -QList scanSetRanges(const QByteArray &json, ScanError *error = nullptr); +QList scanSetRanges(const QByteArray &json, + ScanError *error = nullptr, + const ScanProgressCallback &progress = ScanProgressCallback()); } // namespace RawJson diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index 3834be9ed..66c7bdbe1 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -741,6 +743,135 @@ TEST_F(OracleImporterTest, StartImportParsesSetsLazily) ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull()); } +// ============================================================================ +// Scan progress reporting tests +// ============================================================================ + +TEST(OracleScanProgress, ScanProgressReportsMonotonicBytesToTotal) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + QJsonArray cards; + for (int i = 0; i < 40; ++i) { + QJsonObject card; + card["name"] = QString("Card %1").arg(i); + card["text"] = "Some rules text used to bulk up the card payload."; + card["layout"] = "normal"; + cards.append(card); + } + setObj["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + + QList> reports; + RawJson::ScanError error; + const QList ranges = + RawJson::scanSetRanges(data, &error, [&reports](qsizetype bytesRead, qsizetype totalBytes) { + reports.append({bytesRead, totalBytes}); + }); + + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 1); + ASSERT_FALSE(reports.isEmpty()); + ASSERT_GT(reports.size(), 1); + + qsizetype last = 0; + for (const auto &[bytesRead, totalBytes] : reports) { + ASSERT_EQ(totalBytes, data.size()); + ASSERT_GE(bytesRead, last) << "scan progress must be monotonic"; + ASSERT_LE(bytesRead, totalBytes) << "scan progress must not overshoot the document size"; + last = bytesRead; + } + ASSERT_EQ(reports.constLast().first, data.size()) << "scan must end at 100%"; + ASSERT_LE(reports.size(), 160) << "scan reports must be throttled"; +} + +TEST(OracleScanProgress, ScanWithoutCallbackStillParses) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(data, &error); + + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 1); + ASSERT_EQ(ranges.first().code, "tst"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayEmitsScanProgress) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + QJsonArray cards; + for (int i = 0; i < 40; ++i) { + QJsonObject card; + card["name"] = QString("Card %1").arg(i); + cards.append(card); + } + setObj["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + + QList> emissions; + QObject::connect(importer, &OracleImporter::dataReadProgress, + [&emissions](int bytesRead, int totalBytes) { emissions.append({bytesRead, totalBytes}); }); + + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_FALSE(emissions.isEmpty()); + for (const auto &[bytesRead, totalBytes] : emissions) { + ASSERT_EQ(totalBytes, data.size()); + ASSERT_GE(bytesRead, 0); + ASSERT_LE(bytesRead, totalBytes); + } + ASSERT_EQ(emissions.constLast().first, data.size()); +} + +TEST_F(OracleImporterTest, DisablingProgressReportingSuppressesScanEmissions) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + + int emissions = 0; + QObject::connect(importer, &OracleImporter::dataReadProgress, [&emissions](int, int) { ++emissions; }); + + importer->setProgressReporting(false); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(emissions, 0); + + importer->setProgressReporting(true); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_GT(emissions, 0); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv);