From 23bdc1d9866654edf6b4b2c2efb9489ba5d7c71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sun, 30 Aug 2026 13:39:00 +0200 Subject: [PATCH 01/25] [Oracle] Replace vendored QtJson with native QJson for set import - Drop the vendored oracle/src/qt-json/json.{h,cpp} implementation - Switch SetToDownload and importCardsFromSet from QList to native QJsonArray/QJsonObject - Release set JSON data after import in the save sets page Took 20 minutes --- format.sh | 1 - oracle/CMakeLists.txt | 1 - oracle/src/oracleimporter.cpp | 143 ++++----- oracle/src/oracleimporter.h | 11 +- oracle/src/pages.cpp | 3 + oracle/src/qt-json/AUTHORS | 3 - oracle/src/qt-json/LICENSE | 27 -- oracle/src/qt-json/README | 96 ------ oracle/src/qt-json/json.cpp | 545 ---------------------------------- oracle/src/qt-json/json.h | 204 ------------- 10 files changed, 83 insertions(+), 951 deletions(-) delete mode 100644 oracle/src/qt-json/AUTHORS delete mode 100644 oracle/src/qt-json/LICENSE delete mode 100644 oracle/src/qt-json/README delete mode 100644 oracle/src/qt-json/json.cpp delete mode 100644 oracle/src/qt-json/json.h diff --git a/format.sh b/format.sh index 3fa435be1..ca3557ea7 100755 --- a/format.sh +++ b/format.sh @@ -22,7 +22,6 @@ libcockatrice_* \ exclude=("libcockatrice_rng/libcockatrice/rng/sfmt/" \ "libcockatrice_utility/libcockatrice/utility/peglib.h" \ "oracle/src/lzma/" \ -"oracle/src/qt-json/" \ "oracle/src/zip/" \ "servatrice/src/smtp/") exts=("cpp" "h" "proto") diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 0736db7f5..6a29b6935 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,7 +23,6 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp - src/qt-json/json.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index fdb32bb8d..a531b82d1 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -3,9 +3,10 @@ #include "libcockatrice/interfaces/noop_card_preference_provider.h" #include "libcockatrice/interfaces/noop_card_set_priority_controller.h" #include "parsehelpers.h" -#include "qt-json/json.h" #include +#include +#include #include #include #include @@ -44,24 +45,24 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s bool OracleImporter::readSetsFromByteArray(const QByteArray &data) { - bool ok; - auto setsMap = QtJson::Json::parse(QString(data), ok).toMap().value("data").toMap(); - if (!ok) { - qDebug() << "error: QtJson::Json::parse()"; + QJsonParseError error; + auto doc = QJsonDocument::fromJson(data, &error); + if (error.error != QJsonParseError::NoError) { + qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); return false; } + auto setsObj = doc.object().value("data").toObject(); + QList newSetList; - QListIterator it(setsMap.values()); - - while (it.hasNext()) { - QVariantMap map = it.next().toMap(); - QString shortName = map.value("code").toString().toUpper(); - QString longName = map.value("name").toString(); - QList setCards = map.value("cards").toList(); - QString setType = map.value("type").toString(); - QDate releaseDate = map.value("releaseDate").toDate(); + for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { + QJsonObject setObj = it.value().toObject(); + QString shortName = setObj.value("code").toString().toUpper(); + QString longName = setObj.value("name").toString(); + QJsonArray setCards = setObj.value("cards").toArray(); + QString setType = setObj.value("type").toString(); + QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { @@ -142,12 +143,11 @@ CardInfoPtr OracleImporter::addCard(QString name, // Workaround for card name weirdness name = name.replace("Æ", "AE"); name = name.replace("’", "'"); - if (cards.contains(name)) { - CardInfoPtr card = cards.value(name); + auto existingIt = cards.constFind(name); + if (existingIt != cards.constEnd()) { + CardInfoPtr card = existingIt.value(); card->addToSet(printingInfo.getSet(), printingInfo); - if (card->getProperties().filter(formatRegex).empty()) { - card->combineLegalities(properties); - } + card->combineLegalities(properties); return card; } @@ -222,12 +222,12 @@ CardInfoPtr OracleImporter::addCard(QString name, return newCard; } -static QString getStringPropertyFromMap(const QVariantMap &card, const QString &propertyName) +static QString getJsonString(const QJsonObject &obj, const QString &key) { - return card.contains(propertyName) ? card.value(propertyName).toString() : QString(""); + return obj.value(key).toString(); } -int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList &cardsList) +int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList) { // mtgjson name => xml name static const QMap cardProperties{ @@ -248,7 +248,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList static const QString ptSeparator = "/"; static constexpr bool isToken = false; - static const QList setsWithCardsWithSameNameButDifferentText = {"UST"}; + static const QSet setsWithCardsWithSameNameButDifferentText = {"UST"}; int numCards = 0; @@ -256,16 +256,16 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList QMap, QString>> splitCards; // Keeps track of all names encountered so far - QList allNameProps; + QSet allNameProps; - for (const QVariant &cardVar : cardsList) { - QVariantMap card = cardVar.toMap(); + for (const QJsonValue &cardVal : cardsList) { + QJsonObject card = cardVal.toObject(); /* Currently used layouts are: * augment, double_faced_token, flip, host, leveler, meld, normal, planar, * saga, scheme, split, token, transform, vanguard */ - QString layout = getStringPropertyFromMap(card, "layout"); + QString layout = getJsonString(card, "layout"); // don't import tokens from the json file if (layout == "token") { @@ -273,9 +273,9 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList } // normal cards handling - QString name = getStringPropertyFromMap(card, "name"); - QString text = getStringPropertyFromMap(card, "text"); - QString faceName = getStringPropertyFromMap(card, "faceName"); + QString name = getJsonString(card, "name"); + QString text = getJsonString(card, "text"); + QString faceName = getJsonString(card, "faceName"); if (faceName.isEmpty()) { faceName = name; } @@ -283,39 +283,34 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // card properties QHash properties; for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty); + QString propertyValue = getJsonString(card, i.key()); if (!propertyValue.isEmpty()) { - properties.insert(xmlPropertyName, propertyValue); + properties.insert(i.value(), propertyValue); } } // per-set properties QHash printingProps; for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty); + QString propertyValue = getJsonString(card, i.key()); if (!propertyValue.isEmpty()) { - printingProps.insert(xmlPropertyName, propertyValue); + printingProps.insert(i.value(), propertyValue); } } // handle flavorNames specially due to double-faced cards - QString faceFlavorName = getStringPropertyFromMap(card, "faceFlavorName"); - QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getStringPropertyFromMap(card, "flavorName"); + QString faceFlavorName = getJsonString(card, "faceFlavorName"); + QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName"); if (!flavorName.isEmpty()) { printingProps.insert("flavorName", flavorName); } // Identifiers + QJsonObject identifiers = card.value("identifiers").toObject(); for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card.value("identifiers").toMap(), mtgjsonProperty); + QString propertyValue = getJsonString(identifiers, i.key()); if (!propertyValue.isEmpty()) { - printingProps.insert(xmlPropertyName, propertyValue); + printingProps.insert(i.value(), propertyValue); } } @@ -331,21 +326,20 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) { numComponent = " (" + QString(lastChar).toLower() + ")"; } - allNameProps.append(faceName); + allNameProps.insert(faceName); // special handling properties - QString colors = card.value("colors").toStringList().join(""); + QString colors = card.value("colors").toVariant().toStringList().join(""); if (!colors.isEmpty()) { properties.insert("colors", colors); } - // special handling properties - QString colorIdentity = card.value("colorIdentity").toStringList().join(""); + QString colorIdentity = card.value("colorIdentity").toVariant().toStringList().join(""); if (!colorIdentity.isEmpty()) { properties.insert("coloridentity", colorIdentity); } - const auto &mainCardType = getMainCardType(card.value("types").toStringList()); + const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList()); if (mainCardType.isEmpty()) { qDebug() << "warning: no mainCardType for card:" << name; } else { @@ -354,22 +348,22 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // Depending on whether power and/or toughness are present, the format // is either P/T (most common), P (no toughness), or /T (no power). - QString power = getStringPropertyFromMap(card, "power"); - QString toughness = getStringPropertyFromMap(card, "toughness"); + QString power = getJsonString(card, "power"); + QString toughness = getJsonString(card, "toughness"); if (toughness.isEmpty() && !power.isEmpty()) { properties.insert("pt", power); } else if (!toughness.isEmpty()) { properties.insert("pt", power + ptSeparator + toughness); } - auto legalities = card.value("legalities").toMap(); - for (auto i = legalities.cbegin(), end = legalities.cend(); i != end; ++i) { + auto legalities = card.value("legalities").toObject(); + for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) { properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower()); } // split cards are considered a single card, enqueue for later merging if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") { - auto _faceName = getStringPropertyFromMap(card, "faceName"); + auto _faceName = getJsonString(card, "faceName"); SplitCardPart split(_faceName, text, properties, printingInfo); auto found_iter = splitCards.find(name + numProperty); if (found_iter == splitCards.end()) { @@ -382,11 +376,11 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList QList relatedCards; // add other face for split cards as card relation - if (!getStringPropertyFromMap(card, "side").isEmpty()) { - auto faceManaValue = getStringPropertyFromMap(card, "faceManaValue"); + if (!getJsonString(card, "side").isEmpty()) { + auto faceManaValue = getJsonString(card, "faceManaValue"); if (faceManaValue.isEmpty()) { // check the old name for the property, for backwards compatibility purposes - faceManaValue = getStringPropertyFromMap(card, "faceConvertedManaCost"); + faceManaValue = getJsonString(card, "faceConvertedManaCost"); } properties["cmc"] = faceManaValue; @@ -406,15 +400,15 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList name = faceName; } - // mtgjon related cards - if (card.contains("relatedCards")) { - QVariantMap givenRelated = card.value("relatedCards").toMap(); + // mtgjson related cards + QJsonObject givenRelated = card.value("relatedCards").toObject(); + if (!givenRelated.isEmpty()) { // conjured cards from a spellbook - if (givenRelated.contains("spellbook")) { - auto spbk = givenRelated.value("spellbook").toStringList(); - for (const QString &spbkName : spbk) { - relatedCards.append( - new CardRelation(spbkName, CardRelationType::DoesNotAttach, false, false, 1, true)); + QJsonArray spellbook = givenRelated.value("spellbook").toArray(); + if (!spellbook.isEmpty()) { + for (const QJsonValue &spbkVal : spellbook) { + relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false, + false, 1, true)); } } } @@ -427,7 +421,6 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // split cards handling static const QString splitCardPropSeparator = QString(" // "); static const QString splitCardTextSeparator = QString("\n\n---\n\n"); - static const QList noRelatedCards = {}; QList, QString>> partsAndNames = splitCards.values(); for (auto [splitCardParts, name] : partsAndNames) { @@ -465,20 +458,20 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList } } } - CardInfoPtr newCard = addCard(name, text, isToken, properties, noRelatedCards, printingInfo); + CardInfoPtr newCard = addCard(name, text, isToken, properties, {}, printingInfo); numCards++; } return numCards; } -FormatRulesNameMap OracleImporter::createDefaultMagicFormats() +static FormatRulesNameMap buildDefaultMagicFormats() { // Predefined common exceptions CardCondition superTypeIsBasic; superTypeIsBasic.field = "type"; superTypeIsBasic.matchType = "regex"; - superTypeIsBasic.value = "\bBasic\b[^—]+\bLand\b"; + superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)"; ExceptionRule basicLands; basicLands.conditions.append(superTypeIsBasic); @@ -491,7 +484,6 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats() ExceptionRule mayContainAnyNumber; mayContainAnyNumber.conditions.append(anyNumberAllowed); - // Map to store default rules FormatRulesNameMap defaultFormatRulesNameMap; // ----------------- Helper lambda to create format ----------------- @@ -537,6 +529,12 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats() return defaultFormatRulesNameMap; } +FormatRulesNameMap OracleImporter::createDefaultMagicFormats() +{ + static const FormatRulesNameMap cached = buildDefaultMagicFormats(); + return cached; +} + int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); @@ -576,6 +574,11 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion); } +void OracleImporter::releaseSetData() +{ + allSets.clear(); +} + void OracleImporter::clear() { sets.clear(); diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 99644f9ce..696e35cab 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,8 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include +#include #include #include #include @@ -44,7 +46,7 @@ class SetToDownload { private: QString shortName, longName; - QList cards; + QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; @@ -58,7 +60,7 @@ public: { return longName; } - const QList &getCards() const + const QJsonArray &getCards() const { return cards; } @@ -76,7 +78,7 @@ public: } SetToDownload(QString _shortName, QString _longName, - QList _cards, + QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) @@ -154,7 +156,7 @@ public: bool readSetsFromByteArray(const QByteArray &data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); - int importCardsFromSet(const CardSetPtr ¤tSet, const QList &cardsList); + int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); FormatRulesNameMap createDefaultMagicFormats(); const CardNameMap &getCardList() const { @@ -164,6 +166,7 @@ public: { return allSets; } + void releaseSetData(); void clear(); }; diff --git a/oracle/src/pages.cpp b/oracle/src/pages.cpp index df4d1a98c..94e662ffe 100644 --- a/oracle/src/pages.cpp +++ b/oracle/src/pages.cpp @@ -560,6 +560,9 @@ void SaveSetsPage::initializePage() int setsImported = wizard()->importer->startImport(); + // JSON data no longer needed after CardInfo objects are built + wizard()->importer->releaseSetData(); + if (setsImported == 0) { QMessageBox::critical(this, tr("Error"), tr("No set has been imported.")); } diff --git a/oracle/src/qt-json/AUTHORS b/oracle/src/qt-json/AUTHORS deleted file mode 100644 index 29a85929f..000000000 --- a/oracle/src/qt-json/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -Eeli Reilin -Luis Gustavo S. Barreto -Stephen Kockentiedt diff --git a/oracle/src/qt-json/LICENSE b/oracle/src/qt-json/LICENSE deleted file mode 100644 index 3c42b515a..000000000 --- a/oracle/src/qt-json/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright 2011 Eeli Reilin. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation -are those of the authors and should not be interpreted as representing -official policies, either expressed or implied, of Eeli Reilin. - diff --git a/oracle/src/qt-json/README b/oracle/src/qt-json/README deleted file mode 100644 index b60c1599b..000000000 --- a/oracle/src/qt-json/README +++ /dev/null @@ -1,96 +0,0 @@ -######################################################################## -1. INTRODUCTION - -The Json class is a simple class for parsing JSON data into a QVariant -hierarchies. Now, we can also reverse the process and serialize -QVariant hierarchies into valid JSON data. - - -######################################################################## -2. HOW TO USE - -The parser is really easy to use. Let's say we have the following -QString of JSON data: - ------------------------------------------------------------------------- -{ - "encoding" : "UTF-8", - "plug-ins" : [ - "python", - "c++", - "ruby" - ], - "indent" : { - "length" : 3, - "use_space" : true - } -} ------------------------------------------------------------------------- - -We would first call the parse-method: - ------------------------------------------------------------------------- -//Say that we're using the QtJson namespace -using namespace QtJson; -bool ok; -//json is a QString containing the JSON data -QVariantMap result = Json::parse(json, ok).toMap(); - -if(!ok) { - qFatal("An error occurred during parsing"); - exit(1); -} ------------------------------------------------------------------------- - -Assuming the parsing process completed without errors, we would then -go through the hierarchy: - ------------------------------------------------------------------------- -qDebug() << "encoding:" << result["encoding"].toString(); -qDebug() << "plugins:"; - -foreach(QVariant plugin, result["plug-ins"].toList()) { - qDebug() << "\t-" << plugin.toString(); -} - -QVariantMap nestedMap = result["indent"].toMap(); -qDebug() << "length:" << nestedMap["length"].toInt(); -qDebug() << "use_space:" << nestedMap["use_space"].toBool(); ------------------------------------------------------------------------- - -The previous code would print out the following: - ------------------------------------------------------------------------- -encoding: "UTF-8" -plugins: - - "python" - - "c++" - - "ruby" -length: 3 -use_space: true ------------------------------------------------------------------------- - -To write JSON data from Qt object is as simple as parsing: - ------------------------------------------------------------------------- -QVariantMap map; -map["name"] = "Name"; -map["age"] = 22; - -QByteArray data = Json::serialize(map); ------------------------------------------------------------------------- - -The byte array 'data' contains valid JSON data: - ------------------------------------------------------------------------- -{ - name: "Luis Gustavo", - age: 22, -} ------------------------------------------------------------------------- - - -######################################################################## -4. CONTRIBUTING - -The code is available to download at GitHub. Contribute if you dare! diff --git a/oracle/src/qt-json/json.cpp b/oracle/src/qt-json/json.cpp deleted file mode 100644 index ff739b49d..000000000 --- a/oracle/src/qt-json/json.cpp +++ /dev/null @@ -1,545 +0,0 @@ -/* Copyright 2011 Eeli Reilin. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, - * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * The views and conclusions contained in the software and documentation - * are those of the authors and should not be interpreted as representing - * official policies, either expressed or implied, of Eeli Reilin. - */ - -/** - * \file json.cpp - */ - -#include "json.h" - -#include -#include - -namespace QtJson -{ - -static QString sanitizeString(QString str) -{ - str.replace(QLatin1String("\\"), QLatin1String("\\\\")); - str.replace(QLatin1String("\""), QLatin1String("\\\"")); - str.replace(QLatin1String("\b"), QLatin1String("\\b")); - str.replace(QLatin1String("\f"), QLatin1String("\\f")); - str.replace(QLatin1String("\n"), QLatin1String("\\n")); - str.replace(QLatin1String("\r"), QLatin1String("\\r")); - str.replace(QLatin1String("\t"), QLatin1String("\\t")); - return QString(QLatin1String("\"%1\"")).arg(str); -} - -static QByteArray join(const QList &list, const QByteArray &sep) -{ - QByteArray res; - for (const QByteArray &i : list) { - if (!res.isEmpty()) { - res += sep; - } - res += i; - } - return res; -} - -/** - * parse - */ -QVariant Json::parse(const QString &json) -{ - bool success = true; - return Json::parse(json, success); -} - -/** - * parse - */ -QVariant Json::parse(const QString &json, bool &success) -{ - success = true; - - // Return an empty QVariant if the JSON data is either null or empty - if (!json.isNull() || !json.isEmpty()) { - // We'll start from index 0 - int index = 0; - - // Parse the first value - QVariant value = Json::parseValue(json, index, success); - - // Return the parsed value - return value; - } else { - // Return the empty QVariant - return QVariant(); - } -} - -QByteArray Json::serialize(const QVariant &data) -{ - bool success = true; - return Json::serialize(data, success); -} - -QByteArray Json::serialize(const QVariant &data, bool &success) -{ - QByteArray str; - success = true; - - if (!data.isValid()) // invalid or null? - { - str = "null"; - } - else if ((data.typeId() == QMetaType::Type::QVariantList) || - (data.typeId() == QMetaType::Type::QStringList)) // variant is a list? - { - QList values; - const QVariantList list = data.toList(); - for (const QVariant &v : list) { - QByteArray serializedValue = serialize(v); - if (serializedValue.isNull()) { - success = false; - break; - } - values << serializedValue; - } - - str = "[ " + join(values, ", ") + " ]"; - } - else if ((data.typeId() == QMetaType::Type::QVariantHash)) // variant is a hash? - { - const QVariantHash vhash = data.toHash(); - QHashIterator it(vhash); - str = "{ "; - QList pairs; - - while (it.hasNext()) { - it.next(); - QByteArray serializedValue = serialize(it.value()); - - if (serializedValue.isNull()) { - success = false; - break; - } - - pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue; - } - - str += join(pairs, ", "); - str += " }"; - } - else if ((data.typeId() == QMetaType::Type::QVariantMap)) // variant is a map? - { - const QVariantMap vmap = data.toMap(); - QMapIterator it(vmap); - str = "{ "; - QList pairs; - while (it.hasNext()) { - it.next(); - QByteArray serializedValue = serialize(it.value()); - if (serializedValue.isNull()) { - success = false; - break; - } - pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue; - } - str += join(pairs, ", "); - str += " }"; - } - else if ((data.typeId() == QMetaType::Type::QString) || - (data.typeId() == QMetaType::Type::QByteArray)) // a string or a byte array? - { - str = sanitizeString(data.toString()).toUtf8(); - } - else if (data.typeId() == QMetaType::Type::Double) // double? - { - str = QByteArray::number(data.toDouble(), 'g', 20); - if (!str.contains(".") && !str.contains("e")) { - str += ".0"; - } - } - else if (data.typeId() == QMetaType::Type::Bool) // boolean value? - { - str = data.toBool() ? "true" : "false"; - } - else if (data.typeId() == QMetaType::Type::ULongLong) // large unsigned number? - { - str = QByteArray::number(data.value()); - } else if (data.canConvert()) // any signed number? - { - str = QByteArray::number(data.value()); - } else if (data.canConvert()) { - str = QString::number(data.value()).toUtf8(); - } else if (data.canConvert()) // can value be converted to string? - { - // this will catch QDate, QDateTime, QUrl, ... - str = sanitizeString(data.toString()).toUtf8(); - } else { - success = false; - } - if (success) { - return str; - } else { - return QByteArray(); - } -} - -/** - * parseValue - */ -QVariant Json::parseValue(const QString &json, int &index, bool &success) -{ - // Determine what kind of data we should parse by - // checking out the upcoming token - switch (Json::lookAhead(json, index)) { - case JsonTokenString: - return Json::parseString(json, index, success); - case JsonTokenNumber: - return Json::parseNumber(json, index); - case JsonTokenCurlyOpen: - return Json::parseObject(json, index, success); - case JsonTokenSquaredOpen: - return Json::parseArray(json, index, success); - case JsonTokenTrue: - Json::nextToken(json, index); - return QVariant(true); - case JsonTokenFalse: - Json::nextToken(json, index); - return QVariant(false); - case JsonTokenNull: - Json::nextToken(json, index); - return QVariant(); - case JsonTokenNone: - break; - } - - // If there were no tokens, flag the failure and return an empty QVariant - success = false; - return QVariant(); -} - -/** - * parseObject - */ -QVariant Json::parseObject(const QString &json, int &index, bool &success) -{ - QVariantMap map; - int token; - - // Get rid of the whitespace and increment index - Json::nextToken(json, index); - - // Loop through all of the key/value pairs of the object - bool done = false; - while (!done) { - // Get the upcoming token - token = Json::lookAhead(json, index); - - if (token == JsonTokenNone) { - success = false; - return QVariantMap(); - } else if (token == JsonTokenComma) { - Json::nextToken(json, index); - } else if (token == JsonTokenCurlyClose) { - Json::nextToken(json, index); - return map; - } else { - // Parse the key/value pair's name - QString name = Json::parseString(json, index, success).toString(); - - if (!success) { - return QVariantMap(); - } - - // Get the next token - token = Json::nextToken(json, index); - - // If the next token is not a colon, flag the failure - // return an empty QVariant - if (token != JsonTokenColon) { - success = false; - return QVariant(QVariantMap()); - } - - // Parse the key/value pair's value - QVariant value = Json::parseValue(json, index, success); - - if (!success) { - return QVariantMap(); - } - - // Assign the value to the key in the map - map[name] = value; - } - } - - // Return the map successfully - return QVariant(map); -} - -/** - * parseArray - */ -QVariant Json::parseArray(const QString &json, int &index, bool &success) -{ - QVariantList list; - - Json::nextToken(json, index); - - bool done = false; - while (!done) { - int token = Json::lookAhead(json, index); - - if (token == JsonTokenNone) { - success = false; - return QVariantList(); - } else if (token == JsonTokenComma) { - Json::nextToken(json, index); - } else if (token == JsonTokenSquaredClose) { - Json::nextToken(json, index); - break; - } else { - QVariant value = Json::parseValue(json, index, success); - - if (!success) { - return QVariantList(); - } - - list.push_back(value); - } - } - - return QVariant(list); -} - -/** - * parseString - */ -QVariant Json::parseString(const QString &json, int &index, bool &success) -{ - QString s; - QChar c; - - Json::eatWhitespace(json, index); - - c = json[index++]; - - bool complete = false; - while (!complete) { - if (index == json.size()) { - break; - } - - c = json[index++]; - - if (c == '\"') { - complete = true; - break; - } else if (c == '\\') { - if (index == json.size()) { - break; - } - - c = json[index++]; - - if (c == '\"') { - s.append('\"'); - } else if (c == '\\') { - s.append('\\'); - } else if (c == '/') { - s.append('/'); - } else if (c == 'b') { - s.append('\b'); - } else if (c == 'f') { - s.append('\f'); - } else if (c == 'n') { - s.append('\n'); - } else if (c == 'r') { - s.append('\r'); - } else if (c == 't') { - s.append('\t'); - } else if (c == 'u') { - int remainingLength = json.size() - index; - - if (remainingLength >= 4) { - QString unicodeStr = json.mid(index, 4); - - int symbol = unicodeStr.toInt(0, 16); - - s.append(QChar(symbol)); - - index += 4; - } else { - break; - } - } - } else { - s.append(c); - } - } - - if (!complete) { - success = false; - return QVariant(); - } - - return QVariant(s); -} - -/** - * parseNumber - */ -QVariant Json::parseNumber(const QString &json, int &index) -{ - Json::eatWhitespace(json, index); - - int lastIndex = Json::lastIndexOfNumber(json, index); - int charLength = (lastIndex - index) + 1; - QString numberStr; - - numberStr = json.mid(index, charLength); - - index = lastIndex + 1; - - if (numberStr.contains('.')) { - return QVariant(numberStr.toDouble(NULL)); - } else if (numberStr.startsWith('-')) { - return QVariant(numberStr.toLongLong(NULL)); - } else { - return QVariant(numberStr.toULongLong(NULL)); - } -} - -/** - * lastIndexOfNumber - */ -int Json::lastIndexOfNumber(const QString &json, int index) -{ - static const QString numericCharacters("0123456789+-.eE"); - int lastIndex; - - for (lastIndex = index; lastIndex < json.size(); lastIndex++) { - if (numericCharacters.indexOf(json[lastIndex]) == -1) { - break; - } - } - - return lastIndex - 1; -} - -/** - * eatWhitespace - */ -void Json::eatWhitespace(const QString &json, int &index) -{ - static const QString whitespaceChars(" \t\n\r"); - for (; index < json.size(); index++) { - if (whitespaceChars.indexOf(json[index]) == -1) { - break; - } - } -} - -/** - * lookAhead - */ -int Json::lookAhead(const QString &json, int index) -{ - int saveIndex = index; - return Json::nextToken(json, saveIndex); -} - -/** - * nextToken - */ -int Json::nextToken(const QString &json, int &index) -{ - Json::eatWhitespace(json, index); - - if (index == json.size()) { - return JsonTokenNone; - } - - QChar c = json[index]; - index++; - switch (c.toLatin1()) { - case '{': - return JsonTokenCurlyOpen; - case '}': - return JsonTokenCurlyClose; - case '[': - return JsonTokenSquaredOpen; - case ']': - return JsonTokenSquaredClose; - case ',': - return JsonTokenComma; - case '"': - return JsonTokenString; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - return JsonTokenNumber; - case ':': - return JsonTokenColon; - } - - index--; - - int remainingLength = json.size() - index; - - // True - if (remainingLength >= 4) { - if (json[index] == 't' && json[index + 1] == 'r' && json[index + 2] == 'u' && json[index + 3] == 'e') { - index += 4; - return JsonTokenTrue; - } - } - - // False - if (remainingLength >= 5) { - if (json[index] == 'f' && json[index + 1] == 'a' && json[index + 2] == 'l' && json[index + 3] == 's' && - json[index + 4] == 'e') { - index += 5; - return JsonTokenFalse; - } - } - - // Null - if (remainingLength >= 4) { - if (json[index] == 'n' && json[index + 1] == 'u' && json[index + 2] == 'l' && json[index + 3] == 'l') { - index += 4; - return JsonTokenNull; - } - } - - return JsonTokenNone; -} - -} // namespace QtJson diff --git a/oracle/src/qt-json/json.h b/oracle/src/qt-json/json.h deleted file mode 100644 index cf0499d4e..000000000 --- a/oracle/src/qt-json/json.h +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2011 Eeli Reilin. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, - * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * The views and conclusions contained in the software and documentation - * are those of the authors and should not be interpreted as representing - * official policies, either expressed or implied, of Eeli Reilin. - */ - -/** - * \file json.h - */ - -#ifndef JSON_H -#define JSON_H - -#include -#include - -namespace QtJson -{ - -/** - * \enum JsonToken - */ -enum JsonToken -{ - JsonTokenNone = 0, - JsonTokenCurlyOpen = 1, - JsonTokenCurlyClose = 2, - JsonTokenSquaredOpen = 3, - JsonTokenSquaredClose = 4, - JsonTokenColon = 5, - JsonTokenComma = 6, - JsonTokenString = 7, - JsonTokenNumber = 8, - JsonTokenTrue = 9, - JsonTokenFalse = 10, - JsonTokenNull = 11 -}; - -/** - * \class Json - * \brief A JSON data parser - * - * Json parses a JSON data into a QVariant hierarchy. - */ -class Json -{ - public: - /** - * Parse a JSON string - * - * \param json The JSON data - */ - static QVariant parse(const QString &json); - - /** - * Parse a JSON string - * - * \param json The JSON data - * \param success The success of the parsing - */ - static QVariant parse(const QString &json, bool &success); - - /** - * This method generates a textual JSON representation - * - * \param data The JSON data generated by the parser. - * \param success The success of the serialization - */ - static QByteArray serialize(const QVariant &data); - - /** - * This method generates a textual JSON representation - * - * \param data The JSON data generated by the parser. - * \param success The success of the serialization - * - * \return QByteArray Textual JSON representation - */ - static QByteArray serialize(const QVariant &data, bool &success); - - private: - /** - * Parses a value starting from index - * - * \param json The JSON data - * \param index The start index - * \param success The success of the parse process - * - * \return QVariant The parsed value - */ - static QVariant parseValue(const QString &json, int &index, - bool &success); - - /** - * Parses an object starting from index - * - * \param json The JSON data - * \param index The start index - * \param success The success of the object parse - * - * \return QVariant The parsed object map - */ - static QVariant parseObject(const QString &json, int &index, - bool &success); - - /** - * Parses an array starting from index - * - * \param json The JSON data - * \param index The starting index - * \param success The success of the array parse - * - * \return QVariant The parsed variant array - */ - static QVariant parseArray(const QString &json, int &index, - bool &success); - - /** - * Parses a string starting from index - * - * \param json The JSON data - * \param index The starting index - * \param success The success of the string parse - * - * \return QVariant The parsed string - */ - static QVariant parseString(const QString &json, int &index, - bool &success); - - /** - * Parses a number starting from index - * - * \param json The JSON data - * \param index The starting index - * - * \return QVariant The parsed number - */ - static QVariant parseNumber(const QString &json, int &index); - - /** - * Get the last index of a number starting from index - * - * \param json The JSON data - * \param index The starting index - * - * \return The last index of the number - */ - static int lastIndexOfNumber(const QString &json, int index); - - /** - * Skip unwanted whitespace symbols starting from index - * - * \param json The JSON data - * \param index The start index - */ - static void eatWhitespace(const QString &json, int &index); - - /** - * Check what token lies ahead - * - * \param json The JSON data - * \param index The starting index - * - * \return int The upcoming token - */ - static int lookAhead(const QString &json, int index); - - /** - * Get the next JSON token - * - * \param json The JSON data - * \param index The starting index - * - * \return int The next JSON token - */ - static int nextToken(const QString &json, int &index); -}; - - -} //end namespace - -#endif //JSON_H From b16497327fffbcbbe83b6b4026ec27d58b4f7dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sun, 30 Aug 2026 22:54:12 +0200 Subject: [PATCH 02/25] [Oracle] Restore property coercion and legality merge in native JSON import --- oracle/src/oracleimporter.cpp | 23 ++++++++++++++++++----- oracle/src/oracleimporter.h | 5 ++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index a531b82d1..752317512 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -147,7 +147,11 @@ CardInfoPtr OracleImporter::addCard(QString name, if (existingIt != cards.constEnd()) { CardInfoPtr card = existingIt.value(); card->addToSet(printingInfo.getSet(), printingInfo); - card->combineLegalities(properties); + // Only merge legalities when the card has none yet, so multi-format + // printings don't overwrite each other's legality lists. + if (card->getProperties().filter(formatRegex).empty()) { + card->combineLegalities(properties); + } return card; } @@ -224,7 +228,10 @@ CardInfoPtr OracleImporter::addCard(QString name, static QString getJsonString(const QJsonObject &obj, const QString &key) { - return obj.value(key).toString(); + // QVariant coerces numbers and booleans to text, while QJsonValue::toString() + // returns a null string for them — some MTGJSON fields (manaValue, + // convertedManaCost, isOnlineOnly, isRebalanced) carry those types. + return obj.value(key).toVariant().toString(); } int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList) @@ -329,12 +336,18 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson allNameProps.insert(faceName); // special handling properties - QString colors = card.value("colors").toVariant().toStringList().join(""); + QString colors; + for (const QJsonValue &color : card.value("colors").toArray()) { + colors += color.toString(); + } if (!colors.isEmpty()) { properties.insert("colors", colors); } - QString colorIdentity = card.value("colorIdentity").toVariant().toStringList().join(""); + QString colorIdentity; + for (const QJsonValue &color : card.value("colorIdentity").toArray()) { + colorIdentity += color.toString(); + } if (!colorIdentity.isEmpty()) { properties.insert("coloridentity", colorIdentity); } @@ -529,7 +542,7 @@ static FormatRulesNameMap buildDefaultMagicFormats() return defaultFormatRulesNameMap; } -FormatRulesNameMap OracleImporter::createDefaultMagicFormats() +const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats() { static const FormatRulesNameMap cached = buildDefaultMagicFormats(); return cached; diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 696e35cab..52a7cd349 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -157,7 +157,10 @@ public: int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); - FormatRulesNameMap createDefaultMagicFormats(); + /** + * @brief Returns the default format rules. The result is memoized on first use and must be treated as immutable. + */ + const FormatRulesNameMap &createDefaultMagicFormats(); const CardNameMap &getCardList() const { return cards; From 770282c5b5d8544117ad4753165573e9f947b437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sun, 30 Aug 2026 13:39:09 +0200 Subject: [PATCH 03/25] [Oracle] Add oracle importer tests and fix set parsing details - Add oracle_importer_test and oracle_importer_benchmark_test targets - Preserve the first printing's legalities when an existing card is reused - Concatenate split-card coloridentity and sort/dedupe card colors - Use a raw string for the Basic Land format regex - Pre-allocate the card hash and micro-optimize string handling Took 2 minutes --- oracle/src/oracleimporter.cpp | 41 +- tests/oracle/CMakeLists.txt | 32 ++ .../oracle/oracle_importer_benchmark_test.cpp | 264 +++++++++ tests/oracle/oracle_importer_test.cpp | 525 ++++++++++++++++++ 4 files changed, 847 insertions(+), 15 deletions(-) create mode 100644 tests/oracle/oracle_importer_benchmark_test.cpp create mode 100644 tests/oracle/oracle_importer_test.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 752317512..f0dc3b1ae 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,8 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) // capitalize set type if (setType.length() > 0) { // basic grammar for words that aren't capitalized, like in "From the Vault" - const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", "of", "in", "and", "with", "or"}; + static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", + "of", "in", "and", "with", "or"}; QStringList words = setType.split("_"); setType.clear(); bool first = false; @@ -75,7 +77,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) if (first && noCapitalize.contains(item)) { setType += item + QString(" "); } else { - setType += item[0].toUpper() + item.mid(1, -1) + QString(" "); + setType += item[0].toUpper() + item.mid(1) + QString(" "); first = true; } } @@ -123,14 +125,8 @@ static void sortAndReduceColors(QString &colors) std::sort(colors.begin(), colors.end(), [](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); }); // reduce - QChar lastChar = '\0'; - for (int i = 0; i < colors.size(); ++i) { - if (colors.at(i) == lastChar) { - colors.remove(i, 1); - } else { - lastChar = colors.at(i); - } - } + auto last = std::unique(colors.begin(), colors.end()); + colors.erase(last, colors.end()); } CardInfoPtr OracleImporter::addCard(QString name, @@ -186,8 +182,9 @@ CardInfoPtr OracleImporter::addCard(QString name, // DETECT CARD POSITIONING INFO - bool landscapeOrientation = properties.value("maintype") == "Battle" || properties.value("layout") == "split" || - properties.value("layout") == "planar"; + QString layoutVal = properties.value("layout"); + bool landscapeOrientation = + properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar"; // cards that enter the field tapped bool cipt = parseCipt(name, text) || landscapeOrientation; @@ -426,7 +423,8 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } - CardInfoPtr newCard = addCard(name + numComponent, text, isToken, properties, relatedCards, printingInfo); + CardInfoPtr newCard = + addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo); numCards++; } } @@ -459,7 +457,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) { if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty properties.insert(prop, thisCardPropertyValue); - } else if (prop == "colors") { // the card is both colors + } else if (prop == "colors" || prop == "coloridentity") { // the card is both colors properties.insert(prop, originalPropertyValue + thisCardPropertyValue); } else if (prop == "maintype") { // don't create maintypes with //es in them continue; @@ -471,7 +469,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } } - CardInfoPtr newCard = addCard(name, text, isToken, properties, {}, printingInfo); + CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo); numCards++; } @@ -552,6 +550,19 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); + // Pre-allocate the cards hash to avoid rehashing during import. The hash + // is keyed by distinct card name rather than by printings: AllPrintings + // ships ~100k printings for ~35k names, so reserving the printing count + // would overallocate ~3x (against this stack's RAM goal). Collecting + // distinct names is cheap — one pass over the already-parsed name fields. + QSet distinctNames; + for (const SetToDownload &curSetToParse : allSets) { + for (const QJsonValue &cardValue : curSetToParse.getCards()) { + distinctNames.insert(cardValue.toObject().value("name").toString()); + } + } + cards.reserve(distinctNames.size()); + // add an empty set for tokens CardSetPtr tokenSet = CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens"); diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index c5c1e9097..d126390c7 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -7,3 +7,35 @@ endif() target_link_libraries(parse_cipt_test Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}) add_test(NAME parse_cipt_test COMMAND parse_cipt_test) + +# Oracle importer unit tests +add_executable( + oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + oracle_importer_test.cpp +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_test gtest) +endif() + +target_link_libraries( + oracle_importer_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} +) + +add_test(NAME oracle_importer_test COMMAND oracle_importer_test) + +# Oracle importer benchmark tests (manual, not run in CI) +add_executable( + oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp + ../../oracle/src/parsehelpers.cpp oracle_importer_benchmark_test.cpp +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_benchmark_test gtest) +endif() + +target_link_libraries( + oracle_importer_benchmark_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} +) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp new file mode 100644 index 000000000..b3cc55319 --- /dev/null +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -0,0 +1,264 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include + +// Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set +static QByteArray buildSyntheticData(int numSets, int cardsPerSet) +{ + QJsonObject dataObj; + for (int s = 0; s < numSets; ++s) { + QJsonArray cardsArray; + for (int c = 0; c < cardsPerSet; ++c) { + QJsonObject card; + card["name"] = QString("Card %1").arg(s * cardsPerSet + c); + card["text"] = "This is a test card with some rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["power"] = "2"; + card["toughness"] = "2"; + card["colors"] = QJsonArray{"W"}; + card["colorIdentity"] = QJsonArray{"W"}; + card["types"] = QJsonArray{"Creature"}; + // Real MTGJSON types: floats and booleans, not strings. This + // exercises the QVariant coercion in the property reader. + card["convertedManaCost"] = 1.0; + card["manaValue"] = 1.0; + card["isOnlineOnly"] = false; + card["isRebalanced"] = false; + + QJsonObject legalities; + legalities["standard"] = "legal"; + legalities["modern"] = "legal"; + legalities["legacy"] = "legal"; + legalities["vintage"] = "legal"; + legalities["commander"] = "legal"; + card["legalities"] = legalities; + + QJsonObject identifiers; + identifiers["scryfallId"] = QString("id-%1-%2").arg(s).arg(c); + card["identifiers"] = identifiers; + + // In AllPrintings, number and rarity are flat fields on the card + // object, exactly as set below. + card["number"] = QString::number(c + 1); + card["rarity"] = "common"; + + cardsArray.append(card); + } + + QJsonObject setObj; + setObj["code"] = QString("T%1").arg(s, 2, 10, QChar('0')); + setObj["name"] = QString("Test Set %1").arg(s); + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = cardsArray; + + dataObj[QString("T%1").arg(s, 2, 10, QChar('0'))] = setObj; + } + + QJsonObject root; + root["data"] = dataObj; + return QJsonDocument(root).toJson(QJsonDocument::Compact); +} + +// ============================================================================ +// Import throughput benchmark +// ============================================================================ + +TEST(OracleBenchmark, ImportThroughput) +{ + static constexpr int numSets = 10; + static constexpr int cardsPerSet = 500; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + OracleImporter importer; + + // Phase 1: Parse JSON + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(data); + ASSERT_TRUE(ok); + qint64 parseMs = timer.elapsed(); + + // Phase 2: Import cards + timer.restart(); + int importedSets = importer.startImport(); + qint64 importMs = timer.elapsed(); + + int totalImported = 0; + for (const auto &card : importer.getCardList()) { + Q_UNUSED(card); + totalImported++; + } + + // The fixture generates globally unique card names, so the expected + // counts are exact: a regression here means cards were dropped. + ASSERT_EQ(importedSets, numSets); + ASSERT_EQ(totalImported, numSets * cardsPerSet); + // Real-data probe: numeric convertedManaCost must be coerced to text + // (regression for the QJsonValue::toString() reader in #7214). + auto probeCard = importer.getCardList().value("Card 0"); + ASSERT_FALSE(probeCard.isNull()); + ASSERT_EQ(probeCard->getProperty("cmc"), "1"); + + qDebug().noquote() + << QString("Oracle Import Benchmark: %1 sets, %2 unique cards").arg(importedSets).arg(totalImported); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + qDebug().noquote() << QString(" Total: %1 ms").arg(parseMs + importMs); + if (importMs > 0) { + qDebug().noquote() << QString(" Throughput: %1 cards/sec") + .arg(static_cast(totalImported) / importMs * 1000.0, 0, 'f', 0); + } +} + +// ============================================================================ +// readSetsFromByteArray benchmark +// ============================================================================ + +TEST(OracleBenchmark, ParseJsonThroughput) +{ + static constexpr int numSets = 20; + static constexpr int cardsPerSet = 1000; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + // Run 5 iterations and report average + static constexpr int iterations = 5; + qint64 totalMs = 0; + + for (int i = 0; i < iterations; ++i) { + OracleImporter importer; + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(data); + ASSERT_TRUE(ok); + totalMs += timer.elapsed(); + } + + qint64 avgMs = totalMs / iterations; + qDebug().noquote() << QString("Parse Benchmark (%1 iterations): avg %2 ms for %3 sets x %4 cards") + .arg(iterations) + .arg(avgMs) + .arg(numSets) + .arg(cardsPerSet); +} + +// ============================================================================ +// Split card merging benchmark +// ============================================================================ + +TEST(OracleBenchmark, SplitCardMerging) +{ + static constexpr int numSplitCards = 1000; + + QJsonArray cardsList; + for (int i = 0; i < numSplitCards; ++i) { + QJsonObject face1; + face1["name"] = QString("Fire %1 // Ice %1").arg(i); + face1["text"] = "Fire side text."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = QString("Fire %1").arg(i); + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face1["identifiers"] = QJsonObject{{"scryfallId", QString("f-%1").arg(i)}}; + face1["number"] = QString::number(i + 1); + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = QString("Fire %1 // Ice %1").arg(i); + face2["text"] = "Ice side text."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = QString("Ice %1").arg(i); + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face2["identifiers"] = QJsonObject{{"scryfallId", QString("i-%1").arg(i)}}; + face2["number"] = QString::number(i + 1); + face2["rarity"] = "uncommon"; + + cardsList.append(face1); + cardsList.append(face2); + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Split Test"); + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numSplitCards); + qDebug().noquote() << QString("Split Card Merge Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +// ============================================================================ +// sortAndReduceColors microbenchmark +// ============================================================================ + +// We can't call sortAndReduceColors directly (it's static), so we benchmark +// through importCardsFromSet with color properties. + +TEST(OracleBenchmark, ImportCardsWithColors) +{ + static constexpr int numCards = 10000; + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Color Test"); + + QJsonArray cardsList; + for (int i = 0; i < numCards; ++i) { + QJsonObject card; + card["name"] = QString("Color Card %1").arg(i); + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["colors"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["colorIdentity"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["number"] = QString::number(i + 1); + card["rarity"] = "common"; + card["legalities"] = QJsonObject{{"standard", "legal"}}; + card["identifiers"] = QJsonObject{{"scryfallId", QString("c-%1").arg(i)}}; + cardsList.append(card); + } + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numCards); + qDebug().noquote() << QString("Import with Colors Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp new file mode 100644 index 000000000..19f66e5e6 --- /dev/null +++ b/tests/oracle/oracle_importer_test.cpp @@ -0,0 +1,525 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include + +class OracleImporterTest : public ::testing::Test +{ +protected: + void SetUp() override + { + controller = new NoopCardSetPriorityController(); + importer = new OracleImporter(); + set = CardSet::newInstance(controller, "TST", "Test Set"); + } + + void TearDown() override + { + delete importer; + delete controller; + } + + // Helper: build a minimal card JSON object + QJsonObject makeCard(const QString &name, + const QString &colors = "", + const QString &colorIdentity = "", + const QVariantMap &legalities = {}) + { + QJsonObject card; + card["name"] = name; + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["number"] = "1"; + card["rarity"] = "common"; + + if (!colors.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colors) { + arr.append(QString(c)); + } + card["colors"] = arr; + } + if (!colorIdentity.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colorIdentity) { + arr.append(QString(c)); + } + card["colorIdentity"] = arr; + } + if (!legalities.isEmpty()) { + QJsonObject legalObj; + for (auto it = legalities.constBegin(); it != legalities.constEnd(); ++it) { + legalObj[it.key()] = it.value().toString(); + } + card["legalities"] = legalObj; + } + + QJsonObject identifiers; + identifiers["scryfallId"] = QUuid::createUuid().toString(QUuid::WithoutBraces); + card["identifiers"] = identifiers; + + return card; + } + + NoopCardSetPriorityController *controller; + OracleImporter *importer; + CardSetPtr set; +}; + +// ============================================================================ +// sortAndReduceColors tests (tested via importCardsFromSet) +// ============================================================================ + +TEST_F(OracleImporterTest, SortAndReduceColorsSingleColor) +{ + QJsonArray cards{makeCard("Red Card", "R", "R")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Red Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "R"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsDeduplicates) +{ + QJsonArray cards{makeCard("Dedup Card", "WWUUB", "WU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Dedup Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUB"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsSortsWUBRG) +{ + QJsonArray cards{makeCard("Sort Card", "RGW", "RGW")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Sort Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsAllFive) +{ + QJsonArray cards{makeCard("Five Color", "BRGWU", "BRGWU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Five Color"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUBRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorIdentity) +{ + QJsonArray cards{makeCard("Color Id Card", "W", "GWR")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Color Id Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("coloridentity"), "WRG"); +} + +TEST_F(OracleImporterTest, SingleColorNotSorted) +{ + QJsonArray cards{makeCard("Single Card", "B", "B")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Single Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "B"); +} + +// ============================================================================ +// Legality guard tests +// ============================================================================ + +TEST_F(OracleImporterTest, NewCardKeepsLegalityProperties) +{ + // Verifies that format-* properties survive addCard on a fresh card + // (not the combineLegalities guard, which only runs on existing printings). + QVariantMap leg; + leg["standard"] = "legal"; + leg["modern"] = "legal"; + QJsonArray cards{makeCard("Legal Card", "", "", leg)}; + + importer->importCardsFromSet(set, cards); + auto card = importer->getCardList().value("Legal Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityMergeAllowedWhenCardHasNoLegalities) +{ + // First printing carries no legalities at all, so the guard's + // `properties.filter(formatRegex).empty()` predicate is true and the + // second printing's legalities must be merged in. + QJsonArray cards1{makeCard("Unmerged Card")}; + importer->importCardsFromSet(set, cards1); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg; + leg["standard"] = "legal"; + QJsonArray cards2{makeCard("Unmerged Card", "", "", leg)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Unmerged Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityGuardPreservesFirstPrinting) +{ + // First printing: standard=legal, modern=legal + QVariantMap leg1; + leg1["standard"] = "legal"; + leg1["modern"] = "legal"; + QJsonArray cards1{makeCard("Guarded Card", "", "", leg1)}; + importer->importCardsFromSet(set, cards1); + + // Second printing: standard=banned, modern=not_legal + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg2; + leg2["standard"] = "banned"; + leg2["modern"] = "not_legal"; + QJsonArray cards2{makeCard("Guarded Card", "", "", leg2)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Guarded Card"); + ASSERT_FALSE(card.isNull()); + // Guard should preserve first printing's legalities + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +// ============================================================================ +// createDefaultMagicFormats tests +// ============================================================================ + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsContainsExpectedFormats) +{ + auto formats = importer->createDefaultMagicFormats(); + ASSERT_TRUE(formats.contains("standard")); + ASSERT_TRUE(formats.contains("modern")); + ASSERT_TRUE(formats.contains("legacy")); + ASSERT_TRUE(formats.contains("vintage")); + ASSERT_TRUE(formats.contains("commander")); + ASSERT_TRUE(formats.contains("pauper")); + ASSERT_TRUE(formats.contains("pioneer")); + ASSERT_TRUE(formats.contains("brawl")); + ASSERT_TRUE(formats.contains("historic")); + ASSERT_TRUE(formats.contains("timeless")); + ASSERT_TRUE(formats.contains("duel")); + ASSERT_TRUE(formats.contains("oathbreaker")); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsSingletonDeckSizes) +{ + auto formats = importer->createDefaultMagicFormats(); + auto commander = formats.value("commander"); + ASSERT_FALSE(commander.isNull()); + ASSERT_EQ(commander->minDeckSize, 100); + ASSERT_EQ(commander->maxDeckSize, 100); + ASSERT_EQ(commander->maxSideboardSize, 15); + + auto brawl = formats.value("brawl"); + ASSERT_FALSE(brawl.isNull()); + ASSERT_EQ(brawl->minDeckSize, 60); + ASSERT_EQ(brawl->maxDeckSize, 60); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsVintageHasRestricted) +{ + auto formats = importer->createDefaultMagicFormats(); + auto vintage = formats.value("vintage"); + ASSERT_FALSE(vintage.isNull()); + bool hasRestricted = false; + for (const auto &ac : vintage->allowedCounts) { + if (ac.label == "restricted") { + hasRestricted = true; + ASSERT_EQ(ac.max, 1); + } + } + ASSERT_TRUE(hasRestricted); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsRegexMatchesBasicLands) +{ + auto formats = importer->createDefaultMagicFormats(); + auto standard = formats.value("standard"); + ASSERT_FALSE(standard.isNull()); + ASSERT_FALSE(standard->exceptions.isEmpty()); + + auto &basicLandsException = standard->exceptions.first(); + ASSERT_FALSE(basicLandsException.conditions.isEmpty()); + + auto &condition = basicLandsException.conditions.first(); + ASSERT_EQ(condition.field, "type"); + ASSERT_EQ(condition.matchType, "regex"); + + // Verify the regex actually works (was broken before: \b = backspace, not word boundary) + QRegularExpression regex(condition.value); + ASSERT_TRUE(regex.isValid()); + ASSERT_TRUE(regex.match("Basic Land — Forest").hasMatch()); + ASSERT_TRUE(regex.match("Basic Snow Land — Mountain").hasMatch()); + ASSERT_FALSE(regex.match("Creature — Elf Warrior").hasMatch()); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsCaching) +{ + // The memoized map returns the same FormatRulesPtr instances, so the + // shared pointers must be identical across calls. This is the only + // observable effect of the cache: contents would match either way. + auto first = importer->createDefaultMagicFormats(); + auto second = importer->createDefaultMagicFormats(); + ASSERT_EQ(first.value("standard").data(), second.value("standard").data()); +} + +// ============================================================================ +// readSetsFromByteArray tests +// ============================================================================ + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayValidJson) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().size(), 1); + ASSERT_EQ(importer->getSets().first().getShortName(), "TST"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayInvalidJson) +{ + QByteArray data = "not valid json"; + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayEmptyData) +{ + QJsonObject root; + root["data"] = QJsonObject(); + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayCapitalizesSetType) +{ + QJsonObject setObj; + setObj["code"] = "ftv"; + setObj["name"] = "From The Vault"; + setObj["type"] = "from_the_vault"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"FTV", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().first().getSetType(), "From the Vault"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArraySortsSetsByName) +{ + // QJsonObject iterates keys in lexicographic order ("AAA" before "ZZZ"), + // so leaving the natural order matching the alphabetical sort makes the + // assertion pass trivially. Inverting it keeps the sort meaningful: + // iteration yields "AAA" (Zeta Set) first, then the sort by name must + // promote "ZZZ" (Alpha Set) to the front. + QJsonObject setA; + setA["code"] = "aaa"; + setA["name"] = "Zeta Set"; + setA["type"] = "expansion"; + setA["releaseDate"] = "2024-01-01"; + setA["cards"] = QJsonArray(); + + QJsonObject setB; + setB["code"] = "zzz"; + setB["name"] = "Alpha Set"; + setB["type"] = "expansion"; + setB["releaseDate"] = "2024-01-01"; + setB["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"AAA", setA}, {"ZZZ", setB}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + auto sets = importer->getSets(); + ASSERT_GE(sets.size(), 2); + ASSERT_EQ(sets.first().getShortName(), "ZZZ"); +} + +// ============================================================================ +// Split card coloridentity tests +// ============================================================================ + +TEST_F(OracleImporterTest, SplitCardColorIdentityConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + int count = importer->importCardsFromSet(set, cardsList); + ASSERT_EQ(count, 1); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + // coloridentity should be "RU" (concatenated), then sorted to "UR" + // by sortAndReduceColors when it reaches addCard + ASSERT_EQ(card->getProperty("coloridentity"), "UR"); +} + +TEST_F(OracleImporterTest, SplitCardColorsConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + importer->importCardsFromSet(set, cardsList); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + QString colors = card->getProperty("colors"); + ASSERT_FALSE(colors.contains("//")) << "colors should not contain '//', got: " << colors.toStdString(); + ASSERT_TRUE(colors.contains("R")); + ASSERT_TRUE(colors.contains("U")); +} + +// ============================================================================ +// Mana cost formatting tests +// ============================================================================ + +TEST_F(OracleImporterTest, ManaCostStripsBraces) +{ + QJsonObject card = makeCard("Mana Card"); + card["manaCost"] = "{2}{W}{B}"; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Mana Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("manacost"), "2WB"); +} + +// ============================================================================ +// Card deduplication tests +// ============================================================================ + +TEST_F(OracleImporterTest, DuplicateCardNameReturnsExisting) +{ + QJsonArray cards{makeCard("Dupe Card")}; + importer->importCardsFromSet(set, cards); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QJsonArray cards2{makeCard("Dupe Card")}; + importer->importCardsFromSet(set2, cards2); + + ASSERT_EQ(importer->getCardList().size(), 1); +} + +TEST_F(OracleImporterTest, AELigatureReplaced) +{ + QJsonObject card = makeCard(QString::fromUtf8("\xC3\x86ther Vial")); // Æther Vial + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + // Æ is replaced with AE, resulting in "AEther Vial" + ASSERT_FALSE(importer->getCardList().contains(QString::fromUtf8("\xC3\x86ther Vial"))); + ASSERT_TRUE(importer->getCardList().contains("AEther Vial")); +} + +TEST_F(OracleImporterTest, ApostropheNormalized) +{ + QJsonObject card = makeCard(QString::fromUtf8("Jace\u2019s Ingenuity")); + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 3dc9dba67a51dcdcb822373b855ed7364c053809 Mon Sep 17 00:00:00 2001 From: RickyRister <42636155+RickyRister@users.noreply.github.com> Date: Sun, 30 Aug 2026 14:43:23 -0700 Subject: [PATCH 04/25] [SettingsPage] Refactor: Clean up order of variables (#7184) * [SettingsPage] Refactor: Clean up order of variables * fixes --- .../appearance_settings_page.cpp | 95 ++++++++++--------- .../settings_page/appearance_settings_page.h | 61 +++++++----- .../settings_page/general_settings_page.cpp | 71 +++++++------- .../settings_page/general_settings_page.h | 58 +++++------ .../user_interface_settings_page.cpp | 45 +++++---- .../user_interface_settings_page.h | 11 ++- 6 files changed, 182 insertions(+), 159 deletions(-) diff --git a/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.cpp b/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.cpp index 881c54167..c8494f095 100644 --- a/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.cpp +++ b/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.cpp @@ -154,6 +154,48 @@ AppearanceSettingsPage::AppearanceSettingsPage() homeTabGroupBox = new QGroupBox; homeTabGroupBox->setLayout(homeTabGrid); + // Playmat settings + playmatVisibilityCombo.addItem(tr("Show all playmats"), PlaymatVisibilityAll); + playmatVisibilityCombo.addItem(tr("Show own playmat only"), PlaymatVisibilityOwnOnly); + playmatVisibilityCombo.addItem(tr("Don't use playmats"), PlaymatVisibilityNone); + int visIdx = playmatVisibilityCombo.findData(settings.userInterface().getPlaymatVisibility()); + if (visIdx >= 0) { + playmatVisibilityCombo.setCurrentIndex(visIdx); + } + connect(&playmatVisibilityCombo, qOverload(&QComboBox::currentIndexChanged), this, [this](int index) { + SettingsCache::instance().userInterface().setPlaymatVisibility(playmatVisibilityCombo.itemData(index).toInt()); + }); + playmatVisibilityLabel.setBuddy(&playmatVisibilityCombo); + + // Playmat mode: Override / Fallback / Deck-only + playmatModeCombo.addItem(tr("Override deck playmat"), PlaymatModeOverrideDeck); + playmatModeCombo.addItem(tr("Fallback if deck has none"), PlaymatModeFallback); + playmatModeCombo.addItem(tr("Deck only, ignore collection"), PlaymatModeDeckOnly); + int modeIdx = playmatModeCombo.findData(settings.userInterface().getPlaymatMode()); + if (modeIdx >= 0) { + playmatModeCombo.setCurrentIndex(modeIdx); + } + connect(&playmatModeCombo, qOverload(&QComboBox::currentIndexChanged), this, [this](int index) { + SettingsCache::instance().userInterface().setPlaymatMode(playmatModeCombo.itemData(index).toInt()); + }); + playmatModeLabel.setBuddy(&playmatModeCombo); + + // User-level playmat settings: fallback collection. + connect(&playmatDefaultEditButton, &QPushButton::clicked, this, + &AppearanceSettingsPage::openPlaymatCollectionDialog); + + auto *playmatGrid = new QGridLayout; + playmatGrid->addWidget(&playmatVisibilityLabel, 0, 0, 1, 1); + playmatGrid->addWidget(&playmatVisibilityCombo, 0, 1, 1, 1); + playmatGrid->addWidget(&playmatModeLabel, 1, 0, 1, 1); + playmatGrid->addWidget(&playmatModeCombo, 1, 1, 1, 1); + playmatGrid->addWidget(&playmatDefaultLabel, 2, 0, 1, 1); + playmatGrid->addWidget(&playmatDefaultEditButton, 2, 1, 1, 1); + + playmatGroupBox = new QGroupBox; + playmatGroupBox->setLayout(playmatGrid); + + // Styling settings styleUserListCheckBox.setChecked(settings.appearance().getStyleUserList()); connect(&styleUserListCheckBox, &QCheckBox::QT_STATE_CHANGED, &settings.appearance(), &AppearanceSettings::setStyleUserList); @@ -259,7 +301,6 @@ AppearanceSettingsPage::AppearanceSettingsPage() cardLayoutGroupBox->setLayout(cardLayoutGrid); // Card counter colors - auto *cardCounterColorsLayout = new QGridLayout; cardCounterColorsLayout->setColumnStretch(1, 1); cardCounterColorsLayout->setColumnStretch(3, 1); @@ -339,47 +380,6 @@ AppearanceSettingsPage::AppearanceSettingsPage() tableGroupBox = new QGroupBox; tableGroupBox->setLayout(tableGrid); - // Playmat settings - playmatVisibilityCombo.addItem(tr("Show all playmats"), PlaymatVisibilityAll); - playmatVisibilityCombo.addItem(tr("Show own playmat only"), PlaymatVisibilityOwnOnly); - playmatVisibilityCombo.addItem(tr("Don't use playmats"), PlaymatVisibilityNone); - int visIdx = playmatVisibilityCombo.findData(settings.userInterface().getPlaymatVisibility()); - if (visIdx >= 0) { - playmatVisibilityCombo.setCurrentIndex(visIdx); - } - connect(&playmatVisibilityCombo, qOverload(&QComboBox::currentIndexChanged), this, [this](int index) { - SettingsCache::instance().userInterface().setPlaymatVisibility(playmatVisibilityCombo.itemData(index).toInt()); - }); - playmatVisibilityLabel.setBuddy(&playmatVisibilityCombo); - - // Playmat mode: Override / Fallback / Deck-only - playmatModeCombo.addItem(tr("Override deck playmat"), PlaymatModeOverrideDeck); - playmatModeCombo.addItem(tr("Fallback if deck has none"), PlaymatModeFallback); - playmatModeCombo.addItem(tr("Deck only, ignore collection"), PlaymatModeDeckOnly); - int modeIdx = playmatModeCombo.findData(settings.userInterface().getPlaymatMode()); - if (modeIdx >= 0) { - playmatModeCombo.setCurrentIndex(modeIdx); - } - connect(&playmatModeCombo, qOverload(&QComboBox::currentIndexChanged), this, [this](int index) { - SettingsCache::instance().userInterface().setPlaymatMode(playmatModeCombo.itemData(index).toInt()); - }); - playmatModeLabel.setBuddy(&playmatModeCombo); - - // User-level playmat settings: fallback collection. - connect(&playmatDefaultEditButton, &QPushButton::clicked, this, - &AppearanceSettingsPage::openPlaymatCollectionDialog); - - auto *playmatGrid = new QGridLayout; - playmatGrid->addWidget(&playmatVisibilityLabel, 0, 0, 1, 1); - playmatGrid->addWidget(&playmatVisibilityCombo, 0, 1, 1, 1); - playmatGrid->addWidget(&playmatModeLabel, 1, 0, 1, 1); - playmatGrid->addWidget(&playmatModeCombo, 1, 1, 1, 1); - playmatGrid->addWidget(&playmatDefaultLabel, 2, 0, 1, 1); - playmatGrid->addWidget(&playmatDefaultEditButton, 2, 1, 1, 1); - - playmatGroupBox = new QGroupBox; - playmatGroupBox->setLayout(playmatGrid); - // putting it all together auto *mainLayout = new QVBoxLayout; mainLayout->addWidget(themeGroupBox); @@ -512,6 +512,12 @@ void AppearanceSettingsPage::retranslateUi() homeTabButtonColorSourceBox.setToolTip( tr("Automatic: extract from background if present, otherwise use theme default")); + playmatGroupBox->setTitle(tr("Playmat settings")); + playmatVisibilityLabel.setText(tr("Playmat visibility:")); + playmatModeLabel.setText(tr("Default collection behavior:")); + playmatDefaultLabel.setText(tr("Default playmat collection:")); + playmatDefaultEditButton.setText(tr("Edit...")); + stylingGroupBox->setTitle(tr("Styling settings")); styleUserListCheckBox.setText(tr("Style user list")); @@ -554,9 +560,4 @@ void AppearanceSettingsPage::retranslateUi() tableGroupBox->setTitle(tr("Table grid layout")); invertVerticalCoordinateCheckBox.setText(tr("Invert vertical coordinate")); minPlayersForMultiColumnLayoutLabel.setText(tr("Minimum player count for multi-column layout:")); - playmatGroupBox->setTitle(tr("Playmat settings")); - playmatVisibilityLabel.setText(tr("Playmat visibility:")); - playmatModeLabel.setText(tr("Default collection behavior:")); - playmatDefaultLabel.setText(tr("Default playmat collection:")); - playmatDefaultEditButton.setText(tr("Edit...")); } diff --git a/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.h b/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.h index 6b0369694..8db71ff8f 100644 --- a/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.h +++ b/cockatrice/src/interface/widgets/settings_page/appearance_settings_page.h @@ -44,46 +44,55 @@ private: QLabel homeTabButtonColorSourceLabel; QComboBox homeTabButtonColorSourceBox; - QCheckBox styleUserListCheckBox; - QCheckBox showShortcutsCheckBox; - QCheckBox showGameSelectorFilterToolbarCheckBox; - QLabel minPlayersForMultiColumnLayoutLabel; - QLabel maxFontSizeForCardsLabel; - QCheckBox overrideAllCardArtWithPersonalPreferenceCheckBox; - QCheckBox bumpSetsWithCardsInDeckToTopCheckBox; - QCheckBox displayCardNamesCheckBox; - QCheckBox autoRotateSidewaysLayoutCardsCheckBox; - QCheckBox cardScalingCheckBox; - QCheckBox roundCardCornersCheckBox; - QLabel verticalCardOverlapPercentLabel; - QSpinBox verticalCardOverlapPercentBox; - QLabel cardViewInitialRowsMaxLabel; - QSpinBox cardViewInitialRowsMaxBox; - QLabel cardViewExpandedRowsMaxLabel; - QSpinBox cardViewExpandedRowsMaxBox; - QCheckBox horizontalHandCheckBox; - QCheckBox leftJustifiedHandCheckBox; - QCheckBox invertVerticalCoordinateCheckBox; QLabel playmatVisibilityLabel; QComboBox playmatVisibilityCombo; QLabel playmatModeLabel; QComboBox playmatModeCombo; QLabel playmatDefaultLabel; QPushButton playmatDefaultEditButton; + + QCheckBox styleUserListCheckBox; + + QCheckBox showShortcutsCheckBox; + QCheckBox showGameSelectorFilterToolbarCheckBox; + + QCheckBox overrideAllCardArtWithPersonalPreferenceCheckBox; + QCheckBox bumpSetsWithCardsInDeckToTopCheckBox; + + QCheckBox displayCardNamesCheckBox; + QCheckBox autoRotateSidewaysLayoutCardsCheckBox; + QCheckBox cardScalingCheckBox; + QCheckBox roundCardCornersCheckBox; + QLabel maxFontSizeForCardsLabel; + QSpinBox maxFontSizeForCardsEdit; + + QLabel verticalCardOverlapPercentLabel; + QSpinBox verticalCardOverlapPercentBox; + QLabel cardViewInitialRowsMaxLabel; + QSpinBox cardViewInitialRowsMaxBox; + QLabel cardViewExpandedRowsMaxLabel; + QSpinBox cardViewExpandedRowsMaxBox; + + QList cardCounterNames; + + QCheckBox horizontalHandCheckBox; + QCheckBox leftJustifiedHandCheckBox; + + QCheckBox invertVerticalCoordinateCheckBox; + QLabel minPlayersForMultiColumnLayoutLabel; + QSpinBox minPlayersForMultiColumnLayoutEdit; + QGroupBox *themeGroupBox; QGroupBox *homeTabGroupBox; + QGroupBox *playmatGroupBox; QGroupBox *stylingGroupBox; QGroupBox *menuGroupBox; QGroupBox *printingsGroupBox; QGroupBox *cardsGroupBox; QGroupBox *cardLayoutGroupBox; - QGroupBox *handGroupBox; - QGroupBox *playmatGroupBox; - QGroupBox *tableGroupBox; QGroupBox *cardCountersGroupBox; - QList cardCounterNames; - QSpinBox minPlayersForMultiColumnLayoutEdit; - QSpinBox maxFontSizeForCardsEdit; + QGroupBox *handGroupBox; + QGroupBox *tableGroupBox; public: AppearanceSettingsPage(); diff --git a/cockatrice/src/interface/widgets/settings_page/general_settings_page.cpp b/cockatrice/src/interface/widgets/settings_page/general_settings_page.cpp index a293660f9..62b06fb60 100644 --- a/cockatrice/src/interface/widgets/settings_page/general_settings_page.cpp +++ b/cockatrice/src/interface/widgets/settings_page/general_settings_page.cpp @@ -425,29 +425,28 @@ void GeneralSettingsPage::updateStartupServerControlsVisibility() void GeneralSettingsPage::retranslateUi() { + const auto &settings = SettingsCache::instance(); + languageGroupBox->setTitle(tr("Language settings")); languageLabel.setText(tr("Language:")); - - versionGroupBox->setTitle(tr("Version settings")); - cardDatabaseGroupBox->setTitle(tr("Card database")); - startupGroupBox->setTitle(tr("Startup settings")); - - if (SettingsCache::instance().getIsPortableBuild()) { - pathsGroupBox->setTitle(tr("Paths (editing disabled in portable mode)")); - } else { - pathsGroupBox->setTitle(tr("Paths")); - } advertiseTranslationPageLabel.setText( QString("%2").arg(WIKI_TRANSLATION_FAQ).arg(tr("How to help with translations"))); - deckPathLabel.setText(tr("Decks directory:")); - filtersPathLabel.setText(tr("Filters directory:")); - replaysPathLabel.setText(tr("Replays directory:")); - picsPathLabel.setText(tr("Pictures directory:")); - cardDatabasePathLabel.setText(tr("Card database:")); - customCardDatabasePathLabel.setText(tr("Custom database directory:")); - tokenDatabasePathLabel.setText(tr("Token database:")); + + versionGroupBox->setTitle(tr("Version settings")); updateReleaseChannelLabel.setText(tr("Update channel")); startupUpdateCheckCheckBox.setText(tr("Check for client updates on startup")); + updateNotificationCheckBox.setText(tr("Notify if a feature supported by the server is missing in my client")); + newVersionOracleCheckBox.setText(tr("Automatically run Oracle when running a new version of Cockatrice")); + + // We can't change the strings after they're put into the QComboBox, so this is our workaround + int oldIndex = updateReleaseChannelBox.currentIndex(); + updateReleaseChannelBox.clear(); + for (ReleaseChannel *chan : settings.getUpdateReleaseChannels()) { + updateReleaseChannelBox.addItem(tr(chan->getName().toUtf8())); + } + updateReleaseChannelBox.setCurrentIndex(oldIndex); + + cardDatabaseGroupBox->setTitle(tr("Card database")); startupCardUpdateCheckBehaviorLabel.setText(tr("Check for card database updates on startup")); startupCardUpdateCheckBehaviorSelector.setItemText(startupCardUpdateCheckBehaviorIndexNone, tr("Don't check")); startupCardUpdateCheckBehaviorSelector.setItemText(startupCardUpdateCheckBehaviorIndexPrompt, @@ -456,8 +455,13 @@ void GeneralSettingsPage::retranslateUi() tr("Always update in the background")); cardUpdateCheckIntervalLabel.setText(tr("Check for card database updates every")); cardUpdateCheckIntervalSpinBox.setSuffix(tr(" days")); - updateNotificationCheckBox.setText(tr("Notify if a feature supported by the server is missing in my client")); - newVersionOracleCheckBox.setText(tr("Automatically run Oracle when running a new version of Cockatrice")); + + QDate lastCheckDate = settings.updates().getLastCardUpdateCheck(); + int daysAgo = lastCheckDate.daysTo(QDate::currentDate()); + lastCardUpdateCheckDateLabel.setText( + tr("Last update check on %1 (%2 days ago)").arg(lastCheckDate.toString()).arg(daysAgo)); + + startupGroupBox->setTitle(tr("Startup settings")); showTipsOnStartup.setText(tr("Show tips on startup")); startupTabLabel.setText(tr("Startup tab:")); startupTabSelector.setItemText(StartupTab::StartupTabHome, tr("Home")); @@ -473,21 +477,18 @@ void GeneralSettingsPage::retranslateUi() startupServerLabel.setText(tr("Server:")); startupRoomLabel.setText(tr("Room:")); startupRoomNameEdit->setPlaceholderText(tr("Room name")); - resetAllPathsButton->setText(tr("Reset all paths")); - const auto &settings = SettingsCache::instance(); - - QDate lastCheckDate = settings.updates().getLastCardUpdateCheck(); - int daysAgo = lastCheckDate.daysTo(QDate::currentDate()); - - lastCardUpdateCheckDateLabel.setText( - tr("Last update check on %1 (%2 days ago)").arg(lastCheckDate.toString()).arg(daysAgo)); - - // We can't change the strings after they're put into the QComboBox, so this is our workaround - int oldIndex = updateReleaseChannelBox.currentIndex(); - updateReleaseChannelBox.clear(); - for (ReleaseChannel *chan : settings.getUpdateReleaseChannels()) { - updateReleaseChannelBox.addItem(tr(chan->getName().toUtf8())); + if (settings.getIsPortableBuild()) { + pathsGroupBox->setTitle(tr("Paths (editing disabled in portable mode)")); + } else { + pathsGroupBox->setTitle(tr("Paths")); } - updateReleaseChannelBox.setCurrentIndex(oldIndex); -} \ No newline at end of file + deckPathLabel.setText(tr("Decks directory:")); + filtersPathLabel.setText(tr("Filters directory:")); + replaysPathLabel.setText(tr("Replays directory:")); + picsPathLabel.setText(tr("Pictures directory:")); + cardDatabasePathLabel.setText(tr("Card database:")); + customCardDatabasePathLabel.setText(tr("Custom database directory:")); + tokenDatabasePathLabel.setText(tr("Token database:")); + resetAllPathsButton->setText(tr("Reset all paths")); +} diff --git a/cockatrice/src/interface/widgets/settings_page/general_settings_page.h b/cockatrice/src/interface/widgets/settings_page/general_settings_page.h index 8dd7e8798..e0c1a47bf 100644 --- a/cockatrice/src/interface/widgets/settings_page/general_settings_page.h +++ b/cockatrice/src/interface/widgets/settings_page/general_settings_page.h @@ -42,6 +42,37 @@ private: QGroupBox *startupGroupBox; QGroupBox *pathsGroupBox; + QLabel languageLabel; + QComboBox languageBox; + QLabel advertiseTranslationPageLabel; + + QLabel updateReleaseChannelLabel; + QComboBox updateReleaseChannelBox; + QCheckBox startupUpdateCheckCheckBox; + QCheckBox updateNotificationCheckBox; + QCheckBox newVersionOracleCheckBox; + + QLabel startupCardUpdateCheckBehaviorLabel; + QComboBox startupCardUpdateCheckBehaviorSelector; + QLabel cardUpdateCheckIntervalLabel; + QSpinBox cardUpdateCheckIntervalSpinBox; + QLabel lastCardUpdateCheckDateLabel; + + QCheckBox showTipsOnStartup; + QLabel startupTabLabel; + QComboBox startupTabSelector; + QLabel startupServerLabel; + QComboBox startupServerSelector; + QLabel startupRoomLabel; + QLineEdit *startupRoomNameEdit; + + QLabel deckPathLabel; + QLabel filtersPathLabel; + QLabel replaysPathLabel; + QLabel picsPathLabel; + QLabel cardDatabasePathLabel; + QLabel customCardDatabasePathLabel; + QLabel tokenDatabasePathLabel; QLineEdit *deckPathEdit; QLineEdit *filtersPathEdit; QLineEdit *replaysPathEdit; @@ -51,33 +82,6 @@ private: QLineEdit *tokenDatabasePathEdit; QPushButton *resetAllPathsButton; QLabel *allPathsResetLabel; - QComboBox languageBox; - QCheckBox startupUpdateCheckCheckBox; - QLabel startupCardUpdateCheckBehaviorLabel; - QComboBox startupCardUpdateCheckBehaviorSelector; - QLabel cardUpdateCheckIntervalLabel; - QSpinBox cardUpdateCheckIntervalSpinBox; - QLabel lastCardUpdateCheckDateLabel; - QCheckBox updateNotificationCheckBox; - QCheckBox newVersionOracleCheckBox; - QComboBox updateReleaseChannelBox; - QLabel languageLabel; - QLabel deckPathLabel; - QLabel filtersPathLabel; - QLabel replaysPathLabel; - QLabel picsPathLabel; - QLabel cardDatabasePathLabel; - QLabel customCardDatabasePathLabel; - QLabel tokenDatabasePathLabel; - QLabel updateReleaseChannelLabel; - QLabel advertiseTranslationPageLabel; - QCheckBox showTipsOnStartup; - QLabel startupTabLabel; - QComboBox startupTabSelector; - QLabel startupServerLabel; - QComboBox startupServerSelector; - QLabel startupRoomLabel; - QLineEdit *startupRoomNameEdit; }; #endif // COCKATRICE_GENERAL_SETTINGS_PAGE_H diff --git a/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.cpp b/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.cpp index 182e75aac..2c6e062da 100644 --- a/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.cpp +++ b/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.cpp @@ -20,26 +20,7 @@ enum visualDeckStoragePromptForConversionIndex UserInterfaceSettingsPage::UserInterfaceSettingsPage() { - // general settings and notification settings - notificationsEnabledCheckBox.setChecked(SettingsCache::instance().userInterface().getNotificationsEnabled()); - connect(¬ificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, &SettingsCache::instance().userInterface(), - &InterfaceSettings::setNotificationsEnabled); - connect(¬ificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, this, - &UserInterfaceSettingsPage::setNotificationEnabled); - - specNotificationsEnabledCheckBox.setChecked( - SettingsCache::instance().userInterface().getSpectatorNotificationsEnabled()); - specNotificationsEnabledCheckBox.setEnabled(SettingsCache::instance().userInterface().getNotificationsEnabled()); - connect(&specNotificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, &SettingsCache::instance().userInterface(), - &InterfaceSettings::setSpectatorNotificationsEnabled); - - buddyConnectNotificationsEnabledCheckBox.setChecked( - SettingsCache::instance().userInterface().getBuddyConnectNotificationsEnabled()); - buddyConnectNotificationsEnabledCheckBox.setEnabled( - SettingsCache::instance().userInterface().getNotificationsEnabled()); - connect(&buddyConnectNotificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, - &SettingsCache::instance().userInterface(), &InterfaceSettings::setBuddyConnectNotificationsEnabled); - + // general settings doubleClickToPlayCheckBox.setChecked(SettingsCache::instance().userInterface().getDoubleClickToPlay()); connect(&doubleClickToPlayCheckBox, &QCheckBox::QT_STATE_CHANGED, &SettingsCache::instance().userInterface(), &InterfaceSettings::setDoubleClickToPlay); @@ -103,6 +84,26 @@ UserInterfaceSettingsPage::UserInterfaceSettingsPage() generalGroupBox = new QGroupBox; generalGroupBox->setLayout(generalGrid); + // notification settings + notificationsEnabledCheckBox.setChecked(SettingsCache::instance().userInterface().getNotificationsEnabled()); + connect(¬ificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, &SettingsCache::instance().userInterface(), + &InterfaceSettings::setNotificationsEnabled); + connect(¬ificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, this, + &UserInterfaceSettingsPage::setNotificationEnabled); + + specNotificationsEnabledCheckBox.setChecked( + SettingsCache::instance().userInterface().getSpectatorNotificationsEnabled()); + specNotificationsEnabledCheckBox.setEnabled(SettingsCache::instance().userInterface().getNotificationsEnabled()); + connect(&specNotificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, &SettingsCache::instance().userInterface(), + &InterfaceSettings::setSpectatorNotificationsEnabled); + + buddyConnectNotificationsEnabledCheckBox.setChecked( + SettingsCache::instance().userInterface().getBuddyConnectNotificationsEnabled()); + buddyConnectNotificationsEnabledCheckBox.setEnabled( + SettingsCache::instance().userInterface().getNotificationsEnabled()); + connect(&buddyConnectNotificationsEnabledCheckBox, &QCheckBox::QT_STATE_CHANGED, + &SettingsCache::instance().userInterface(), &InterfaceSettings::setBuddyConnectNotificationsEnabled); + auto *notificationsGrid = new QGridLayout; notificationsGrid->addWidget(¬ificationsEnabledCheckBox, 0, 0); notificationsGrid->addWidget(&specNotificationsEnabledCheckBox, 1, 0); @@ -355,6 +356,7 @@ void UserInterfaceSettingsPage::retranslateUi() notificationsEnabledCheckBox.setText(tr("Enable notifications in taskbar")); specNotificationsEnabledCheckBox.setText(tr("Notify in the taskbar for game events while you are spectating")); buddyConnectNotificationsEnabledCheckBox.setText(tr("Notify in the taskbar when users in your buddy list connect")); + animationGroupBox->setTitle(tr("Animation settings")); enableAllAnimationsButton.setText(tr("&Enable all animations")); disableAllAnimationsButton.setText(tr("&Disable all animations")); @@ -362,6 +364,7 @@ void UserInterfaceSettingsPage::retranslateUi() arrowDrawAnimationCheckBox.setText(tr("&Arrow draw animation")); lifeCounterAnimationsCheckBox.setText(tr("Life counter flash")); battlefieldFlashCheckBox.setText(tr("Battlefield flash on damage")); + deckEditorGroupBox->setTitle(tr("Deck editor/storage settings")); openDeckInNewTabCheckBox.setText(tr("Open deck in new tab by default")); visualDeckStorageInGameCheckBox.setText(tr("Use visual deck storage in game lobby")); @@ -397,8 +400,8 @@ void UserInterfaceSettingsPage::retranslateUi() 0, CommanderBracketNames::CommanderSpellbookBracketNames); commanderSpellbookIntegrationBracketNamingSelector.setItemText( 1, CommanderBracketNames::OfficialCommanderBracketNames); - commanderSpellbookIntegrationUseOfficialBracketNamesExplainer.setToolTip(CommanderBracketNames::Explainer); + replayGroupBox->setTitle(tr("Replay settings")); rewindBufferingMsLabel.setText(tr("Buffer time for backwards skip via shortcut:")); rewindBufferingMsBox.setSuffix(" ms"); diff --git a/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.h b/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.h index 0dc4cf4e8..e8a30fb1f 100644 --- a/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.h +++ b/cockatrice/src/interface/widgets/settings_page/user_interface_settings_page.h @@ -23,9 +23,6 @@ private slots: void updateCommanderSpellbookUiState(); private: - QCheckBox notificationsEnabledCheckBox; - QCheckBox specNotificationsEnabledCheckBox; - QCheckBox buddyConnectNotificationsEnabledCheckBox; QCheckBox doubleClickToPlayCheckBox; QCheckBox clickPlaysAllSelectedCheckBox; QCheckBox playToStackCheckBox; @@ -37,12 +34,18 @@ private: QCheckBox showTotalSelectionCountCheckBox; QCheckBox useTearOffMenusCheckBox; QCheckBox keepGameChatFocusCheckBox; + + QCheckBox notificationsEnabledCheckBox; + QCheckBox specNotificationsEnabledCheckBox; + QCheckBox buddyConnectNotificationsEnabledCheckBox; + QPushButton enableAllAnimationsButton; QPushButton disableAllAnimationsButton; QCheckBox tapAnimationCheckBox; QCheckBox arrowDrawAnimationCheckBox; QCheckBox lifeCounterAnimationsCheckBox; QCheckBox battlefieldFlashCheckBox; + QCheckBox openDeckInNewTabCheckBox; QLabel visualDeckStoragePromptForConversionLabel; QComboBox visualDeckStoragePromptForConversionSelector; @@ -57,8 +60,10 @@ private: QLabel commanderSpellbookIntegrationUseOfficialBracketNamesLabel; QToolButton commanderSpellbookIntegrationUseOfficialBracketNamesExplainer; QComboBox commanderSpellbookIntegrationBracketNamingSelector; + QLabel rewindBufferingMsLabel; QSpinBox rewindBufferingMsBox; + QGroupBox *generalGroupBox; QGroupBox *notificationsGroupBox; QGroupBox *animationGroupBox; From 03de1af6781643040b477d02f0bb48a14b279e59 Mon Sep 17 00:00:00 2001 From: RickyRister <42636155+RickyRister@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:28:50 -0700 Subject: [PATCH 05/25] [VDS] Fix search filter not being applied on refresh (#7229) --- .../visual_deck_storage_folder_display_widget.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cockatrice/src/interface/widgets/visual_deck_storage/visual_deck_storage_folder_display_widget.cpp b/cockatrice/src/interface/widgets/visual_deck_storage/visual_deck_storage_folder_display_widget.cpp index fbaabf90f..22d73b604 100644 --- a/cockatrice/src/interface/widgets/visual_deck_storage/visual_deck_storage_folder_display_widget.cpp +++ b/cockatrice/src/interface/widgets/visual_deck_storage/visual_deck_storage_folder_display_widget.cpp @@ -125,9 +125,7 @@ void VisualDeckStorageFolderDisplayWidget::continueDeckPass() } const bool matches = index.data(VisualDeckStorageRoles::FilterMatchRole).toBool(); - if (matches == deckPreviewWidget->isHidden()) { - deckPreviewWidget->setVisible(matches); - } + deckPreviewWidget->setVisible(matches); if (matches) { ++visibleDeckCount; } From 9bf2202739d94028337f3744b122f0e3242b3886 Mon Sep 17 00:00:00 2001 From: RickyRister <42636155+RickyRister@users.noreply.github.com> Date: Mon, 31 Aug 2026 07:31:02 -0700 Subject: [PATCH 06/25] [Game] Fix dragged card always placed on bottom of stack (#7228) --- cockatrice/src/game_graphics/zones/stack_zone.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cockatrice/src/game_graphics/zones/stack_zone.cpp b/cockatrice/src/game_graphics/zones/stack_zone.cpp index e9b14f13d..e15051885 100644 --- a/cockatrice/src/game_graphics/zones/stack_zone.cpp +++ b/cockatrice/src/game_graphics/zones/stack_zone.cpp @@ -58,17 +58,12 @@ void StackZone::handleDropEvent(const QList &dragItems, } const auto &cards = getLogic()->getCards(); - int index; + int index = calcDropIndexFromY(dropPoint.y(), MIN_CARD_VISIBLE); if (startZone == getLogic()) { - // Reordering within the zone: use drop position - index = calcDropIndexFromY(dropPoint.y(), MIN_CARD_VISIBLE); // Same-zone no-op: don't move a card onto itself if (!cards.isEmpty() && cards.at(index)->getId() == dragItems.at(0)->getId()) { return; } - } else { - // Coming from another zone: append at end (top of stack, rendered on top) - index = static_cast(cards.size()); } Command_MoveCard cmd; From d9745012776211fed1089e3d29061124d022a07b Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:31:11 +0200 Subject: [PATCH 07/25] [GameScene] Sever connections properly. (#7191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [GameScene] Sever connections properly. Took 2 minutes Took 54 minutes * [GameScene] Sever animated item destroy connections at teardown Fix crash when a replay's board is closed (GameScene teardown abort). The old QObject::disconnect(nullptr, nullptr, this, nullptr) is invalid per Qt docs (the sender must never be nullptr), so it never severed the PMF destroyed -> GameScene::removeAnimatedItem connections that fire when QGraphicsScene::~QGraphicsScene -> clear() destroys the remaining items. Store the QMetaObject::Connection handle for each animated item and disconnect them all in ~GameScene via the connection-handle overload. Dedup connections on the connection map rather than animatedItems, since the animation timer clears animatedItems on completion, which let a re-registered item (e.g. a life counter flashed repeatedly) accumulate orphaned duplicate destroyed connections that survived teardown. --------- Co-authored-by: Lukas Brübach --- cockatrice/src/game_graphics/game_scene.cpp | 27 +++++++++++++++------ cockatrice/src/game_graphics/game_scene.h | 8 +++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/cockatrice/src/game_graphics/game_scene.cpp b/cockatrice/src/game_graphics/game_scene.cpp index 87af4c73c..457f1b3f7 100644 --- a/cockatrice/src/game_graphics/game_scene.cpp +++ b/cockatrice/src/game_graphics/game_scene.cpp @@ -44,11 +44,16 @@ GameScene::GameScene(PhasesToolbar *_phasesToolbar, QObject *parent) GameScene::~GameScene() { - // Sever all incoming connections (animated item destroy-tracking) before the - // members below are destroyed: the base QGraphicsScene destructor destroys the - // remaining items, and their destroyed() signals must not reach slots that - // reference members that no longer exist. - QObject::disconnect(nullptr, nullptr, this, nullptr); + // Sever all destroyed->removeAnimatedItem connections before the members below + // are destroyed: the base QGraphicsScene destructor destroys the remaining items, + // and their destroyed() signals must not reach slots that reference members that + // no longer exist. The connection handle overload is used because the string-based + // disconnect(nullptr, nullptr, this, nullptr) is invalid (the sender must never be + // nullptr) and would otherwise fail to sever these pointer-to-member connections. + for (auto it = animationItemConnections.constBegin(); it != animationItemConnections.constEnd(); ++it) { + QObject::disconnect(*it); + } + animationItemConnections.clear(); delete animationTimer; animationTimer = nullptr; @@ -777,8 +782,15 @@ void GameScene::registerAnimationItem(IAnimatedItem *item) if (!object) { return; } - if (!animatedItems.contains(object)) { - connect(object, &QObject::destroyed, this, &GameScene::removeAnimatedItem); + // Guard against duplicate connections using the connection map, not + // animatedItems: the animation timer removes entries from animatedItems when an + // animation completes, but the destroyed->removeAnimatedItem connection must + // persist until the object is destroyed. Relying on animatedItems here would let + // a re-registered item (e.g. a life counter that flashes repeatedly) accumulate + // duplicate destroyed connections, the older ones of which would survive teardown. + if (!animationItemConnections.contains(object)) { + animationItemConnections.insert(object, + connect(object, &QObject::destroyed, this, &GameScene::removeAnimatedItem)); } animatedItems.insert(object, item); if (animationTimer && !animationTimer->isActive()) { @@ -797,6 +809,7 @@ void GameScene::unregisterAnimationItem(IAnimatedItem *item) void GameScene::removeAnimatedItem(QObject *item) { animatedItems.remove(item); + animationItemConnections.remove(item); if (animationTimer && animatedItems.isEmpty()) { animationTimer->stop(); } diff --git a/cockatrice/src/game_graphics/game_scene.h b/cockatrice/src/game_graphics/game_scene.h index c12696189..859d7a6eb 100644 --- a/cockatrice/src/game_graphics/game_scene.h +++ b/cockatrice/src/game_graphics/game_scene.h @@ -54,9 +54,11 @@ private: QPointer hoveredCard; ///< Currently hovered card QBasicTimer *animationTimer; ///< Timer for scene animations QHash animatedItems; ///< Items currently animating - int playerRotation; ///< Rotation offset for player layout - bool rearranging = false; ///< Guard against re-entrant rearrange - bool needsReArrange = false; ///< Pending rearrange requested during a pass + QHash + animationItemConnections; ///< destroyed->removeAnimatedItem handles per animated item + int playerRotation; ///< Rotation offset for player layout + bool rearranging = false; ///< Guard against re-entrant rearrange + bool needsReArrange = false; ///< Pending rearrange requested during a pass /** * @brief Updates which card is currently hovered based on scene coordinates. From 45c7ff6f872d6aa7d6d1739194ff919d1a98916c Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:31:49 +0200 Subject: [PATCH 08/25] [Mods] Properly close card art rules tab on disconnect (#7227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lukas Brübach --- .../src/interface/widgets/tabs/tab_supervisor.cpp | 10 +++++++++- .../interfaces/interface_tabs_settings_provider.h | 1 + .../libcockatrice/settings/tabs_settings.cpp | 10 ++++++++++ .../libcockatrice/settings/tabs_settings.h | 2 ++ tests/settings/settings_defaults_test.cpp | 6 ++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp b/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp index b0dac3e7c..c177dce35 100644 --- a/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp +++ b/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp @@ -245,6 +245,7 @@ void TabSupervisor::retranslateUi() aTabLog->setText(tr("Logs")); aTabReport->setText(tr("Report Queue")); aTabModeration->setText(tr("Moderation")); + aTabCardArtRules->setText(tr("Card Art Rules")); // tabs QList tabs; @@ -256,6 +257,7 @@ void TabSupervisor::retranslateUi() tabs.append(tabLog); tabs.append(tabReport); tabs.append(tabModeration); + tabs.append(tabCardArtRules); QMapIterator roomIterator(roomTabs); while (roomIterator.hasNext()) { tabs.append(roomIterator.next().value()); @@ -520,7 +522,9 @@ void TabSupervisor::start(const ServerInfo_User &_userInfo) if (SettingsCache::instance().tabs().getTabModerationOpen()) { openTabModeration(); } - openTabCardArtRules(); + if (SettingsCache::instance().tabs().getTabCardArtRulesOpen()) { + openTabCardArtRules(); + } } retranslateUi(); @@ -582,6 +586,9 @@ void TabSupervisor::stop() if (tabModeration) { tabModeration->close(); } + if (tabCardArtRules) { + tabCardArtRules->close(); + } } QList tabsToDelete; @@ -775,6 +782,7 @@ void TabSupervisor::openTabAdmin() void TabSupervisor::actTabCardArtRules(bool checked) { + SettingsCache::instance().tabs().setTabCardArtRulesOpen(checked); if (checked && !tabCardArtRules) { openTabCardArtRules(); setCurrentWidget(tabCardArtRules); diff --git a/libcockatrice_interfaces/libcockatrice/interfaces/interface_tabs_settings_provider.h b/libcockatrice_interfaces/libcockatrice/interfaces/interface_tabs_settings_provider.h index a81616cb0..054c4cd72 100644 --- a/libcockatrice_interfaces/libcockatrice/interfaces/interface_tabs_settings_provider.h +++ b/libcockatrice_interfaces/libcockatrice/interfaces/interface_tabs_settings_provider.h @@ -21,6 +21,7 @@ public: [[nodiscard]] virtual bool getTabLogOpen() const = 0; [[nodiscard]] virtual bool getTabReportOpen() const = 0; [[nodiscard]] virtual bool getTabModerationOpen() const = 0; + [[nodiscard]] virtual bool getTabCardArtRulesOpen() const = 0; }; #endif // COCKATRICE_INTERFACE_TABS_SETTINGS_PROVIDER_H diff --git a/libcockatrice_settings/libcockatrice/settings/tabs_settings.cpp b/libcockatrice_settings/libcockatrice/settings/tabs_settings.cpp index 85a1424a6..cf5bfd81a 100644 --- a/libcockatrice_settings/libcockatrice/settings/tabs_settings.cpp +++ b/libcockatrice_settings/libcockatrice/settings/tabs_settings.cpp @@ -106,6 +106,11 @@ bool TabsSettings::getTabModerationOpen() const return getValue("moderation", QString(), QString(), false).toBool(); } +bool TabsSettings::getTabCardArtRulesOpen() const +{ + return getValue("cardArtRules", QString(), QString(), false).toBool(); +} + void TabsSettings::setTabVisualDeckStorageOpen(bool value) { setValue(value, "visualDeckStorage"); @@ -150,3 +155,8 @@ void TabsSettings::setTabModerationOpen(bool value) { setValue(value, "moderation"); } + +void TabsSettings::setTabCardArtRulesOpen(bool value) +{ + setValue(value, "cardArtRules"); +} diff --git a/libcockatrice_settings/libcockatrice/settings/tabs_settings.h b/libcockatrice_settings/libcockatrice/settings/tabs_settings.h index 365d91af7..eb78d311b 100644 --- a/libcockatrice_settings/libcockatrice/settings/tabs_settings.h +++ b/libcockatrice_settings/libcockatrice/settings/tabs_settings.h @@ -43,6 +43,7 @@ public: [[nodiscard]] bool getTabLogOpen() const override; [[nodiscard]] bool getTabReportOpen() const override; [[nodiscard]] bool getTabModerationOpen() const override; + [[nodiscard]] bool getTabCardArtRulesOpen() const override; void setStartupTabIndex(int value); void setStartupServerHost(const QString &host); @@ -57,6 +58,7 @@ public: void setTabLogOpen(bool value); void setTabReportOpen(bool value); void setTabModerationOpen(bool value); + void setTabCardArtRulesOpen(bool value); signals: void startupTabIndexChanged(int index); diff --git a/tests/settings/settings_defaults_test.cpp b/tests/settings/settings_defaults_test.cpp index 6c79d5227..139656f27 100644 --- a/tests/settings/settings_defaults_test.cpp +++ b/tests/settings/settings_defaults_test.cpp @@ -238,6 +238,12 @@ TEST_F(SettingsDefaultsTest, Tabs_ModerationOpen_Default) ASSERT_EQ(s.getTabModerationOpen(), false); } +TEST_F(SettingsDefaultsTest, Tabs_CardArtRulesOpen_Default) +{ + TabsSettings s(settingsPath, nullptr); + ASSERT_EQ(s.getTabCardArtRulesOpen(), false); +} + // --- ChatSettings --- TEST_F(SettingsDefaultsTest, Chat_Mention_Default) From 425b16ea0d33353582ea1ddc9bda2b33d6ad3a19 Mon Sep 17 00:00:00 2001 From: RickyRister <42636155+RickyRister@users.noreply.github.com> Date: Tue, 1 Sep 2026 03:11:31 -0700 Subject: [PATCH 09/25] [Game] Allow dropping cards at bottom of stack zone (#7230) --- cockatrice/src/game_graphics/zones/hand_zone.cpp | 3 ++- cockatrice/src/game_graphics/zones/select_zone.cpp | 5 +++-- cockatrice/src/game_graphics/zones/select_zone.h | 6 +++++- cockatrice/src/game_graphics/zones/stack_zone.cpp | 7 ++++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cockatrice/src/game_graphics/zones/hand_zone.cpp b/cockatrice/src/game_graphics/zones/hand_zone.cpp index b52a4955a..1a8f7a910 100644 --- a/cockatrice/src/game_graphics/zones/hand_zone.cpp +++ b/cockatrice/src/game_graphics/zones/hand_zone.cpp @@ -41,7 +41,8 @@ void HandZone::handleDropEvent(const QList &dragItems, } } } else { - x = calcDropIndexFromY(dropPoint.y()); + bool sameZone = startZone == getLogic(); + x = calcDropIndexFromY(dropPoint.y(), !sameZone); } Command_MoveCard cmd; diff --git a/cockatrice/src/game_graphics/zones/select_zone.cpp b/cockatrice/src/game_graphics/zones/select_zone.cpp index c58c41b92..470c70fcf 100644 --- a/cockatrice/src/game_graphics/zones/select_zone.cpp +++ b/cockatrice/src/game_graphics/zones/select_zone.cpp @@ -83,7 +83,7 @@ SelectZone::StackLayoutParams SelectZone::buildStackParams(qreal minOffset) cons return {cardCount, boundingRect().height(), cardHeight, offset, minOffset}; } -int SelectZone::calcDropIndexFromY(qreal dropY, qreal minOffset) const +int SelectZone::calcDropIndexFromY(qreal dropY, bool allowCountExpand, qreal minOffset) const { const auto &cards = getLogic()->getCards(); if (cards.isEmpty()) { @@ -94,7 +94,8 @@ int SelectZone::calcDropIndexFromY(qreal dropY, qreal minOffset) const if (effectiveOffset <= 0.0) { return 0; } - return qBound(0, qRound((dropY - start) / effectiveOffset), params.cardCount - 1); + int max = allowCountExpand ? params.cardCount : params.cardCount - 1; + return qBound(0, qRound((dropY - start) / effectiveOffset), max); } void SelectZone::restoreStaleEscapedCards() diff --git a/cockatrice/src/game_graphics/zones/select_zone.h b/cockatrice/src/game_graphics/zones/select_zone.h index 7408f29b6..b5d3ca37a 100644 --- a/cockatrice/src/game_graphics/zones/select_zone.h +++ b/cockatrice/src/game_graphics/zones/select_zone.h @@ -104,8 +104,12 @@ protected: /** * @brief Computes the card index at a given y-coordinate within the zone's vertical layout. * Returns 0 if the zone has no cards or the offset is zero. + * + * @param dropY The y-coordinate that the card was dropped at + * @param allowCountExpand If false, clamps the index at the number of cards minus 1 + * @param minOffset Minimum offset to preserve */ - int calcDropIndexFromY(qreal dropY, qreal minOffset = 0.0) const; + int calcDropIndexFromY(qreal dropY, bool allowCountExpand, qreal minOffset = 0.0) const; /** * @brief Positions cards vertically with alternating left/right x-offsets. diff --git a/cockatrice/src/game_graphics/zones/stack_zone.cpp b/cockatrice/src/game_graphics/zones/stack_zone.cpp index e15051885..ff62097c7 100644 --- a/cockatrice/src/game_graphics/zones/stack_zone.cpp +++ b/cockatrice/src/game_graphics/zones/stack_zone.cpp @@ -57,10 +57,11 @@ void StackZone::handleDropEvent(const QList &dragItems, return; } - const auto &cards = getLogic()->getCards(); - int index = calcDropIndexFromY(dropPoint.y(), MIN_CARD_VISIBLE); - if (startZone == getLogic()) { + bool sameZone = startZone == getLogic(); + int index = calcDropIndexFromY(dropPoint.y(), !sameZone, MIN_CARD_VISIBLE); + if (sameZone) { // Same-zone no-op: don't move a card onto itself + const auto &cards = getLogic()->getCards(); if (!cards.isEmpty() && cards.at(index)->getId() == dragItems.at(0)->getId()) { return; } From 4e9d14816371f8310af6c47eaad9d0a0c76baee0 Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:11:02 +0200 Subject: [PATCH 10/25] [TabSupervisor] Initialize all tabs (#7231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lukas Brübach --- cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp b/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp index c177dce35..ed0ddaf06 100644 --- a/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp +++ b/cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp @@ -116,9 +116,10 @@ void CloseButton::paintEvent(QPaintEvent * /*event*/) } TabSupervisor::TabSupervisor(AbstractClient *_client, QMenu *tabsMenu, QWidget *parent) - : QTabWidget(parent), userInfo(nullptr), client(_client), tabsMenu(tabsMenu), tabVisualDeckStorage(nullptr), - tabServer(nullptr), tabAccount(nullptr), tabDeckStorage(nullptr), tabReplays(nullptr), tabAdmin(nullptr), - tabLog(nullptr), tabReport(nullptr), tabModeration(nullptr), isLocalGame(false) + : QTabWidget(parent), userInfo(nullptr), client(_client), tabsMenu(tabsMenu), tabHome(nullptr), + tabVisualDeckStorage(nullptr), tabServer(nullptr), tabAccount(nullptr), tabDeckStorage(nullptr), + tabReplays(nullptr), tabAdmin(nullptr), tabCardArtRules(nullptr), tabLog(nullptr), tabReport(nullptr), + tabModeration(nullptr), isLocalGame(false) { setElideMode(Qt::ElideRight); setMovable(true); From bd1fb69800e9780bdf470d5ec763bec4aaf1bd5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 2 Sep 2026 14:50:58 +0200 Subject: [PATCH 11/25] [Oracle/Tests] Pin cmc coercion in CI run; scope the reserve pass The #7214 coercion assertion lived only in oracle_importer_benchmark_test, which gets no add_test and so never runs under ctest. Add NumericManaValueCoercedToCmc and LegacyConvertedManaCostCoercedToCmc to oracle_importer_test (a CI-ran binary): manaValue/convertedManaCost are JSON numbers in AllPrintings, and QJsonValue::toString() would drop them to an empty cmc without the #7214 coercion fix. Wrap the distinct-name reserve pass in a bare block so the ~35k name QStrings are handed back before the memory-heavy import loop starts. --- oracle/src/oracleimporter.cpp | 14 +++++++++----- tests/oracle/oracle_importer_test.cpp | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index f0dc3b1ae..49577baf0 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -555,13 +555,17 @@ int OracleImporter::startImport() // ships ~100k printings for ~35k names, so reserving the printing count // would overallocate ~3x (against this stack's RAM goal). Collecting // distinct names is cheap — one pass over the already-parsed name fields. - QSet distinctNames; - for (const SetToDownload &curSetToParse : allSets) { - for (const QJsonValue &cardValue : curSetToParse.getCards()) { - distinctNames.insert(cardValue.toObject().value("name").toString()); + { + QSet distinctNames; + for (const SetToDownload &curSetToParse : allSets) { + for (const QJsonValue &cardValue : curSetToParse.getCards()) { + distinctNames.insert(cardValue.toObject().value("name").toString()); + } } + cards.reserve(distinctNames.size()); + // The set goes out of scope here, handing the ~35k name QStrings back + // to the allocator before the (memory-heavy) import loop starts. } - cards.reserve(distinctNames.size()); // add an empty set for tokens CardSetPtr tokenSet = diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index 19f66e5e6..145a2ca0f 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -482,6 +482,33 @@ TEST_F(OracleImporterTest, ManaCostStripsBraces) ASSERT_EQ(result->getProperty("manacost"), "2WB"); } +// cmc comes through as a JSON number ("convertedManaCost"/"manaValue" are +// floats in AllPrintings), so this pins the number-to-text coercion that +// QJsonValue::toString() dropped in #7214. +TEST_F(OracleImporterTest, NumericManaValueCoercedToCmc) +{ + QJsonObject card = makeCard("Cmc Card"); + card["manaValue"] = 3; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Cmc Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("cmc"), "3"); +} + +TEST_F(OracleImporterTest, LegacyConvertedManaCostCoercedToCmc) +{ + QJsonObject card = makeCard("Legacy Cmc Card"); + card["convertedManaCost"] = 3.0; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Legacy Cmc Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("cmc"), "3"); +} + // ============================================================================ // Card deduplication tests // ============================================================================ From 81c9edbf898118fc11ef32e44854212432015085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sat, 29 Aug 2026 21:08:56 +0200 Subject: [PATCH 12/25] [Oracle] Add RAM usage benchmarks for the oracle importer - Measure process peak/current RSS via procfs (Linux) or getrusage (macOS) - Add a synthetic-scale RAM benchmark and an opt-in real AllPrintings run gated by COCKATRICE_ORACLE_RAM_BENCHMARK=1 - Mirror the wizard's magic-byte handling to decompress .xz/.zip payloads - Wire optional ZLIB/LibLZMA into the benchmark target and raise its timeout Took 2 minutes --- tests/oracle/CMakeLists.txt | 36 ++- .../oracle/oracle_importer_benchmark_test.cpp | 300 ++++++++++++++++++ 2 files changed, 332 insertions(+), 4 deletions(-) diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index d126390c7..cbff4f19c 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -25,10 +25,33 @@ target_link_libraries( add_test(NAME oracle_importer_test COMMAND oracle_importer_test) -# Oracle importer benchmark tests (manual, not run in CI) +# Oracle importer benchmark tests (manual, not run in CI, incl. RAM benchmark) +# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark +# can download and decompress whatever AllPrintings format the default URL selects. +find_package(ZLIB) +if(ZLIB_FOUND) + add_definitions("-DHAS_ZLIB") + set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp) + set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES}) + include_directories(${ZLIB_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled") +endif() + +find_package(LibLZMA) +if(LIBLZMA_FOUND) + add_definitions("-DHAS_LZMA") + list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp) + list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES}) + include_directories(${LIBLZMA_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled") +endif() + add_executable( - oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp - ../../oracle/src/parsehelpers.cpp oracle_importer_benchmark_test.cpp + oracle_importer_benchmark_test + ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} ) if(NOT GTEST_FOUND) @@ -36,6 +59,11 @@ if(NOT GTEST_FOUND) endif() target_link_libraries( - oracle_importer_benchmark_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + oracle_importer_benchmark_test + libcockatrice_card + libcockatrice_interfaces + Threads::Threads + ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES} + ${_ORACLE_BENCH_EXTRA_LIBRARIES} ) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index b3cc55319..fe2d0f107 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -1,13 +1,33 @@ #include "../../oracle/src/oracleimporter.h" #include "gtest/gtest.h" +#include +#include #include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include #include +#if defined(HAS_LZMA) +#include "../../oracle/src/lzma/decompress.h" +#endif +#if defined(HAS_ZLIB) +#include "../../oracle/src/zip/unzip.h" +#endif +#if defined(Q_OS_MACOS) +#include +#include +#endif + // Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set static QByteArray buildSyntheticData(int numSets, int cardsPerSet) { @@ -257,8 +277,288 @@ TEST(OracleBenchmark, ImportCardsWithColors) .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); } +// ============================================================================ +// RAM usage measurement +// ============================================================================ + +// Mirrors the default AllPrintings URL selection in oracle/src/pages.cpp. +#if defined(HAS_LZMA) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.xz"); +#elif defined(HAS_ZLIB) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.zip"); +#else +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json"); +#endif + +// Magic bytes also from oracle/src/pages.cpp +static const QByteArray kXzSignature("\xFD\x37\x7A\x58\x5A", 6); +static const QByteArray kZipSignature("PK"); + +struct MemorySnapshot +{ + qint64 peakRssKb = -1; // process high-water mark (VmHWM on Linux, ru_maxrss on macOS) + qint64 rssKb = -1; // current resident set size + bool available = false; + + static MemorySnapshot current() + { + MemorySnapshot snap; +#if defined(Q_OS_LINUX) + QFile statusFile("/proc/self/status"); + if (statusFile.open(QIODevice::ReadOnly | QIODevice::Text)) { + // /proc files report size() == 0, so atEnd() is immediately true: read everything first. + const QList lines = statusFile.readAll().split('\n'); + for (const QByteArray &line : lines) { + if (line.startsWith("VmHWM:")) { + snap.peakRssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } else if (line.startsWith("VmRSS:")) { + snap.rssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } + } + snap.available = snap.peakRssKb >= 0; + } +#elif defined(Q_OS_MACOS) + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + snap.peakRssKb = usage.ru_maxrss / 1024; // bytes -> kB + snap.available = snap.peakRssKb >= 0; + } + // getrusage has no current-RSS equivalent; task_info's resident_size + // is the closest macOS analog to Linux VmRSS. + mach_task_basic_info info = {}; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, reinterpret_cast(&info), &count) == + KERN_SUCCESS) { + snap.rssKb = info.resident_size / 1024; + } +#endif + return snap; + } +}; + +static QString formatKb(qint64 kb) +{ + if (kb < 0) { + return "N/A"; + } + return QString("%1 MB").arg(kb / 1024.0, 0, 'f', 1); +} + +static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, const MemorySnapshot ¤t) +{ + if (!baseline.available || !current.available) { + qDebug().noquote() << QString(" %1: memory stats unavailable on this platform").arg(phase); + return; + } + // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a + // peak-based delta between phases is ~0.0 MB by construction once the + // fixture build has set the process peak. The live signals are current RSS + // and the process peak; the delta is only meaningful where current RSS is + // a per-phase value (see "after releaseSetData()"). + QString rssDelta = "N/A"; + if (current.rssKb >= 0 && baseline.rssKb >= 0) { + rssDelta = formatKb(current.rssKb - baseline.rssKb); + } + qDebug().noquote() << QString(" %1: current RSS %2 | delta vs baseline %3 | process peak %4") + .arg(phase) + .arg(formatKb(current.rssKb)) + .arg(rssDelta) + .arg(formatKb(current.peakRssKb)); +} + +// Decompresses the download payload when the default URL is a compressed build, +// mirroring the wizard's magic-byte handling in oracle/src/pages.cpp. +static QByteArray decompressSetsData(const QByteArray &payload) +{ + if (payload.startsWith(kXzSignature)) { +#if defined(HAS_LZMA) + QBuffer inBuffer(const_cast(&payload)); + QByteArray out; + QBuffer outBuffer(&out); + inBuffer.open(QIODevice::ReadOnly); + outBuffer.open(QIODevice::WriteOnly); + XzDecompressor xz; + if (!xz.decompress(&inBuffer, &outBuffer)) { + qDebug() << "RAM benchmark: xz decompression failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is xz-compressed but this build has no LZMA support"; + return {}; +#endif + } + if (payload.startsWith(kZipSignature)) { +#if defined(HAS_ZLIB) + QBuffer inBuffer(const_cast(&payload)); + inBuffer.open(QIODevice::ReadOnly); + UnZip unzip; + if (unzip.openArchive(&inBuffer) != UnZip::Ok) { + qDebug() << "RAM benchmark: zip archive open failed"; + return {}; + } + if (unzip.fileList().size() != 1) { + qDebug() << "RAM benchmark: zip archive doesn't contain exactly one file"; + return {}; + } + QByteArray out; + QBuffer outBuffer(&out); + outBuffer.open(QIODevice::WriteOnly); + const auto errorCode = unzip.extractFile(unzip.fileList().value(0), &outBuffer); + unzip.closeArchive(); + if (errorCode != UnZip::Ok) { + qDebug() << "RAM benchmark: zip extraction failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is zip-compressed but this build has no zlib support"; + return {}; +#endif + } + return payload; +} + +TEST(OracleBenchmark, ImportRamUsage) +{ + static constexpr int numSets = 30; + static constexpr int cardsPerSet = 2000; // ~60k cards, roughly AllPrintings scale + + // Baseline must precede the fixture build: a high-water mark set while + // generating the synthetic JSON would otherwise mask the importer phases. + // Where memory stats are unavailable (Windows), skip before doing the + // 60k-card fixture build, which would otherwise be pure wasted work. + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + const QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(data)); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (synthetic): %1 sets, %2 cards, %3 MB JSON") + .arg(importedSets) + .arg(totalCards) + .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("parse", baseline, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + +TEST(OracleBenchmark, ImportRamUsageAllPrintings) +{ + // Only "1" enables the download: unset (the default and the CI setup) and + // an explicit "0" both disable it. + bool envOk = false; + const int enabled = qEnvironmentVariableIntValue("COCKATRICE_ORACLE_RAM_BENCHMARK", &envOk); + if (!envOk || enabled == 0) { + GTEST_SKIP() << "Set COCKATRICE_ORACLE_RAM_BENCHMARK=1 to download the real AllPrintings dataset for this " + "RAM benchmark. Default URL: " + << kDefaultAllPrintingsUrl.toDisplayString().toStdString(); + } + + // Baseline must precede the request so the phase covers the download + + // decompress step, including the payload materialized by readAll(). + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + QNetworkAccessManager nam; + QNetworkRequest request(kDefaultAllPrintingsUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "Cockatrice Oracle RAM benchmark"); + QNetworkReply *reply = nam.get(request); + + QEventLoop loop; + QTimer timeoutTimer; + timeoutTimer.setSingleShot(true); + bool timedOut = false; + QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeoutTimer, &QTimer::timeout, &loop, [&] { + timedOut = true; + reply->abort(); + }); + timeoutTimer.start(10 * 60 * 1000); + loop.exec(); + timeoutTimer.stop(); + + // abort() leaves reply->error() as OperationCanceledError, so a timed-out + // download takes the same GTEST_SKIP path as any other network error + // instead of reading a truncated body and failing the parse below. + if (timedOut || reply->error() != QNetworkReply::NoError) { + GTEST_SKIP() << "Download failed: " << reply->errorString().toStdString(); + } + const QByteArray payload = reply->readAll(); + reply->deleteLater(); + + // mtgjson can answer 200 with an HTML page (mirrors the wizard's '<' check + // in pages.cpp); reject it before trying to decompress/parse. + if (payload.startsWith("<")) { + GTEST_SKIP() << "Download returned a non-JSON body (HTML page instead of data), skipping"; + } + + const QByteArray setsData = decompressSetsData(payload); + const MemorySnapshot afterDownload = MemorySnapshot::current(); + if (setsData.isEmpty()) { + GTEST_SKIP() << "No data to import (download or decompression failed)"; + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(setsData)); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (real AllPrintings): %1 sets, %2 unique cards") + .arg(importedSets) + .arg(totalCards); + qDebug().noquote() << QString(" URL: %1").arg(kDefaultAllPrintingsUrl.toDisplayString()); + qDebug().noquote() << QString(" Downloaded: %1 MB, decompressed: %2 MB") + .arg(payload.size() / (1024.0 * 1024.0), 0, 'f', 1) + .arg(setsData.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("download+decompress", baseline, afterDownload); + logRamPhase("parse", afterDownload, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + int main(int argc, char **argv) { + // Required for the event loop used by the real-AllPrintings download benchmark + QCoreApplication app(argc, argv); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } From 64e5c0de193a8182ccd18f92c7d367981f61c27a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 2 Sep 2026 14:55:52 +0200 Subject: [PATCH 13/25] [Oracle/Tests] Measure parse against post-fixture baseline; assert release empties sets --- .../oracle/oracle_importer_benchmark_test.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index fe2d0f107..567519a48 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -353,8 +353,9 @@ static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, co // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a // peak-based delta between phases is ~0.0 MB by construction once the // fixture build has set the process peak. The live signals are current RSS - // and the process peak; the delta is only meaningful where current RSS is - // a per-phase value (see "after releaseSetData()"). + // and the process peak; the delta is meaningful only where the baseline was + // taken immediately before the phase it measures (e.g. the import phase, + // which compares afterParse against afterImport). QString rssDelta = "N/A"; if (current.rssKb >= 0 && baseline.rssKb >= 0) { rssDelta = formatKb(current.rssKb - baseline.rssKb); @@ -435,6 +436,13 @@ TEST(OracleBenchmark, ImportRamUsage) const QByteArray data = buildSyntheticData(numSets, cardsPerSet); + // The fixture build leaves freed-but-unreturned arenas behind (current RSS + // rarely falls once glibc allocates). Baseline immediately after it so the + // parse phase measures only the importer's own growth (~40 MB) rather than + // swallowing the fixture builder's spike. + const MemorySnapshot afterFixture = MemorySnapshot::current(); + logRamPhase("fixture build", baseline, afterFixture); + NoopCardSetPriorityController controller; OracleImporter importer; @@ -459,9 +467,14 @@ TEST(OracleBenchmark, ImportRamUsage) .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1); qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); - logRamPhase("parse", baseline, afterParse); + logRamPhase("parse", afterFixture, afterParse); logRamPhase("import", afterParse, afterImport); logRamPhase("after releaseSetData()", afterImport, afterRelease); + + // Freeing the parsed tree rarely moves current RSS (allocator reuse), so the + // meaningful signal that release actually dropped the buffers is emptiness, + // not an RSS delta. + ASSERT_TRUE(importer.getSets().isEmpty()); } TEST(OracleBenchmark, ImportRamUsageAllPrintings) From b6e74928d4b12096d11c5dbe674803d8925c5f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sun, 30 Aug 2026 13:39:40 +0200 Subject: [PATCH 14/25] [Oracle] Parse sets lazily to slash importer peak memory - Add a raw JSON scanner that splits the document into per-set byte ranges without materializing the JSON tree - Keep only the raw document bytes and parse one set at a time in startImport - Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied - Clear the retained raw data in releaseSetData()/clear() - Cover the scanner and lazy parsing with tests Took 2 minutes --- oracle/CMakeLists.txt | 1 + oracle/src/oracleimporter.cpp | 93 ++- oracle/src/oracleimporter.h | 40 +- oracle/src/raw_json_scanner.cpp | 621 ++++++++++++++++++ oracle/src/raw_json_scanner.h | 76 +++ tests/oracle/CMakeLists.txt | 4 +- .../oracle/oracle_importer_benchmark_test.cpp | 7 +- tests/oracle/oracle_importer_test.cpp | 167 +++++ 8 files changed, 964 insertions(+), 45 deletions(-) create mode 100644 oracle/src/raw_json_scanner.cpp create mode 100644 oracle/src/raw_json_scanner.h diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 6a29b6935..bb1795caf 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,6 +23,7 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp + src/raw_json_scanner.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 49577baf0..6a5633215 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -44,26 +45,23 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s return priority; } -bool OracleImporter::readSetsFromByteArray(const QByteArray &data) +bool OracleImporter::readSetsFromByteArray(QByteArray data) { - QJsonParseError error; - auto doc = QJsonDocument::fromJson(data, &error); - if (error.error != QJsonParseError::NoError) { - qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(data, &error); + if (error.isError()) { + qDebug() << "error: RawJson::scanSetRanges():" << error.message; return false; } - auto setsObj = doc.object().value("data").toObject(); - QList newSetList; + newSetList.reserve(ranges.size()); - for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { - QJsonObject setObj = it.value().toObject(); - QString shortName = setObj.value("code").toString().toUpper(); - QString longName = setObj.value("name").toString(); - QJsonArray setCards = setObj.value("cards").toArray(); - QString setType = setObj.value("type").toString(); - QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); + for (const RawJson::SetRange &range : ranges) { + QString shortName = range.code.toUpper(); + QString longName = range.name; + QString setType = range.type; + QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { @@ -83,7 +81,9 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) } setType = setType.trimmed(); } - newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate)); + SetToDownload set(shortName, longName, priority, setType, releaseDate); + set.setRawRange(range.dataRange); + newSetList.append(set); } std::sort(newSetList.begin(), newSetList.end()); @@ -92,6 +92,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) return false; } allSets = newSetList; + rawSetsData = std::move(data); return true; } @@ -550,22 +551,17 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); - // Pre-allocate the cards hash to avoid rehashing during import. The hash - // is keyed by distinct card name rather than by printings: AllPrintings - // ships ~100k printings for ~35k names, so reserving the printing count - // would overallocate ~3x (against this stack's RAM goal). Collecting - // distinct names is cheap — one pass over the already-parsed name fields. - { - QSet distinctNames; - for (const SetToDownload &curSetToParse : allSets) { - for (const QJsonValue &cardValue : curSetToParse.getCards()) { - distinctNames.insert(cardValue.toObject().value("name").toString()); - } - } - cards.reserve(distinctNames.size()); - // The set goes out of scope here, handing the ~35k name QStrings back - // to the allocator before the (memory-heavy) import loop starts. +<<<<<<< HEAD + // Pre-allocate the cards hash to avoid rehashing during import. Keys are + // distinct card names while raw ranges only count printings (AllPrintings + // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact + // distinct-name count would require eagerly parsing, which the lazy reader + // deliberately avoids. It's a capacity hint, so the overshoot is harmless. + int estimatedCards = 0; + for (const SetToDownload &curSetToParse : allSets) { + estimatedCards += curSetToParse.getRawRange().cardCount; } + cards.reserve(estimatedCards); // add an empty set for tokens CardSetPtr tokenSet = @@ -578,11 +574,44 @@ int OracleImporter::startImport() CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(), curSetToParse.getLongName(), curSetToParse.getSetType(), curSetToParse.getReleaseDate(), curSetToParse.getPriority()); + + // parse only this set's slice of the raw document so the whole JSON tree is + // never kept in memory at once + const RawJson::SetDataRange &rawRange = curSetToParse.getRawRange(); + const qsizetype rangeEnd = rawRange.start + rawRange.length; + if (rawRange.start < 0 || rawRange.length <= 0 || rangeEnd > rawSetsData.size()) { + // rawSetsData is cleared by releaseSetData() while SetToDownload copies + // taken from getSets() keep their ranges, and nothing else enforces the + // pairing — so never index past the buffer on stale/mismatched ranges. + qWarning() << "error: out-of-bounds raw range for set" << curSetToParse.getShortName() << "skipping"; + ++setIndex; + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + // sliced() shares the buffer instead of deep-copying the slice; the largest + // sets in AllPrintings are tens of MB, so the copy is worth avoiding here. + const QByteArray setBytes = rawSetsData.sliced(rawRange.start, rawRange.length); + QJsonParseError parseError; + const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError); + if (parseError.error != QJsonParseError::NoError) { + qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":" + << parseError.errorString(); + ++setIndex; + // Keep the progress accounting honest: a set that failed to parse + // still advanced the index, so report it (with zero imported cards) + // rather than letting SaveSetsPage's bar stall per failed set. + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + + // Only add the set to the database once its slice parsed cleanly; + // a set that fails here must not persist as an empty set in cards.xml. if (!sets.contains(newSet->getShortName())) { sets.insert(newSet->getShortName(), newSet); } - int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards()); + const QJsonArray setCards = setDoc.object().value("cards").toArray(); + int numCardsInSet = importCardsFromSet(newSet, setCards); ++setIndex; @@ -605,6 +634,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr void OracleImporter::releaseSetData() { allSets.clear(); + rawSetsData.clear(); } void OracleImporter::clear() @@ -612,4 +642,5 @@ void OracleImporter::clear() sets.clear(); cards.clear(); allSets.clear(); + rawSetsData.clear(); } diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 52a7cd349..8cb30ca40 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,9 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include "raw_json_scanner.h" + +#include #include #include #include @@ -46,10 +49,12 @@ class SetToDownload { private: QString shortName, longName; - QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; + // Byte range of this set's object within the importer's raw JSON text. Parsing + // one set at a time keeps peak memory low instead of holding the whole document. + RawJson::SetDataRange rawRange; public: const QString &getShortName() const @@ -60,10 +65,6 @@ public: { return longName; } - const QJsonArray &getCards() const - { - return cards; - } const QString &getSetType() const { return setType; @@ -76,16 +77,23 @@ public: { return priority; } + const RawJson::SetDataRange &getRawRange() const + { + return rawRange; + } SetToDownload(QString _shortName, QString _longName, - QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) - : shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)), - releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority) + : shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate), + setType(std::move(_setType)), priority(_priority) { } + void setRawRange(const RawJson::SetDataRange &_rawRange) + { + rawRange = _rawRange; + } bool operator<(const SetToDownload &set) const { return longName.compare(set.longName, Qt::CaseInsensitive) < 0; @@ -141,6 +149,12 @@ private: QList allSets; + /** + * The raw JSON text of the source document, retained for lazy per-set + * parsing during startImport(). Frees the card data as each set is imported. + */ + QByteArray rawSetsData; + CardInfoPtr addCard(QString name, const QString &text, bool isToken, @@ -153,7 +167,11 @@ signals: public: explicit OracleImporter(QObject *parent = nullptr); - bool readSetsFromByteArray(const QByteArray &data); + /** + * Scans the given JSON document for set metadata. Takes the data by value so + * the wizard can hand over its decompressed buffer without copying it. + */ + bool readSetsFromByteArray(QByteArray data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); @@ -169,6 +187,10 @@ public: { return allSets; } + const QByteArray &getRawSetsData() const + { + return rawSetsData; + } void releaseSetData(); void clear(); }; diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp new file mode 100644 index 000000000..01ea3fab5 --- /dev/null +++ b/oracle/src/raw_json_scanner.cpp @@ -0,0 +1,621 @@ +#include "raw_json_scanner.h" + +#include + +namespace +{ + +// Nesting cap matching QJsonDocument's limit, so a pathologically deep document +// fails shallowly instead of overflowing the stack through the recursive +// skipValue/skipArray/skipObject walk (Qt's parser caps at 1024 for the same +// reason and reports DeepNesting). +constexpr int kMaxNestingDepth = 1024; + +inline bool isWhitespace(char c) +{ + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +const char *skipWhitespace(const char *p, const char *end) +{ + while (p < end && isWhitespace(*p)) { + ++p; + } + return p; +} + +inline bool isHexDigit(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +inline quint8 hexValue(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; +} + +/** + * @brief Skips past a JSON string without decoding it, validating escapes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool skipString(const char *&p, const char *end) +{ + ++p; // opening quote + for (;;) { + const void *quote = memchr(p, '"', static_cast(end - p)); + if (!quote) { + return false; // unterminated string + } + // Backslash escapes can only appear before the closing quote, so bound + // the scan to the string extent instead of the rest of the document. + const void *backslash = memchr(p, '\\', static_cast(static_cast(quote) - p)); + if (!backslash) { + p = static_cast(quote) + 1; + return true; + } + const char *b = static_cast(backslash); + if (end - b < 2) { + return false; + } + const char escaped = b[1]; + if (escaped == 'u') { + if (end - b < 6) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(b[2 + i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(b[2 + i]); + } + p = b + 6; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect the low-surrogate escape for the second half + if (end - p < 6 || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } + continue; + } + switch (escaped) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + p = b + 2; + continue; + default: + return false; // invalid escape + } + } +} + +/** + * @brief Decodes a JSON string into @p out, validating it as it goes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool decodeString(const char *&p, const char *end, QString &out) +{ + out.clear(); + QByteArray utf8; + auto flush = [&out, &utf8]() { + if (!utf8.isEmpty()) { + out += QString::fromUtf8(utf8); + utf8.clear(); + } + }; + + ++p; // opening quote + while (p < end) { + const char c = *p; + if (c == '\\') { + flush(); + ++p; // escaped character + if (p >= end) { + return false; + } + const char escaped = *p; + if (escaped == 'u') { + ++p; // first hex digit + if (p + 4 > end) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(p[i]); + } + p += 4; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect a low-surrogate escape for the second half + if (p + 6 > end || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + out += QChar(codepoint); + out += QChar(low); + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } else { + out += QChar(codepoint); + } + continue; + } + switch (escaped) { + case '"': + out += '"'; + break; + case '\\': + out += '\\'; + break; + case '/': + out += '/'; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + default: + return false; + } + ++p; + continue; + } + if (c == '"') { + ++p; + flush(); + return true; + } + if (static_cast(c) < 0x20) { + return false; // unescaped control character is invalid JSON + } + utf8 += c; + ++p; + } + return false; +} + +/** + * @brief Reads a set-metadata field, tolerating null and non-string values. + * + * A set's metadata may carry null or non-string values in otherwise-valid + * payloads ("releaseDate": null, "type": 7). The token itself was already + * structurally validated by skipValue, so a non-string value is accepted and + * leaves @p out at its default (empty) — one bad set must not abort the + * import of every other set in the document. + */ +bool decodeStringMember(const char *&fs, const char *&fe, QString &out) +{ + if (fs >= fe) { + return false; + } + if (*fs != '"') { + return true; + } + return decodeString(fs, fe, out); +} + +bool matchLiteral(const char *&p, const char *end, const char *literal, int length) +{ + if (end - p < length || memcmp(p, literal, static_cast(length)) != 0) { + return false; + } + const char *after = p + length; + if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) { + return false; + } + p = after; + return true; +} + +bool skipNumber(const char *&p, const char *end) +{ + // JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? + if (p < end && *p == '-') { + ++p; + } + if (p < end && *p == '0') { + ++p; + } else if (p < end && *p >= '1' && *p <= '9') { + ++p; + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } else { + return false; + } + if (p < end && *p == '.') { + ++p; + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + if (p < end && (*p == 'e' || *p == 'E')) { + ++p; + if (p < end && (*p == '+' || *p == '-')) { + ++p; + } + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + return true; +} + +bool skipValue(const char *&p, const char *end, int depth); +bool skipObject(const char *&p, const char *end, int depth); +bool skipArray(const char *&p, const char *end, int depth); + +bool skipPrimitive(const char *&p, const char *end) +{ + if (p >= end) { + return false; + } + const char c = *p; + if (c == '"') { + return skipString(p, end); + } + if (c == 't') { + return matchLiteral(p, end, "true", 4); + } + if (c == 'f') { + return matchLiteral(p, end, "false", 5); + } + if (c == 'n') { + return matchLiteral(p, end, "null", 4); + } + if (c == '-' || (c >= '0' && c <= '9')) { + return skipNumber(p, end); + } + return false; +} + +bool skipObject(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + if (!skipString(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +bool skipArray(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + if (p < end && *p == ']') { + ++p; + return true; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + ++p; + return true; + } + return false; + } +} + +bool skipValue(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + const char c = *p; + if (c == '{') { + return skipObject(p, end, depth - 1); + } + if (c == '[') { + return skipArray(p, end, depth - 1); + } + return skipPrimitive(p, end); +} + +/** + * @brief Iterates the members of the object starting at @p p. + * + * For each member invokes @p memberCallback with the key and the byte range of + * its value. Advancing @p p is unaffected by the callback. + */ +template bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + QString key; + if (!decodeString(p, end, key)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + const char *valueStart = skipWhitespace(p, end); + const char *valueEnd = valueStart; + if (!skipValue(valueEnd, end, depth - 1)) { + return false; + } + if (!memberCallback(key, valueStart, valueEnd)) { + return false; + } + p = valueEnd; + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +// Counts the direct elements of an array value; returns -1 if the array is malformed. +int countArrayElements(const char *p, const char *end, int depth) +{ + if (depth <= 0) { + return -1; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + int count = 0; + if (p < end && *p == ']') { + return 0; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return -1; + } + ++count; + p = skipWhitespace(p, end); + if (p >= end) { + return -1; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + return count; + } + return -1; + } +} + +} // namespace + +namespace RawJson +{ + +QList scanSetRanges(const QByteArray &json, ScanError *error) +{ + QList ranges; + if (error) { + *error = ScanError{}; + } + + const auto fail = [&](const QString &message) -> QList { + if (error) { + error->message = message; + } + return {}; + }; + + const char *begin = json.constData(); + const char *end = begin + json.size(); + if (begin >= end) { + return fail(QStringLiteral("empty JSON document")); + } + + const char *p = skipWhitespace(begin, end); + if (p >= end || *p != '{') { + return fail(QStringLiteral("top-level JSON must be an object")); + } + + bool foundData = false; + bool malformedSetData = false; + + const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) { + if (key == QStringLiteral("data")) { + foundData = true; + if (valueStart >= valueEnd || *valueStart != '{') { + malformedSetData = true; + return false; + } + const char *setP = valueStart; + const bool ok = forEachObjectMember(setP, valueEnd, kMaxNestingDepth - 1, + [&](const QString &setCode, const char *setStart, const char *setEnd) { + if (setStart >= setEnd || *setStart != '{') { + malformedSetData = true; + return false; + } + SetRange range; + range.dataRange.start = setStart - begin; + range.dataRange.length = setEnd - setStart; + range.code = setCode; + + const char *memberP = setStart; + const bool metaOk = forEachObjectMember( + memberP, setEnd, kMaxNestingDepth - 2, + [&](const QString &field, const char *fs, const char *fe) { + if (field == QStringLiteral("code")) { + return decodeStringMember(fs, fe, range.code); + } + if (field == QStringLiteral("name")) { + return decodeStringMember(fs, fe, range.name); + } + if (field == QStringLiteral("type")) { + return decodeStringMember(fs, fe, range.type); + } + if (field == QStringLiteral("releaseDate")) { + return decodeStringMember(fs, fe, range.releaseDate); + } + if (field == QStringLiteral("cards")) { + if (fs >= fe) { + return false; + } + if (*fs != '[') { + // e.g. "cards": null — treat as an empty array, + // matching Qt's tolerance. + return true; + } + range.dataRange.cardCount = + countArrayElements(fs, fe, kMaxNestingDepth - 2); + return range.dataRange.cardCount >= 0; + } + return true; + }); + if (!metaOk) { + malformedSetData = true; + return false; + } + ranges.append(range); + return true; + }); + if (!ok) { + malformedSetData = true; + return false; + } + } + return true; + }; + + if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback)) { + return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")); + } + p = skipWhitespace(p, end); + if (p != end) { + return fail(QStringLiteral("trailing content after top-level JSON object")); + } + if (!foundData) { + return fail(QStringLiteral("missing \"data\" object")); + } + if (ranges.isEmpty()) { + return fail(QStringLiteral("no sets found in \"data\"")); + } + return ranges; +} + +} // namespace RawJson \ No newline at end of file diff --git a/oracle/src/raw_json_scanner.h b/oracle/src/raw_json_scanner.h new file mode 100644 index 000000000..f6e3a4647 --- /dev/null +++ b/oracle/src/raw_json_scanner.h @@ -0,0 +1,76 @@ +#ifndef RAW_JSON_SCANNER_H +#define RAW_JSON_SCANNER_H + +#include +#include +#include + +namespace RawJson +{ + +/** + * @brief The byte extent of a set's object inside the scanned document, plus + * the size of its cards array. This is the slice SetToDownload needs for lazy + * per-set parsing; the metadata strings live in SetRange alongside it. + */ +struct SetDataRange +{ + /** @brief Byte offset of the set's object within the scanned buffer. */ + qsizetype start = -1; + /** @brief Byte length of the set's object, including the surrounding braces. */ + qsizetype length = 0; + /** @brief Number of entries in the set's "cards" array. */ + int cardCount = 0; +}; + +struct SetRange +{ + /** @brief The byte slice of this set within the document. */ + SetDataRange dataRange; + QString code; + QString name; + QString type; + QString releaseDate; +}; + +struct ScanError +{ + bool isError() const + { + return !message.isEmpty(); + } + QString message; +}; + +/** + * @brief Scans a full MTGJSON document without materializing the JSON tree. + * + * Splits the top-level "data" object into per-set byte ranges and reads each + * set's metadata directly from the raw bytes. The oracle importer can then + * parse one set at a time during import, keeping peak memory far below a single + * QJsonDocument::fromJson() over the whole file. + * + * The whole document is structurally validated while scanning (strings, + * escapes, braces, and a trailing-content check) and nesting depth is capped at + * 1024 to match QJsonDocument, so pathologically deep documents fail shallowly + * instead of exhausting the stack. Verdicts agree with QJsonDocument::fromJson + * on structurally malformed input; unlike Qt, string metadata fields + * ("name", "type", "releaseDate", "code") tolerate null / non-string values by + * defaulting to empty rather than rejecting the whole document, so one broken + * set cannot abort the import of the rest. + * + * Following QJsonDocument::fromJson's convention, the parsed ranges are + * returned by value and any failure is reported through the @p error out + * parameter. + * + * @param json The raw MTGJSON document bytes. + * @param error Out parameter. Set to an error ScanError when the document + * cannot be parsed, otherwise left empty. Passing a null + * pointer disables error reporting. + * @return The detected per-set ranges, or an empty list on failure. + */ +QList scanSetRanges(const QByteArray &json, ScanError *error = nullptr); + +} // namespace RawJson + +#endif // RAW_JSON_SCANNER_H \ No newline at end of file diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index cbff4f19c..9bc5ee5be 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -11,7 +11,7 @@ add_test(NAME parse_cipt_test COMMAND parse_cipt_test) # Oracle importer unit tests add_executable( oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_test.cpp + ../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp ) if(NOT GTEST_FOUND) @@ -51,7 +51,7 @@ endif() add_executable( oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} + ../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} ) if(NOT GTEST_FOUND) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index 567519a48..633c9e40a 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -157,9 +157,10 @@ TEST(OracleBenchmark, ParseJsonThroughput) for (int i = 0; i < iterations; ++i) { OracleImporter importer; + QByteArray source = data; QElapsedTimer timer; timer.start(); - bool ok = importer.readSetsFromByteArray(data); + bool ok = importer.readSetsFromByteArray(std::move(source)); ASSERT_TRUE(ok); totalMs += timer.elapsed(); } @@ -448,7 +449,7 @@ TEST(OracleBenchmark, ImportRamUsage) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(data)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); @@ -540,7 +541,7 @@ TEST(OracleBenchmark, ImportRamUsageAllPrintings) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(setsData)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index 145a2ca0f..c42840f7e 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -545,6 +545,173 @@ TEST_F(OracleImporterTest, ApostropheNormalized) ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); } +// ============================================================================ +// RawJson scanner tests +// ============================================================================ + +TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse) +{ + QJsonObject root; + QJsonObject data; + data["AAA"] = makeCard("Alpha Card"); + data["BBB"] = makeCard("Beta Card"); + root["data"] = data; + + const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact); + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(bytes, &error); + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 2); + + const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject(); + for (const RawJson::SetRange &range : ranges) { + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(bytes.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError) + << range.code.toStdString() << ": " << parseError.errorString().toStdString(); + ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards) +{ + const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\"," + "\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\"," + "\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}"; + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(json, &error); + ASSERT_FALSE(error.isError()); + ASSERT_EQ(ranges.size(), 1); + + const RawJson::SetRange &range = ranges.first(); + ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key + const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00); + ASSERT_EQ(range.name, expectedName); + ASSERT_EQ(range.type, "expansion"); + ASSERT_EQ(range.releaseDate, "2024-01-05"); + ASSERT_EQ(range.dataRange.cardCount, 3); + + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(json.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError); + ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName); + ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3); +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson) +{ + const QList invalid = {"not json", + "[]", + "{\"data\":[]}", + "{\"data\":{}}", + "{\"other\":{}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\"," + "\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"}; + + for (const QByteArray &json : invalid) { + RawJson::ScanError error; + RawJson::scanSetRanges(json, &error); + EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesMatchesFullJsonParseVerdicts) +{ + // Verdicts must agree with QJsonDocument::fromJson for the inputs below — + // including the metadata quirks ("name": null, "type": 7, "releaseDate": null, + // "cards": null) that used to make the scanner reject sets Qt accepts. + const QList inputs = { + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[{" + "\"n\":1}]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":null,\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":null,\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":7,\"releaseDate\":\"2024-01-01\",\"cards\":null}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"releaseDate\":\"2024-01-01\",\"cards\":[1,2,3]}}}", + // structurally invalid JSON (both parsers must reject) + "not json", + "{\"data\":{\"A\":{\"name\":\"unterminated}}", + }; + + for (const QByteArray &input : inputs) { + QJsonParseError qtError; + QJsonDocument::fromJson(input, &qtError); + const bool qtOk = qtError.error == QJsonParseError::NoError; + + RawJson::ScanError scanError; + const QList ranges = RawJson::scanSetRanges(input, &scanError); + EXPECT_EQ(qtOk, !scanError.isError()) << "verdict mismatch for: " << input.constData(); + if (scanError.isError()) { + continue; + } + for (const RawJson::SetRange &range : ranges) { + QJsonParseError sliceError; + QJsonDocument::fromJson(QByteArray(input.constData() + range.dataRange.start, range.dataRange.length), + &sliceError); + EXPECT_EQ(sliceError.error, QJsonParseError::NoError) << "bad range slice for: " << input.constData(); + } + } +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsDeepNesting) +{ + // Far beyond the shared 1024 container cap: Qt reports DeepNesting and the + // scanner must reject too, without overflowing the stack through its + // recursive skipValue walk. + QString nesting; + nesting.reserve(10000); + for (int i = 0; i < 5000; ++i) { + nesting += '['; + } + for (int i = 0; i < 5000; ++i) { + nesting += ']'; + } + const QByteArray json = ("{\"data\":{\"A\":{\"code\":\"a\",\"cards\":" + nesting + "}}}").toUtf8(); + + QJsonParseError qtError; + QJsonDocument::fromJson(json, &qtError); + ASSERT_NE(qtError.error, QJsonParseError::NoError) << "expected Qt to reject deep nesting"; + + RawJson::ScanError scanError; + RawJson::scanSetRanges(json, &scanError); + ASSERT_TRUE(scanError.isError()) << "scanner accepted a document Qt rejects as too deeply nested"; +} + +// ============================================================================ +// Lazy per-set parsing tests +// ============================================================================ + +TEST_F(OracleImporterTest, StartImportParsesSetsLazily) +{ + QJsonObject setObj = makeCard("Lazy Import Card"); + QJsonArray cards; + cards.append(setObj); + QJsonObject dataSet; + dataSet["code"] = "tst"; + dataSet["name"] = "Test Set"; + dataSet["type"] = "expansion"; + dataSet["releaseDate"] = "2024-01-01"; + dataSet["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", dataSet}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_FALSE(importer->getRawSetsData().isEmpty()); + + const int importedSets = importer->startImport(); + ASSERT_EQ(importedSets, 1); + ASSERT_EQ(importer->getCardList().size(), 1); + ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull()); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); From f7263e128713fcc2906f6f894296a21561014ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 2 Sep 2026 15:04:38 +0200 Subject: [PATCH 15/25] [Oracle] Fix nesting-depth cap, tolerate unescaped control chars, lazy-parse review fixes --- oracle/src/oracleimporter.cpp | 1 - oracle/src/raw_json_scanner.cpp | 16 +++++++-------- tests/oracle/oracle_importer_test.cpp | 29 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 6a5633215..d745b250c 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -551,7 +551,6 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); -<<<<<<< HEAD // Pre-allocate the cards hash to avoid rehashing during import. Keys are // distinct card names while raw ranges only count printings (AllPrintings // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp index 01ea3fab5..db972491c 100644 --- a/oracle/src/raw_json_scanner.cpp +++ b/oracle/src/raw_json_scanner.cpp @@ -213,9 +213,10 @@ bool decodeString(const char *&p, const char *end, QString &out) flush(); return true; } - if (static_cast(c) < 0x20) { - return false; // unescaped control character is invalid JSON - } + // Deliberately accept unescaped control characters (e.g. a tab inside + // a set name): QJsonDocument and skipString accept them too, so + // rejecting them here would fail the whole document on a byte that + // Qt is fine with — the very total-failure mode this scanner avoids. utf8 += c; ++p; } @@ -399,20 +400,19 @@ bool skipArray(const char *&p, const char *end, int depth) bool skipValue(const char *&p, const char *end, int depth) { - if (depth <= 0) { - return false; // nest deeper than the cap - } p = skipWhitespace(p, end); if (p >= end) { return false; } const char c = *p; if (c == '{') { - return skipObject(p, end, depth - 1); + // pass depth through: skipObject consumes the single decrement for this level + return skipObject(p, end, depth); } if (c == '[') { - return skipArray(p, end, depth - 1); + return skipArray(p, end, depth); } + // a primitive is a leaf, so it never wastes a nesting level return skipPrimitive(p, end); } diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index c42840f7e..3834be9ed 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -634,6 +634,9 @@ TEST_F(OracleImporterTest, ScanSetRangesMatchesFullJsonParseVerdicts) "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":null,\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":7,\"releaseDate\":\"2024-01-01\",\"cards\":null}}}", "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"releaseDate\":\"2024-01-01\",\"cards\":[1,2,3]}}}", + // unescaped control character inside a string: QJsonDocument and + // skipString both accept it, so the scanner must not reject the whole doc + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"N\tX\",\"releaseDate\":\"2024-01-01\",\"cards\":[{\"n\":1}]}}}", // structurally invalid JSON (both parsers must reject) "not json", "{\"data\":{\"A\":{\"name\":\"unterminated}}", @@ -683,6 +686,32 @@ TEST_F(OracleImporterTest, ScanSetRangesRejectsDeepNesting) ASSERT_TRUE(scanError.isError()) << "scanner accepted a document Qt rejects as too deeply nested"; } +TEST_F(OracleImporterTest, ScanSetRangesAcceptsQtMaxNesting) +{ + // Pins the boundary rather than only the far-past case: a depth Qt still + // accepts must be accepted by the scanner too. Before the fix the scanner's + // cap was roughly half of Qt's (each level cost two decrements), so a + // depth of 1000 here was rejected even though QJsonDocument parses it. + constexpr int depth = 1000; + QString nesting; + nesting.reserve(2 * depth); + for (int i = 0; i < depth; ++i) { + nesting += '['; + } + for (int i = 0; i < depth; ++i) { + nesting += ']'; + } + const QByteArray json = ("{\"data\":{\"A\":{\"code\":\"a\",\"cards\":" + nesting + "}}}").toUtf8(); + + QJsonParseError qtError; + QJsonDocument::fromJson(json, &qtError); + ASSERT_EQ(qtError.error, QJsonParseError::NoError) << "expected Qt to accept depth " << depth; + + RawJson::ScanError scanError; + RawJson::scanSetRanges(json, &scanError); + ASSERT_FALSE(scanError.isError()) << "scanner rejected a document Qt accepts at depth " << depth; +} + // ============================================================================ // Lazy per-set parsing tests // ============================================================================ From 35ebae8d7fa09de5157a522f6d6c96fa341f91cb Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 05:20:52 +0200 Subject: [PATCH 16/25] [Build] Bump cmake_minimum_required from 3.10 to 3.16 (#7232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3.16 is already required by Qt6 (and enforced at find_package time). This unlocks native target_precompile_headers(), better AUTOMOC/AUTORCC handling, and qt6_finalize_project() without a version guard. Co-authored-by: Lukas Brübach --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bac46c2bc..3db871e2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,8 +5,8 @@ # This file sets all the variables shared between the projects # like the installation path, compilation flags etc.. -# cmake 3.16 is required if using qt6 -cmake_minimum_required(VERSION 3.10) +# 3.16 required for Qt6 and target_precompile_headers() +cmake_minimum_required(VERSION 3.16) # Use compiler cache (ccache) option(USE_CCACHE "Cache the build results with ccache" OFF) From fcfb14cf569c1025e942e12133f977ca655af8bb Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 05:20:53 +0200 Subject: [PATCH 17/25] [Build] Use pipes for GCC/Clang compilation (#7233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass -pipe so GCC/Clang transfer intermediate representation between compiler stages over pipes instead of temporary files, reducing build I/O. Co-authored-by: Lukas Brübach --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3db871e2c..5ef0f5573 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,9 @@ elseif(CMAKE_COMPILER_IS_GNUCXX) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${FLAG}") endif() endforeach() + + # Reduce compiler I/O by using pipes between stages instead of temp files + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe") else() # other: osx/llvm, bsd/llvm set(CMAKE_CXX_FLAGS_RELEASE "-O2") @@ -192,6 +195,9 @@ else() else() set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra") endif() + + # Reduce compiler I/O by using pipes between stages instead of temp files + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pipe") endif() # GNU systems need to define the Mersenne exponent for the RNG to compile w/o warning From 3ec62df3e7c1dc5eefe5a004926689d85a4fa918 Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 05:20:53 +0200 Subject: [PATCH 18/25] [Protocol] Remove duplicate event_game_state_changed.proto entry (#7234) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .proto file appeared twice in the PROTO_FILES list, causing protoc to process it twice on every build. Keep a single entry. Co-authored-by: Lukas Brübach --- libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt b/libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt index 3a193ae3c..f22828f46 100644 --- a/libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt +++ b/libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt @@ -90,7 +90,6 @@ set(PROTO_FILES event_game_log_notice.proto event_game_say.proto event_game_state_changed.proto - event_game_state_changed.proto event_join.proto event_join_room.proto event_kicked.proto From d6fbfb32a1e71ad07433405595a9cc0ab4c7928f Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:49:18 +0200 Subject: [PATCH 19/25] [Security] Use a CSPRNG for salts, tokens, and RNG seeding (#7192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Security] Use a CSPRNG for salts, tokens, and RNG seeding Password salts and activation tokens were generated with the global SFMT RNG, which was seeded from a 32-bit timestamp, making registration salts and activation tokens predictable. The game RNG used the same timestamp seed across restarts. Add CryptoUtil backed by OpenSSL RAND_bytes and use it for salt/token generation and to seed RNG_SFMT with a 64-bit CSPRNG value in both the client and server. Link libcockatrice_utility against OpenSSL::Crypto. Took 30 seconds Took 25 minutes * Lint. Took 4 minutes Took 36 seconds --------- Co-authored-by: Lukas Brübach --- .ci/Arch/Dockerfile | 1 + .ci/Debian12/Dockerfile | 1 + .ci/Debian13/Dockerfile | 1 + .ci/Fedora43/Dockerfile | 1 + .ci/Fedora44/Dockerfile | 1 + .ci/Servatrice_Debian12/Dockerfile | 1 + .ci/Ubuntu24.04/Dockerfile | 1 + .ci/Ubuntu26.04/Dockerfile | 1 + CMakeLists.txt | 5 --- Dockerfile | 2 + cockatrice/src/main.cpp | 3 +- .../libcockatrice/rng/rng_sfmt.cpp | 8 ++-- .../libcockatrice/rng/rng_sfmt.h | 2 +- libcockatrice_utility/CMakeLists.txt | 10 +++-- .../libcockatrice/utility/cryptoutil.cpp | 25 +++++++++++ .../libcockatrice/utility/cryptoutil.h | 13 ++++++ .../libcockatrice/utility/passwordhasher.cpp | 26 +++++++++--- servatrice/src/main.cpp | 3 +- tests/password_hash_test.cpp | 41 +++++++++++-------- 19 files changed, 109 insertions(+), 37 deletions(-) create mode 100644 libcockatrice_utility/libcockatrice/utility/cryptoutil.cpp create mode 100644 libcockatrice_utility/libcockatrice/utility/cryptoutil.h diff --git a/.ci/Arch/Dockerfile b/.ci/Arch/Dockerfile index f37315262..b08e568f3 100644 --- a/.ci/Arch/Dockerfile +++ b/.ci/Arch/Dockerfile @@ -8,6 +8,7 @@ RUN pacman --sync --refresh --sysupgrade --needed --noconfirm \ gtest \ mariadb-libs \ ninja \ + openssl \ protobuf \ qt6-base \ qt6-declarative \ diff --git a/.ci/Debian12/Dockerfile b/.ci/Debian12/Dockerfile index 0fa227d6f..e3df94ab5 100644 --- a/.ci/Debian12/Dockerfile +++ b/.ci/Debian12/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && \ libprotobuf-dev \ libqt6multimedia6 \ libqt6sql6-mysql \ + libssl-dev \ ninja-build \ protobuf-compiler \ qt6-image-formats-plugins \ diff --git a/.ci/Debian13/Dockerfile b/.ci/Debian13/Dockerfile index 13e8b35c7..60e490c98 100644 --- a/.ci/Debian13/Dockerfile +++ b/.ci/Debian13/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && \ libprotobuf-dev \ libqt6multimedia6 \ libqt6sql6-mysql \ + libssl-dev \ ninja-build \ protobuf-compiler \ qt6-image-formats-plugins \ diff --git a/.ci/Fedora43/Dockerfile b/.ci/Fedora43/Dockerfile index 68e894543..4005bbf67 100644 --- a/.ci/Fedora43/Dockerfile +++ b/.ci/Fedora43/Dockerfile @@ -7,6 +7,7 @@ RUN dnf install -y \ git \ mariadb-devel \ ninja-build \ + openssl-devel \ protobuf-devel \ qt6-{qtdeclarative,qtshadertools,qttools,qtsvg,qtmultimedia,qtwebsockets}-devel \ qt6-qtimageformats \ diff --git a/.ci/Fedora44/Dockerfile b/.ci/Fedora44/Dockerfile index ffd7c1b9b..e0224cdc6 100644 --- a/.ci/Fedora44/Dockerfile +++ b/.ci/Fedora44/Dockerfile @@ -7,6 +7,7 @@ RUN dnf install -y \ git \ mariadb-devel \ ninja-build \ + openssl-devel \ protobuf-devel \ qt6-{qtdeclarative,qtshadertools,qttools,qtsvg,qtmultimedia,qtwebsockets}-devel \ qt6-qtimageformats \ diff --git a/.ci/Servatrice_Debian12/Dockerfile b/.ci/Servatrice_Debian12/Dockerfile index 21f6a036e..321aa7c0f 100644 --- a/.ci/Servatrice_Debian12/Dockerfile +++ b/.ci/Servatrice_Debian12/Dockerfile @@ -12,6 +12,7 @@ RUN apt-get update && \ libmariadb-dev-compat \ libprotobuf-dev \ libqt6sql6-mysql \ + libssl-dev \ ninja-build \ protobuf-compiler \ qt6-tools-dev \ diff --git a/.ci/Ubuntu24.04/Dockerfile b/.ci/Ubuntu24.04/Dockerfile index 12320c276..10adc5e64 100644 --- a/.ci/Ubuntu24.04/Dockerfile +++ b/.ci/Ubuntu24.04/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && \ libprotobuf-dev \ libqt6multimedia6 \ libqt6sql6-mysql \ + libssl-dev \ ninja-build \ protobuf-compiler \ qt6-image-formats-plugins \ diff --git a/.ci/Ubuntu26.04/Dockerfile b/.ci/Ubuntu26.04/Dockerfile index ce3d9cd6c..1b6cf825f 100644 --- a/.ci/Ubuntu26.04/Dockerfile +++ b/.ci/Ubuntu26.04/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && \ libprotobuf-dev \ libqt6multimedia6 \ libqt6sql6-mysql \ + libssl-dev \ ninja-build \ protobuf-compiler \ qt6-image-formats-plugins \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ef0f5573..35eb8111b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,11 +245,6 @@ if(WIN32) find_package(OpenSSL REQUIRED) if(OPENSSL_FOUND) include_directories(${OPENSSL_INCLUDE_DIRS}) - else() - message( - WARNING - "Could not find OpenSSL runtime libraries. They are not required for compiling, but needs to be available at runtime." - ) endif() endif() diff --git a/Dockerfile b/Dockerfile index 382309d47..7d3deb5fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update \ libmariadb-dev-compat \ libprotobuf-dev \ libqt6sql6-mysql \ + libssl-dev \ qt6-websockets-dev \ protobuf-compiler \ qt6-tools-dev \ @@ -42,6 +43,7 @@ RUN apt-get update \ libprotobuf32t64 \ libqt6sql6-mysql \ libqt6websockets6 \ + libssl3 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/cockatrice/src/main.cpp b/cockatrice/src/main.cpp index 84d5d175f..d8aa1cd08 100644 --- a/cockatrice/src/main.cpp +++ b/cockatrice/src/main.cpp @@ -53,6 +53,7 @@ #include #include #include +#include QTranslator *translator, *qtTranslator; RNG_Abstract *rng; @@ -292,7 +293,7 @@ int main(int argc, char *argv[]) } } - rng = new RNG_SFMT; + rng = new RNG_SFMT(CryptoUtil::randomUInt64()); themeManager = new ThemeManager; soundEngine = new SoundEngine; diff --git a/libcockatrice_rng/libcockatrice/rng/rng_sfmt.cpp b/libcockatrice_rng/libcockatrice/rng/rng_sfmt.cpp index 5b38deb3f..4c578b4e4 100644 --- a/libcockatrice_rng/libcockatrice/rng/rng_sfmt.cpp +++ b/libcockatrice_rng/libcockatrice/rng/rng_sfmt.cpp @@ -1,6 +1,5 @@ #include "rng_sfmt.h" -#include #include #include #include @@ -11,10 +10,11 @@ #define UINT64_MAX (~(uint64_t)0) #endif -RNG_SFMT::RNG_SFMT(QObject *parent) : RNG_Abstract(parent) +RNG_SFMT::RNG_SFMT(uint64_t seed, QObject *parent) : RNG_Abstract(parent) { - // initialize the random number generator with a 32bit integer seed (timestamp) - sfmt_init_gen_rand(&sfmt, QDateTime::currentDateTime().toSecsSinceEpoch()); + // initialize the random number generator with a 64bit seed, e.g. from a CSPRNG + uint32_t seedArray[2] = {static_cast(seed), static_cast(seed >> 32)}; + sfmt_init_by_array(&sfmt, seedArray, 2); } /** diff --git a/libcockatrice_rng/libcockatrice/rng/rng_sfmt.h b/libcockatrice_rng/libcockatrice/rng/rng_sfmt.h index 7e9f53df3..a180dad99 100644 --- a/libcockatrice_rng/libcockatrice/rng/rng_sfmt.h +++ b/libcockatrice_rng/libcockatrice/rng/rng_sfmt.h @@ -36,7 +36,7 @@ private: unsigned int cdf(unsigned int min, unsigned int max); public: - explicit RNG_SFMT(QObject *parent = nullptr); + explicit RNG_SFMT(uint64_t seed, QObject *parent = nullptr); unsigned int rand(int min, int max) override; }; diff --git a/libcockatrice_utility/CMakeLists.txt b/libcockatrice_utility/CMakeLists.txt index c6411ea76..db23f7951 100644 --- a/libcockatrice_utility/CMakeLists.txt +++ b/libcockatrice_utility/CMakeLists.txt @@ -6,13 +6,15 @@ set(CMAKE_AUTOUIC ON) set(CMAKE_AUTORCC ON) set(UTILITY_SOURCES - libcockatrice/utility/expression.cpp libcockatrice/utility/levenshtein.cpp libcockatrice/utility/passwordhasher.cpp - libcockatrice/utility/server_rate_limiter.cpp libcockatrice/utility/warning_categories.cpp + libcockatrice/utility/cryptoutil.cpp libcockatrice/utility/expression.cpp libcockatrice/utility/levenshtein.cpp + libcockatrice/utility/passwordhasher.cpp libcockatrice/utility/server_rate_limiter.cpp + libcockatrice/utility/warning_categories.cpp ) set(UTILITY_HEADERS libcockatrice/utility/card_ref.h libcockatrice/utility/color.h + libcockatrice/utility/cryptoutil.h libcockatrice/utility/expression.h libcockatrice/utility/levenshtein.h libcockatrice/utility/macros.h @@ -32,7 +34,9 @@ add_library(libcockatrice_utility STATIC ${UTILITY_SOURCES} ${UTILITY_HEADERS}) target_include_directories(libcockatrice_utility PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(libcockatrice_utility PUBLIC libcockatrice_rng ${QT_CORE_MODULE}) +find_package(OpenSSL REQUIRED) + +target_link_libraries(libcockatrice_utility PUBLIC libcockatrice_rng OpenSSL::Crypto ${QT_CORE_MODULE}) set(ORACLE_LIBS) diff --git a/libcockatrice_utility/libcockatrice/utility/cryptoutil.cpp b/libcockatrice_utility/libcockatrice/utility/cryptoutil.cpp new file mode 100644 index 000000000..416ef261b --- /dev/null +++ b/libcockatrice_utility/libcockatrice/utility/cryptoutil.cpp @@ -0,0 +1,25 @@ +#include "cryptoutil.h" + +#include + +namespace CryptoUtil +{ +QByteArray randomBytes(int count) +{ + QByteArray bytes(count, '\0'); + if (RAND_bytes(reinterpret_cast(bytes.data()), count) != 1) { + // Randomness failure is fatal: never fall back to a predictable source. + qFatal("CryptoUtil::randomBytes: RAND_bytes failed"); + } + return bytes; +} + +quint64 randomUInt64() +{ + quint64 value; + if (RAND_bytes(reinterpret_cast(&value), sizeof(value)) != 1) { + qFatal("CryptoUtil::randomUInt64: RAND_bytes failed"); + } + return value; +} +} // namespace CryptoUtil diff --git a/libcockatrice_utility/libcockatrice/utility/cryptoutil.h b/libcockatrice_utility/libcockatrice/utility/cryptoutil.h new file mode 100644 index 000000000..dba9dc37d --- /dev/null +++ b/libcockatrice_utility/libcockatrice/utility/cryptoutil.h @@ -0,0 +1,13 @@ +#ifndef CRYPTOUTIL_H +#define CRYPTOUTIL_H + +#include +#include + +namespace CryptoUtil +{ +QByteArray randomBytes(int count); +quint64 randomUInt64(); +} // namespace CryptoUtil + +#endif diff --git a/libcockatrice_utility/libcockatrice/utility/passwordhasher.cpp b/libcockatrice_utility/libcockatrice/utility/passwordhasher.cpp index c40c5f94f..1c22fdcfa 100644 --- a/libcockatrice_utility/libcockatrice/utility/passwordhasher.cpp +++ b/libcockatrice_utility/libcockatrice/utility/passwordhasher.cpp @@ -1,7 +1,7 @@ #include "passwordhasher.h" #include -#include +#include QString PasswordHasher::computeHash(const QString &password, const QString &salt) { @@ -21,12 +21,28 @@ QString PasswordHasher::generateRandomSalt(const int len) static const char alphanum[] = "0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; + const int size = sizeof(alphanum) - 1; + + // Two bytes per character, corrected for modulo bias via rejection sampling. + const int bucketSize = 65536 / size; + const int limit = bucketSize * size; QString ret; - int size = sizeof(alphanum) - 1; - + ret.reserve(len); + QByteArray random = CryptoUtil::randomBytes(len * 2); + int bytesUsed = 0; for (int i = 0; i < len; ++i) { - ret.append(alphanum[rng->rand(0, size)]); + unsigned int value; + do { + if (bytesUsed >= random.size()) { + random = CryptoUtil::randomBytes(len * 2); + bytesUsed = 0; + } + value = static_cast(static_cast(random.at(bytesUsed))) << 8 | + static_cast(static_cast(random.at(bytesUsed + 1))); + bytesUsed += 2; + } while (value >= limit); + ret.append(alphanum[value / bucketSize]); } return ret; @@ -34,5 +50,5 @@ QString PasswordHasher::generateRandomSalt(const int len) QString PasswordHasher::generateActivationToken() { - return QCryptographicHash::hash(generateRandomSalt().toUtf8(), QCryptographicHash::Md5).toBase64().left(16); + return QString(CryptoUtil::randomBytes(16).toBase64().left(16)); } diff --git a/servatrice/src/main.cpp b/servatrice/src/main.cpp index 9e7fe38d9..13bf95a82 100644 --- a/servatrice/src/main.cpp +++ b/servatrice/src/main.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include RNG_Abstract *rng; @@ -169,7 +170,7 @@ int main(int argc, char *argv[]) signalhandler = new SignalHandler(); - rng = new RNG_SFMT; + rng = new RNG_SFMT(CryptoUtil::randomUInt64()); std::cerr << "Servatrice " << VERSION_STRING << " starting." << std::endl; std::cerr << "-------------------------" << std::endl; diff --git a/tests/password_hash_test.cpp b/tests/password_hash_test.cpp index 38d9b6315..2b8f8bdb7 100644 --- a/tests/password_hash_test.cpp +++ b/tests/password_hash_test.cpp @@ -1,25 +1,9 @@ #include "gtest/gtest.h" -#include -#include +#include #include -RNG_Abstract *rng; - namespace { -class PasswordHashTest : public ::testing::Test -{ -protected: - void SetUp() override - { - rng = new RNG_SFMT; - } - - void TearDown() override - { - delete rng; - } -}; TEST(PasswordHashTest, RegressionTest) { @@ -29,6 +13,29 @@ TEST(PasswordHashTest, RegressionTest) QString hash = PasswordHasher::computeHash(password, salt); ASSERT_EQ(hash, salt + expected) << "The computed hash value remains the same"; } + +TEST(PasswordHashTest, SaltUsesAlphanumericCharset) +{ + static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + const QString salt = PasswordHasher::generateRandomSalt(); + ASSERT_EQ(salt.size(), 16); + for (const QChar &c : salt) { + ASSERT_NE(strchr(alphanum, c.toLatin1()), nullptr); + } +} + +TEST(PasswordHashTest, SaltsAreUnique) +{ + const QString salt1 = PasswordHasher::generateRandomSalt(); + const QString salt2 = PasswordHasher::generateRandomSalt(); + ASSERT_NE(salt1, salt2); +} + +TEST(PasswordHashTest, TokenHasExpectedLength) +{ + const QString token = PasswordHasher::generateActivationToken(); + ASSERT_EQ(token.size(), 16); +} } // namespace int main(int argc, char **argv) From 4d4ddd427869f0e8318b5c6c8ff325b8fe120723 Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:14:16 +0200 Subject: [PATCH 20/25] [Oracle] Replace vendored qt-json with native QJson for set import (#7214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Oracle] Replace vendored QtJson with native QJson for set import - Drop the vendored oracle/src/qt-json/json.{h,cpp} implementation - Switch SetToDownload and importCardsFromSet from QList to native QJsonArray/QJsonObject - Release set JSON data after import in the save sets page Took 20 minutes * [Oracle] Restore property coercion and legality merge in native JSON import --------- Co-authored-by: Lukas Brübach --- format.sh | 1 - oracle/CMakeLists.txt | 1 - oracle/src/oracleimporter.cpp | 150 +++++----- oracle/src/oracleimporter.h | 16 +- oracle/src/pages.cpp | 3 + oracle/src/qt-json/AUTHORS | 3 - oracle/src/qt-json/LICENSE | 27 -- oracle/src/qt-json/README | 96 ------ oracle/src/qt-json/json.cpp | 545 ---------------------------------- oracle/src/qt-json/json.h | 204 ------------- 10 files changed, 97 insertions(+), 949 deletions(-) delete mode 100644 oracle/src/qt-json/AUTHORS delete mode 100644 oracle/src/qt-json/LICENSE delete mode 100644 oracle/src/qt-json/README delete mode 100644 oracle/src/qt-json/json.cpp delete mode 100644 oracle/src/qt-json/json.h diff --git a/format.sh b/format.sh index 3fa435be1..ca3557ea7 100755 --- a/format.sh +++ b/format.sh @@ -22,7 +22,6 @@ libcockatrice_* \ exclude=("libcockatrice_rng/libcockatrice/rng/sfmt/" \ "libcockatrice_utility/libcockatrice/utility/peglib.h" \ "oracle/src/lzma/" \ -"oracle/src/qt-json/" \ "oracle/src/zip/" \ "servatrice/src/smtp/") exts=("cpp" "h" "proto") diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 0736db7f5..6a29b6935 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,7 +23,6 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp - src/qt-json/json.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index fdb32bb8d..752317512 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -3,9 +3,10 @@ #include "libcockatrice/interfaces/noop_card_preference_provider.h" #include "libcockatrice/interfaces/noop_card_set_priority_controller.h" #include "parsehelpers.h" -#include "qt-json/json.h" #include +#include +#include #include #include #include @@ -44,24 +45,24 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s bool OracleImporter::readSetsFromByteArray(const QByteArray &data) { - bool ok; - auto setsMap = QtJson::Json::parse(QString(data), ok).toMap().value("data").toMap(); - if (!ok) { - qDebug() << "error: QtJson::Json::parse()"; + QJsonParseError error; + auto doc = QJsonDocument::fromJson(data, &error); + if (error.error != QJsonParseError::NoError) { + qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); return false; } + auto setsObj = doc.object().value("data").toObject(); + QList newSetList; - QListIterator it(setsMap.values()); - - while (it.hasNext()) { - QVariantMap map = it.next().toMap(); - QString shortName = map.value("code").toString().toUpper(); - QString longName = map.value("name").toString(); - QList setCards = map.value("cards").toList(); - QString setType = map.value("type").toString(); - QDate releaseDate = map.value("releaseDate").toDate(); + for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { + QJsonObject setObj = it.value().toObject(); + QString shortName = setObj.value("code").toString().toUpper(); + QString longName = setObj.value("name").toString(); + QJsonArray setCards = setObj.value("cards").toArray(); + QString setType = setObj.value("type").toString(); + QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { @@ -142,9 +143,12 @@ CardInfoPtr OracleImporter::addCard(QString name, // Workaround for card name weirdness name = name.replace("Æ", "AE"); name = name.replace("’", "'"); - if (cards.contains(name)) { - CardInfoPtr card = cards.value(name); + auto existingIt = cards.constFind(name); + if (existingIt != cards.constEnd()) { + CardInfoPtr card = existingIt.value(); card->addToSet(printingInfo.getSet(), printingInfo); + // Only merge legalities when the card has none yet, so multi-format + // printings don't overwrite each other's legality lists. if (card->getProperties().filter(formatRegex).empty()) { card->combineLegalities(properties); } @@ -222,12 +226,15 @@ CardInfoPtr OracleImporter::addCard(QString name, return newCard; } -static QString getStringPropertyFromMap(const QVariantMap &card, const QString &propertyName) +static QString getJsonString(const QJsonObject &obj, const QString &key) { - return card.contains(propertyName) ? card.value(propertyName).toString() : QString(""); + // QVariant coerces numbers and booleans to text, while QJsonValue::toString() + // returns a null string for them — some MTGJSON fields (manaValue, + // convertedManaCost, isOnlineOnly, isRebalanced) carry those types. + return obj.value(key).toVariant().toString(); } -int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList &cardsList) +int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList) { // mtgjson name => xml name static const QMap cardProperties{ @@ -248,7 +255,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList static const QString ptSeparator = "/"; static constexpr bool isToken = false; - static const QList setsWithCardsWithSameNameButDifferentText = {"UST"}; + static const QSet setsWithCardsWithSameNameButDifferentText = {"UST"}; int numCards = 0; @@ -256,16 +263,16 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList QMap, QString>> splitCards; // Keeps track of all names encountered so far - QList allNameProps; + QSet allNameProps; - for (const QVariant &cardVar : cardsList) { - QVariantMap card = cardVar.toMap(); + for (const QJsonValue &cardVal : cardsList) { + QJsonObject card = cardVal.toObject(); /* Currently used layouts are: * augment, double_faced_token, flip, host, leveler, meld, normal, planar, * saga, scheme, split, token, transform, vanguard */ - QString layout = getStringPropertyFromMap(card, "layout"); + QString layout = getJsonString(card, "layout"); // don't import tokens from the json file if (layout == "token") { @@ -273,9 +280,9 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList } // normal cards handling - QString name = getStringPropertyFromMap(card, "name"); - QString text = getStringPropertyFromMap(card, "text"); - QString faceName = getStringPropertyFromMap(card, "faceName"); + QString name = getJsonString(card, "name"); + QString text = getJsonString(card, "text"); + QString faceName = getJsonString(card, "faceName"); if (faceName.isEmpty()) { faceName = name; } @@ -283,39 +290,34 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // card properties QHash properties; for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty); + QString propertyValue = getJsonString(card, i.key()); if (!propertyValue.isEmpty()) { - properties.insert(xmlPropertyName, propertyValue); + properties.insert(i.value(), propertyValue); } } // per-set properties QHash printingProps; for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty); + QString propertyValue = getJsonString(card, i.key()); if (!propertyValue.isEmpty()) { - printingProps.insert(xmlPropertyName, propertyValue); + printingProps.insert(i.value(), propertyValue); } } // handle flavorNames specially due to double-faced cards - QString faceFlavorName = getStringPropertyFromMap(card, "faceFlavorName"); - QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getStringPropertyFromMap(card, "flavorName"); + QString faceFlavorName = getJsonString(card, "faceFlavorName"); + QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName"); if (!flavorName.isEmpty()) { printingProps.insert("flavorName", flavorName); } // Identifiers + QJsonObject identifiers = card.value("identifiers").toObject(); for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) { - QString mtgjsonProperty = i.key(); - QString xmlPropertyName = i.value(); - QString propertyValue = getStringPropertyFromMap(card.value("identifiers").toMap(), mtgjsonProperty); + QString propertyValue = getJsonString(identifiers, i.key()); if (!propertyValue.isEmpty()) { - printingProps.insert(xmlPropertyName, propertyValue); + printingProps.insert(i.value(), propertyValue); } } @@ -331,21 +333,26 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) { numComponent = " (" + QString(lastChar).toLower() + ")"; } - allNameProps.append(faceName); + allNameProps.insert(faceName); // special handling properties - QString colors = card.value("colors").toStringList().join(""); + QString colors; + for (const QJsonValue &color : card.value("colors").toArray()) { + colors += color.toString(); + } if (!colors.isEmpty()) { properties.insert("colors", colors); } - // special handling properties - QString colorIdentity = card.value("colorIdentity").toStringList().join(""); + QString colorIdentity; + for (const QJsonValue &color : card.value("colorIdentity").toArray()) { + colorIdentity += color.toString(); + } if (!colorIdentity.isEmpty()) { properties.insert("coloridentity", colorIdentity); } - const auto &mainCardType = getMainCardType(card.value("types").toStringList()); + const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList()); if (mainCardType.isEmpty()) { qDebug() << "warning: no mainCardType for card:" << name; } else { @@ -354,22 +361,22 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // Depending on whether power and/or toughness are present, the format // is either P/T (most common), P (no toughness), or /T (no power). - QString power = getStringPropertyFromMap(card, "power"); - QString toughness = getStringPropertyFromMap(card, "toughness"); + QString power = getJsonString(card, "power"); + QString toughness = getJsonString(card, "toughness"); if (toughness.isEmpty() && !power.isEmpty()) { properties.insert("pt", power); } else if (!toughness.isEmpty()) { properties.insert("pt", power + ptSeparator + toughness); } - auto legalities = card.value("legalities").toMap(); - for (auto i = legalities.cbegin(), end = legalities.cend(); i != end; ++i) { + auto legalities = card.value("legalities").toObject(); + for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) { properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower()); } // split cards are considered a single card, enqueue for later merging if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") { - auto _faceName = getStringPropertyFromMap(card, "faceName"); + auto _faceName = getJsonString(card, "faceName"); SplitCardPart split(_faceName, text, properties, printingInfo); auto found_iter = splitCards.find(name + numProperty); if (found_iter == splitCards.end()) { @@ -382,11 +389,11 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList QList relatedCards; // add other face for split cards as card relation - if (!getStringPropertyFromMap(card, "side").isEmpty()) { - auto faceManaValue = getStringPropertyFromMap(card, "faceManaValue"); + if (!getJsonString(card, "side").isEmpty()) { + auto faceManaValue = getJsonString(card, "faceManaValue"); if (faceManaValue.isEmpty()) { // check the old name for the property, for backwards compatibility purposes - faceManaValue = getStringPropertyFromMap(card, "faceConvertedManaCost"); + faceManaValue = getJsonString(card, "faceConvertedManaCost"); } properties["cmc"] = faceManaValue; @@ -406,15 +413,15 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList name = faceName; } - // mtgjon related cards - if (card.contains("relatedCards")) { - QVariantMap givenRelated = card.value("relatedCards").toMap(); + // mtgjson related cards + QJsonObject givenRelated = card.value("relatedCards").toObject(); + if (!givenRelated.isEmpty()) { // conjured cards from a spellbook - if (givenRelated.contains("spellbook")) { - auto spbk = givenRelated.value("spellbook").toStringList(); - for (const QString &spbkName : spbk) { - relatedCards.append( - new CardRelation(spbkName, CardRelationType::DoesNotAttach, false, false, 1, true)); + QJsonArray spellbook = givenRelated.value("spellbook").toArray(); + if (!spellbook.isEmpty()) { + for (const QJsonValue &spbkVal : spellbook) { + relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false, + false, 1, true)); } } } @@ -427,7 +434,6 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList // split cards handling static const QString splitCardPropSeparator = QString(" // "); static const QString splitCardTextSeparator = QString("\n\n---\n\n"); - static const QList noRelatedCards = {}; QList, QString>> partsAndNames = splitCards.values(); for (auto [splitCardParts, name] : partsAndNames) { @@ -465,20 +471,20 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList } } } - CardInfoPtr newCard = addCard(name, text, isToken, properties, noRelatedCards, printingInfo); + CardInfoPtr newCard = addCard(name, text, isToken, properties, {}, printingInfo); numCards++; } return numCards; } -FormatRulesNameMap OracleImporter::createDefaultMagicFormats() +static FormatRulesNameMap buildDefaultMagicFormats() { // Predefined common exceptions CardCondition superTypeIsBasic; superTypeIsBasic.field = "type"; superTypeIsBasic.matchType = "regex"; - superTypeIsBasic.value = "\bBasic\b[^—]+\bLand\b"; + superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)"; ExceptionRule basicLands; basicLands.conditions.append(superTypeIsBasic); @@ -491,7 +497,6 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats() ExceptionRule mayContainAnyNumber; mayContainAnyNumber.conditions.append(anyNumberAllowed); - // Map to store default rules FormatRulesNameMap defaultFormatRulesNameMap; // ----------------- Helper lambda to create format ----------------- @@ -537,6 +542,12 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats() return defaultFormatRulesNameMap; } +const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats() +{ + static const FormatRulesNameMap cached = buildDefaultMagicFormats(); + return cached; +} + int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); @@ -576,6 +587,11 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion); } +void OracleImporter::releaseSetData() +{ + allSets.clear(); +} + void OracleImporter::clear() { sets.clear(); diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 99644f9ce..52a7cd349 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,8 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include +#include #include #include #include @@ -44,7 +46,7 @@ class SetToDownload { private: QString shortName, longName; - QList cards; + QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; @@ -58,7 +60,7 @@ public: { return longName; } - const QList &getCards() const + const QJsonArray &getCards() const { return cards; } @@ -76,7 +78,7 @@ public: } SetToDownload(QString _shortName, QString _longName, - QList _cards, + QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) @@ -154,8 +156,11 @@ public: bool readSetsFromByteArray(const QByteArray &data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); - int importCardsFromSet(const CardSetPtr ¤tSet, const QList &cardsList); - FormatRulesNameMap createDefaultMagicFormats(); + int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); + /** + * @brief Returns the default format rules. The result is memoized on first use and must be treated as immutable. + */ + const FormatRulesNameMap &createDefaultMagicFormats(); const CardNameMap &getCardList() const { return cards; @@ -164,6 +169,7 @@ public: { return allSets; } + void releaseSetData(); void clear(); }; diff --git a/oracle/src/pages.cpp b/oracle/src/pages.cpp index df4d1a98c..94e662ffe 100644 --- a/oracle/src/pages.cpp +++ b/oracle/src/pages.cpp @@ -560,6 +560,9 @@ void SaveSetsPage::initializePage() int setsImported = wizard()->importer->startImport(); + // JSON data no longer needed after CardInfo objects are built + wizard()->importer->releaseSetData(); + if (setsImported == 0) { QMessageBox::critical(this, tr("Error"), tr("No set has been imported.")); } diff --git a/oracle/src/qt-json/AUTHORS b/oracle/src/qt-json/AUTHORS deleted file mode 100644 index 29a85929f..000000000 --- a/oracle/src/qt-json/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -Eeli Reilin -Luis Gustavo S. Barreto -Stephen Kockentiedt diff --git a/oracle/src/qt-json/LICENSE b/oracle/src/qt-json/LICENSE deleted file mode 100644 index 3c42b515a..000000000 --- a/oracle/src/qt-json/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright 2011 Eeli Reilin. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation -are those of the authors and should not be interpreted as representing -official policies, either expressed or implied, of Eeli Reilin. - diff --git a/oracle/src/qt-json/README b/oracle/src/qt-json/README deleted file mode 100644 index b60c1599b..000000000 --- a/oracle/src/qt-json/README +++ /dev/null @@ -1,96 +0,0 @@ -######################################################################## -1. INTRODUCTION - -The Json class is a simple class for parsing JSON data into a QVariant -hierarchies. Now, we can also reverse the process and serialize -QVariant hierarchies into valid JSON data. - - -######################################################################## -2. HOW TO USE - -The parser is really easy to use. Let's say we have the following -QString of JSON data: - ------------------------------------------------------------------------- -{ - "encoding" : "UTF-8", - "plug-ins" : [ - "python", - "c++", - "ruby" - ], - "indent" : { - "length" : 3, - "use_space" : true - } -} ------------------------------------------------------------------------- - -We would first call the parse-method: - ------------------------------------------------------------------------- -//Say that we're using the QtJson namespace -using namespace QtJson; -bool ok; -//json is a QString containing the JSON data -QVariantMap result = Json::parse(json, ok).toMap(); - -if(!ok) { - qFatal("An error occurred during parsing"); - exit(1); -} ------------------------------------------------------------------------- - -Assuming the parsing process completed without errors, we would then -go through the hierarchy: - ------------------------------------------------------------------------- -qDebug() << "encoding:" << result["encoding"].toString(); -qDebug() << "plugins:"; - -foreach(QVariant plugin, result["plug-ins"].toList()) { - qDebug() << "\t-" << plugin.toString(); -} - -QVariantMap nestedMap = result["indent"].toMap(); -qDebug() << "length:" << nestedMap["length"].toInt(); -qDebug() << "use_space:" << nestedMap["use_space"].toBool(); ------------------------------------------------------------------------- - -The previous code would print out the following: - ------------------------------------------------------------------------- -encoding: "UTF-8" -plugins: - - "python" - - "c++" - - "ruby" -length: 3 -use_space: true ------------------------------------------------------------------------- - -To write JSON data from Qt object is as simple as parsing: - ------------------------------------------------------------------------- -QVariantMap map; -map["name"] = "Name"; -map["age"] = 22; - -QByteArray data = Json::serialize(map); ------------------------------------------------------------------------- - -The byte array 'data' contains valid JSON data: - ------------------------------------------------------------------------- -{ - name: "Luis Gustavo", - age: 22, -} ------------------------------------------------------------------------- - - -######################################################################## -4. CONTRIBUTING - -The code is available to download at GitHub. Contribute if you dare! diff --git a/oracle/src/qt-json/json.cpp b/oracle/src/qt-json/json.cpp deleted file mode 100644 index ff739b49d..000000000 --- a/oracle/src/qt-json/json.cpp +++ /dev/null @@ -1,545 +0,0 @@ -/* Copyright 2011 Eeli Reilin. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, - * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * The views and conclusions contained in the software and documentation - * are those of the authors and should not be interpreted as representing - * official policies, either expressed or implied, of Eeli Reilin. - */ - -/** - * \file json.cpp - */ - -#include "json.h" - -#include -#include - -namespace QtJson -{ - -static QString sanitizeString(QString str) -{ - str.replace(QLatin1String("\\"), QLatin1String("\\\\")); - str.replace(QLatin1String("\""), QLatin1String("\\\"")); - str.replace(QLatin1String("\b"), QLatin1String("\\b")); - str.replace(QLatin1String("\f"), QLatin1String("\\f")); - str.replace(QLatin1String("\n"), QLatin1String("\\n")); - str.replace(QLatin1String("\r"), QLatin1String("\\r")); - str.replace(QLatin1String("\t"), QLatin1String("\\t")); - return QString(QLatin1String("\"%1\"")).arg(str); -} - -static QByteArray join(const QList &list, const QByteArray &sep) -{ - QByteArray res; - for (const QByteArray &i : list) { - if (!res.isEmpty()) { - res += sep; - } - res += i; - } - return res; -} - -/** - * parse - */ -QVariant Json::parse(const QString &json) -{ - bool success = true; - return Json::parse(json, success); -} - -/** - * parse - */ -QVariant Json::parse(const QString &json, bool &success) -{ - success = true; - - // Return an empty QVariant if the JSON data is either null or empty - if (!json.isNull() || !json.isEmpty()) { - // We'll start from index 0 - int index = 0; - - // Parse the first value - QVariant value = Json::parseValue(json, index, success); - - // Return the parsed value - return value; - } else { - // Return the empty QVariant - return QVariant(); - } -} - -QByteArray Json::serialize(const QVariant &data) -{ - bool success = true; - return Json::serialize(data, success); -} - -QByteArray Json::serialize(const QVariant &data, bool &success) -{ - QByteArray str; - success = true; - - if (!data.isValid()) // invalid or null? - { - str = "null"; - } - else if ((data.typeId() == QMetaType::Type::QVariantList) || - (data.typeId() == QMetaType::Type::QStringList)) // variant is a list? - { - QList values; - const QVariantList list = data.toList(); - for (const QVariant &v : list) { - QByteArray serializedValue = serialize(v); - if (serializedValue.isNull()) { - success = false; - break; - } - values << serializedValue; - } - - str = "[ " + join(values, ", ") + " ]"; - } - else if ((data.typeId() == QMetaType::Type::QVariantHash)) // variant is a hash? - { - const QVariantHash vhash = data.toHash(); - QHashIterator it(vhash); - str = "{ "; - QList pairs; - - while (it.hasNext()) { - it.next(); - QByteArray serializedValue = serialize(it.value()); - - if (serializedValue.isNull()) { - success = false; - break; - } - - pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue; - } - - str += join(pairs, ", "); - str += " }"; - } - else if ((data.typeId() == QMetaType::Type::QVariantMap)) // variant is a map? - { - const QVariantMap vmap = data.toMap(); - QMapIterator it(vmap); - str = "{ "; - QList pairs; - while (it.hasNext()) { - it.next(); - QByteArray serializedValue = serialize(it.value()); - if (serializedValue.isNull()) { - success = false; - break; - } - pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue; - } - str += join(pairs, ", "); - str += " }"; - } - else if ((data.typeId() == QMetaType::Type::QString) || - (data.typeId() == QMetaType::Type::QByteArray)) // a string or a byte array? - { - str = sanitizeString(data.toString()).toUtf8(); - } - else if (data.typeId() == QMetaType::Type::Double) // double? - { - str = QByteArray::number(data.toDouble(), 'g', 20); - if (!str.contains(".") && !str.contains("e")) { - str += ".0"; - } - } - else if (data.typeId() == QMetaType::Type::Bool) // boolean value? - { - str = data.toBool() ? "true" : "false"; - } - else if (data.typeId() == QMetaType::Type::ULongLong) // large unsigned number? - { - str = QByteArray::number(data.value()); - } else if (data.canConvert()) // any signed number? - { - str = QByteArray::number(data.value()); - } else if (data.canConvert()) { - str = QString::number(data.value()).toUtf8(); - } else if (data.canConvert()) // can value be converted to string? - { - // this will catch QDate, QDateTime, QUrl, ... - str = sanitizeString(data.toString()).toUtf8(); - } else { - success = false; - } - if (success) { - return str; - } else { - return QByteArray(); - } -} - -/** - * parseValue - */ -QVariant Json::parseValue(const QString &json, int &index, bool &success) -{ - // Determine what kind of data we should parse by - // checking out the upcoming token - switch (Json::lookAhead(json, index)) { - case JsonTokenString: - return Json::parseString(json, index, success); - case JsonTokenNumber: - return Json::parseNumber(json, index); - case JsonTokenCurlyOpen: - return Json::parseObject(json, index, success); - case JsonTokenSquaredOpen: - return Json::parseArray(json, index, success); - case JsonTokenTrue: - Json::nextToken(json, index); - return QVariant(true); - case JsonTokenFalse: - Json::nextToken(json, index); - return QVariant(false); - case JsonTokenNull: - Json::nextToken(json, index); - return QVariant(); - case JsonTokenNone: - break; - } - - // If there were no tokens, flag the failure and return an empty QVariant - success = false; - return QVariant(); -} - -/** - * parseObject - */ -QVariant Json::parseObject(const QString &json, int &index, bool &success) -{ - QVariantMap map; - int token; - - // Get rid of the whitespace and increment index - Json::nextToken(json, index); - - // Loop through all of the key/value pairs of the object - bool done = false; - while (!done) { - // Get the upcoming token - token = Json::lookAhead(json, index); - - if (token == JsonTokenNone) { - success = false; - return QVariantMap(); - } else if (token == JsonTokenComma) { - Json::nextToken(json, index); - } else if (token == JsonTokenCurlyClose) { - Json::nextToken(json, index); - return map; - } else { - // Parse the key/value pair's name - QString name = Json::parseString(json, index, success).toString(); - - if (!success) { - return QVariantMap(); - } - - // Get the next token - token = Json::nextToken(json, index); - - // If the next token is not a colon, flag the failure - // return an empty QVariant - if (token != JsonTokenColon) { - success = false; - return QVariant(QVariantMap()); - } - - // Parse the key/value pair's value - QVariant value = Json::parseValue(json, index, success); - - if (!success) { - return QVariantMap(); - } - - // Assign the value to the key in the map - map[name] = value; - } - } - - // Return the map successfully - return QVariant(map); -} - -/** - * parseArray - */ -QVariant Json::parseArray(const QString &json, int &index, bool &success) -{ - QVariantList list; - - Json::nextToken(json, index); - - bool done = false; - while (!done) { - int token = Json::lookAhead(json, index); - - if (token == JsonTokenNone) { - success = false; - return QVariantList(); - } else if (token == JsonTokenComma) { - Json::nextToken(json, index); - } else if (token == JsonTokenSquaredClose) { - Json::nextToken(json, index); - break; - } else { - QVariant value = Json::parseValue(json, index, success); - - if (!success) { - return QVariantList(); - } - - list.push_back(value); - } - } - - return QVariant(list); -} - -/** - * parseString - */ -QVariant Json::parseString(const QString &json, int &index, bool &success) -{ - QString s; - QChar c; - - Json::eatWhitespace(json, index); - - c = json[index++]; - - bool complete = false; - while (!complete) { - if (index == json.size()) { - break; - } - - c = json[index++]; - - if (c == '\"') { - complete = true; - break; - } else if (c == '\\') { - if (index == json.size()) { - break; - } - - c = json[index++]; - - if (c == '\"') { - s.append('\"'); - } else if (c == '\\') { - s.append('\\'); - } else if (c == '/') { - s.append('/'); - } else if (c == 'b') { - s.append('\b'); - } else if (c == 'f') { - s.append('\f'); - } else if (c == 'n') { - s.append('\n'); - } else if (c == 'r') { - s.append('\r'); - } else if (c == 't') { - s.append('\t'); - } else if (c == 'u') { - int remainingLength = json.size() - index; - - if (remainingLength >= 4) { - QString unicodeStr = json.mid(index, 4); - - int symbol = unicodeStr.toInt(0, 16); - - s.append(QChar(symbol)); - - index += 4; - } else { - break; - } - } - } else { - s.append(c); - } - } - - if (!complete) { - success = false; - return QVariant(); - } - - return QVariant(s); -} - -/** - * parseNumber - */ -QVariant Json::parseNumber(const QString &json, int &index) -{ - Json::eatWhitespace(json, index); - - int lastIndex = Json::lastIndexOfNumber(json, index); - int charLength = (lastIndex - index) + 1; - QString numberStr; - - numberStr = json.mid(index, charLength); - - index = lastIndex + 1; - - if (numberStr.contains('.')) { - return QVariant(numberStr.toDouble(NULL)); - } else if (numberStr.startsWith('-')) { - return QVariant(numberStr.toLongLong(NULL)); - } else { - return QVariant(numberStr.toULongLong(NULL)); - } -} - -/** - * lastIndexOfNumber - */ -int Json::lastIndexOfNumber(const QString &json, int index) -{ - static const QString numericCharacters("0123456789+-.eE"); - int lastIndex; - - for (lastIndex = index; lastIndex < json.size(); lastIndex++) { - if (numericCharacters.indexOf(json[lastIndex]) == -1) { - break; - } - } - - return lastIndex - 1; -} - -/** - * eatWhitespace - */ -void Json::eatWhitespace(const QString &json, int &index) -{ - static const QString whitespaceChars(" \t\n\r"); - for (; index < json.size(); index++) { - if (whitespaceChars.indexOf(json[index]) == -1) { - break; - } - } -} - -/** - * lookAhead - */ -int Json::lookAhead(const QString &json, int index) -{ - int saveIndex = index; - return Json::nextToken(json, saveIndex); -} - -/** - * nextToken - */ -int Json::nextToken(const QString &json, int &index) -{ - Json::eatWhitespace(json, index); - - if (index == json.size()) { - return JsonTokenNone; - } - - QChar c = json[index]; - index++; - switch (c.toLatin1()) { - case '{': - return JsonTokenCurlyOpen; - case '}': - return JsonTokenCurlyClose; - case '[': - return JsonTokenSquaredOpen; - case ']': - return JsonTokenSquaredClose; - case ',': - return JsonTokenComma; - case '"': - return JsonTokenString; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - return JsonTokenNumber; - case ':': - return JsonTokenColon; - } - - index--; - - int remainingLength = json.size() - index; - - // True - if (remainingLength >= 4) { - if (json[index] == 't' && json[index + 1] == 'r' && json[index + 2] == 'u' && json[index + 3] == 'e') { - index += 4; - return JsonTokenTrue; - } - } - - // False - if (remainingLength >= 5) { - if (json[index] == 'f' && json[index + 1] == 'a' && json[index + 2] == 'l' && json[index + 3] == 's' && - json[index + 4] == 'e') { - index += 5; - return JsonTokenFalse; - } - } - - // Null - if (remainingLength >= 4) { - if (json[index] == 'n' && json[index + 1] == 'u' && json[index + 2] == 'l' && json[index + 3] == 'l') { - index += 4; - return JsonTokenNull; - } - } - - return JsonTokenNone; -} - -} // namespace QtJson diff --git a/oracle/src/qt-json/json.h b/oracle/src/qt-json/json.h deleted file mode 100644 index cf0499d4e..000000000 --- a/oracle/src/qt-json/json.h +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2011 Eeli Reilin. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY ''AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, - * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * The views and conclusions contained in the software and documentation - * are those of the authors and should not be interpreted as representing - * official policies, either expressed or implied, of Eeli Reilin. - */ - -/** - * \file json.h - */ - -#ifndef JSON_H -#define JSON_H - -#include -#include - -namespace QtJson -{ - -/** - * \enum JsonToken - */ -enum JsonToken -{ - JsonTokenNone = 0, - JsonTokenCurlyOpen = 1, - JsonTokenCurlyClose = 2, - JsonTokenSquaredOpen = 3, - JsonTokenSquaredClose = 4, - JsonTokenColon = 5, - JsonTokenComma = 6, - JsonTokenString = 7, - JsonTokenNumber = 8, - JsonTokenTrue = 9, - JsonTokenFalse = 10, - JsonTokenNull = 11 -}; - -/** - * \class Json - * \brief A JSON data parser - * - * Json parses a JSON data into a QVariant hierarchy. - */ -class Json -{ - public: - /** - * Parse a JSON string - * - * \param json The JSON data - */ - static QVariant parse(const QString &json); - - /** - * Parse a JSON string - * - * \param json The JSON data - * \param success The success of the parsing - */ - static QVariant parse(const QString &json, bool &success); - - /** - * This method generates a textual JSON representation - * - * \param data The JSON data generated by the parser. - * \param success The success of the serialization - */ - static QByteArray serialize(const QVariant &data); - - /** - * This method generates a textual JSON representation - * - * \param data The JSON data generated by the parser. - * \param success The success of the serialization - * - * \return QByteArray Textual JSON representation - */ - static QByteArray serialize(const QVariant &data, bool &success); - - private: - /** - * Parses a value starting from index - * - * \param json The JSON data - * \param index The start index - * \param success The success of the parse process - * - * \return QVariant The parsed value - */ - static QVariant parseValue(const QString &json, int &index, - bool &success); - - /** - * Parses an object starting from index - * - * \param json The JSON data - * \param index The start index - * \param success The success of the object parse - * - * \return QVariant The parsed object map - */ - static QVariant parseObject(const QString &json, int &index, - bool &success); - - /** - * Parses an array starting from index - * - * \param json The JSON data - * \param index The starting index - * \param success The success of the array parse - * - * \return QVariant The parsed variant array - */ - static QVariant parseArray(const QString &json, int &index, - bool &success); - - /** - * Parses a string starting from index - * - * \param json The JSON data - * \param index The starting index - * \param success The success of the string parse - * - * \return QVariant The parsed string - */ - static QVariant parseString(const QString &json, int &index, - bool &success); - - /** - * Parses a number starting from index - * - * \param json The JSON data - * \param index The starting index - * - * \return QVariant The parsed number - */ - static QVariant parseNumber(const QString &json, int &index); - - /** - * Get the last index of a number starting from index - * - * \param json The JSON data - * \param index The starting index - * - * \return The last index of the number - */ - static int lastIndexOfNumber(const QString &json, int index); - - /** - * Skip unwanted whitespace symbols starting from index - * - * \param json The JSON data - * \param index The start index - */ - static void eatWhitespace(const QString &json, int &index); - - /** - * Check what token lies ahead - * - * \param json The JSON data - * \param index The starting index - * - * \return int The upcoming token - */ - static int lookAhead(const QString &json, int index); - - /** - * Get the next JSON token - * - * \param json The JSON data - * \param index The starting index - * - * \return int The next JSON token - */ - static int nextToken(const QString &json, int &index); -}; - - -} //end namespace - -#endif //JSON_H From 61e6a9913e4096303b8be61be727fbd7015b6a24 Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:14:16 +0200 Subject: [PATCH 21/25] [Oracle] Add oracle importer tests and fix set parsing details (#7215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Oracle] Add oracle importer tests and fix set parsing details - Add oracle_importer_test and oracle_importer_benchmark_test targets - Preserve the first printing's legalities when an existing card is reused - Concatenate split-card coloridentity and sort/dedupe card colors - Use a raw string for the Basic Land format regex - Pre-allocate the card hash and micro-optimize string handling Took 2 minutes * [Oracle/Tests] Pin cmc coercion in CI run; scope the reserve pass The #7214 coercion assertion lived only in oracle_importer_benchmark_test, which gets no add_test and so never runs under ctest. Add NumericManaValueCoercedToCmc and LegacyConvertedManaCostCoercedToCmc to oracle_importer_test (a CI-ran binary): manaValue/convertedManaCost are JSON numbers in AllPrintings, and QJsonValue::toString() would drop them to an empty cmc without the #7214 coercion fix. Wrap the distinct-name reserve pass in a bare block so the ~35k name QStrings are handed back before the memory-heavy import loop starts. --------- Co-authored-by: Lukas Brübach --- oracle/src/oracleimporter.cpp | 45 +- tests/oracle/CMakeLists.txt | 32 + .../oracle/oracle_importer_benchmark_test.cpp | 264 +++++++++ tests/oracle/oracle_importer_test.cpp | 552 ++++++++++++++++++ 4 files changed, 878 insertions(+), 15 deletions(-) create mode 100644 tests/oracle/oracle_importer_benchmark_test.cpp create mode 100644 tests/oracle/oracle_importer_test.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 752317512..49577baf0 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,8 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) // capitalize set type if (setType.length() > 0) { // basic grammar for words that aren't capitalized, like in "From the Vault" - const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", "of", "in", "and", "with", "or"}; + static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", + "of", "in", "and", "with", "or"}; QStringList words = setType.split("_"); setType.clear(); bool first = false; @@ -75,7 +77,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) if (first && noCapitalize.contains(item)) { setType += item + QString(" "); } else { - setType += item[0].toUpper() + item.mid(1, -1) + QString(" "); + setType += item[0].toUpper() + item.mid(1) + QString(" "); first = true; } } @@ -123,14 +125,8 @@ static void sortAndReduceColors(QString &colors) std::sort(colors.begin(), colors.end(), [](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); }); // reduce - QChar lastChar = '\0'; - for (int i = 0; i < colors.size(); ++i) { - if (colors.at(i) == lastChar) { - colors.remove(i, 1); - } else { - lastChar = colors.at(i); - } - } + auto last = std::unique(colors.begin(), colors.end()); + colors.erase(last, colors.end()); } CardInfoPtr OracleImporter::addCard(QString name, @@ -186,8 +182,9 @@ CardInfoPtr OracleImporter::addCard(QString name, // DETECT CARD POSITIONING INFO - bool landscapeOrientation = properties.value("maintype") == "Battle" || properties.value("layout") == "split" || - properties.value("layout") == "planar"; + QString layoutVal = properties.value("layout"); + bool landscapeOrientation = + properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar"; // cards that enter the field tapped bool cipt = parseCipt(name, text) || landscapeOrientation; @@ -426,7 +423,8 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } - CardInfoPtr newCard = addCard(name + numComponent, text, isToken, properties, relatedCards, printingInfo); + CardInfoPtr newCard = + addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo); numCards++; } } @@ -459,7 +457,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) { if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty properties.insert(prop, thisCardPropertyValue); - } else if (prop == "colors") { // the card is both colors + } else if (prop == "colors" || prop == "coloridentity") { // the card is both colors properties.insert(prop, originalPropertyValue + thisCardPropertyValue); } else if (prop == "maintype") { // don't create maintypes with //es in them continue; @@ -471,7 +469,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } } - CardInfoPtr newCard = addCard(name, text, isToken, properties, {}, printingInfo); + CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo); numCards++; } @@ -552,6 +550,23 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); + // Pre-allocate the cards hash to avoid rehashing during import. The hash + // is keyed by distinct card name rather than by printings: AllPrintings + // ships ~100k printings for ~35k names, so reserving the printing count + // would overallocate ~3x (against this stack's RAM goal). Collecting + // distinct names is cheap — one pass over the already-parsed name fields. + { + QSet distinctNames; + for (const SetToDownload &curSetToParse : allSets) { + for (const QJsonValue &cardValue : curSetToParse.getCards()) { + distinctNames.insert(cardValue.toObject().value("name").toString()); + } + } + cards.reserve(distinctNames.size()); + // The set goes out of scope here, handing the ~35k name QStrings back + // to the allocator before the (memory-heavy) import loop starts. + } + // add an empty set for tokens CardSetPtr tokenSet = CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens"); diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index c5c1e9097..d126390c7 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -7,3 +7,35 @@ endif() target_link_libraries(parse_cipt_test Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}) add_test(NAME parse_cipt_test COMMAND parse_cipt_test) + +# Oracle importer unit tests +add_executable( + oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + oracle_importer_test.cpp +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_test gtest) +endif() + +target_link_libraries( + oracle_importer_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} +) + +add_test(NAME oracle_importer_test COMMAND oracle_importer_test) + +# Oracle importer benchmark tests (manual, not run in CI) +add_executable( + oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp + ../../oracle/src/parsehelpers.cpp oracle_importer_benchmark_test.cpp +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_benchmark_test gtest) +endif() + +target_link_libraries( + oracle_importer_benchmark_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} +) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp new file mode 100644 index 000000000..b3cc55319 --- /dev/null +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -0,0 +1,264 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include + +// Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set +static QByteArray buildSyntheticData(int numSets, int cardsPerSet) +{ + QJsonObject dataObj; + for (int s = 0; s < numSets; ++s) { + QJsonArray cardsArray; + for (int c = 0; c < cardsPerSet; ++c) { + QJsonObject card; + card["name"] = QString("Card %1").arg(s * cardsPerSet + c); + card["text"] = "This is a test card with some rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["power"] = "2"; + card["toughness"] = "2"; + card["colors"] = QJsonArray{"W"}; + card["colorIdentity"] = QJsonArray{"W"}; + card["types"] = QJsonArray{"Creature"}; + // Real MTGJSON types: floats and booleans, not strings. This + // exercises the QVariant coercion in the property reader. + card["convertedManaCost"] = 1.0; + card["manaValue"] = 1.0; + card["isOnlineOnly"] = false; + card["isRebalanced"] = false; + + QJsonObject legalities; + legalities["standard"] = "legal"; + legalities["modern"] = "legal"; + legalities["legacy"] = "legal"; + legalities["vintage"] = "legal"; + legalities["commander"] = "legal"; + card["legalities"] = legalities; + + QJsonObject identifiers; + identifiers["scryfallId"] = QString("id-%1-%2").arg(s).arg(c); + card["identifiers"] = identifiers; + + // In AllPrintings, number and rarity are flat fields on the card + // object, exactly as set below. + card["number"] = QString::number(c + 1); + card["rarity"] = "common"; + + cardsArray.append(card); + } + + QJsonObject setObj; + setObj["code"] = QString("T%1").arg(s, 2, 10, QChar('0')); + setObj["name"] = QString("Test Set %1").arg(s); + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = cardsArray; + + dataObj[QString("T%1").arg(s, 2, 10, QChar('0'))] = setObj; + } + + QJsonObject root; + root["data"] = dataObj; + return QJsonDocument(root).toJson(QJsonDocument::Compact); +} + +// ============================================================================ +// Import throughput benchmark +// ============================================================================ + +TEST(OracleBenchmark, ImportThroughput) +{ + static constexpr int numSets = 10; + static constexpr int cardsPerSet = 500; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + OracleImporter importer; + + // Phase 1: Parse JSON + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(data); + ASSERT_TRUE(ok); + qint64 parseMs = timer.elapsed(); + + // Phase 2: Import cards + timer.restart(); + int importedSets = importer.startImport(); + qint64 importMs = timer.elapsed(); + + int totalImported = 0; + for (const auto &card : importer.getCardList()) { + Q_UNUSED(card); + totalImported++; + } + + // The fixture generates globally unique card names, so the expected + // counts are exact: a regression here means cards were dropped. + ASSERT_EQ(importedSets, numSets); + ASSERT_EQ(totalImported, numSets * cardsPerSet); + // Real-data probe: numeric convertedManaCost must be coerced to text + // (regression for the QJsonValue::toString() reader in #7214). + auto probeCard = importer.getCardList().value("Card 0"); + ASSERT_FALSE(probeCard.isNull()); + ASSERT_EQ(probeCard->getProperty("cmc"), "1"); + + qDebug().noquote() + << QString("Oracle Import Benchmark: %1 sets, %2 unique cards").arg(importedSets).arg(totalImported); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + qDebug().noquote() << QString(" Total: %1 ms").arg(parseMs + importMs); + if (importMs > 0) { + qDebug().noquote() << QString(" Throughput: %1 cards/sec") + .arg(static_cast(totalImported) / importMs * 1000.0, 0, 'f', 0); + } +} + +// ============================================================================ +// readSetsFromByteArray benchmark +// ============================================================================ + +TEST(OracleBenchmark, ParseJsonThroughput) +{ + static constexpr int numSets = 20; + static constexpr int cardsPerSet = 1000; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + // Run 5 iterations and report average + static constexpr int iterations = 5; + qint64 totalMs = 0; + + for (int i = 0; i < iterations; ++i) { + OracleImporter importer; + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(data); + ASSERT_TRUE(ok); + totalMs += timer.elapsed(); + } + + qint64 avgMs = totalMs / iterations; + qDebug().noquote() << QString("Parse Benchmark (%1 iterations): avg %2 ms for %3 sets x %4 cards") + .arg(iterations) + .arg(avgMs) + .arg(numSets) + .arg(cardsPerSet); +} + +// ============================================================================ +// Split card merging benchmark +// ============================================================================ + +TEST(OracleBenchmark, SplitCardMerging) +{ + static constexpr int numSplitCards = 1000; + + QJsonArray cardsList; + for (int i = 0; i < numSplitCards; ++i) { + QJsonObject face1; + face1["name"] = QString("Fire %1 // Ice %1").arg(i); + face1["text"] = "Fire side text."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = QString("Fire %1").arg(i); + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face1["identifiers"] = QJsonObject{{"scryfallId", QString("f-%1").arg(i)}}; + face1["number"] = QString::number(i + 1); + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = QString("Fire %1 // Ice %1").arg(i); + face2["text"] = "Ice side text."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = QString("Ice %1").arg(i); + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face2["identifiers"] = QJsonObject{{"scryfallId", QString("i-%1").arg(i)}}; + face2["number"] = QString::number(i + 1); + face2["rarity"] = "uncommon"; + + cardsList.append(face1); + cardsList.append(face2); + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Split Test"); + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numSplitCards); + qDebug().noquote() << QString("Split Card Merge Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +// ============================================================================ +// sortAndReduceColors microbenchmark +// ============================================================================ + +// We can't call sortAndReduceColors directly (it's static), so we benchmark +// through importCardsFromSet with color properties. + +TEST(OracleBenchmark, ImportCardsWithColors) +{ + static constexpr int numCards = 10000; + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Color Test"); + + QJsonArray cardsList; + for (int i = 0; i < numCards; ++i) { + QJsonObject card; + card["name"] = QString("Color Card %1").arg(i); + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["colors"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["colorIdentity"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["number"] = QString::number(i + 1); + card["rarity"] = "common"; + card["legalities"] = QJsonObject{{"standard", "legal"}}; + card["identifiers"] = QJsonObject{{"scryfallId", QString("c-%1").arg(i)}}; + cardsList.append(card); + } + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numCards); + qDebug().noquote() << QString("Import with Colors Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp new file mode 100644 index 000000000..145a2ca0f --- /dev/null +++ b/tests/oracle/oracle_importer_test.cpp @@ -0,0 +1,552 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include + +class OracleImporterTest : public ::testing::Test +{ +protected: + void SetUp() override + { + controller = new NoopCardSetPriorityController(); + importer = new OracleImporter(); + set = CardSet::newInstance(controller, "TST", "Test Set"); + } + + void TearDown() override + { + delete importer; + delete controller; + } + + // Helper: build a minimal card JSON object + QJsonObject makeCard(const QString &name, + const QString &colors = "", + const QString &colorIdentity = "", + const QVariantMap &legalities = {}) + { + QJsonObject card; + card["name"] = name; + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["number"] = "1"; + card["rarity"] = "common"; + + if (!colors.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colors) { + arr.append(QString(c)); + } + card["colors"] = arr; + } + if (!colorIdentity.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colorIdentity) { + arr.append(QString(c)); + } + card["colorIdentity"] = arr; + } + if (!legalities.isEmpty()) { + QJsonObject legalObj; + for (auto it = legalities.constBegin(); it != legalities.constEnd(); ++it) { + legalObj[it.key()] = it.value().toString(); + } + card["legalities"] = legalObj; + } + + QJsonObject identifiers; + identifiers["scryfallId"] = QUuid::createUuid().toString(QUuid::WithoutBraces); + card["identifiers"] = identifiers; + + return card; + } + + NoopCardSetPriorityController *controller; + OracleImporter *importer; + CardSetPtr set; +}; + +// ============================================================================ +// sortAndReduceColors tests (tested via importCardsFromSet) +// ============================================================================ + +TEST_F(OracleImporterTest, SortAndReduceColorsSingleColor) +{ + QJsonArray cards{makeCard("Red Card", "R", "R")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Red Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "R"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsDeduplicates) +{ + QJsonArray cards{makeCard("Dedup Card", "WWUUB", "WU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Dedup Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUB"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsSortsWUBRG) +{ + QJsonArray cards{makeCard("Sort Card", "RGW", "RGW")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Sort Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsAllFive) +{ + QJsonArray cards{makeCard("Five Color", "BRGWU", "BRGWU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Five Color"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUBRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorIdentity) +{ + QJsonArray cards{makeCard("Color Id Card", "W", "GWR")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Color Id Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("coloridentity"), "WRG"); +} + +TEST_F(OracleImporterTest, SingleColorNotSorted) +{ + QJsonArray cards{makeCard("Single Card", "B", "B")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Single Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "B"); +} + +// ============================================================================ +// Legality guard tests +// ============================================================================ + +TEST_F(OracleImporterTest, NewCardKeepsLegalityProperties) +{ + // Verifies that format-* properties survive addCard on a fresh card + // (not the combineLegalities guard, which only runs on existing printings). + QVariantMap leg; + leg["standard"] = "legal"; + leg["modern"] = "legal"; + QJsonArray cards{makeCard("Legal Card", "", "", leg)}; + + importer->importCardsFromSet(set, cards); + auto card = importer->getCardList().value("Legal Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityMergeAllowedWhenCardHasNoLegalities) +{ + // First printing carries no legalities at all, so the guard's + // `properties.filter(formatRegex).empty()` predicate is true and the + // second printing's legalities must be merged in. + QJsonArray cards1{makeCard("Unmerged Card")}; + importer->importCardsFromSet(set, cards1); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg; + leg["standard"] = "legal"; + QJsonArray cards2{makeCard("Unmerged Card", "", "", leg)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Unmerged Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityGuardPreservesFirstPrinting) +{ + // First printing: standard=legal, modern=legal + QVariantMap leg1; + leg1["standard"] = "legal"; + leg1["modern"] = "legal"; + QJsonArray cards1{makeCard("Guarded Card", "", "", leg1)}; + importer->importCardsFromSet(set, cards1); + + // Second printing: standard=banned, modern=not_legal + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg2; + leg2["standard"] = "banned"; + leg2["modern"] = "not_legal"; + QJsonArray cards2{makeCard("Guarded Card", "", "", leg2)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Guarded Card"); + ASSERT_FALSE(card.isNull()); + // Guard should preserve first printing's legalities + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +// ============================================================================ +// createDefaultMagicFormats tests +// ============================================================================ + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsContainsExpectedFormats) +{ + auto formats = importer->createDefaultMagicFormats(); + ASSERT_TRUE(formats.contains("standard")); + ASSERT_TRUE(formats.contains("modern")); + ASSERT_TRUE(formats.contains("legacy")); + ASSERT_TRUE(formats.contains("vintage")); + ASSERT_TRUE(formats.contains("commander")); + ASSERT_TRUE(formats.contains("pauper")); + ASSERT_TRUE(formats.contains("pioneer")); + ASSERT_TRUE(formats.contains("brawl")); + ASSERT_TRUE(formats.contains("historic")); + ASSERT_TRUE(formats.contains("timeless")); + ASSERT_TRUE(formats.contains("duel")); + ASSERT_TRUE(formats.contains("oathbreaker")); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsSingletonDeckSizes) +{ + auto formats = importer->createDefaultMagicFormats(); + auto commander = formats.value("commander"); + ASSERT_FALSE(commander.isNull()); + ASSERT_EQ(commander->minDeckSize, 100); + ASSERT_EQ(commander->maxDeckSize, 100); + ASSERT_EQ(commander->maxSideboardSize, 15); + + auto brawl = formats.value("brawl"); + ASSERT_FALSE(brawl.isNull()); + ASSERT_EQ(brawl->minDeckSize, 60); + ASSERT_EQ(brawl->maxDeckSize, 60); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsVintageHasRestricted) +{ + auto formats = importer->createDefaultMagicFormats(); + auto vintage = formats.value("vintage"); + ASSERT_FALSE(vintage.isNull()); + bool hasRestricted = false; + for (const auto &ac : vintage->allowedCounts) { + if (ac.label == "restricted") { + hasRestricted = true; + ASSERT_EQ(ac.max, 1); + } + } + ASSERT_TRUE(hasRestricted); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsRegexMatchesBasicLands) +{ + auto formats = importer->createDefaultMagicFormats(); + auto standard = formats.value("standard"); + ASSERT_FALSE(standard.isNull()); + ASSERT_FALSE(standard->exceptions.isEmpty()); + + auto &basicLandsException = standard->exceptions.first(); + ASSERT_FALSE(basicLandsException.conditions.isEmpty()); + + auto &condition = basicLandsException.conditions.first(); + ASSERT_EQ(condition.field, "type"); + ASSERT_EQ(condition.matchType, "regex"); + + // Verify the regex actually works (was broken before: \b = backspace, not word boundary) + QRegularExpression regex(condition.value); + ASSERT_TRUE(regex.isValid()); + ASSERT_TRUE(regex.match("Basic Land — Forest").hasMatch()); + ASSERT_TRUE(regex.match("Basic Snow Land — Mountain").hasMatch()); + ASSERT_FALSE(regex.match("Creature — Elf Warrior").hasMatch()); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsCaching) +{ + // The memoized map returns the same FormatRulesPtr instances, so the + // shared pointers must be identical across calls. This is the only + // observable effect of the cache: contents would match either way. + auto first = importer->createDefaultMagicFormats(); + auto second = importer->createDefaultMagicFormats(); + ASSERT_EQ(first.value("standard").data(), second.value("standard").data()); +} + +// ============================================================================ +// readSetsFromByteArray tests +// ============================================================================ + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayValidJson) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().size(), 1); + ASSERT_EQ(importer->getSets().first().getShortName(), "TST"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayInvalidJson) +{ + QByteArray data = "not valid json"; + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayEmptyData) +{ + QJsonObject root; + root["data"] = QJsonObject(); + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayCapitalizesSetType) +{ + QJsonObject setObj; + setObj["code"] = "ftv"; + setObj["name"] = "From The Vault"; + setObj["type"] = "from_the_vault"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"FTV", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().first().getSetType(), "From the Vault"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArraySortsSetsByName) +{ + // QJsonObject iterates keys in lexicographic order ("AAA" before "ZZZ"), + // so leaving the natural order matching the alphabetical sort makes the + // assertion pass trivially. Inverting it keeps the sort meaningful: + // iteration yields "AAA" (Zeta Set) first, then the sort by name must + // promote "ZZZ" (Alpha Set) to the front. + QJsonObject setA; + setA["code"] = "aaa"; + setA["name"] = "Zeta Set"; + setA["type"] = "expansion"; + setA["releaseDate"] = "2024-01-01"; + setA["cards"] = QJsonArray(); + + QJsonObject setB; + setB["code"] = "zzz"; + setB["name"] = "Alpha Set"; + setB["type"] = "expansion"; + setB["releaseDate"] = "2024-01-01"; + setB["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"AAA", setA}, {"ZZZ", setB}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + auto sets = importer->getSets(); + ASSERT_GE(sets.size(), 2); + ASSERT_EQ(sets.first().getShortName(), "ZZZ"); +} + +// ============================================================================ +// Split card coloridentity tests +// ============================================================================ + +TEST_F(OracleImporterTest, SplitCardColorIdentityConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + int count = importer->importCardsFromSet(set, cardsList); + ASSERT_EQ(count, 1); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + // coloridentity should be "RU" (concatenated), then sorted to "UR" + // by sortAndReduceColors when it reaches addCard + ASSERT_EQ(card->getProperty("coloridentity"), "UR"); +} + +TEST_F(OracleImporterTest, SplitCardColorsConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + importer->importCardsFromSet(set, cardsList); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + QString colors = card->getProperty("colors"); + ASSERT_FALSE(colors.contains("//")) << "colors should not contain '//', got: " << colors.toStdString(); + ASSERT_TRUE(colors.contains("R")); + ASSERT_TRUE(colors.contains("U")); +} + +// ============================================================================ +// Mana cost formatting tests +// ============================================================================ + +TEST_F(OracleImporterTest, ManaCostStripsBraces) +{ + QJsonObject card = makeCard("Mana Card"); + card["manaCost"] = "{2}{W}{B}"; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Mana Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("manacost"), "2WB"); +} + +// cmc comes through as a JSON number ("convertedManaCost"/"manaValue" are +// floats in AllPrintings), so this pins the number-to-text coercion that +// QJsonValue::toString() dropped in #7214. +TEST_F(OracleImporterTest, NumericManaValueCoercedToCmc) +{ + QJsonObject card = makeCard("Cmc Card"); + card["manaValue"] = 3; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Cmc Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("cmc"), "3"); +} + +TEST_F(OracleImporterTest, LegacyConvertedManaCostCoercedToCmc) +{ + QJsonObject card = makeCard("Legacy Cmc Card"); + card["convertedManaCost"] = 3.0; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Legacy Cmc Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("cmc"), "3"); +} + +// ============================================================================ +// Card deduplication tests +// ============================================================================ + +TEST_F(OracleImporterTest, DuplicateCardNameReturnsExisting) +{ + QJsonArray cards{makeCard("Dupe Card")}; + importer->importCardsFromSet(set, cards); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QJsonArray cards2{makeCard("Dupe Card")}; + importer->importCardsFromSet(set2, cards2); + + ASSERT_EQ(importer->getCardList().size(), 1); +} + +TEST_F(OracleImporterTest, AELigatureReplaced) +{ + QJsonObject card = makeCard(QString::fromUtf8("\xC3\x86ther Vial")); // Æther Vial + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + // Æ is replaced with AE, resulting in "AEther Vial" + ASSERT_FALSE(importer->getCardList().contains(QString::fromUtf8("\xC3\x86ther Vial"))); + ASSERT_TRUE(importer->getCardList().contains("AEther Vial")); +} + +TEST_F(OracleImporterTest, ApostropheNormalized) +{ + QJsonObject card = makeCard(QString::fromUtf8("Jace\u2019s Ingenuity")); + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 7d8514ec34652540dc704bd2ba271f471476dbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sat, 29 Aug 2026 21:08:56 +0200 Subject: [PATCH 22/25] [Oracle] Add RAM usage benchmarks for the oracle importer - Measure process peak/current RSS via procfs (Linux) or getrusage (macOS) - Add a synthetic-scale RAM benchmark and an opt-in real AllPrintings run gated by COCKATRICE_ORACLE_RAM_BENCHMARK=1 - Mirror the wizard's magic-byte handling to decompress .xz/.zip payloads - Wire optional ZLIB/LibLZMA into the benchmark target and raise its timeout Took 2 minutes --- tests/oracle/CMakeLists.txt | 36 ++- .../oracle/oracle_importer_benchmark_test.cpp | 300 ++++++++++++++++++ 2 files changed, 332 insertions(+), 4 deletions(-) diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index d126390c7..cbff4f19c 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -25,10 +25,33 @@ target_link_libraries( add_test(NAME oracle_importer_test COMMAND oracle_importer_test) -# Oracle importer benchmark tests (manual, not run in CI) +# Oracle importer benchmark tests (manual, not run in CI, incl. RAM benchmark) +# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark +# can download and decompress whatever AllPrintings format the default URL selects. +find_package(ZLIB) +if(ZLIB_FOUND) + add_definitions("-DHAS_ZLIB") + set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp) + set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES}) + include_directories(${ZLIB_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled") +endif() + +find_package(LibLZMA) +if(LIBLZMA_FOUND) + add_definitions("-DHAS_LZMA") + list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp) + list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES}) + include_directories(${LIBLZMA_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled") +endif() + add_executable( - oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp - ../../oracle/src/parsehelpers.cpp oracle_importer_benchmark_test.cpp + oracle_importer_benchmark_test + ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} ) if(NOT GTEST_FOUND) @@ -36,6 +59,11 @@ if(NOT GTEST_FOUND) endif() target_link_libraries( - oracle_importer_benchmark_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + oracle_importer_benchmark_test + libcockatrice_card + libcockatrice_interfaces + Threads::Threads + ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES} + ${_ORACLE_BENCH_EXTRA_LIBRARIES} ) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index b3cc55319..fe2d0f107 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -1,13 +1,33 @@ #include "../../oracle/src/oracleimporter.h" #include "gtest/gtest.h" +#include +#include #include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include #include +#if defined(HAS_LZMA) +#include "../../oracle/src/lzma/decompress.h" +#endif +#if defined(HAS_ZLIB) +#include "../../oracle/src/zip/unzip.h" +#endif +#if defined(Q_OS_MACOS) +#include +#include +#endif + // Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set static QByteArray buildSyntheticData(int numSets, int cardsPerSet) { @@ -257,8 +277,288 @@ TEST(OracleBenchmark, ImportCardsWithColors) .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); } +// ============================================================================ +// RAM usage measurement +// ============================================================================ + +// Mirrors the default AllPrintings URL selection in oracle/src/pages.cpp. +#if defined(HAS_LZMA) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.xz"); +#elif defined(HAS_ZLIB) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.zip"); +#else +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json"); +#endif + +// Magic bytes also from oracle/src/pages.cpp +static const QByteArray kXzSignature("\xFD\x37\x7A\x58\x5A", 6); +static const QByteArray kZipSignature("PK"); + +struct MemorySnapshot +{ + qint64 peakRssKb = -1; // process high-water mark (VmHWM on Linux, ru_maxrss on macOS) + qint64 rssKb = -1; // current resident set size + bool available = false; + + static MemorySnapshot current() + { + MemorySnapshot snap; +#if defined(Q_OS_LINUX) + QFile statusFile("/proc/self/status"); + if (statusFile.open(QIODevice::ReadOnly | QIODevice::Text)) { + // /proc files report size() == 0, so atEnd() is immediately true: read everything first. + const QList lines = statusFile.readAll().split('\n'); + for (const QByteArray &line : lines) { + if (line.startsWith("VmHWM:")) { + snap.peakRssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } else if (line.startsWith("VmRSS:")) { + snap.rssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } + } + snap.available = snap.peakRssKb >= 0; + } +#elif defined(Q_OS_MACOS) + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + snap.peakRssKb = usage.ru_maxrss / 1024; // bytes -> kB + snap.available = snap.peakRssKb >= 0; + } + // getrusage has no current-RSS equivalent; task_info's resident_size + // is the closest macOS analog to Linux VmRSS. + mach_task_basic_info info = {}; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, reinterpret_cast(&info), &count) == + KERN_SUCCESS) { + snap.rssKb = info.resident_size / 1024; + } +#endif + return snap; + } +}; + +static QString formatKb(qint64 kb) +{ + if (kb < 0) { + return "N/A"; + } + return QString("%1 MB").arg(kb / 1024.0, 0, 'f', 1); +} + +static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, const MemorySnapshot ¤t) +{ + if (!baseline.available || !current.available) { + qDebug().noquote() << QString(" %1: memory stats unavailable on this platform").arg(phase); + return; + } + // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a + // peak-based delta between phases is ~0.0 MB by construction once the + // fixture build has set the process peak. The live signals are current RSS + // and the process peak; the delta is only meaningful where current RSS is + // a per-phase value (see "after releaseSetData()"). + QString rssDelta = "N/A"; + if (current.rssKb >= 0 && baseline.rssKb >= 0) { + rssDelta = formatKb(current.rssKb - baseline.rssKb); + } + qDebug().noquote() << QString(" %1: current RSS %2 | delta vs baseline %3 | process peak %4") + .arg(phase) + .arg(formatKb(current.rssKb)) + .arg(rssDelta) + .arg(formatKb(current.peakRssKb)); +} + +// Decompresses the download payload when the default URL is a compressed build, +// mirroring the wizard's magic-byte handling in oracle/src/pages.cpp. +static QByteArray decompressSetsData(const QByteArray &payload) +{ + if (payload.startsWith(kXzSignature)) { +#if defined(HAS_LZMA) + QBuffer inBuffer(const_cast(&payload)); + QByteArray out; + QBuffer outBuffer(&out); + inBuffer.open(QIODevice::ReadOnly); + outBuffer.open(QIODevice::WriteOnly); + XzDecompressor xz; + if (!xz.decompress(&inBuffer, &outBuffer)) { + qDebug() << "RAM benchmark: xz decompression failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is xz-compressed but this build has no LZMA support"; + return {}; +#endif + } + if (payload.startsWith(kZipSignature)) { +#if defined(HAS_ZLIB) + QBuffer inBuffer(const_cast(&payload)); + inBuffer.open(QIODevice::ReadOnly); + UnZip unzip; + if (unzip.openArchive(&inBuffer) != UnZip::Ok) { + qDebug() << "RAM benchmark: zip archive open failed"; + return {}; + } + if (unzip.fileList().size() != 1) { + qDebug() << "RAM benchmark: zip archive doesn't contain exactly one file"; + return {}; + } + QByteArray out; + QBuffer outBuffer(&out); + outBuffer.open(QIODevice::WriteOnly); + const auto errorCode = unzip.extractFile(unzip.fileList().value(0), &outBuffer); + unzip.closeArchive(); + if (errorCode != UnZip::Ok) { + qDebug() << "RAM benchmark: zip extraction failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is zip-compressed but this build has no zlib support"; + return {}; +#endif + } + return payload; +} + +TEST(OracleBenchmark, ImportRamUsage) +{ + static constexpr int numSets = 30; + static constexpr int cardsPerSet = 2000; // ~60k cards, roughly AllPrintings scale + + // Baseline must precede the fixture build: a high-water mark set while + // generating the synthetic JSON would otherwise mask the importer phases. + // Where memory stats are unavailable (Windows), skip before doing the + // 60k-card fixture build, which would otherwise be pure wasted work. + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + const QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(data)); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (synthetic): %1 sets, %2 cards, %3 MB JSON") + .arg(importedSets) + .arg(totalCards) + .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("parse", baseline, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + +TEST(OracleBenchmark, ImportRamUsageAllPrintings) +{ + // Only "1" enables the download: unset (the default and the CI setup) and + // an explicit "0" both disable it. + bool envOk = false; + const int enabled = qEnvironmentVariableIntValue("COCKATRICE_ORACLE_RAM_BENCHMARK", &envOk); + if (!envOk || enabled == 0) { + GTEST_SKIP() << "Set COCKATRICE_ORACLE_RAM_BENCHMARK=1 to download the real AllPrintings dataset for this " + "RAM benchmark. Default URL: " + << kDefaultAllPrintingsUrl.toDisplayString().toStdString(); + } + + // Baseline must precede the request so the phase covers the download + + // decompress step, including the payload materialized by readAll(). + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + QNetworkAccessManager nam; + QNetworkRequest request(kDefaultAllPrintingsUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "Cockatrice Oracle RAM benchmark"); + QNetworkReply *reply = nam.get(request); + + QEventLoop loop; + QTimer timeoutTimer; + timeoutTimer.setSingleShot(true); + bool timedOut = false; + QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeoutTimer, &QTimer::timeout, &loop, [&] { + timedOut = true; + reply->abort(); + }); + timeoutTimer.start(10 * 60 * 1000); + loop.exec(); + timeoutTimer.stop(); + + // abort() leaves reply->error() as OperationCanceledError, so a timed-out + // download takes the same GTEST_SKIP path as any other network error + // instead of reading a truncated body and failing the parse below. + if (timedOut || reply->error() != QNetworkReply::NoError) { + GTEST_SKIP() << "Download failed: " << reply->errorString().toStdString(); + } + const QByteArray payload = reply->readAll(); + reply->deleteLater(); + + // mtgjson can answer 200 with an HTML page (mirrors the wizard's '<' check + // in pages.cpp); reject it before trying to decompress/parse. + if (payload.startsWith("<")) { + GTEST_SKIP() << "Download returned a non-JSON body (HTML page instead of data), skipping"; + } + + const QByteArray setsData = decompressSetsData(payload); + const MemorySnapshot afterDownload = MemorySnapshot::current(); + if (setsData.isEmpty()) { + GTEST_SKIP() << "No data to import (download or decompression failed)"; + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(setsData)); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (real AllPrintings): %1 sets, %2 unique cards") + .arg(importedSets) + .arg(totalCards); + qDebug().noquote() << QString(" URL: %1").arg(kDefaultAllPrintingsUrl.toDisplayString()); + qDebug().noquote() << QString(" Downloaded: %1 MB, decompressed: %2 MB") + .arg(payload.size() / (1024.0 * 1024.0), 0, 'f', 1) + .arg(setsData.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("download+decompress", baseline, afterDownload); + logRamPhase("parse", afterDownload, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + int main(int argc, char **argv) { + // Required for the event loop used by the real-AllPrintings download benchmark + QCoreApplication app(argc, argv); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } From 34d96d9fe55b49baa56338fb8d698bb003273f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 2 Sep 2026 14:55:52 +0200 Subject: [PATCH 23/25] [Oracle/Tests] Measure parse against post-fixture baseline; assert release empties sets --- .../oracle/oracle_importer_benchmark_test.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index fe2d0f107..567519a48 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -353,8 +353,9 @@ static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, co // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a // peak-based delta between phases is ~0.0 MB by construction once the // fixture build has set the process peak. The live signals are current RSS - // and the process peak; the delta is only meaningful where current RSS is - // a per-phase value (see "after releaseSetData()"). + // and the process peak; the delta is meaningful only where the baseline was + // taken immediately before the phase it measures (e.g. the import phase, + // which compares afterParse against afterImport). QString rssDelta = "N/A"; if (current.rssKb >= 0 && baseline.rssKb >= 0) { rssDelta = formatKb(current.rssKb - baseline.rssKb); @@ -435,6 +436,13 @@ TEST(OracleBenchmark, ImportRamUsage) const QByteArray data = buildSyntheticData(numSets, cardsPerSet); + // The fixture build leaves freed-but-unreturned arenas behind (current RSS + // rarely falls once glibc allocates). Baseline immediately after it so the + // parse phase measures only the importer's own growth (~40 MB) rather than + // swallowing the fixture builder's spike. + const MemorySnapshot afterFixture = MemorySnapshot::current(); + logRamPhase("fixture build", baseline, afterFixture); + NoopCardSetPriorityController controller; OracleImporter importer; @@ -459,9 +467,14 @@ TEST(OracleBenchmark, ImportRamUsage) .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1); qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); - logRamPhase("parse", baseline, afterParse); + logRamPhase("parse", afterFixture, afterParse); logRamPhase("import", afterParse, afterImport); logRamPhase("after releaseSetData()", afterImport, afterRelease); + + // Freeing the parsed tree rarely moves current RSS (allocator reuse), so the + // meaningful signal that release actually dropped the buffers is emptiness, + // not an RSS delta. + ASSERT_TRUE(importer.getSets().isEmpty()); } TEST(OracleBenchmark, ImportRamUsageAllPrintings) From 99d8db3398df6d6498b2c6b14501958b316ca5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Sun, 30 Aug 2026 13:39:40 +0200 Subject: [PATCH 24/25] [Oracle] Parse sets lazily to slash importer peak memory - Add a raw JSON scanner that splits the document into per-set byte ranges without materializing the JSON tree - Keep only the raw document bytes and parse one set at a time in startImport - Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied - Clear the retained raw data in releaseSetData()/clear() - Cover the scanner and lazy parsing with tests Took 2 minutes --- oracle/CMakeLists.txt | 1 + oracle/src/oracleimporter.cpp | 93 ++- oracle/src/oracleimporter.h | 40 +- oracle/src/raw_json_scanner.cpp | 621 ++++++++++++++++++ oracle/src/raw_json_scanner.h | 76 +++ tests/oracle/CMakeLists.txt | 4 +- .../oracle/oracle_importer_benchmark_test.cpp | 7 +- tests/oracle/oracle_importer_test.cpp | 167 +++++ 8 files changed, 964 insertions(+), 45 deletions(-) create mode 100644 oracle/src/raw_json_scanner.cpp create mode 100644 oracle/src/raw_json_scanner.h diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 6a29b6935..bb1795caf 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,6 +23,7 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp + src/raw_json_scanner.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 49577baf0..6a5633215 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -44,26 +45,23 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s return priority; } -bool OracleImporter::readSetsFromByteArray(const QByteArray &data) +bool OracleImporter::readSetsFromByteArray(QByteArray data) { - QJsonParseError error; - auto doc = QJsonDocument::fromJson(data, &error); - if (error.error != QJsonParseError::NoError) { - qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(data, &error); + if (error.isError()) { + qDebug() << "error: RawJson::scanSetRanges():" << error.message; return false; } - auto setsObj = doc.object().value("data").toObject(); - QList newSetList; + newSetList.reserve(ranges.size()); - for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { - QJsonObject setObj = it.value().toObject(); - QString shortName = setObj.value("code").toString().toUpper(); - QString longName = setObj.value("name").toString(); - QJsonArray setCards = setObj.value("cards").toArray(); - QString setType = setObj.value("type").toString(); - QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); + for (const RawJson::SetRange &range : ranges) { + QString shortName = range.code.toUpper(); + QString longName = range.name; + QString setType = range.type; + QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { @@ -83,7 +81,9 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) } setType = setType.trimmed(); } - newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate)); + SetToDownload set(shortName, longName, priority, setType, releaseDate); + set.setRawRange(range.dataRange); + newSetList.append(set); } std::sort(newSetList.begin(), newSetList.end()); @@ -92,6 +92,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) return false; } allSets = newSetList; + rawSetsData = std::move(data); return true; } @@ -550,22 +551,17 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); - // Pre-allocate the cards hash to avoid rehashing during import. The hash - // is keyed by distinct card name rather than by printings: AllPrintings - // ships ~100k printings for ~35k names, so reserving the printing count - // would overallocate ~3x (against this stack's RAM goal). Collecting - // distinct names is cheap — one pass over the already-parsed name fields. - { - QSet distinctNames; - for (const SetToDownload &curSetToParse : allSets) { - for (const QJsonValue &cardValue : curSetToParse.getCards()) { - distinctNames.insert(cardValue.toObject().value("name").toString()); - } - } - cards.reserve(distinctNames.size()); - // The set goes out of scope here, handing the ~35k name QStrings back - // to the allocator before the (memory-heavy) import loop starts. +<<<<<<< HEAD + // Pre-allocate the cards hash to avoid rehashing during import. Keys are + // distinct card names while raw ranges only count printings (AllPrintings + // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact + // distinct-name count would require eagerly parsing, which the lazy reader + // deliberately avoids. It's a capacity hint, so the overshoot is harmless. + int estimatedCards = 0; + for (const SetToDownload &curSetToParse : allSets) { + estimatedCards += curSetToParse.getRawRange().cardCount; } + cards.reserve(estimatedCards); // add an empty set for tokens CardSetPtr tokenSet = @@ -578,11 +574,44 @@ int OracleImporter::startImport() CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(), curSetToParse.getLongName(), curSetToParse.getSetType(), curSetToParse.getReleaseDate(), curSetToParse.getPriority()); + + // parse only this set's slice of the raw document so the whole JSON tree is + // never kept in memory at once + const RawJson::SetDataRange &rawRange = curSetToParse.getRawRange(); + const qsizetype rangeEnd = rawRange.start + rawRange.length; + if (rawRange.start < 0 || rawRange.length <= 0 || rangeEnd > rawSetsData.size()) { + // rawSetsData is cleared by releaseSetData() while SetToDownload copies + // taken from getSets() keep their ranges, and nothing else enforces the + // pairing — so never index past the buffer on stale/mismatched ranges. + qWarning() << "error: out-of-bounds raw range for set" << curSetToParse.getShortName() << "skipping"; + ++setIndex; + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + // sliced() shares the buffer instead of deep-copying the slice; the largest + // sets in AllPrintings are tens of MB, so the copy is worth avoiding here. + const QByteArray setBytes = rawSetsData.sliced(rawRange.start, rawRange.length); + QJsonParseError parseError; + const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError); + if (parseError.error != QJsonParseError::NoError) { + qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":" + << parseError.errorString(); + ++setIndex; + // Keep the progress accounting honest: a set that failed to parse + // still advanced the index, so report it (with zero imported cards) + // rather than letting SaveSetsPage's bar stall per failed set. + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + + // Only add the set to the database once its slice parsed cleanly; + // a set that fails here must not persist as an empty set in cards.xml. if (!sets.contains(newSet->getShortName())) { sets.insert(newSet->getShortName(), newSet); } - int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards()); + const QJsonArray setCards = setDoc.object().value("cards").toArray(); + int numCardsInSet = importCardsFromSet(newSet, setCards); ++setIndex; @@ -605,6 +634,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr void OracleImporter::releaseSetData() { allSets.clear(); + rawSetsData.clear(); } void OracleImporter::clear() @@ -612,4 +642,5 @@ void OracleImporter::clear() sets.clear(); cards.clear(); allSets.clear(); + rawSetsData.clear(); } diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 52a7cd349..8cb30ca40 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,9 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include "raw_json_scanner.h" + +#include #include #include #include @@ -46,10 +49,12 @@ class SetToDownload { private: QString shortName, longName; - QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; + // Byte range of this set's object within the importer's raw JSON text. Parsing + // one set at a time keeps peak memory low instead of holding the whole document. + RawJson::SetDataRange rawRange; public: const QString &getShortName() const @@ -60,10 +65,6 @@ public: { return longName; } - const QJsonArray &getCards() const - { - return cards; - } const QString &getSetType() const { return setType; @@ -76,16 +77,23 @@ public: { return priority; } + const RawJson::SetDataRange &getRawRange() const + { + return rawRange; + } SetToDownload(QString _shortName, QString _longName, - QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) - : shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)), - releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority) + : shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate), + setType(std::move(_setType)), priority(_priority) { } + void setRawRange(const RawJson::SetDataRange &_rawRange) + { + rawRange = _rawRange; + } bool operator<(const SetToDownload &set) const { return longName.compare(set.longName, Qt::CaseInsensitive) < 0; @@ -141,6 +149,12 @@ private: QList allSets; + /** + * The raw JSON text of the source document, retained for lazy per-set + * parsing during startImport(). Frees the card data as each set is imported. + */ + QByteArray rawSetsData; + CardInfoPtr addCard(QString name, const QString &text, bool isToken, @@ -153,7 +167,11 @@ signals: public: explicit OracleImporter(QObject *parent = nullptr); - bool readSetsFromByteArray(const QByteArray &data); + /** + * Scans the given JSON document for set metadata. Takes the data by value so + * the wizard can hand over its decompressed buffer without copying it. + */ + bool readSetsFromByteArray(QByteArray data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); @@ -169,6 +187,10 @@ public: { return allSets; } + const QByteArray &getRawSetsData() const + { + return rawSetsData; + } void releaseSetData(); void clear(); }; diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp new file mode 100644 index 000000000..01ea3fab5 --- /dev/null +++ b/oracle/src/raw_json_scanner.cpp @@ -0,0 +1,621 @@ +#include "raw_json_scanner.h" + +#include + +namespace +{ + +// Nesting cap matching QJsonDocument's limit, so a pathologically deep document +// fails shallowly instead of overflowing the stack through the recursive +// skipValue/skipArray/skipObject walk (Qt's parser caps at 1024 for the same +// reason and reports DeepNesting). +constexpr int kMaxNestingDepth = 1024; + +inline bool isWhitespace(char c) +{ + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +const char *skipWhitespace(const char *p, const char *end) +{ + while (p < end && isWhitespace(*p)) { + ++p; + } + return p; +} + +inline bool isHexDigit(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +inline quint8 hexValue(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; +} + +/** + * @brief Skips past a JSON string without decoding it, validating escapes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool skipString(const char *&p, const char *end) +{ + ++p; // opening quote + for (;;) { + const void *quote = memchr(p, '"', static_cast(end - p)); + if (!quote) { + return false; // unterminated string + } + // Backslash escapes can only appear before the closing quote, so bound + // the scan to the string extent instead of the rest of the document. + const void *backslash = memchr(p, '\\', static_cast(static_cast(quote) - p)); + if (!backslash) { + p = static_cast(quote) + 1; + return true; + } + const char *b = static_cast(backslash); + if (end - b < 2) { + return false; + } + const char escaped = b[1]; + if (escaped == 'u') { + if (end - b < 6) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(b[2 + i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(b[2 + i]); + } + p = b + 6; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect the low-surrogate escape for the second half + if (end - p < 6 || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } + continue; + } + switch (escaped) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + p = b + 2; + continue; + default: + return false; // invalid escape + } + } +} + +/** + * @brief Decodes a JSON string into @p out, validating it as it goes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool decodeString(const char *&p, const char *end, QString &out) +{ + out.clear(); + QByteArray utf8; + auto flush = [&out, &utf8]() { + if (!utf8.isEmpty()) { + out += QString::fromUtf8(utf8); + utf8.clear(); + } + }; + + ++p; // opening quote + while (p < end) { + const char c = *p; + if (c == '\\') { + flush(); + ++p; // escaped character + if (p >= end) { + return false; + } + const char escaped = *p; + if (escaped == 'u') { + ++p; // first hex digit + if (p + 4 > end) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(p[i]); + } + p += 4; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect a low-surrogate escape for the second half + if (p + 6 > end || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + out += QChar(codepoint); + out += QChar(low); + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } else { + out += QChar(codepoint); + } + continue; + } + switch (escaped) { + case '"': + out += '"'; + break; + case '\\': + out += '\\'; + break; + case '/': + out += '/'; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + default: + return false; + } + ++p; + continue; + } + if (c == '"') { + ++p; + flush(); + return true; + } + if (static_cast(c) < 0x20) { + return false; // unescaped control character is invalid JSON + } + utf8 += c; + ++p; + } + return false; +} + +/** + * @brief Reads a set-metadata field, tolerating null and non-string values. + * + * A set's metadata may carry null or non-string values in otherwise-valid + * payloads ("releaseDate": null, "type": 7). The token itself was already + * structurally validated by skipValue, so a non-string value is accepted and + * leaves @p out at its default (empty) — one bad set must not abort the + * import of every other set in the document. + */ +bool decodeStringMember(const char *&fs, const char *&fe, QString &out) +{ + if (fs >= fe) { + return false; + } + if (*fs != '"') { + return true; + } + return decodeString(fs, fe, out); +} + +bool matchLiteral(const char *&p, const char *end, const char *literal, int length) +{ + if (end - p < length || memcmp(p, literal, static_cast(length)) != 0) { + return false; + } + const char *after = p + length; + if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) { + return false; + } + p = after; + return true; +} + +bool skipNumber(const char *&p, const char *end) +{ + // JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? + if (p < end && *p == '-') { + ++p; + } + if (p < end && *p == '0') { + ++p; + } else if (p < end && *p >= '1' && *p <= '9') { + ++p; + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } else { + return false; + } + if (p < end && *p == '.') { + ++p; + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + if (p < end && (*p == 'e' || *p == 'E')) { + ++p; + if (p < end && (*p == '+' || *p == '-')) { + ++p; + } + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + return true; +} + +bool skipValue(const char *&p, const char *end, int depth); +bool skipObject(const char *&p, const char *end, int depth); +bool skipArray(const char *&p, const char *end, int depth); + +bool skipPrimitive(const char *&p, const char *end) +{ + if (p >= end) { + return false; + } + const char c = *p; + if (c == '"') { + return skipString(p, end); + } + if (c == 't') { + return matchLiteral(p, end, "true", 4); + } + if (c == 'f') { + return matchLiteral(p, end, "false", 5); + } + if (c == 'n') { + return matchLiteral(p, end, "null", 4); + } + if (c == '-' || (c >= '0' && c <= '9')) { + return skipNumber(p, end); + } + return false; +} + +bool skipObject(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + if (!skipString(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +bool skipArray(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + if (p < end && *p == ']') { + ++p; + return true; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + ++p; + return true; + } + return false; + } +} + +bool skipValue(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + const char c = *p; + if (c == '{') { + return skipObject(p, end, depth - 1); + } + if (c == '[') { + return skipArray(p, end, depth - 1); + } + return skipPrimitive(p, end); +} + +/** + * @brief Iterates the members of the object starting at @p p. + * + * For each member invokes @p memberCallback with the key and the byte range of + * its value. Advancing @p p is unaffected by the callback. + */ +template bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + QString key; + if (!decodeString(p, end, key)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + const char *valueStart = skipWhitespace(p, end); + const char *valueEnd = valueStart; + if (!skipValue(valueEnd, end, depth - 1)) { + return false; + } + if (!memberCallback(key, valueStart, valueEnd)) { + return false; + } + p = valueEnd; + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +// Counts the direct elements of an array value; returns -1 if the array is malformed. +int countArrayElements(const char *p, const char *end, int depth) +{ + if (depth <= 0) { + return -1; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + int count = 0; + if (p < end && *p == ']') { + return 0; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return -1; + } + ++count; + p = skipWhitespace(p, end); + if (p >= end) { + return -1; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + return count; + } + return -1; + } +} + +} // namespace + +namespace RawJson +{ + +QList scanSetRanges(const QByteArray &json, ScanError *error) +{ + QList ranges; + if (error) { + *error = ScanError{}; + } + + const auto fail = [&](const QString &message) -> QList { + if (error) { + error->message = message; + } + return {}; + }; + + const char *begin = json.constData(); + const char *end = begin + json.size(); + if (begin >= end) { + return fail(QStringLiteral("empty JSON document")); + } + + const char *p = skipWhitespace(begin, end); + if (p >= end || *p != '{') { + return fail(QStringLiteral("top-level JSON must be an object")); + } + + bool foundData = false; + bool malformedSetData = false; + + const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) { + if (key == QStringLiteral("data")) { + foundData = true; + if (valueStart >= valueEnd || *valueStart != '{') { + malformedSetData = true; + return false; + } + const char *setP = valueStart; + const bool ok = forEachObjectMember(setP, valueEnd, kMaxNestingDepth - 1, + [&](const QString &setCode, const char *setStart, const char *setEnd) { + if (setStart >= setEnd || *setStart != '{') { + malformedSetData = true; + return false; + } + SetRange range; + range.dataRange.start = setStart - begin; + range.dataRange.length = setEnd - setStart; + range.code = setCode; + + const char *memberP = setStart; + const bool metaOk = forEachObjectMember( + memberP, setEnd, kMaxNestingDepth - 2, + [&](const QString &field, const char *fs, const char *fe) { + if (field == QStringLiteral("code")) { + return decodeStringMember(fs, fe, range.code); + } + if (field == QStringLiteral("name")) { + return decodeStringMember(fs, fe, range.name); + } + if (field == QStringLiteral("type")) { + return decodeStringMember(fs, fe, range.type); + } + if (field == QStringLiteral("releaseDate")) { + return decodeStringMember(fs, fe, range.releaseDate); + } + if (field == QStringLiteral("cards")) { + if (fs >= fe) { + return false; + } + if (*fs != '[') { + // e.g. "cards": null — treat as an empty array, + // matching Qt's tolerance. + return true; + } + range.dataRange.cardCount = + countArrayElements(fs, fe, kMaxNestingDepth - 2); + return range.dataRange.cardCount >= 0; + } + return true; + }); + if (!metaOk) { + malformedSetData = true; + return false; + } + ranges.append(range); + return true; + }); + if (!ok) { + malformedSetData = true; + return false; + } + } + return true; + }; + + if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback)) { + return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")); + } + p = skipWhitespace(p, end); + if (p != end) { + return fail(QStringLiteral("trailing content after top-level JSON object")); + } + if (!foundData) { + return fail(QStringLiteral("missing \"data\" object")); + } + if (ranges.isEmpty()) { + return fail(QStringLiteral("no sets found in \"data\"")); + } + return ranges; +} + +} // namespace RawJson \ No newline at end of file diff --git a/oracle/src/raw_json_scanner.h b/oracle/src/raw_json_scanner.h new file mode 100644 index 000000000..f6e3a4647 --- /dev/null +++ b/oracle/src/raw_json_scanner.h @@ -0,0 +1,76 @@ +#ifndef RAW_JSON_SCANNER_H +#define RAW_JSON_SCANNER_H + +#include +#include +#include + +namespace RawJson +{ + +/** + * @brief The byte extent of a set's object inside the scanned document, plus + * the size of its cards array. This is the slice SetToDownload needs for lazy + * per-set parsing; the metadata strings live in SetRange alongside it. + */ +struct SetDataRange +{ + /** @brief Byte offset of the set's object within the scanned buffer. */ + qsizetype start = -1; + /** @brief Byte length of the set's object, including the surrounding braces. */ + qsizetype length = 0; + /** @brief Number of entries in the set's "cards" array. */ + int cardCount = 0; +}; + +struct SetRange +{ + /** @brief The byte slice of this set within the document. */ + SetDataRange dataRange; + QString code; + QString name; + QString type; + QString releaseDate; +}; + +struct ScanError +{ + bool isError() const + { + return !message.isEmpty(); + } + QString message; +}; + +/** + * @brief Scans a full MTGJSON document without materializing the JSON tree. + * + * Splits the top-level "data" object into per-set byte ranges and reads each + * set's metadata directly from the raw bytes. The oracle importer can then + * parse one set at a time during import, keeping peak memory far below a single + * QJsonDocument::fromJson() over the whole file. + * + * The whole document is structurally validated while scanning (strings, + * escapes, braces, and a trailing-content check) and nesting depth is capped at + * 1024 to match QJsonDocument, so pathologically deep documents fail shallowly + * instead of exhausting the stack. Verdicts agree with QJsonDocument::fromJson + * on structurally malformed input; unlike Qt, string metadata fields + * ("name", "type", "releaseDate", "code") tolerate null / non-string values by + * defaulting to empty rather than rejecting the whole document, so one broken + * set cannot abort the import of the rest. + * + * Following QJsonDocument::fromJson's convention, the parsed ranges are + * returned by value and any failure is reported through the @p error out + * parameter. + * + * @param json The raw MTGJSON document bytes. + * @param error Out parameter. Set to an error ScanError when the document + * cannot be parsed, otherwise left empty. Passing a null + * pointer disables error reporting. + * @return The detected per-set ranges, or an empty list on failure. + */ +QList scanSetRanges(const QByteArray &json, ScanError *error = nullptr); + +} // namespace RawJson + +#endif // RAW_JSON_SCANNER_H \ No newline at end of file diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index cbff4f19c..9bc5ee5be 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -11,7 +11,7 @@ add_test(NAME parse_cipt_test COMMAND parse_cipt_test) # Oracle importer unit tests add_executable( oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_test.cpp + ../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp ) if(NOT GTEST_FOUND) @@ -51,7 +51,7 @@ endif() add_executable( oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} + ../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} ) if(NOT GTEST_FOUND) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index 567519a48..633c9e40a 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -157,9 +157,10 @@ TEST(OracleBenchmark, ParseJsonThroughput) for (int i = 0; i < iterations; ++i) { OracleImporter importer; + QByteArray source = data; QElapsedTimer timer; timer.start(); - bool ok = importer.readSetsFromByteArray(data); + bool ok = importer.readSetsFromByteArray(std::move(source)); ASSERT_TRUE(ok); totalMs += timer.elapsed(); } @@ -448,7 +449,7 @@ TEST(OracleBenchmark, ImportRamUsage) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(data)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); @@ -540,7 +541,7 @@ TEST(OracleBenchmark, ImportRamUsageAllPrintings) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(setsData)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index 145a2ca0f..c42840f7e 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -545,6 +545,173 @@ TEST_F(OracleImporterTest, ApostropheNormalized) ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); } +// ============================================================================ +// RawJson scanner tests +// ============================================================================ + +TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse) +{ + QJsonObject root; + QJsonObject data; + data["AAA"] = makeCard("Alpha Card"); + data["BBB"] = makeCard("Beta Card"); + root["data"] = data; + + const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact); + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(bytes, &error); + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 2); + + const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject(); + for (const RawJson::SetRange &range : ranges) { + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(bytes.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError) + << range.code.toStdString() << ": " << parseError.errorString().toStdString(); + ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards) +{ + const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\"," + "\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\"," + "\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}"; + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(json, &error); + ASSERT_FALSE(error.isError()); + ASSERT_EQ(ranges.size(), 1); + + const RawJson::SetRange &range = ranges.first(); + ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key + const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00); + ASSERT_EQ(range.name, expectedName); + ASSERT_EQ(range.type, "expansion"); + ASSERT_EQ(range.releaseDate, "2024-01-05"); + ASSERT_EQ(range.dataRange.cardCount, 3); + + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(json.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError); + ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName); + ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3); +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson) +{ + const QList invalid = {"not json", + "[]", + "{\"data\":[]}", + "{\"data\":{}}", + "{\"other\":{}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\"," + "\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"}; + + for (const QByteArray &json : invalid) { + RawJson::ScanError error; + RawJson::scanSetRanges(json, &error); + EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesMatchesFullJsonParseVerdicts) +{ + // Verdicts must agree with QJsonDocument::fromJson for the inputs below — + // including the metadata quirks ("name": null, "type": 7, "releaseDate": null, + // "cards": null) that used to make the scanner reject sets Qt accepts. + const QList inputs = { + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[{" + "\"n\":1}]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":null,\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":null,\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":7,\"releaseDate\":\"2024-01-01\",\"cards\":null}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"releaseDate\":\"2024-01-01\",\"cards\":[1,2,3]}}}", + // structurally invalid JSON (both parsers must reject) + "not json", + "{\"data\":{\"A\":{\"name\":\"unterminated}}", + }; + + for (const QByteArray &input : inputs) { + QJsonParseError qtError; + QJsonDocument::fromJson(input, &qtError); + const bool qtOk = qtError.error == QJsonParseError::NoError; + + RawJson::ScanError scanError; + const QList ranges = RawJson::scanSetRanges(input, &scanError); + EXPECT_EQ(qtOk, !scanError.isError()) << "verdict mismatch for: " << input.constData(); + if (scanError.isError()) { + continue; + } + for (const RawJson::SetRange &range : ranges) { + QJsonParseError sliceError; + QJsonDocument::fromJson(QByteArray(input.constData() + range.dataRange.start, range.dataRange.length), + &sliceError); + EXPECT_EQ(sliceError.error, QJsonParseError::NoError) << "bad range slice for: " << input.constData(); + } + } +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsDeepNesting) +{ + // Far beyond the shared 1024 container cap: Qt reports DeepNesting and the + // scanner must reject too, without overflowing the stack through its + // recursive skipValue walk. + QString nesting; + nesting.reserve(10000); + for (int i = 0; i < 5000; ++i) { + nesting += '['; + } + for (int i = 0; i < 5000; ++i) { + nesting += ']'; + } + const QByteArray json = ("{\"data\":{\"A\":{\"code\":\"a\",\"cards\":" + nesting + "}}}").toUtf8(); + + QJsonParseError qtError; + QJsonDocument::fromJson(json, &qtError); + ASSERT_NE(qtError.error, QJsonParseError::NoError) << "expected Qt to reject deep nesting"; + + RawJson::ScanError scanError; + RawJson::scanSetRanges(json, &scanError); + ASSERT_TRUE(scanError.isError()) << "scanner accepted a document Qt rejects as too deeply nested"; +} + +// ============================================================================ +// Lazy per-set parsing tests +// ============================================================================ + +TEST_F(OracleImporterTest, StartImportParsesSetsLazily) +{ + QJsonObject setObj = makeCard("Lazy Import Card"); + QJsonArray cards; + cards.append(setObj); + QJsonObject dataSet; + dataSet["code"] = "tst"; + dataSet["name"] = "Test Set"; + dataSet["type"] = "expansion"; + dataSet["releaseDate"] = "2024-01-01"; + dataSet["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", dataSet}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_FALSE(importer->getRawSetsData().isEmpty()); + + const int importedSets = importer->startImport(); + ASSERT_EQ(importedSets, 1); + ASSERT_EQ(importer->getCardList().size(), 1); + ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull()); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); From 3b1d808c73db4a4c17c4e7d2fb2c62e676840999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Br=C3=BCbach?= Date: Wed, 2 Sep 2026 15:04:38 +0200 Subject: [PATCH 25/25] [Oracle] Fix nesting-depth cap, tolerate unescaped control chars, lazy-parse review fixes --- oracle/src/oracleimporter.cpp | 1 - oracle/src/raw_json_scanner.cpp | 16 +++++++-------- tests/oracle/oracle_importer_test.cpp | 29 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 6a5633215..d745b250c 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -551,7 +551,6 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); -<<<<<<< HEAD // Pre-allocate the cards hash to avoid rehashing during import. Keys are // distinct card names while raw ranges only count printings (AllPrintings // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp index 01ea3fab5..db972491c 100644 --- a/oracle/src/raw_json_scanner.cpp +++ b/oracle/src/raw_json_scanner.cpp @@ -213,9 +213,10 @@ bool decodeString(const char *&p, const char *end, QString &out) flush(); return true; } - if (static_cast(c) < 0x20) { - return false; // unescaped control character is invalid JSON - } + // Deliberately accept unescaped control characters (e.g. a tab inside + // a set name): QJsonDocument and skipString accept them too, so + // rejecting them here would fail the whole document on a byte that + // Qt is fine with — the very total-failure mode this scanner avoids. utf8 += c; ++p; } @@ -399,20 +400,19 @@ bool skipArray(const char *&p, const char *end, int depth) bool skipValue(const char *&p, const char *end, int depth) { - if (depth <= 0) { - return false; // nest deeper than the cap - } p = skipWhitespace(p, end); if (p >= end) { return false; } const char c = *p; if (c == '{') { - return skipObject(p, end, depth - 1); + // pass depth through: skipObject consumes the single decrement for this level + return skipObject(p, end, depth); } if (c == '[') { - return skipArray(p, end, depth - 1); + return skipArray(p, end, depth); } + // a primitive is a leaf, so it never wastes a nesting level return skipPrimitive(p, end); } diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index c42840f7e..3834be9ed 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -634,6 +634,9 @@ TEST_F(OracleImporterTest, ScanSetRangesMatchesFullJsonParseVerdicts) "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":null,\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":7,\"releaseDate\":\"2024-01-01\",\"cards\":null}}}", "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"releaseDate\":\"2024-01-01\",\"cards\":[1,2,3]}}}", + // unescaped control character inside a string: QJsonDocument and + // skipString both accept it, so the scanner must not reject the whole doc + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"N\tX\",\"releaseDate\":\"2024-01-01\",\"cards\":[{\"n\":1}]}}}", // structurally invalid JSON (both parsers must reject) "not json", "{\"data\":{\"A\":{\"name\":\"unterminated}}", @@ -683,6 +686,32 @@ TEST_F(OracleImporterTest, ScanSetRangesRejectsDeepNesting) ASSERT_TRUE(scanError.isError()) << "scanner accepted a document Qt rejects as too deeply nested"; } +TEST_F(OracleImporterTest, ScanSetRangesAcceptsQtMaxNesting) +{ + // Pins the boundary rather than only the far-past case: a depth Qt still + // accepts must be accepted by the scanner too. Before the fix the scanner's + // cap was roughly half of Qt's (each level cost two decrements), so a + // depth of 1000 here was rejected even though QJsonDocument parses it. + constexpr int depth = 1000; + QString nesting; + nesting.reserve(2 * depth); + for (int i = 0; i < depth; ++i) { + nesting += '['; + } + for (int i = 0; i < depth; ++i) { + nesting += ']'; + } + const QByteArray json = ("{\"data\":{\"A\":{\"code\":\"a\",\"cards\":" + nesting + "}}}").toUtf8(); + + QJsonParseError qtError; + QJsonDocument::fromJson(json, &qtError); + ASSERT_EQ(qtError.error, QJsonParseError::NoError) << "expected Qt to accept depth " << depth; + + RawJson::ScanError scanError; + RawJson::scanSetRanges(json, &scanError); + ASSERT_FALSE(scanError.isError()) << "scanner rejected a document Qt accepts at depth " << depth; +} + // ============================================================================ // Lazy per-set parsing tests // ============================================================================