diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 6a29b6935..bb1795caf 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,6 +23,7 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp + src/raw_json_scanner.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index a531b82d1..29f866164 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -43,31 +45,29 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s return priority; } -bool OracleImporter::readSetsFromByteArray(const QByteArray &data) +bool OracleImporter::readSetsFromByteArray(QByteArray data) { - QJsonParseError error; - auto doc = QJsonDocument::fromJson(data, &error); - if (error.error != QJsonParseError::NoError) { - qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(data, &error); + if (error.isError()) { + qDebug() << "error: RawJson::scanSetRanges():" << error.message; return false; } - auto setsObj = doc.object().value("data").toObject(); - QList newSetList; + newSetList.reserve(ranges.size()); - for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { - QJsonObject setObj = it.value().toObject(); - QString shortName = setObj.value("code").toString().toUpper(); - QString longName = setObj.value("name").toString(); - QJsonArray setCards = setObj.value("cards").toArray(); - QString setType = setObj.value("type").toString(); - QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); + for (const RawJson::SetRange &range : ranges) { + QString shortName = range.code.toUpper(); + QString longName = range.name; + QString setType = range.type; + QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { // basic grammar for words that aren't capitalized, like in "From the Vault" - const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", "of", "in", "and", "with", "or"}; + static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", + "of", "in", "and", "with", "or"}; QStringList words = setType.split("_"); setType.clear(); bool first = false; @@ -75,13 +75,15 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) if (first && noCapitalize.contains(item)) { setType += item + QString(" "); } else { - setType += item[0].toUpper() + item.mid(1, -1) + QString(" "); + setType += item[0].toUpper() + item.mid(1) + QString(" "); first = true; } } setType = setType.trimmed(); } - newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate)); + SetToDownload set(shortName, longName, priority, setType, releaseDate); + set.setRawRange(range.dataRange); + newSetList.append(set); } std::sort(newSetList.begin(), newSetList.end()); @@ -90,6 +92,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) return false; } allSets = newSetList; + rawSetsData = std::move(data); return true; } @@ -123,14 +126,8 @@ static void sortAndReduceColors(QString &colors) std::sort(colors.begin(), colors.end(), [](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); }); // reduce - QChar lastChar = '\0'; - for (int i = 0; i < colors.size(); ++i) { - if (colors.at(i) == lastChar) { - colors.remove(i, 1); - } else { - lastChar = colors.at(i); - } - } + auto last = std::unique(colors.begin(), colors.end()); + colors.erase(last, colors.end()); } CardInfoPtr OracleImporter::addCard(QString name, @@ -147,7 +144,11 @@ CardInfoPtr OracleImporter::addCard(QString name, if (existingIt != cards.constEnd()) { CardInfoPtr card = existingIt.value(); card->addToSet(printingInfo.getSet(), printingInfo); - card->combineLegalities(properties); + // Only merge legalities when the card has none yet, so multi-format + // printings don't overwrite each other's legality lists. + if (card->getProperties().filter(formatRegex).empty()) { + card->combineLegalities(properties); + } return card; } @@ -182,8 +183,9 @@ CardInfoPtr OracleImporter::addCard(QString name, // DETECT CARD POSITIONING INFO - bool landscapeOrientation = properties.value("maintype") == "Battle" || properties.value("layout") == "split" || - properties.value("layout") == "planar"; + QString layoutVal = properties.value("layout"); + bool landscapeOrientation = + properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar"; // cards that enter the field tapped bool cipt = parseCipt(name, text) || landscapeOrientation; @@ -224,7 +226,10 @@ CardInfoPtr OracleImporter::addCard(QString name, static QString getJsonString(const QJsonObject &obj, const QString &key) { - return obj.value(key).toString(); + // QVariant coerces numbers and booleans to text, while QJsonValue::toString() + // returns a null string for them — some MTGJSON fields (manaValue, + // convertedManaCost, isOnlineOnly, isRebalanced) carry those types. + return obj.value(key).toVariant().toString(); } int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList) @@ -329,12 +334,18 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson allNameProps.insert(faceName); // special handling properties - QString colors = card.value("colors").toVariant().toStringList().join(""); + QString colors; + for (const QJsonValue &color : card.value("colors").toArray()) { + colors += color.toString(); + } if (!colors.isEmpty()) { properties.insert("colors", colors); } - QString colorIdentity = card.value("colorIdentity").toVariant().toStringList().join(""); + QString colorIdentity; + for (const QJsonValue &color : card.value("colorIdentity").toArray()) { + colorIdentity += color.toString(); + } if (!colorIdentity.isEmpty()) { properties.insert("coloridentity", colorIdentity); } @@ -413,7 +424,8 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } - CardInfoPtr newCard = addCard(name + numComponent, text, isToken, properties, relatedCards, printingInfo); + CardInfoPtr newCard = + addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo); numCards++; } } @@ -446,7 +458,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) { if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty properties.insert(prop, thisCardPropertyValue); - } else if (prop == "colors") { // the card is both colors + } else if (prop == "colors" || prop == "coloridentity") { // the card is both colors properties.insert(prop, originalPropertyValue + thisCardPropertyValue); } else if (prop == "maintype") { // don't create maintypes with //es in them continue; @@ -458,7 +470,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJson } } } - CardInfoPtr newCard = addCard(name, text, isToken, properties, {}, printingInfo); + CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo); numCards++; } @@ -529,7 +541,7 @@ static FormatRulesNameMap buildDefaultMagicFormats() return defaultFormatRulesNameMap; } -FormatRulesNameMap OracleImporter::createDefaultMagicFormats() +const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats() { static const FormatRulesNameMap cached = buildDefaultMagicFormats(); return cached; @@ -539,6 +551,17 @@ int OracleImporter::startImport() { static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController(); + // Pre-allocate the cards hash to avoid rehashing during import. Keys are + // distinct card names while raw ranges only count printings (AllPrintings + // ~100k printings vs ~35k names), so this over-reserves somewhat; an exact + // distinct-name count would require eagerly parsing, which the lazy reader + // deliberately avoids. It's a capacity hint, so the overshoot is harmless. + int estimatedCards = 0; + for (const SetToDownload &curSetToParse : allSets) { + estimatedCards += curSetToParse.getRawRange().cardCount; + } + cards.reserve(estimatedCards); + // add an empty set for tokens CardSetPtr tokenSet = CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens"); @@ -554,7 +577,35 @@ int OracleImporter::startImport() sets.insert(newSet->getShortName(), newSet); } - int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards()); + // parse only this set's slice of the raw document so the whole JSON tree is + // never kept in memory at once + const RawJson::SetDataRange &rawRange = curSetToParse.getRawRange(); + const qsizetype rangeEnd = rawRange.start + rawRange.length; + if (rawRange.start < 0 || rawRange.length <= 0 || rangeEnd > rawSetsData.size()) { + // rawSetsData is cleared by releaseSetData() while SetToDownload copies + // taken from getSets() keep their ranges, and nothing else enforces the + // pairing — so never index past the buffer on stale/mismatched ranges. + qWarning() << "error: out-of-bounds raw range for set" << curSetToParse.getShortName() << "skipping"; + ++setIndex; + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + const QByteArray setBytes(rawSetsData.constData() + rawRange.start, rawRange.length); + QJsonParseError parseError; + const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError); + if (parseError.error != QJsonParseError::NoError) { + qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":" + << parseError.errorString(); + ++setIndex; + // Keep the progress accounting honest: a set that failed to parse + // still advanced the index, so report it (with zero imported cards) + // rather than letting SaveSetsPage's bar stall per failed set. + emit setIndexChanged(0, setIndex, curSetToParse.getLongName()); + continue; + } + + const QJsonArray setCards = setDoc.object().value("cards").toArray(); + int numCardsInSet = importCardsFromSet(newSet, setCards); ++setIndex; @@ -577,6 +628,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr void OracleImporter::releaseSetData() { allSets.clear(); + rawSetsData.clear(); } void OracleImporter::clear() @@ -584,4 +636,5 @@ void OracleImporter::clear() sets.clear(); cards.clear(); allSets.clear(); + rawSetsData.clear(); } diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 696e35cab..8cb30ca40 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,9 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include "raw_json_scanner.h" + +#include #include #include #include @@ -46,10 +49,12 @@ class SetToDownload { private: QString shortName, longName; - QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; + // Byte range of this set's object within the importer's raw JSON text. Parsing + // one set at a time keeps peak memory low instead of holding the whole document. + RawJson::SetDataRange rawRange; public: const QString &getShortName() const @@ -60,10 +65,6 @@ public: { return longName; } - const QJsonArray &getCards() const - { - return cards; - } const QString &getSetType() const { return setType; @@ -76,16 +77,23 @@ public: { return priority; } + const RawJson::SetDataRange &getRawRange() const + { + return rawRange; + } SetToDownload(QString _shortName, QString _longName, - QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) - : shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)), - releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority) + : shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate), + setType(std::move(_setType)), priority(_priority) { } + void setRawRange(const RawJson::SetDataRange &_rawRange) + { + rawRange = _rawRange; + } bool operator<(const SetToDownload &set) const { return longName.compare(set.longName, Qt::CaseInsensitive) < 0; @@ -141,6 +149,12 @@ private: QList allSets; + /** + * The raw JSON text of the source document, retained for lazy per-set + * parsing during startImport(). Frees the card data as each set is imported. + */ + QByteArray rawSetsData; + CardInfoPtr addCard(QString name, const QString &text, bool isToken, @@ -153,11 +167,18 @@ signals: public: explicit OracleImporter(QObject *parent = nullptr); - bool readSetsFromByteArray(const QByteArray &data); + /** + * Scans the given JSON document for set metadata. Takes the data by value so + * the wizard can hand over its decompressed buffer without copying it. + */ + bool readSetsFromByteArray(QByteArray data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); - FormatRulesNameMap createDefaultMagicFormats(); + /** + * @brief Returns the default format rules. The result is memoized on first use and must be treated as immutable. + */ + const FormatRulesNameMap &createDefaultMagicFormats(); const CardNameMap &getCardList() const { return cards; @@ -166,6 +187,10 @@ public: { return allSets; } + const QByteArray &getRawSetsData() const + { + return rawSetsData; + } void releaseSetData(); void clear(); }; diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp new file mode 100644 index 000000000..01ea3fab5 --- /dev/null +++ b/oracle/src/raw_json_scanner.cpp @@ -0,0 +1,621 @@ +#include "raw_json_scanner.h" + +#include + +namespace +{ + +// Nesting cap matching QJsonDocument's limit, so a pathologically deep document +// fails shallowly instead of overflowing the stack through the recursive +// skipValue/skipArray/skipObject walk (Qt's parser caps at 1024 for the same +// reason and reports DeepNesting). +constexpr int kMaxNestingDepth = 1024; + +inline bool isWhitespace(char c) +{ + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +const char *skipWhitespace(const char *p, const char *end) +{ + while (p < end && isWhitespace(*p)) { + ++p; + } + return p; +} + +inline bool isHexDigit(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +inline quint8 hexValue(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; +} + +/** + * @brief Skips past a JSON string without decoding it, validating escapes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool skipString(const char *&p, const char *end) +{ + ++p; // opening quote + for (;;) { + const void *quote = memchr(p, '"', static_cast(end - p)); + if (!quote) { + return false; // unterminated string + } + // Backslash escapes can only appear before the closing quote, so bound + // the scan to the string extent instead of the rest of the document. + const void *backslash = memchr(p, '\\', static_cast(static_cast(quote) - p)); + if (!backslash) { + p = static_cast(quote) + 1; + return true; + } + const char *b = static_cast(backslash); + if (end - b < 2) { + return false; + } + const char escaped = b[1]; + if (escaped == 'u') { + if (end - b < 6) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(b[2 + i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(b[2 + i]); + } + p = b + 6; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect the low-surrogate escape for the second half + if (end - p < 6 || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } + continue; + } + switch (escaped) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + p = b + 2; + continue; + default: + return false; // invalid escape + } + } +} + +/** + * @brief Decodes a JSON string into @p out, validating it as it goes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool decodeString(const char *&p, const char *end, QString &out) +{ + out.clear(); + QByteArray utf8; + auto flush = [&out, &utf8]() { + if (!utf8.isEmpty()) { + out += QString::fromUtf8(utf8); + utf8.clear(); + } + }; + + ++p; // opening quote + while (p < end) { + const char c = *p; + if (c == '\\') { + flush(); + ++p; // escaped character + if (p >= end) { + return false; + } + const char escaped = *p; + if (escaped == 'u') { + ++p; // first hex digit + if (p + 4 > end) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(p[i]); + } + p += 4; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect a low-surrogate escape for the second half + if (p + 6 > end || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + out += QChar(codepoint); + out += QChar(low); + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } else { + out += QChar(codepoint); + } + continue; + } + switch (escaped) { + case '"': + out += '"'; + break; + case '\\': + out += '\\'; + break; + case '/': + out += '/'; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + default: + return false; + } + ++p; + continue; + } + if (c == '"') { + ++p; + flush(); + return true; + } + if (static_cast(c) < 0x20) { + return false; // unescaped control character is invalid JSON + } + utf8 += c; + ++p; + } + return false; +} + +/** + * @brief Reads a set-metadata field, tolerating null and non-string values. + * + * A set's metadata may carry null or non-string values in otherwise-valid + * payloads ("releaseDate": null, "type": 7). The token itself was already + * structurally validated by skipValue, so a non-string value is accepted and + * leaves @p out at its default (empty) — one bad set must not abort the + * import of every other set in the document. + */ +bool decodeStringMember(const char *&fs, const char *&fe, QString &out) +{ + if (fs >= fe) { + return false; + } + if (*fs != '"') { + return true; + } + return decodeString(fs, fe, out); +} + +bool matchLiteral(const char *&p, const char *end, const char *literal, int length) +{ + if (end - p < length || memcmp(p, literal, static_cast(length)) != 0) { + return false; + } + const char *after = p + length; + if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) { + return false; + } + p = after; + return true; +} + +bool skipNumber(const char *&p, const char *end) +{ + // JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? + if (p < end && *p == '-') { + ++p; + } + if (p < end && *p == '0') { + ++p; + } else if (p < end && *p >= '1' && *p <= '9') { + ++p; + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } else { + return false; + } + if (p < end && *p == '.') { + ++p; + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + if (p < end && (*p == 'e' || *p == 'E')) { + ++p; + if (p < end && (*p == '+' || *p == '-')) { + ++p; + } + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + return true; +} + +bool skipValue(const char *&p, const char *end, int depth); +bool skipObject(const char *&p, const char *end, int depth); +bool skipArray(const char *&p, const char *end, int depth); + +bool skipPrimitive(const char *&p, const char *end) +{ + if (p >= end) { + return false; + } + const char c = *p; + if (c == '"') { + return skipString(p, end); + } + if (c == 't') { + return matchLiteral(p, end, "true", 4); + } + if (c == 'f') { + return matchLiteral(p, end, "false", 5); + } + if (c == 'n') { + return matchLiteral(p, end, "null", 4); + } + if (c == '-' || (c >= '0' && c <= '9')) { + return skipNumber(p, end); + } + return false; +} + +bool skipObject(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + if (!skipString(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +bool skipArray(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + if (p < end && *p == ']') { + ++p; + return true; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + ++p; + return true; + } + return false; + } +} + +bool skipValue(const char *&p, const char *end, int depth) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + const char c = *p; + if (c == '{') { + return skipObject(p, end, depth - 1); + } + if (c == '[') { + return skipArray(p, end, depth - 1); + } + return skipPrimitive(p, end); +} + +/** + * @brief Iterates the members of the object starting at @p p. + * + * For each member invokes @p memberCallback with the key and the byte range of + * its value. Advancing @p p is unaffected by the callback. + */ +template bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback) +{ + if (depth <= 0) { + return false; // nest deeper than the cap + } + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + QString key; + if (!decodeString(p, end, key)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + const char *valueStart = skipWhitespace(p, end); + const char *valueEnd = valueStart; + if (!skipValue(valueEnd, end, depth - 1)) { + return false; + } + if (!memberCallback(key, valueStart, valueEnd)) { + return false; + } + p = valueEnd; + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +// Counts the direct elements of an array value; returns -1 if the array is malformed. +int countArrayElements(const char *p, const char *end, int depth) +{ + if (depth <= 0) { + return -1; // nest deeper than the cap + } + ++p; // '[' + p = skipWhitespace(p, end); + int count = 0; + if (p < end && *p == ']') { + return 0; + } + for (;;) { + if (!skipValue(p, end, depth - 1)) { + return -1; + } + ++count; + p = skipWhitespace(p, end); + if (p >= end) { + return -1; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + return count; + } + return -1; + } +} + +} // namespace + +namespace RawJson +{ + +QList scanSetRanges(const QByteArray &json, ScanError *error) +{ + QList ranges; + if (error) { + *error = ScanError{}; + } + + const auto fail = [&](const QString &message) -> QList { + if (error) { + error->message = message; + } + return {}; + }; + + const char *begin = json.constData(); + const char *end = begin + json.size(); + if (begin >= end) { + return fail(QStringLiteral("empty JSON document")); + } + + const char *p = skipWhitespace(begin, end); + if (p >= end || *p != '{') { + return fail(QStringLiteral("top-level JSON must be an object")); + } + + bool foundData = false; + bool malformedSetData = false; + + const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) { + if (key == QStringLiteral("data")) { + foundData = true; + if (valueStart >= valueEnd || *valueStart != '{') { + malformedSetData = true; + return false; + } + const char *setP = valueStart; + const bool ok = forEachObjectMember(setP, valueEnd, kMaxNestingDepth - 1, + [&](const QString &setCode, const char *setStart, const char *setEnd) { + if (setStart >= setEnd || *setStart != '{') { + malformedSetData = true; + return false; + } + SetRange range; + range.dataRange.start = setStart - begin; + range.dataRange.length = setEnd - setStart; + range.code = setCode; + + const char *memberP = setStart; + const bool metaOk = forEachObjectMember( + memberP, setEnd, kMaxNestingDepth - 2, + [&](const QString &field, const char *fs, const char *fe) { + if (field == QStringLiteral("code")) { + return decodeStringMember(fs, fe, range.code); + } + if (field == QStringLiteral("name")) { + return decodeStringMember(fs, fe, range.name); + } + if (field == QStringLiteral("type")) { + return decodeStringMember(fs, fe, range.type); + } + if (field == QStringLiteral("releaseDate")) { + return decodeStringMember(fs, fe, range.releaseDate); + } + if (field == QStringLiteral("cards")) { + if (fs >= fe) { + return false; + } + if (*fs != '[') { + // e.g. "cards": null — treat as an empty array, + // matching Qt's tolerance. + return true; + } + range.dataRange.cardCount = + countArrayElements(fs, fe, kMaxNestingDepth - 2); + return range.dataRange.cardCount >= 0; + } + return true; + }); + if (!metaOk) { + malformedSetData = true; + return false; + } + ranges.append(range); + return true; + }); + if (!ok) { + malformedSetData = true; + return false; + } + } + return true; + }; + + if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback)) { + return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")); + } + p = skipWhitespace(p, end); + if (p != end) { + return fail(QStringLiteral("trailing content after top-level JSON object")); + } + if (!foundData) { + return fail(QStringLiteral("missing \"data\" object")); + } + if (ranges.isEmpty()) { + return fail(QStringLiteral("no sets found in \"data\"")); + } + return ranges; +} + +} // namespace RawJson \ No newline at end of file diff --git a/oracle/src/raw_json_scanner.h b/oracle/src/raw_json_scanner.h new file mode 100644 index 000000000..f6e3a4647 --- /dev/null +++ b/oracle/src/raw_json_scanner.h @@ -0,0 +1,76 @@ +#ifndef RAW_JSON_SCANNER_H +#define RAW_JSON_SCANNER_H + +#include +#include +#include + +namespace RawJson +{ + +/** + * @brief The byte extent of a set's object inside the scanned document, plus + * the size of its cards array. This is the slice SetToDownload needs for lazy + * per-set parsing; the metadata strings live in SetRange alongside it. + */ +struct SetDataRange +{ + /** @brief Byte offset of the set's object within the scanned buffer. */ + qsizetype start = -1; + /** @brief Byte length of the set's object, including the surrounding braces. */ + qsizetype length = 0; + /** @brief Number of entries in the set's "cards" array. */ + int cardCount = 0; +}; + +struct SetRange +{ + /** @brief The byte slice of this set within the document. */ + SetDataRange dataRange; + QString code; + QString name; + QString type; + QString releaseDate; +}; + +struct ScanError +{ + bool isError() const + { + return !message.isEmpty(); + } + QString message; +}; + +/** + * @brief Scans a full MTGJSON document without materializing the JSON tree. + * + * Splits the top-level "data" object into per-set byte ranges and reads each + * set's metadata directly from the raw bytes. The oracle importer can then + * parse one set at a time during import, keeping peak memory far below a single + * QJsonDocument::fromJson() over the whole file. + * + * The whole document is structurally validated while scanning (strings, + * escapes, braces, and a trailing-content check) and nesting depth is capped at + * 1024 to match QJsonDocument, so pathologically deep documents fail shallowly + * instead of exhausting the stack. Verdicts agree with QJsonDocument::fromJson + * on structurally malformed input; unlike Qt, string metadata fields + * ("name", "type", "releaseDate", "code") tolerate null / non-string values by + * defaulting to empty rather than rejecting the whole document, so one broken + * set cannot abort the import of the rest. + * + * Following QJsonDocument::fromJson's convention, the parsed ranges are + * returned by value and any failure is reported through the @p error out + * parameter. + * + * @param json The raw MTGJSON document bytes. + * @param error Out parameter. Set to an error ScanError when the document + * cannot be parsed, otherwise left empty. Passing a null + * pointer disables error reporting. + * @return The detected per-set ranges, or an empty list on failure. + */ +QList scanSetRanges(const QByteArray &json, ScanError *error = nullptr); + +} // namespace RawJson + +#endif // RAW_JSON_SCANNER_H \ No newline at end of file diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index c5c1e9097..9bc5ee5be 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -7,3 +7,63 @@ endif() target_link_libraries(parse_cipt_test Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}) add_test(NAME parse_cipt_test COMMAND parse_cipt_test) + +# Oracle importer unit tests +add_executable( + oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + ../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_test gtest) +endif() + +target_link_libraries( + oracle_importer_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} +) + +add_test(NAME oracle_importer_test COMMAND oracle_importer_test) + +# Oracle importer benchmark tests (manual, not run in CI, incl. RAM benchmark) +# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark +# can download and decompress whatever AllPrintings format the default URL selects. +find_package(ZLIB) +if(ZLIB_FOUND) + add_definitions("-DHAS_ZLIB") + set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp) + set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES}) + include_directories(${ZLIB_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled") +endif() + +find_package(LibLZMA) +if(LIBLZMA_FOUND) + add_definitions("-DHAS_LZMA") + list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp) + list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES}) + include_directories(${LIBLZMA_INCLUDE_DIRS}) +else() + message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled") +endif() + +add_executable( + oracle_importer_benchmark_test + ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp + ../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} +) + +if(NOT GTEST_FOUND) + add_dependencies(oracle_importer_benchmark_test gtest) +endif() + +target_link_libraries( + oracle_importer_benchmark_test + libcockatrice_card + libcockatrice_interfaces + Threads::Threads + ${GTEST_BOTH_LIBRARIES} + ${TEST_QT_MODULES} + ${_ORACLE_BENCH_EXTRA_LIBRARIES} +) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp new file mode 100644 index 000000000..e3890bba7 --- /dev/null +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -0,0 +1,565 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(HAS_LZMA) +#include "../../oracle/src/lzma/decompress.h" +#endif +#if defined(HAS_ZLIB) +#include "../../oracle/src/zip/unzip.h" +#endif +#if defined(Q_OS_MACOS) +#include +#include +#endif + +// Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set +static QByteArray buildSyntheticData(int numSets, int cardsPerSet) +{ + QJsonObject dataObj; + for (int s = 0; s < numSets; ++s) { + QJsonArray cardsArray; + for (int c = 0; c < cardsPerSet; ++c) { + QJsonObject card; + card["name"] = QString("Card %1").arg(s * cardsPerSet + c); + card["text"] = "This is a test card with some rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["power"] = "2"; + card["toughness"] = "2"; + card["colors"] = QJsonArray{"W"}; + card["colorIdentity"] = QJsonArray{"W"}; + card["types"] = QJsonArray{"Creature"}; + // Real MTGJSON types: floats and booleans, not strings. This + // exercises the QVariant coercion in the property reader. + card["convertedManaCost"] = 1.0; + card["manaValue"] = 1.0; + card["isOnlineOnly"] = false; + card["isRebalanced"] = false; + + QJsonObject legalities; + legalities["standard"] = "legal"; + legalities["modern"] = "legal"; + legalities["legacy"] = "legal"; + legalities["vintage"] = "legal"; + legalities["commander"] = "legal"; + card["legalities"] = legalities; + + QJsonObject identifiers; + identifiers["scryfallId"] = QString("id-%1-%2").arg(s).arg(c); + card["identifiers"] = identifiers; + + // In AllPrintings, number and rarity are flat fields on the card + // object, exactly as set below. + card["number"] = QString::number(c + 1); + card["rarity"] = "common"; + + cardsArray.append(card); + } + + QJsonObject setObj; + setObj["code"] = QString("T%1").arg(s, 2, 10, QChar('0')); + setObj["name"] = QString("Test Set %1").arg(s); + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = cardsArray; + + dataObj[QString("T%1").arg(s, 2, 10, QChar('0'))] = setObj; + } + + QJsonObject root; + root["data"] = dataObj; + return QJsonDocument(root).toJson(QJsonDocument::Compact); +} + +// ============================================================================ +// Import throughput benchmark +// ============================================================================ + +TEST(OracleBenchmark, ImportThroughput) +{ + static constexpr int numSets = 10; + static constexpr int cardsPerSet = 500; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + OracleImporter importer; + + // Phase 1: Parse JSON + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(data); + ASSERT_TRUE(ok); + qint64 parseMs = timer.elapsed(); + + // Phase 2: Import cards + timer.restart(); + int importedSets = importer.startImport(); + qint64 importMs = timer.elapsed(); + + int totalImported = 0; + for (const auto &card : importer.getCardList()) { + Q_UNUSED(card); + totalImported++; + } + + // The fixture generates globally unique card names, so the expected + // counts are exact: a regression here means cards were dropped. + ASSERT_EQ(importedSets, numSets); + ASSERT_EQ(totalImported, numSets * cardsPerSet); + // Real-data probe: numeric convertedManaCost must be coerced to text + // (regression for the QJsonValue::toString() reader in #7214). + auto probeCard = importer.getCardList().value("Card 0"); + ASSERT_FALSE(probeCard.isNull()); + ASSERT_EQ(probeCard->getProperty("cmc"), "1"); + + qDebug().noquote() + << QString("Oracle Import Benchmark: %1 sets, %2 unique cards").arg(importedSets).arg(totalImported); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + qDebug().noquote() << QString(" Total: %1 ms").arg(parseMs + importMs); + if (importMs > 0) { + qDebug().noquote() << QString(" Throughput: %1 cards/sec") + .arg(static_cast(totalImported) / importMs * 1000.0, 0, 'f', 0); + } +} + +// ============================================================================ +// readSetsFromByteArray benchmark +// ============================================================================ + +TEST(OracleBenchmark, ParseJsonThroughput) +{ + static constexpr int numSets = 20; + static constexpr int cardsPerSet = 1000; + + QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + // Run 5 iterations and report average + static constexpr int iterations = 5; + qint64 totalMs = 0; + + for (int i = 0; i < iterations; ++i) { + OracleImporter importer; + QByteArray source = data; + QElapsedTimer timer; + timer.start(); + bool ok = importer.readSetsFromByteArray(std::move(source)); + ASSERT_TRUE(ok); + totalMs += timer.elapsed(); + } + + qint64 avgMs = totalMs / iterations; + qDebug().noquote() << QString("Parse Benchmark (%1 iterations): avg %2 ms for %3 sets x %4 cards") + .arg(iterations) + .arg(avgMs) + .arg(numSets) + .arg(cardsPerSet); +} + +// ============================================================================ +// Split card merging benchmark +// ============================================================================ + +TEST(OracleBenchmark, SplitCardMerging) +{ + static constexpr int numSplitCards = 1000; + + QJsonArray cardsList; + for (int i = 0; i < numSplitCards; ++i) { + QJsonObject face1; + face1["name"] = QString("Fire %1 // Ice %1").arg(i); + face1["text"] = "Fire side text."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = QString("Fire %1").arg(i); + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face1["identifiers"] = QJsonObject{{"scryfallId", QString("f-%1").arg(i)}}; + face1["number"] = QString::number(i + 1); + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = QString("Fire %1 // Ice %1").arg(i); + face2["text"] = "Ice side text."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = QString("Ice %1").arg(i); + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = QJsonObject{{"standard", "not_legal"}}; + face2["identifiers"] = QJsonObject{{"scryfallId", QString("i-%1").arg(i)}}; + face2["number"] = QString::number(i + 1); + face2["rarity"] = "uncommon"; + + cardsList.append(face1); + cardsList.append(face2); + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Split Test"); + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numSplitCards); + qDebug().noquote() << QString("Split Card Merge Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +// ============================================================================ +// sortAndReduceColors microbenchmark +// ============================================================================ + +// We can't call sortAndReduceColors directly (it's static), so we benchmark +// through importCardsFromSet with color properties. + +TEST(OracleBenchmark, ImportCardsWithColors) +{ + static constexpr int numCards = 10000; + + NoopCardSetPriorityController controller; + OracleImporter importer; + CardSetPtr set = CardSet::newInstance(&controller, "TST", "Color Test"); + + QJsonArray cardsList; + for (int i = 0; i < numCards; ++i) { + QJsonObject card; + card["name"] = QString("Color Card %1").arg(i); + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["colors"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["colorIdentity"] = QJsonArray{"B", "R", "G", "W", "U"}; + card["number"] = QString::number(i + 1); + card["rarity"] = "common"; + card["legalities"] = QJsonObject{{"standard", "legal"}}; + card["identifiers"] = QJsonObject{{"scryfallId", QString("c-%1").arg(i)}}; + cardsList.append(card); + } + + QElapsedTimer timer; + timer.start(); + int count = importer.importCardsFromSet(set, cardsList); + qint64 ms = timer.elapsed(); + + ASSERT_EQ(count, numCards); + qDebug().noquote() << QString("Import with Colors Benchmark: %1 cards in %2 ms (%3 cards/sec)") + .arg(count) + .arg(ms) + .arg(ms > 0 ? static_cast(count) / ms * 1000.0 : 0.0, 0, 'f', 0); +} + +// ============================================================================ +// RAM usage measurement +// ============================================================================ + +// Mirrors the default AllPrintings URL selection in oracle/src/pages.cpp. +#if defined(HAS_LZMA) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.xz"); +#elif defined(HAS_ZLIB) +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.zip"); +#else +static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json"); +#endif + +// Magic bytes also from oracle/src/pages.cpp +static const QByteArray kXzSignature("\xFD\x37\x7A\x58\x5A", 6); +static const QByteArray kZipSignature("PK"); + +struct MemorySnapshot +{ + qint64 peakRssKb = -1; // process high-water mark (VmHWM on Linux, ru_maxrss on macOS) + qint64 rssKb = -1; // current resident set size + bool available = false; + + static MemorySnapshot current() + { + MemorySnapshot snap; +#if defined(Q_OS_LINUX) + QFile statusFile("/proc/self/status"); + if (statusFile.open(QIODevice::ReadOnly | QIODevice::Text)) { + // /proc files report size() == 0, so atEnd() is immediately true: read everything first. + const QList lines = statusFile.readAll().split('\n'); + for (const QByteArray &line : lines) { + if (line.startsWith("VmHWM:")) { + snap.peakRssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } else if (line.startsWith("VmRSS:")) { + snap.rssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong(); + } + } + snap.available = snap.peakRssKb >= 0; + } +#elif defined(Q_OS_MACOS) + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + snap.peakRssKb = usage.ru_maxrss / 1024; // bytes -> kB + snap.available = snap.peakRssKb >= 0; + } + // getrusage has no current-RSS equivalent; task_info's resident_size + // is the closest macOS analog to Linux VmRSS. + mach_task_basic_info info = {}; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, reinterpret_cast(&info), &count) == + KERN_SUCCESS) { + snap.rssKb = info.resident_size / 1024; + } +#endif + return snap; + } +}; + +static QString formatKb(qint64 kb) +{ + if (kb < 0) { + return "N/A"; + } + return QString("%1 MB").arg(kb / 1024.0, 0, 'f', 1); +} + +static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, const MemorySnapshot ¤t) +{ + if (!baseline.available || !current.available) { + qDebug().noquote() << QString(" %1: memory stats unavailable on this platform").arg(phase); + return; + } + // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a + // peak-based delta between phases is ~0.0 MB by construction once the + // fixture build has set the process peak. The live signals are current RSS + // and the process peak; the delta is only meaningful where current RSS is + // a per-phase value (see "after releaseSetData()"). + QString rssDelta = "N/A"; + if (current.rssKb >= 0 && baseline.rssKb >= 0) { + rssDelta = formatKb(current.rssKb - baseline.rssKb); + } + qDebug().noquote() << QString(" %1: current RSS %2 | delta vs baseline %3 | process peak %4") + .arg(phase) + .arg(formatKb(current.rssKb)) + .arg(rssDelta) + .arg(formatKb(current.peakRssKb)); +} + +// Decompresses the download payload when the default URL is a compressed build, +// mirroring the wizard's magic-byte handling in oracle/src/pages.cpp. +static QByteArray decompressSetsData(const QByteArray &payload) +{ + if (payload.startsWith(kXzSignature)) { +#if defined(HAS_LZMA) + QBuffer inBuffer(const_cast(&payload)); + QByteArray out; + QBuffer outBuffer(&out); + inBuffer.open(QIODevice::ReadOnly); + outBuffer.open(QIODevice::WriteOnly); + XzDecompressor xz; + if (!xz.decompress(&inBuffer, &outBuffer)) { + qDebug() << "RAM benchmark: xz decompression failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is xz-compressed but this build has no LZMA support"; + return {}; +#endif + } + if (payload.startsWith(kZipSignature)) { +#if defined(HAS_ZLIB) + QBuffer inBuffer(const_cast(&payload)); + inBuffer.open(QIODevice::ReadOnly); + UnZip unzip; + if (unzip.openArchive(&inBuffer) != UnZip::Ok) { + qDebug() << "RAM benchmark: zip archive open failed"; + return {}; + } + if (unzip.fileList().size() != 1) { + qDebug() << "RAM benchmark: zip archive doesn't contain exactly one file"; + return {}; + } + QByteArray out; + QBuffer outBuffer(&out); + outBuffer.open(QIODevice::WriteOnly); + const auto errorCode = unzip.extractFile(unzip.fileList().value(0), &outBuffer); + unzip.closeArchive(); + if (errorCode != UnZip::Ok) { + qDebug() << "RAM benchmark: zip extraction failed"; + return {}; + } + return out; +#else + qDebug() << "RAM benchmark: download is zip-compressed but this build has no zlib support"; + return {}; +#endif + } + return payload; +} + +TEST(OracleBenchmark, ImportRamUsage) +{ + static constexpr int numSets = 30; + static constexpr int cardsPerSet = 2000; // ~60k cards, roughly AllPrintings scale + + // Baseline must precede the fixture build: a high-water mark set while + // generating the synthetic JSON would otherwise mask the importer phases. + // Where memory stats are unavailable (Windows), skip before doing the + // 60k-card fixture build, which would otherwise be pure wasted work. + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + const QByteArray data = buildSyntheticData(numSets, cardsPerSet); + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data))); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (synthetic): %1 sets, %2 cards, %3 MB JSON") + .arg(importedSets) + .arg(totalCards) + .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("parse", baseline, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + +TEST(OracleBenchmark, ImportRamUsageAllPrintings) +{ + // Only "1" enables the download: unset (the default and the CI setup) and + // an explicit "0" both disable it. + bool envOk = false; + const int enabled = qEnvironmentVariableIntValue("COCKATRICE_ORACLE_RAM_BENCHMARK", &envOk); + if (!envOk || enabled == 0) { + GTEST_SKIP() << "Set COCKATRICE_ORACLE_RAM_BENCHMARK=1 to download the real AllPrintings dataset for this " + "RAM benchmark. Default URL: " + << kDefaultAllPrintingsUrl.toDisplayString().toStdString(); + } + + // Baseline must precede the request so the phase covers the download + + // decompress step, including the payload materialized by readAll(). + const MemorySnapshot baseline = MemorySnapshot::current(); + if (!baseline.available) { + GTEST_SKIP() << "Memory stats unavailable on this platform"; + } + + QNetworkAccessManager nam; + QNetworkRequest request(kDefaultAllPrintingsUrl); + request.setHeader(QNetworkRequest::UserAgentHeader, "Cockatrice Oracle RAM benchmark"); + QNetworkReply *reply = nam.get(request); + + QEventLoop loop; + QTimer timeoutTimer; + timeoutTimer.setSingleShot(true); + bool timedOut = false; + QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeoutTimer, &QTimer::timeout, &loop, [&] { + timedOut = true; + reply->abort(); + }); + timeoutTimer.start(10 * 60 * 1000); + loop.exec(); + timeoutTimer.stop(); + + // abort() leaves reply->error() as OperationCanceledError, so a timed-out + // download takes the same GTEST_SKIP path as any other network error + // instead of reading a truncated body and failing the parse below. + if (timedOut || reply->error() != QNetworkReply::NoError) { + GTEST_SKIP() << "Download failed: " << reply->errorString().toStdString(); + } + const QByteArray payload = reply->readAll(); + reply->deleteLater(); + + // mtgjson can answer 200 with an HTML page (mirrors the wizard's '<' check + // in pages.cpp); reject it before trying to decompress/parse. + if (payload.startsWith("<")) { + GTEST_SKIP() << "Download returned a non-JSON body (HTML page instead of data), skipping"; + } + + const QByteArray setsData = decompressSetsData(payload); + const MemorySnapshot afterDownload = MemorySnapshot::current(); + if (setsData.isEmpty()) { + GTEST_SKIP() << "No data to import (download or decompression failed)"; + } + + NoopCardSetPriorityController controller; + OracleImporter importer; + + QElapsedTimer timer; + timer.start(); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData))); + const qint64 parseMs = timer.elapsed(); + const MemorySnapshot afterParse = MemorySnapshot::current(); + + timer.restart(); + const int importedSets = importer.startImport(); + const qint64 importMs = timer.elapsed(); + const MemorySnapshot afterImport = MemorySnapshot::current(); + + importer.releaseSetData(); + const MemorySnapshot afterRelease = MemorySnapshot::current(); + + const int totalCards = importer.getCardList().size(); + qDebug().noquote() << QString("Oracle RAM Benchmark (real AllPrintings): %1 sets, %2 unique cards") + .arg(importedSets) + .arg(totalCards); + qDebug().noquote() << QString(" URL: %1").arg(kDefaultAllPrintingsUrl.toDisplayString()); + qDebug().noquote() << QString(" Downloaded: %1 MB, decompressed: %2 MB") + .arg(payload.size() / (1024.0 * 1024.0), 0, 'f', 1) + .arg(setsData.size() / (1024.0 * 1024.0), 0, 'f', 1); + qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs); + qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs); + logRamPhase("download+decompress", baseline, afterDownload); + logRamPhase("parse", afterDownload, afterParse); + logRamPhase("import", afterParse, afterImport); + logRamPhase("after releaseSetData()", afterImport, afterRelease); +} + +int main(int argc, char **argv) +{ + // Required for the event loop used by the real-AllPrintings download benchmark + QCoreApplication app(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp new file mode 100644 index 000000000..30f4ed92a --- /dev/null +++ b/tests/oracle/oracle_importer_test.cpp @@ -0,0 +1,692 @@ +#include "../../oracle/src/oracleimporter.h" + +#include "gtest/gtest.h" +#include +#include +#include +#include +#include +#include +#include + +class OracleImporterTest : public ::testing::Test +{ +protected: + void SetUp() override + { + controller = new NoopCardSetPriorityController(); + importer = new OracleImporter(); + set = CardSet::newInstance(controller, "TST", "Test Set"); + } + + void TearDown() override + { + delete importer; + delete controller; + } + + // Helper: build a minimal card JSON object + QJsonObject makeCard(const QString &name, + const QString &colors = "", + const QString &colorIdentity = "", + const QVariantMap &legalities = {}) + { + QJsonObject card; + card["name"] = name; + card["text"] = "Rules text."; + card["layout"] = "normal"; + card["manaCost"] = "{W}"; + card["type"] = "Creature — Human"; + card["types"] = QJsonArray{"Creature"}; + card["number"] = "1"; + card["rarity"] = "common"; + + if (!colors.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colors) { + arr.append(QString(c)); + } + card["colors"] = arr; + } + if (!colorIdentity.isEmpty()) { + QJsonArray arr; + for (const QChar &c : colorIdentity) { + arr.append(QString(c)); + } + card["colorIdentity"] = arr; + } + if (!legalities.isEmpty()) { + QJsonObject legalObj; + for (auto it = legalities.constBegin(); it != legalities.constEnd(); ++it) { + legalObj[it.key()] = it.value().toString(); + } + card["legalities"] = legalObj; + } + + QJsonObject identifiers; + identifiers["scryfallId"] = QUuid::createUuid().toString(QUuid::WithoutBraces); + card["identifiers"] = identifiers; + + return card; + } + + NoopCardSetPriorityController *controller; + OracleImporter *importer; + CardSetPtr set; +}; + +// ============================================================================ +// sortAndReduceColors tests (tested via importCardsFromSet) +// ============================================================================ + +TEST_F(OracleImporterTest, SortAndReduceColorsSingleColor) +{ + QJsonArray cards{makeCard("Red Card", "R", "R")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Red Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "R"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsDeduplicates) +{ + QJsonArray cards{makeCard("Dedup Card", "WWUUB", "WU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Dedup Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUB"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsSortsWUBRG) +{ + QJsonArray cards{makeCard("Sort Card", "RGW", "RGW")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Sort Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorsAllFive) +{ + QJsonArray cards{makeCard("Five Color", "BRGWU", "BRGWU")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Five Color"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "WUBRG"); +} + +TEST_F(OracleImporterTest, SortAndReduceColorIdentity) +{ + QJsonArray cards{makeCard("Color Id Card", "W", "GWR")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Color Id Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("coloridentity"), "WRG"); +} + +TEST_F(OracleImporterTest, SingleColorNotSorted) +{ + QJsonArray cards{makeCard("Single Card", "B", "B")}; + importer->importCardsFromSet(set, cards); + + auto card = importer->getCardList().value("Single Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("colors"), "B"); +} + +// ============================================================================ +// Legality guard tests +// ============================================================================ + +TEST_F(OracleImporterTest, NewCardKeepsLegalityProperties) +{ + // Verifies that format-* properties survive addCard on a fresh card + // (not the combineLegalities guard, which only runs on existing printings). + QVariantMap leg; + leg["standard"] = "legal"; + leg["modern"] = "legal"; + QJsonArray cards{makeCard("Legal Card", "", "", leg)}; + + importer->importCardsFromSet(set, cards); + auto card = importer->getCardList().value("Legal Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityMergeAllowedWhenCardHasNoLegalities) +{ + // First printing carries no legalities at all, so the guard's + // `properties.filter(formatRegex).empty()` predicate is true and the + // second printing's legalities must be merged in. + QJsonArray cards1{makeCard("Unmerged Card")}; + importer->importCardsFromSet(set, cards1); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg; + leg["standard"] = "legal"; + QJsonArray cards2{makeCard("Unmerged Card", "", "", leg)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Unmerged Card"); + ASSERT_FALSE(card.isNull()); + ASSERT_EQ(card->getProperty("format-standard"), "legal"); +} + +TEST_F(OracleImporterTest, LegalityGuardPreservesFirstPrinting) +{ + // First printing: standard=legal, modern=legal + QVariantMap leg1; + leg1["standard"] = "legal"; + leg1["modern"] = "legal"; + QJsonArray cards1{makeCard("Guarded Card", "", "", leg1)}; + importer->importCardsFromSet(set, cards1); + + // Second printing: standard=banned, modern=not_legal + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QVariantMap leg2; + leg2["standard"] = "banned"; + leg2["modern"] = "not_legal"; + QJsonArray cards2{makeCard("Guarded Card", "", "", leg2)}; + importer->importCardsFromSet(set2, cards2); + + auto card = importer->getCardList().value("Guarded Card"); + ASSERT_FALSE(card.isNull()); + // Guard should preserve first printing's legalities + ASSERT_EQ(card->getProperty("format-standard"), "legal"); + ASSERT_EQ(card->getProperty("format-modern"), "legal"); +} + +// ============================================================================ +// createDefaultMagicFormats tests +// ============================================================================ + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsContainsExpectedFormats) +{ + auto formats = importer->createDefaultMagicFormats(); + ASSERT_TRUE(formats.contains("standard")); + ASSERT_TRUE(formats.contains("modern")); + ASSERT_TRUE(formats.contains("legacy")); + ASSERT_TRUE(formats.contains("vintage")); + ASSERT_TRUE(formats.contains("commander")); + ASSERT_TRUE(formats.contains("pauper")); + ASSERT_TRUE(formats.contains("pioneer")); + ASSERT_TRUE(formats.contains("brawl")); + ASSERT_TRUE(formats.contains("historic")); + ASSERT_TRUE(formats.contains("timeless")); + ASSERT_TRUE(formats.contains("duel")); + ASSERT_TRUE(formats.contains("oathbreaker")); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsSingletonDeckSizes) +{ + auto formats = importer->createDefaultMagicFormats(); + auto commander = formats.value("commander"); + ASSERT_FALSE(commander.isNull()); + ASSERT_EQ(commander->minDeckSize, 100); + ASSERT_EQ(commander->maxDeckSize, 100); + ASSERT_EQ(commander->maxSideboardSize, 15); + + auto brawl = formats.value("brawl"); + ASSERT_FALSE(brawl.isNull()); + ASSERT_EQ(brawl->minDeckSize, 60); + ASSERT_EQ(brawl->maxDeckSize, 60); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsVintageHasRestricted) +{ + auto formats = importer->createDefaultMagicFormats(); + auto vintage = formats.value("vintage"); + ASSERT_FALSE(vintage.isNull()); + bool hasRestricted = false; + for (const auto &ac : vintage->allowedCounts) { + if (ac.label == "restricted") { + hasRestricted = true; + ASSERT_EQ(ac.max, 1); + } + } + ASSERT_TRUE(hasRestricted); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsRegexMatchesBasicLands) +{ + auto formats = importer->createDefaultMagicFormats(); + auto standard = formats.value("standard"); + ASSERT_FALSE(standard.isNull()); + ASSERT_FALSE(standard->exceptions.isEmpty()); + + auto &basicLandsException = standard->exceptions.first(); + ASSERT_FALSE(basicLandsException.conditions.isEmpty()); + + auto &condition = basicLandsException.conditions.first(); + ASSERT_EQ(condition.field, "type"); + ASSERT_EQ(condition.matchType, "regex"); + + // Verify the regex actually works (was broken before: \b = backspace, not word boundary) + QRegularExpression regex(condition.value); + ASSERT_TRUE(regex.isValid()); + ASSERT_TRUE(regex.match("Basic Land — Forest").hasMatch()); + ASSERT_TRUE(regex.match("Basic Snow Land — Mountain").hasMatch()); + ASSERT_FALSE(regex.match("Creature — Elf Warrior").hasMatch()); +} + +TEST_F(OracleImporterTest, CreateDefaultMagicFormatsCaching) +{ + // The memoized map returns the same FormatRulesPtr instances, so the + // shared pointers must be identical across calls. This is the only + // observable effect of the cache: contents would match either way. + auto first = importer->createDefaultMagicFormats(); + auto second = importer->createDefaultMagicFormats(); + ASSERT_EQ(first.value("standard").data(), second.value("standard").data()); +} + +// ============================================================================ +// readSetsFromByteArray tests +// ============================================================================ + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayValidJson) +{ + QJsonObject setObj; + setObj["code"] = "tst"; + setObj["name"] = "Test Set"; + setObj["type"] = "expansion"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"TST", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().size(), 1); + ASSERT_EQ(importer->getSets().first().getShortName(), "TST"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayInvalidJson) +{ + QByteArray data = "not valid json"; + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayEmptyData) +{ + QJsonObject root; + root["data"] = QJsonObject(); + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_FALSE(importer->readSetsFromByteArray(data)); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArrayCapitalizesSetType) +{ + QJsonObject setObj; + setObj["code"] = "ftv"; + setObj["name"] = "From The Vault"; + setObj["type"] = "from_the_vault"; + setObj["releaseDate"] = "2024-01-01"; + setObj["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"FTV", setObj}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_EQ(importer->getSets().first().getSetType(), "From the Vault"); +} + +TEST_F(OracleImporterTest, ReadSetsFromByteArraySortsSetsByName) +{ + // QJsonObject iterates keys in lexicographic order ("AAA" before "ZZZ"), + // so leaving the natural order matching the alphabetical sort makes the + // assertion pass trivially. Inverting it keeps the sort meaningful: + // iteration yields "AAA" (Zeta Set) first, then the sort by name must + // promote "ZZZ" (Alpha Set) to the front. + QJsonObject setA; + setA["code"] = "aaa"; + setA["name"] = "Zeta Set"; + setA["type"] = "expansion"; + setA["releaseDate"] = "2024-01-01"; + setA["cards"] = QJsonArray(); + + QJsonObject setB; + setB["code"] = "zzz"; + setB["name"] = "Alpha Set"; + setB["type"] = "expansion"; + setB["releaseDate"] = "2024-01-01"; + setB["cards"] = QJsonArray(); + + QJsonObject root; + root["data"] = QJsonObject{{"AAA", setA}, {"ZZZ", setB}}; + + QByteArray data = QJsonDocument(root).toJson(); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + auto sets = importer->getSets(); + ASSERT_GE(sets.size(), 2); + ASSERT_EQ(sets.first().getShortName(), "ZZZ"); +} + +// ============================================================================ +// Split card coloridentity tests +// ============================================================================ + +TEST_F(OracleImporterTest, SplitCardColorIdentityConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + int count = importer->importCardsFromSet(set, cardsList); + ASSERT_EQ(count, 1); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + // coloridentity should be "RU" (concatenated), then sorted to "UR" + // by sortAndReduceColors when it reaches addCard + ASSERT_EQ(card->getProperty("coloridentity"), "UR"); +} + +TEST_F(OracleImporterTest, SplitCardColorsConcatenated) +{ + QJsonObject leg{{"standard", "not_legal"}}; + + QJsonObject face1; + face1["name"] = "Fire // Ice"; + face1["text"] = "Fire deals 2 damage."; + face1["layout"] = "split"; + face1["side"] = "a"; + face1["faceName"] = "Fire"; + face1["colors"] = QJsonArray{"R"}; + face1["colorIdentity"] = QJsonArray{"R"}; + face1["types"] = QJsonArray{"Instant"}; + face1["manaCost"] = "{R}"; + face1["legalities"] = leg; + face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}}; + face1["number"] = "1"; + face1["rarity"] = "uncommon"; + + QJsonObject face2; + face2["name"] = "Fire // Ice"; + face2["text"] = "Ice taps target artifact."; + face2["layout"] = "split"; + face2["side"] = "b"; + face2["faceName"] = "Ice"; + face2["colors"] = QJsonArray{"U"}; + face2["colorIdentity"] = QJsonArray{"U"}; + face2["types"] = QJsonArray{"Instant"}; + face2["manaCost"] = "{U}"; + face2["legalities"] = leg; + face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}}; + face2["number"] = "1"; + face2["rarity"] = "uncommon"; + + QJsonArray cardsList{face1, face2}; + importer->importCardsFromSet(set, cardsList); + + auto card = importer->getCardList().value("Fire // Ice"); + ASSERT_FALSE(card.isNull()); + + QString colors = card->getProperty("colors"); + ASSERT_FALSE(colors.contains("//")) << "colors should not contain '//', got: " << colors.toStdString(); + ASSERT_TRUE(colors.contains("R")); + ASSERT_TRUE(colors.contains("U")); +} + +// ============================================================================ +// Mana cost formatting tests +// ============================================================================ + +TEST_F(OracleImporterTest, ManaCostStripsBraces) +{ + QJsonObject card = makeCard("Mana Card"); + card["manaCost"] = "{2}{W}{B}"; + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + auto result = importer->getCardList().value("Mana Card"); + ASSERT_FALSE(result.isNull()); + ASSERT_EQ(result->getProperty("manacost"), "2WB"); +} + +// ============================================================================ +// Card deduplication tests +// ============================================================================ + +TEST_F(OracleImporterTest, DuplicateCardNameReturnsExisting) +{ + QJsonArray cards{makeCard("Dupe Card")}; + importer->importCardsFromSet(set, cards); + + CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set"); + QJsonArray cards2{makeCard("Dupe Card")}; + importer->importCardsFromSet(set2, cards2); + + ASSERT_EQ(importer->getCardList().size(), 1); +} + +TEST_F(OracleImporterTest, AELigatureReplaced) +{ + QJsonObject card = makeCard(QString::fromUtf8("\xC3\x86ther Vial")); // Æther Vial + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + // Æ is replaced with AE, resulting in "AEther Vial" + ASSERT_FALSE(importer->getCardList().contains(QString::fromUtf8("\xC3\x86ther Vial"))); + ASSERT_TRUE(importer->getCardList().contains("AEther Vial")); +} + +TEST_F(OracleImporterTest, ApostropheNormalized) +{ + QJsonObject card = makeCard(QString::fromUtf8("Jace\u2019s Ingenuity")); + QJsonArray cards{card}; + + importer->importCardsFromSet(set, cards); + ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); +} + +// ============================================================================ +// RawJson scanner tests +// ============================================================================ + +TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse) +{ + QJsonObject root; + QJsonObject data; + data["AAA"] = makeCard("Alpha Card"); + data["BBB"] = makeCard("Beta Card"); + root["data"] = data; + + const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact); + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(bytes, &error); + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 2); + + const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject(); + for (const RawJson::SetRange &range : ranges) { + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(bytes.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError) + << range.code.toStdString() << ": " << parseError.errorString().toStdString(); + ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards) +{ + const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\"," + "\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\"," + "\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}"; + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(json, &error); + ASSERT_FALSE(error.isError()); + ASSERT_EQ(ranges.size(), 1); + + const RawJson::SetRange &range = ranges.first(); + ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key + const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00); + ASSERT_EQ(range.name, expectedName); + ASSERT_EQ(range.type, "expansion"); + ASSERT_EQ(range.releaseDate, "2024-01-05"); + ASSERT_EQ(range.dataRange.cardCount, 3); + + QJsonParseError parseError; + const QJsonDocument sliceDoc = QJsonDocument::fromJson( + QByteArray(json.constData() + range.dataRange.start, range.dataRange.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError); + ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName); + ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3); +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson) +{ + const QList invalid = {"not json", + "[]", + "{\"data\":[]}", + "{\"data\":{}}", + "{\"other\":{}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\"," + "\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"}; + + for (const QByteArray &json : invalid) { + RawJson::ScanError error; + RawJson::scanSetRanges(json, &error); + EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesMatchesFullJsonParseVerdicts) +{ + // Verdicts must agree with QJsonDocument::fromJson for the inputs below — + // including the metadata quirks ("name": null, "type": 7, "releaseDate": null, + // "cards": null) that used to make the scanner reject sets Qt accepts. + const QList inputs = { + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[{" + "\"n\":1}]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":null,\"type\":\"x\",\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":null,\"releaseDate\":\"2024-01-01\",\"cards\":[]}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":7,\"releaseDate\":\"2024-01-01\",\"cards\":null}}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"releaseDate\":\"2024-01-01\",\"cards\":[1,2,3]}}}", + // structurally invalid JSON (both parsers must reject) + "not json", + "{\"data\":{\"A\":{\"name\":\"unterminated}}", + }; + + for (const QByteArray &input : inputs) { + QJsonParseError qtError; + QJsonDocument::fromJson(input, &qtError); + const bool qtOk = qtError.error == QJsonParseError::NoError; + + RawJson::ScanError scanError; + const QList ranges = RawJson::scanSetRanges(input, &scanError); + EXPECT_EQ(qtOk, !scanError.isError()) << "verdict mismatch for: " << input.constData(); + if (scanError.isError()) { + continue; + } + for (const RawJson::SetRange &range : ranges) { + QJsonParseError sliceError; + QJsonDocument::fromJson(QByteArray(input.constData() + range.dataRange.start, range.dataRange.length), + &sliceError); + EXPECT_EQ(sliceError.error, QJsonParseError::NoError) << "bad range slice for: " << input.constData(); + } + } +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsDeepNesting) +{ + // Far beyond the shared 1024 container cap: Qt reports DeepNesting and the + // scanner must reject too, without overflowing the stack through its + // recursive skipValue walk. + QString nesting; + nesting.reserve(10000); + for (int i = 0; i < 5000; ++i) { + nesting += '['; + } + for (int i = 0; i < 5000; ++i) { + nesting += ']'; + } + const QByteArray json = ("{\"data\":{\"A\":{\"code\":\"a\",\"cards\":" + nesting + "}}}").toUtf8(); + + QJsonParseError qtError; + QJsonDocument::fromJson(json, &qtError); + ASSERT_NE(qtError.error, QJsonParseError::NoError) << "expected Qt to reject deep nesting"; + + RawJson::ScanError scanError; + RawJson::scanSetRanges(json, &scanError); + ASSERT_TRUE(scanError.isError()) << "scanner accepted a document Qt rejects as too deeply nested"; +} + +// ============================================================================ +// Lazy per-set parsing tests +// ============================================================================ + +TEST_F(OracleImporterTest, StartImportParsesSetsLazily) +{ + QJsonObject setObj = makeCard("Lazy Import Card"); + QJsonArray cards; + cards.append(setObj); + QJsonObject dataSet; + dataSet["code"] = "tst"; + dataSet["name"] = "Test Set"; + dataSet["type"] = "expansion"; + dataSet["releaseDate"] = "2024-01-01"; + dataSet["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", dataSet}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_FALSE(importer->getRawSetsData().isEmpty()); + + const int importedSets = importer->startImport(); + ASSERT_EQ(importedSets, 1); + ASSERT_EQ(importer->getCardList().size(), 1); + ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull()); +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}