diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 6a29b6935..bb1795caf 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -23,6 +23,7 @@ set(oracle_SOURCES src/pages.cpp src/pagetemplates.cpp src/parsehelpers.cpp + src/raw_json_scanner.cpp ../cockatrice/src/client/settings/cache_settings.cpp ../cockatrice/src/client/settings/card_counter_settings.cpp ../cockatrice/src/client/settings/shortcuts_settings.cpp diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index c7ae3f2f8..897264987 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -43,26 +44,23 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s return priority; } -bool OracleImporter::readSetsFromByteArray(const QByteArray &data) +bool OracleImporter::readSetsFromByteArray(QByteArray data) { - QJsonParseError error; - auto doc = QJsonDocument::fromJson(data, &error); - if (error.error != QJsonParseError::NoError) { - qDebug() << "error: QJsonDocument::fromJson():" << error.errorString(); + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(data, &error); + if (error.isError()) { + qDebug() << "error: RawJson::scanSetRanges():" << error.message; return false; } - auto setsObj = doc.object().value("data").toObject(); - QList newSetList; + newSetList.reserve(ranges.size()); - for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) { - QJsonObject setObj = it.value().toObject(); - QString shortName = setObj.value("code").toString().toUpper(); - QString longName = setObj.value("name").toString(); - QJsonArray setCards = setObj.value("cards").toArray(); - QString setType = setObj.value("type").toString(); - QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate); + for (const RawJson::SetRange &range : ranges) { + QString shortName = range.code.toUpper(); + QString longName = range.name; + QString setType = range.type; + QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate); CardSet::Priority priority = getSetPriority(setType, shortName); // capitalize set type if (setType.length() > 0) { @@ -82,7 +80,9 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) } setType = setType.trimmed(); } - newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate)); + SetToDownload set(shortName, longName, priority, setType, releaseDate); + set.setRawRange(range); + newSetList.append(set); } std::sort(newSetList.begin(), newSetList.end()); @@ -91,6 +91,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) return false; } allSets = newSetList; + rawSetsData = data; return true; } @@ -541,7 +542,7 @@ int OracleImporter::startImport() // Pre-allocate cards hash to avoid rehashing during import int estimatedCards = 0; for (const SetToDownload &curSetToParse : allSets) { - estimatedCards += curSetToParse.getCards().size(); + estimatedCards += curSetToParse.getRawRange().cardCount; } cards.reserve(estimatedCards); @@ -560,7 +561,21 @@ int OracleImporter::startImport() sets.insert(newSet->getShortName(), newSet); } - int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards()); + // parse only this set's slice of the raw document so the whole JSON tree is + // never kept in memory at once + const RawJson::SetRange &rawRange = curSetToParse.getRawRange(); + const QByteArray setBytes(rawSetsData.constData() + rawRange.start, rawRange.length); + QJsonParseError parseError; + const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError); + if (parseError.error != QJsonParseError::NoError) { + qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":" + << parseError.errorString(); + ++setIndex; + continue; + } + + const QJsonArray setCards = setDoc.object().value("cards").toArray(); + int numCardsInSet = importCardsFromSet(newSet, setCards); ++setIndex; @@ -583,6 +598,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr void OracleImporter::releaseSetData() { allSets.clear(); + rawSetsData.clear(); } void OracleImporter::clear() @@ -590,4 +606,5 @@ void OracleImporter::clear() sets.clear(); cards.clear(); allSets.clear(); + rawSetsData.clear(); } diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 696e35cab..1450e2d99 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,6 +1,9 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H +#include "raw_json_scanner.h" + +#include #include #include #include @@ -46,10 +49,12 @@ class SetToDownload { private: QString shortName, longName; - QJsonArray cards; QDate releaseDate; QString setType; CardSet::Priority priority; + // Byte range of this set's object within the importer's raw JSON text. Parsing + // one set at a time keeps peak memory low instead of holding the whole document. + RawJson::SetRange rawRange; public: const QString &getShortName() const @@ -60,10 +65,6 @@ public: { return longName; } - const QJsonArray &getCards() const - { - return cards; - } const QString &getSetType() const { return setType; @@ -76,16 +77,23 @@ public: { return priority; } + const RawJson::SetRange &getRawRange() const + { + return rawRange; + } SetToDownload(QString _shortName, QString _longName, - QJsonArray _cards, CardSet::Priority _priority, QString _setType = QString(), const QDate &_releaseDate = QDate()) - : shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)), - releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority) + : shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate), + setType(std::move(_setType)), priority(_priority) { } + void setRawRange(const RawJson::SetRange &_rawRange) + { + rawRange = _rawRange; + } bool operator<(const SetToDownload &set) const { return longName.compare(set.longName, Qt::CaseInsensitive) < 0; @@ -141,6 +149,12 @@ private: QList allSets; + /** + * The raw JSON text of the source document, retained for lazy per-set + * parsing during startImport(). Frees the card data as each set is imported. + */ + QByteArray rawSetsData; + CardInfoPtr addCard(QString name, const QString &text, bool isToken, @@ -153,7 +167,11 @@ signals: public: explicit OracleImporter(QObject *parent = nullptr); - bool readSetsFromByteArray(const QByteArray &data); + /** + * Scans the given JSON document for set metadata. Takes the data by value so + * the wizard can hand over its decompressed buffer without copying it. + */ + bool readSetsFromByteArray(QByteArray data); int startImport(); bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion); int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList); @@ -166,6 +184,10 @@ public: { return allSets; } + const QByteArray &getRawSetsData() const + { + return rawSetsData; + } void releaseSetData(); void clear(); }; diff --git a/oracle/src/raw_json_scanner.cpp b/oracle/src/raw_json_scanner.cpp new file mode 100644 index 000000000..0311605f3 --- /dev/null +++ b/oracle/src/raw_json_scanner.cpp @@ -0,0 +1,573 @@ +#include "raw_json_scanner.h" + +#include + +namespace +{ + +inline bool isWhitespace(char c) +{ + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +const char *skipWhitespace(const char *p, const char *end) +{ + while (p < end && isWhitespace(*p)) { + ++p; + } + return p; +} + +inline bool isHexDigit(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +inline quint8 hexValue(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return c - 'A' + 10; +} + +/** + * @brief Skips past a JSON string without decoding it, validating escapes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool skipString(const char *&p, const char *end) +{ + ++p; // opening quote + for (;;) { + const void *quote = memchr(p, '"', static_cast(end - p)); + if (!quote) { + return false; // unterminated string + } + // Backslash escapes can only appear before the closing quote, so bound + // the scan to the string extent instead of the rest of the document. + const void *backslash = memchr(p, '\\', static_cast(static_cast(quote) - p)); + if (!backslash) { + p = static_cast(quote) + 1; + return true; + } + const char *b = static_cast(backslash); + if (end - b < 2) { + return false; + } + const char escaped = b[1]; + if (escaped == 'u') { + if (end - b < 6) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(b[2 + i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(b[2 + i]); + } + p = b + 6; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect the low-surrogate escape for the second half + if (end - p < 6 || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } + continue; + } + switch (escaped) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + p = b + 2; + continue; + default: + return false; // invalid escape + } + } +} + +/** + * @brief Decodes a JSON string into @p out, validating it as it goes. + * @param p In: pointing at the opening quote. Out: pointing past the closing quote. + */ +bool decodeString(const char *&p, const char *end, QString &out) +{ + out.clear(); + QByteArray utf8; + auto flush = [&out, &utf8]() { + if (!utf8.isEmpty()) { + out += QString::fromUtf8(utf8); + utf8.clear(); + } + }; + + ++p; // opening quote + while (p < end) { + const char c = *p; + if (c == '\\') { + flush(); + ++p; // escaped character + if (p >= end) { + return false; + } + const char escaped = *p; + if (escaped == 'u') { + ++p; // first hex digit + if (p + 4 > end) { + return false; + } + quint32 codepoint = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[i])) { + return false; + } + codepoint = codepoint * 16 + hexValue(p[i]); + } + p += 4; + if (codepoint >= 0xD800 && codepoint <= 0xDBFF) { + // expect a low-surrogate escape for the second half + if (p + 6 > end || p[0] != '\\' || p[1] != 'u') { + return false; // unpaired high surrogate + } + quint32 low = 0; + for (int i = 0; i < 4; ++i) { + if (!isHexDigit(p[2 + i])) { + return false; + } + low = low * 16 + hexValue(p[2 + i]); + } + if (low < 0xDC00 || low > 0xDFFF) { + return false; + } + out += QChar(codepoint); + out += QChar(low); + p += 6; + } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) { + return false; // unpaired low surrogate + } else { + out += QChar(codepoint); + } + continue; + } + switch (escaped) { + case '"': + out += '"'; + break; + case '\\': + out += '\\'; + break; + case '/': + out += '/'; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + default: + return false; + } + ++p; + continue; + } + if (c == '"') { + ++p; + flush(); + return true; + } + if (static_cast(c) < 0x20) { + return false; // unescaped control character is invalid JSON + } + utf8 += c; + ++p; + } + return false; +} + +bool matchLiteral(const char *&p, const char *end, const char *literal, int length) +{ + if (end - p < length || memcmp(p, literal, static_cast(length)) != 0) { + return false; + } + const char *after = p + length; + if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) { + return false; + } + p = after; + return true; +} + +bool skipNumber(const char *&p, const char *end) +{ + // JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? + if (p < end && *p == '-') { + ++p; + } + if (p < end && *p == '0') { + ++p; + } else if (p < end && *p >= '1' && *p <= '9') { + ++p; + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } else { + return false; + } + if (p < end && *p == '.') { + ++p; + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + if (p < end && (*p == 'e' || *p == 'E')) { + ++p; + if (p < end && (*p == '+' || *p == '-')) { + ++p; + } + if (p >= end || !QChar::isDigit(*p)) { + return false; + } + while (p < end && QChar::isDigit(*p)) { + ++p; + } + } + return true; +} + +bool skipValue(const char *&p, const char *end); +bool skipObject(const char *&p, const char *end); +bool skipArray(const char *&p, const char *end); + +bool skipPrimitive(const char *&p, const char *end) +{ + if (p >= end) { + return false; + } + const char c = *p; + if (c == '"') { + return skipString(p, end); + } + if (c == 't') { + return matchLiteral(p, end, "true", 4); + } + if (c == 'f') { + return matchLiteral(p, end, "false", 5); + } + if (c == 'n') { + return matchLiteral(p, end, "null", 4); + } + if (c == '-' || (c >= '0' && c <= '9')) { + return skipNumber(p, end); + } + return false; +} + +bool skipObject(const char *&p, const char *end) +{ + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + if (!skipString(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + if (!skipValue(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +bool skipArray(const char *&p, const char *end) +{ + ++p; // '[' + p = skipWhitespace(p, end); + if (p < end && *p == ']') { + ++p; + return true; + } + for (;;) { + if (!skipValue(p, end)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + ++p; + return true; + } + return false; + } +} + +bool skipValue(const char *&p, const char *end) +{ + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + const char c = *p; + if (c == '{') { + return skipObject(p, end); + } + if (c == '[') { + return skipArray(p, end); + } + return skipPrimitive(p, end); +} + +/** + * @brief Iterates the members of the object starting at @p p. + * + * For each member invokes @p memberCallback with the key and the byte range of + * its value. Advancing @p p is unaffected by the callback. + */ +template bool forEachObjectMember(const char *&p, const char *end, F &&memberCallback) +{ + ++p; // '{' + p = skipWhitespace(p, end); + if (p < end && *p == '}') { + ++p; + return true; + } + for (;;) { + p = skipWhitespace(p, end); + if (p >= end || *p != '"') { + return false; + } + QString key; + if (!decodeString(p, end, key)) { + return false; + } + p = skipWhitespace(p, end); + if (p >= end || *p != ':') { + return false; + } + ++p; + const char *valueStart = skipWhitespace(p, end); + const char *valueEnd = valueStart; + if (!skipValue(valueEnd, end)) { + return false; + } + if (!memberCallback(key, valueStart, valueEnd)) { + return false; + } + p = valueEnd; + p = skipWhitespace(p, end); + if (p >= end) { + return false; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == '}') { + ++p; + return true; + } + return false; + } +} + +// Counts the direct elements of an array value; returns -1 if the array is malformed. +int countArrayElements(const char *p, const char *end) +{ + ++p; // '[' + p = skipWhitespace(p, end); + int count = 0; + if (p < end && *p == ']') { + return 0; + } + for (;;) { + if (!skipValue(p, end)) { + return -1; + } + ++count; + p = skipWhitespace(p, end); + if (p >= end) { + return -1; + } + if (*p == ',') { + ++p; + continue; + } + if (*p == ']') { + return count; + } + return -1; + } +} + +} // namespace + +namespace RawJson +{ + +QList scanSetRanges(const QByteArray &json, ScanError *error) +{ + QList ranges; + if (error) { + *error = ScanError{}; + } + + const auto fail = [&](const QString &message) -> QList { + if (error) { + error->message = message; + } + return {}; + }; + + const char *begin = json.constData(); + const char *end = begin + json.size(); + if (begin >= end) { + return fail(QStringLiteral("empty JSON document")); + } + + const char *p = skipWhitespace(begin, end); + if (p >= end || *p != '{') { + return fail(QStringLiteral("top-level JSON must be an object")); + } + + bool foundData = false; + bool malformedSetData = false; + + const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) { + if (key == QStringLiteral("data")) { + foundData = true; + if (valueStart >= valueEnd || *valueStart != '{') { + malformedSetData = true; + return false; + } + const char *setP = valueStart; + const bool ok = forEachObjectMember( + setP, valueEnd, [&](const QString &setCode, const char *setStart, const char *setEnd) { + if (setStart >= setEnd || *setStart != '{') { + malformedSetData = true; + return false; + } + SetRange range; + range.start = setStart - begin; + range.length = setEnd - setStart; + range.code = setCode; + + const char *memberP = setStart; + const bool metaOk = + forEachObjectMember(memberP, setEnd, [&](const QString &field, const char *fs, const char *fe) { + if (field == QStringLiteral("code")) { + return fs < fe && *fs == '"' && decodeString(fs, fe, range.code); + } + if (field == QStringLiteral("name")) { + return fs < fe && *fs == '"' && decodeString(fs, fe, range.name); + } + if (field == QStringLiteral("type")) { + return fs < fe && *fs == '"' && decodeString(fs, fe, range.type); + } + if (field == QStringLiteral("releaseDate")) { + return fs < fe && *fs == '"' && decodeString(fs, fe, range.releaseDate); + } + if (field == QStringLiteral("cards")) { + if (fs >= fe || *fs != '[') { + return false; + } + range.cardCount = countArrayElements(fs, fe); + return range.cardCount >= 0; + } + return true; + }); + if (!metaOk) { + malformedSetData = true; + return false; + } + ranges.append(range); + return true; + }); + if (!ok) { + malformedSetData = true; + return false; + } + } + return true; + }; + + if (!forEachObjectMember(p, end, topLevelCallback)) { + return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")); + } + p = skipWhitespace(p, end); + if (p != end) { + return fail(QStringLiteral("trailing content after top-level JSON object")); + } + if (!foundData) { + return fail(QStringLiteral("missing \"data\" object")); + } + if (ranges.isEmpty()) { + return fail(QStringLiteral("no sets found in \"data\"")); + } + return ranges; +} + +} // namespace RawJson \ No newline at end of file diff --git a/oracle/src/raw_json_scanner.h b/oracle/src/raw_json_scanner.h new file mode 100644 index 000000000..f4c1ba0ab --- /dev/null +++ b/oracle/src/raw_json_scanner.h @@ -0,0 +1,60 @@ +#ifndef RAW_JSON_SCANNER_H +#define RAW_JSON_SCANNER_H + +#include +#include +#include + +namespace RawJson +{ + +struct SetRange +{ + /** @brief Byte offset of the set's object within the scanned buffer. */ + qsizetype start = -1; + /** @brief Byte length of the set's object, including the surrounding braces. */ + qsizetype length = 0; + /** @brief Number of entries in the set's "cards" array. */ + int cardCount = 0; + QString code; + QString name; + QString type; + QString releaseDate; +}; + +struct ScanError +{ + bool isError() const + { + return !message.isEmpty(); + } + QString message; +}; + +/** + * @brief Scans a full MTGJSON document without materializing the JSON tree. + * + * Splits the top-level "data" object into per-set byte ranges and reads each + * set's metadata directly from the raw bytes. The oracle importer can then + * parse one set at a time during import, keeping peak memory far below a single + * QJsonDocument::fromJson() over the whole file. + * + * The whole document is structurally validated while scanning (strings, + * escapes, braces, and a trailing-content check), so malformed input is + * rejected just like QJsonDocument::fromJson would. + * + * Following QJsonDocument::fromJson's convention, the parsed ranges are + * returned by value and any failure is reported through the @p error out + * parameter. + * + * @param json The raw MTGJSON document bytes. + * @param error Out parameter. Set to an error ScanError when the document + * cannot be parsed, otherwise left empty. Passing a null + * pointer disables error reporting. + * @return The detected per-set ranges, or an empty list on failure. + */ +QList scanSetRanges(const QByteArray &json, ScanError *error = nullptr); + +} // namespace RawJson + +#endif // RAW_JSON_SCANNER_H \ No newline at end of file diff --git a/tests/oracle/CMakeLists.txt b/tests/oracle/CMakeLists.txt index 0088f37c9..f25a76417 100644 --- a/tests/oracle/CMakeLists.txt +++ b/tests/oracle/CMakeLists.txt @@ -11,7 +11,7 @@ add_test(NAME parse_cipt_test COMMAND parse_cipt_test) # Oracle importer unit tests add_executable( oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_test.cpp + ../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp ) if(NOT GTEST_FOUND) @@ -51,7 +51,7 @@ endif() add_executable( oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp - oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} + ../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES} ) if(NOT GTEST_FOUND) diff --git a/tests/oracle/oracle_importer_benchmark_test.cpp b/tests/oracle/oracle_importer_benchmark_test.cpp index a52d4f332..cee00e1d4 100644 --- a/tests/oracle/oracle_importer_benchmark_test.cpp +++ b/tests/oracle/oracle_importer_benchmark_test.cpp @@ -146,9 +146,10 @@ TEST(OracleBenchmark, ParseJsonThroughput) for (int i = 0; i < iterations; ++i) { OracleImporter importer; + QByteArray source = data; QElapsedTimer timer; timer.start(); - bool ok = importer.readSetsFromByteArray(data); + bool ok = importer.readSetsFromByteArray(std::move(source)); ASSERT_TRUE(ok); totalMs += timer.elapsed(); } @@ -404,7 +405,7 @@ TEST(OracleBenchmark, ImportRamUsage) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(data)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); @@ -468,7 +469,7 @@ TEST(OracleBenchmark, ImportRamUsageAllPrintings) QElapsedTimer timer; timer.start(); - ASSERT_TRUE(importer.readSetsFromByteArray(setsData)); + ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData))); const qint64 parseMs = timer.elapsed(); const MemorySnapshot afterParse = MemorySnapshot::current(); diff --git a/tests/oracle/oracle_importer_test.cpp b/tests/oracle/oracle_importer_test.cpp index 80f7794e6..9f4b0ab7b 100644 --- a/tests/oracle/oracle_importer_test.cpp +++ b/tests/oracle/oracle_importer_test.cpp @@ -488,6 +488,112 @@ TEST_F(OracleImporterTest, ApostropheNormalized) ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity")); } +// ============================================================================ +// RawJson scanner tests +// ============================================================================ + +TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse) +{ + QJsonObject root; + QJsonObject data; + data["AAA"] = makeCard("Alpha Card"); + data["BBB"] = makeCard("Beta Card"); + root["data"] = data; + + const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact); + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(bytes, &error); + ASSERT_FALSE(error.isError()) << error.message.toStdString(); + ASSERT_EQ(ranges.size(), 2); + + const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject(); + for (const RawJson::SetRange &range : ranges) { + QJsonParseError parseError; + const QJsonDocument sliceDoc = + QJsonDocument::fromJson(QByteArray(bytes.constData() + range.start, range.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError) + << range.code.toStdString() << ": " << parseError.errorString().toStdString(); + ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString(); + } +} + +TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards) +{ + const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\"," + "\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\"," + "\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}"; + + RawJson::ScanError error; + const QList ranges = RawJson::scanSetRanges(json, &error); + ASSERT_FALSE(error.isError()); + ASSERT_EQ(ranges.size(), 1); + + const RawJson::SetRange &range = ranges.first(); + ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key + const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00); + ASSERT_EQ(range.name, expectedName); + ASSERT_EQ(range.type, "expansion"); + ASSERT_EQ(range.releaseDate, "2024-01-05"); + ASSERT_EQ(range.cardCount, 3); + + QJsonParseError parseError; + const QJsonDocument sliceDoc = + QJsonDocument::fromJson(QByteArray(json.constData() + range.start, range.length), &parseError); + ASSERT_EQ(parseError.error, QJsonParseError::NoError); + ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName); + ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3); +} + +TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson) +{ + const QList invalid = {"not json", + "[]", + "{\"data\":[]}", + "{\"data\":{}}", + "{\"other\":{}}", + "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\"," + "\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}", + "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"}; + + for (const QByteArray &json : invalid) { + RawJson::ScanError error; + RawJson::scanSetRanges(json, &error); + EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData(); + } +} + +// ============================================================================ +// Lazy per-set parsing tests +// ============================================================================ + +TEST_F(OracleImporterTest, StartImportParsesSetsLazily) +{ + QJsonObject setObj = makeCard("Lazy Import Card"); + QJsonArray cards; + cards.append(setObj); + QJsonObject dataSet; + dataSet["code"] = "tst"; + dataSet["name"] = "Test Set"; + dataSet["type"] = "expansion"; + dataSet["releaseDate"] = "2024-01-01"; + dataSet["cards"] = cards; + + QJsonObject root; + root["data"] = QJsonObject{{"TST", dataSet}}; + + const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact); + ASSERT_TRUE(importer->readSetsFromByteArray(data)); + ASSERT_FALSE(importer->getRawSetsData().isEmpty()); + + const int importedSets = importer->startImport(); + ASSERT_EQ(importedSets, 1); + ASSERT_EQ(importer->getCardList().size(), 1); + ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull()); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv);