[Oracle] Parse sets lazily to slash importer peak memory

- Add a raw JSON scanner that splits the document into per-set byte ranges
  without materializing the JSON tree
- Keep only the raw document bytes and parse one set at a time in startImport
- Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied
- Clear the retained raw data in releaseSetData()/clear()
- Cover the scanner and lazy parsing with tests

Took 2 minutes

oracle/CMakeLists.txt

@@ -23,6 +23,7 @@ set(oracle_SOURCES
     src/pages.cpp
     src/pagetemplates.cpp
     src/parsehelpers.cpp
+    src/raw_json_scanner.cpp
     ../cockatrice/src/client/settings/cache_settings.cpp
     ../cockatrice/src/client/settings/card_counter_settings.cpp
     ../cockatrice/src/client/settings/shortcuts_settings.cpp

oracle/src/oracleimporter.cpp

@@ -7,6 +7,7 @@
 #include <QDebug>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QJsonParseError>
 #include <QRegularExpression>
 #include <algorithm>
 #include <climits>
@@ -43,26 +44,23 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s
     return priority;
 }
 
-bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
+bool OracleImporter::readSetsFromByteArray(QByteArray data)
 {
-    QJsonParseError error;
+    RawJson::ScanError error;
-    auto doc = QJsonDocument::fromJson(data, &error);
+    const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error);
-    if (error.error != QJsonParseError::NoError) {
+    if (error.isError()) {
-        qDebug() << "error: QJsonDocument::fromJson():" << error.errorString();
+        qDebug() << "error: RawJson::scanSetRanges():" << error.message;
         return false;
     }
 
-    auto setsObj = doc.object().value("data").toObject();
-
     QList<SetToDownload> newSetList;
+    newSetList.reserve(ranges.size());
 
-    for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) {
+    for (const RawJson::SetRange &range : ranges) {
-        QJsonObject setObj = it.value().toObject();
+        QString shortName = range.code.toUpper();
-        QString shortName = setObj.value("code").toString().toUpper();
+        QString longName = range.name;
-        QString longName = setObj.value("name").toString();
+        QString setType = range.type;
-        QJsonArray setCards = setObj.value("cards").toArray();
+        QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate);
-        QString setType = setObj.value("type").toString();
-        QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate);
         CardSet::Priority priority = getSetPriority(setType, shortName);
         // capitalize set type
         if (setType.length() > 0) {
@@ -82,7 +80,9 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
             }
             setType = setType.trimmed();
         }
-        newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate));
+        SetToDownload set(shortName, longName, priority, setType, releaseDate);
+        set.setRawRange(range);
+        newSetList.append(set);
     }
 
     std::sort(newSetList.begin(), newSetList.end());
@@ -91,6 +91,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
         return false;
     }
     allSets = newSetList;
+    rawSetsData = data;
     return true;
 }
 
@@ -541,7 +542,7 @@ int OracleImporter::startImport()
     // Pre-allocate cards hash to avoid rehashing during import
     int estimatedCards = 0;
     for (const SetToDownload &curSetToParse : allSets) {
-        estimatedCards += curSetToParse.getCards().size();
+        estimatedCards += curSetToParse.getRawRange().cardCount;
     }
     cards.reserve(estimatedCards);
 
@@ -560,7 +561,21 @@ int OracleImporter::startImport()
             sets.insert(newSet->getShortName(), newSet);
         }
 
-        int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards());
+        // parse only this set's slice of the raw document so the whole JSON tree is
+        // never kept in memory at once
+        const RawJson::SetRange &rawRange = curSetToParse.getRawRange();
+        const QByteArray setBytes(rawSetsData.constData() + rawRange.start, rawRange.length);
+        QJsonParseError parseError;
+        const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError);
+        if (parseError.error != QJsonParseError::NoError) {
+            qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":"
+                       << parseError.errorString();
+            ++setIndex;
+            continue;
+        }
+
+        const QJsonArray setCards = setDoc.object().value("cards").toArray();
+        int numCardsInSet = importCardsFromSet(newSet, setCards);
 
         ++setIndex;
 
@@ -583,6 +598,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr
 void OracleImporter::releaseSetData()
 {
     allSets.clear();
+    rawSetsData.clear();
 }
 
 void OracleImporter::clear()
@@ -590,4 +606,5 @@ void OracleImporter::clear()
     sets.clear();
     cards.clear();
     allSets.clear();
+    rawSetsData.clear();
 }

oracle/src/oracleimporter.h

@@ -1,6 +1,9 @@
 #ifndef ORACLEIMPORTER_H
 #define ORACLEIMPORTER_H
 
+#include "raw_json_scanner.h"
+
+#include <QByteArray>
 #include <QJsonArray>
 #include <QJsonObject>
 #include <QMap>
@@ -46,10 +49,12 @@ class SetToDownload
 {
 private:
     QString shortName, longName;
-    QJsonArray cards;
     QDate releaseDate;
     QString setType;
     CardSet::Priority priority;
+    // Byte range of this set's object within the importer's raw JSON text. Parsing
+    // one set at a time keeps peak memory low instead of holding the whole document.
+    RawJson::SetRange rawRange;
 
 public:
     const QString &getShortName() const
@@ -60,10 +65,6 @@ public:
     {
         return longName;
     }
-    const QJsonArray &getCards() const
-    {
-        return cards;
-    }
     const QString &getSetType() const
     {
         return setType;
@@ -76,16 +77,23 @@ public:
     {
         return priority;
     }
+    const RawJson::SetRange &getRawRange() const
+    {
+        return rawRange;
+    }
     SetToDownload(QString _shortName,
                   QString _longName,
-                  QJsonArray _cards,
                   CardSet::Priority _priority,
                   QString _setType = QString(),
                   const QDate &_releaseDate = QDate())
-        : shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)),
+        : shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate),
-          releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority)
+          setType(std::move(_setType)), priority(_priority)
     {
     }
+    void setRawRange(const RawJson::SetRange &_rawRange)
+    {
+        rawRange = _rawRange;
+    }
     bool operator<(const SetToDownload &set) const
     {
         return longName.compare(set.longName, Qt::CaseInsensitive) < 0;
@@ -141,6 +149,12 @@ private:
 
     QList<SetToDownload> allSets;
 
+    /**
+     * The raw JSON text of the source document, retained for lazy per-set
+     * parsing during startImport(). Frees the card data as each set is imported.
+     */
+    QByteArray rawSetsData;
+
     CardInfoPtr addCard(QString name,
                         const QString &text,
                         bool isToken,
@@ -153,7 +167,11 @@ signals:
 
 public:
     explicit OracleImporter(QObject *parent = nullptr);
-    bool readSetsFromByteArray(const QByteArray &data);
+    /**
+     * Scans the given JSON document for set metadata. Takes the data by value so
+     * the wizard can hand over its decompressed buffer without copying it.
+     */
+    bool readSetsFromByteArray(QByteArray data);
     int startImport();
     bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion);
     int importCardsFromSet(const CardSetPtr &currentSet, const QJsonArray &cardsList);
@@ -166,6 +184,10 @@ public:
     {
         return allSets;
     }
+    const QByteArray &getRawSetsData() const
+    {
+        return rawSetsData;
+    }
     void releaseSetData();
     void clear();
 };

oracle/src/raw_json_scanner.cpp

@@ -0,0 +1,573 @@
+#include "raw_json_scanner.h"
+
+#include <cstring>
+
+namespace
+{
+
+inline bool isWhitespace(char c)
+{
+    return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+}
+
+const char *skipWhitespace(const char *p, const char *end)
+{
+    while (p < end && isWhitespace(*p)) {
+        ++p;
+    }
+    return p;
+}
+
+inline bool isHexDigit(char c)
+{
+    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
+
+inline quint8 hexValue(char c)
+{
+    if (c >= '0' && c <= '9') {
+        return c - '0';
+    }
+    if (c >= 'a' && c <= 'f') {
+        return c - 'a' + 10;
+    }
+    return c - 'A' + 10;
+}
+
+/**
+ * @brief Skips past a JSON string without decoding it, validating escapes.
+ * @param p In: pointing at the opening quote. Out: pointing past the closing quote.
+ */
+bool skipString(const char *&p, const char *end)
+{
+    ++p; // opening quote
+    for (;;) {
+        const void *quote = memchr(p, '"', static_cast<size_t>(end - p));
+        if (!quote) {
+            return false; // unterminated string
+        }
+        // Backslash escapes can only appear before the closing quote, so bound
+        // the scan to the string extent instead of the rest of the document.
+        const void *backslash = memchr(p, '\\', static_cast<size_t>(static_cast<const char *>(quote) - p));
+        if (!backslash) {
+            p = static_cast<const char *>(quote) + 1;
+            return true;
+        }
+        const char *b = static_cast<const char *>(backslash);
+        if (end - b < 2) {
+            return false;
+        }
+        const char escaped = b[1];
+        if (escaped == 'u') {
+            if (end - b < 6) {
+                return false;
+            }
+            quint32 codepoint = 0;
+            for (int i = 0; i < 4; ++i) {
+                if (!isHexDigit(b[2 + i])) {
+                    return false;
+                }
+                codepoint = codepoint * 16 + hexValue(b[2 + i]);
+            }
+            p = b + 6;
+            if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
+                // expect the low-surrogate escape for the second half
+                if (end - p < 6 || p[0] != '\\' || p[1] != 'u') {
+                    return false; // unpaired high surrogate
+                }
+                quint32 low = 0;
+                for (int i = 0; i < 4; ++i) {
+                    if (!isHexDigit(p[2 + i])) {
+                        return false;
+                    }
+                    low = low * 16 + hexValue(p[2 + i]);
+                }
+                if (low < 0xDC00 || low > 0xDFFF) {
+                    return false;
+                }
+                p += 6;
+            } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
+                return false; // unpaired low surrogate
+            }
+            continue;
+        }
+        switch (escaped) {
+            case '"':
+            case '\\':
+            case '/':
+            case 'b':
+            case 'f':
+            case 'n':
+            case 'r':
+            case 't':
+                p = b + 2;
+                continue;
+            default:
+                return false; // invalid escape
+        }
+    }
+}
+
+/**
+ * @brief Decodes a JSON string into @p out, validating it as it goes.
+ * @param p In: pointing at the opening quote. Out: pointing past the closing quote.
+ */
+bool decodeString(const char *&p, const char *end, QString &out)
+{
+    out.clear();
+    QByteArray utf8;
+    auto flush = [&out, &utf8]() {
+        if (!utf8.isEmpty()) {
+            out += QString::fromUtf8(utf8);
+            utf8.clear();
+        }
+    };
+
+    ++p; // opening quote
+    while (p < end) {
+        const char c = *p;
+        if (c == '\\') {
+            flush();
+            ++p; // escaped character
+            if (p >= end) {
+                return false;
+            }
+            const char escaped = *p;
+            if (escaped == 'u') {
+                ++p; // first hex digit
+                if (p + 4 > end) {
+                    return false;
+                }
+                quint32 codepoint = 0;
+                for (int i = 0; i < 4; ++i) {
+                    if (!isHexDigit(p[i])) {
+                        return false;
+                    }
+                    codepoint = codepoint * 16 + hexValue(p[i]);
+                }
+                p += 4;
+                if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
+                    // expect a low-surrogate escape for the second half
+                    if (p + 6 > end || p[0] != '\\' || p[1] != 'u') {
+                        return false; // unpaired high surrogate
+                    }
+                    quint32 low = 0;
+                    for (int i = 0; i < 4; ++i) {
+                        if (!isHexDigit(p[2 + i])) {
+                            return false;
+                        }
+                        low = low * 16 + hexValue(p[2 + i]);
+                    }
+                    if (low < 0xDC00 || low > 0xDFFF) {
+                        return false;
+                    }
+                    out += QChar(codepoint);
+                    out += QChar(low);
+                    p += 6;
+                } else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
+                    return false; // unpaired low surrogate
+                } else {
+                    out += QChar(codepoint);
+                }
+                continue;
+            }
+            switch (escaped) {
+                case '"':
+                    out += '"';
+                    break;
+                case '\\':
+                    out += '\\';
+                    break;
+                case '/':
+                    out += '/';
+                    break;
+                case 'b':
+                    out += '\b';
+                    break;
+                case 'f':
+                    out += '\f';
+                    break;
+                case 'n':
+                    out += '\n';
+                    break;
+                case 'r':
+                    out += '\r';
+                    break;
+                case 't':
+                    out += '\t';
+                    break;
+                default:
+                    return false;
+            }
+            ++p;
+            continue;
+        }
+        if (c == '"') {
+            ++p;
+            flush();
+            return true;
+        }
+        if (static_cast<unsigned char>(c) < 0x20) {
+            return false; // unescaped control character is invalid JSON
+        }
+        utf8 += c;
+        ++p;
+    }
+    return false;
+}
+
+bool matchLiteral(const char *&p, const char *end, const char *literal, int length)
+{
+    if (end - p < length || memcmp(p, literal, static_cast<size_t>(length)) != 0) {
+        return false;
+    }
+    const char *after = p + length;
+    if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) {
+        return false;
+    }
+    p = after;
+    return true;
+}
+
+bool skipNumber(const char *&p, const char *end)
+{
+    // JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
+    if (p < end && *p == '-') {
+        ++p;
+    }
+    if (p < end && *p == '0') {
+        ++p;
+    } else if (p < end && *p >= '1' && *p <= '9') {
+        ++p;
+        while (p < end && QChar::isDigit(*p)) {
+            ++p;
+        }
+    } else {
+        return false;
+    }
+    if (p < end && *p == '.') {
+        ++p;
+        if (p >= end || !QChar::isDigit(*p)) {
+            return false;
+        }
+        while (p < end && QChar::isDigit(*p)) {
+            ++p;
+        }
+    }
+    if (p < end && (*p == 'e' || *p == 'E')) {
+        ++p;
+        if (p < end && (*p == '+' || *p == '-')) {
+            ++p;
+        }
+        if (p >= end || !QChar::isDigit(*p)) {
+            return false;
+        }
+        while (p < end && QChar::isDigit(*p)) {
+            ++p;
+        }
+    }
+    return true;
+}
+
+bool skipValue(const char *&p, const char *end);
+bool skipObject(const char *&p, const char *end);
+bool skipArray(const char *&p, const char *end);
+
+bool skipPrimitive(const char *&p, const char *end)
+{
+    if (p >= end) {
+        return false;
+    }
+    const char c = *p;
+    if (c == '"') {
+        return skipString(p, end);
+    }
+    if (c == 't') {
+        return matchLiteral(p, end, "true", 4);
+    }
+    if (c == 'f') {
+        return matchLiteral(p, end, "false", 5);
+    }
+    if (c == 'n') {
+        return matchLiteral(p, end, "null", 4);
+    }
+    if (c == '-' || (c >= '0' && c <= '9')) {
+        return skipNumber(p, end);
+    }
+    return false;
+}
+
+bool skipObject(const char *&p, const char *end)
+{
+    ++p; // '{'
+    p = skipWhitespace(p, end);
+    if (p < end && *p == '}') {
+        ++p;
+        return true;
+    }
+    for (;;) {
+        p = skipWhitespace(p, end);
+        if (p >= end || *p != '"') {
+            return false;
+        }
+        if (!skipString(p, end)) {
+            return false;
+        }
+        p = skipWhitespace(p, end);
+        if (p >= end || *p != ':') {
+            return false;
+        }
+        ++p;
+        if (!skipValue(p, end)) {
+            return false;
+        }
+        p = skipWhitespace(p, end);
+        if (p >= end) {
+            return false;
+        }
+        if (*p == ',') {
+            ++p;
+            continue;
+        }
+        if (*p == '}') {
+            ++p;
+            return true;
+        }
+        return false;
+    }
+}
+
+bool skipArray(const char *&p, const char *end)
+{
+    ++p; // '['
+    p = skipWhitespace(p, end);
+    if (p < end && *p == ']') {
+        ++p;
+        return true;
+    }
+    for (;;) {
+        if (!skipValue(p, end)) {
+            return false;
+        }
+        p = skipWhitespace(p, end);
+        if (p >= end) {
+            return false;
+        }
+        if (*p == ',') {
+            ++p;
+            continue;
+        }
+        if (*p == ']') {
+            ++p;
+            return true;
+        }
+        return false;
+    }
+}
+
+bool skipValue(const char *&p, const char *end)
+{
+    p = skipWhitespace(p, end);
+    if (p >= end) {
+        return false;
+    }
+    const char c = *p;
+    if (c == '{') {
+        return skipObject(p, end);
+    }
+    if (c == '[') {
+        return skipArray(p, end);
+    }
+    return skipPrimitive(p, end);
+}
+
+/**
+ * @brief Iterates the members of the object starting at @p p.
+ *
+ * For each member invokes @p memberCallback with the key and the byte range of
+ * its value. Advancing @p p is unaffected by the callback.
+ */
+template <typename F> bool forEachObjectMember(const char *&p, const char *end, F &&memberCallback)
+{
+    ++p; // '{'
+    p = skipWhitespace(p, end);
+    if (p < end && *p == '}') {
+        ++p;
+        return true;
+    }
+    for (;;) {
+        p = skipWhitespace(p, end);
+        if (p >= end || *p != '"') {
+            return false;
+        }
+        QString key;
+        if (!decodeString(p, end, key)) {
+            return false;
+        }
+        p = skipWhitespace(p, end);
+        if (p >= end || *p != ':') {
+            return false;
+        }
+        ++p;
+        const char *valueStart = skipWhitespace(p, end);
+        const char *valueEnd = valueStart;
+        if (!skipValue(valueEnd, end)) {
+            return false;
+        }
+        if (!memberCallback(key, valueStart, valueEnd)) {
+            return false;
+        }
+        p = valueEnd;
+        p = skipWhitespace(p, end);
+        if (p >= end) {
+            return false;
+        }
+        if (*p == ',') {
+            ++p;
+            continue;
+        }
+        if (*p == '}') {
+            ++p;
+            return true;
+        }
+        return false;
+    }
+}
+
+// Counts the direct elements of an array value; returns -1 if the array is malformed.
+int countArrayElements(const char *p, const char *end)
+{
+    ++p; // '['
+    p = skipWhitespace(p, end);
+    int count = 0;
+    if (p < end && *p == ']') {
+        return 0;
+    }
+    for (;;) {
+        if (!skipValue(p, end)) {
+            return -1;
+        }
+        ++count;
+        p = skipWhitespace(p, end);
+        if (p >= end) {
+            return -1;
+        }
+        if (*p == ',') {
+            ++p;
+            continue;
+        }
+        if (*p == ']') {
+            return count;
+        }
+        return -1;
+    }
+}
+
+} // namespace
+
+namespace RawJson
+{
+
+QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
+{
+    QList<SetRange> ranges;
+    if (error) {
+        *error = ScanError{};
+    }
+
+    const auto fail = [&](const QString &message) -> QList<SetRange> {
+        if (error) {
+            error->message = message;
+        }
+        return {};
+    };
+
+    const char *begin = json.constData();
+    const char *end = begin + json.size();
+    if (begin >= end) {
+        return fail(QStringLiteral("empty JSON document"));
+    }
+
+    const char *p = skipWhitespace(begin, end);
+    if (p >= end || *p != '{') {
+        return fail(QStringLiteral("top-level JSON must be an object"));
+    }
+
+    bool foundData = false;
+    bool malformedSetData = false;
+
+    const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) {
+        if (key == QStringLiteral("data")) {
+            foundData = true;
+            if (valueStart >= valueEnd || *valueStart != '{') {
+                malformedSetData = true;
+                return false;
+            }
+            const char *setP = valueStart;
+            const bool ok = forEachObjectMember(
+                setP, valueEnd, [&](const QString &setCode, const char *setStart, const char *setEnd) {
+                    if (setStart >= setEnd || *setStart != '{') {
+                        malformedSetData = true;
+                        return false;
+                    }
+                    SetRange range;
+                    range.start = setStart - begin;
+                    range.length = setEnd - setStart;
+                    range.code = setCode;
+
+                    const char *memberP = setStart;
+                    const bool metaOk =
+                        forEachObjectMember(memberP, setEnd, [&](const QString &field, const char *fs, const char *fe) {
+                            if (field == QStringLiteral("code")) {
+                                return fs < fe && *fs == '"' && decodeString(fs, fe, range.code);
+                            }
+                            if (field == QStringLiteral("name")) {
+                                return fs < fe && *fs == '"' && decodeString(fs, fe, range.name);
+                            }
+                            if (field == QStringLiteral("type")) {
+                                return fs < fe && *fs == '"' && decodeString(fs, fe, range.type);
+                            }
+                            if (field == QStringLiteral("releaseDate")) {
+                                return fs < fe && *fs == '"' && decodeString(fs, fe, range.releaseDate);
+                            }
+                            if (field == QStringLiteral("cards")) {
+                                if (fs >= fe || *fs != '[') {
+                                    return false;
+                                }
+                                range.cardCount = countArrayElements(fs, fe);
+                                return range.cardCount >= 0;
+                            }
+                            return true;
+                        });
+                    if (!metaOk) {
+                        malformedSetData = true;
+                        return false;
+                    }
+                    ranges.append(range);
+                    return true;
+                });
+            if (!ok) {
+                malformedSetData = true;
+                return false;
+            }
+        }
+        return true;
+    };
+
+    if (!forEachObjectMember(p, end, topLevelCallback)) {
+        return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON"));
+    }
+    p = skipWhitespace(p, end);
+    if (p != end) {
+        return fail(QStringLiteral("trailing content after top-level JSON object"));
+    }
+    if (!foundData) {
+        return fail(QStringLiteral("missing \"data\" object"));
+    }
+    if (ranges.isEmpty()) {
+        return fail(QStringLiteral("no sets found in \"data\""));
+    }
+    return ranges;
+}
+
+} // namespace RawJson

oracle/src/raw_json_scanner.h

@@ -0,0 +1,60 @@
+#ifndef RAW_JSON_SCANNER_H
+#define RAW_JSON_SCANNER_H
+
+#include <QByteArray>
+#include <QList>
+#include <QString>
+
+namespace RawJson
+{
+
+struct SetRange
+{
+    /** @brief Byte offset of the set's object within the scanned buffer. */
+    qsizetype start = -1;
+    /** @brief Byte length of the set's object, including the surrounding braces. */
+    qsizetype length = 0;
+    /** @brief Number of entries in the set's "cards" array. */
+    int cardCount = 0;
+    QString code;
+    QString name;
+    QString type;
+    QString releaseDate;
+};
+
+struct ScanError
+{
+    bool isError() const
+    {
+        return !message.isEmpty();
+    }
+    QString message;
+};
+
+/**
+ * @brief Scans a full MTGJSON document without materializing the JSON tree.
+ *
+ * Splits the top-level "data" object into per-set byte ranges and reads each
+ * set's metadata directly from the raw bytes. The oracle importer can then
+ * parse one set at a time during import, keeping peak memory far below a single
+ * QJsonDocument::fromJson() over the whole file.
+ *
+ * The whole document is structurally validated while scanning (strings,
+ * escapes, braces, and a trailing-content check), so malformed input is
+ * rejected just like QJsonDocument::fromJson would.
+ *
+ * Following QJsonDocument::fromJson's convention, the parsed ranges are
+ * returned by value and any failure is reported through the @p error out
+ * parameter.
+ *
+ * @param json The raw MTGJSON document bytes.
+ * @param error Out parameter. Set to an error ScanError when the document
+ *              cannot be parsed, otherwise left empty. Passing a null
+ *              pointer disables error reporting.
+ * @return The detected per-set ranges, or an empty list on failure.
+ */
+QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error = nullptr);
+
+} // namespace RawJson
+
+#endif // RAW_JSON_SCANNER_H

tests/oracle/CMakeLists.txt

@@ -11,7 +11,7 @@ add_test(NAME parse_cipt_test COMMAND parse_cipt_test)
 # Oracle importer unit tests
 add_executable(
   oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
-                       oracle_importer_test.cpp
+                       ../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp
 )
 
 if(NOT GTEST_FOUND)
@@ -51,7 +51,7 @@ endif()
 add_executable(
   oracle_importer_benchmark_test
   ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
-  oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
+  ../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
 )
 
 if(NOT GTEST_FOUND)

tests/oracle/oracle_importer_benchmark_test.cpp

@@ -146,9 +146,10 @@ TEST(OracleBenchmark, ParseJsonThroughput)
 
     for (int i = 0; i < iterations; ++i) {
         OracleImporter importer;
+        QByteArray source = data;
         QElapsedTimer timer;
         timer.start();
-        bool ok = importer.readSetsFromByteArray(data);
+        bool ok = importer.readSetsFromByteArray(std::move(source));
         ASSERT_TRUE(ok);
         totalMs += timer.elapsed();
     }
@@ -404,7 +405,7 @@ TEST(OracleBenchmark, ImportRamUsage)
 
     QElapsedTimer timer;
     timer.start();
-    ASSERT_TRUE(importer.readSetsFromByteArray(data));
+    ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data)));
     const qint64 parseMs = timer.elapsed();
     const MemorySnapshot afterParse = MemorySnapshot::current();
 
@@ -468,7 +469,7 @@ TEST(OracleBenchmark, ImportRamUsageAllPrintings)
 
     QElapsedTimer timer;
     timer.start();
-    ASSERT_TRUE(importer.readSetsFromByteArray(setsData));
+    ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData)));
     const qint64 parseMs = timer.elapsed();
     const MemorySnapshot afterParse = MemorySnapshot::current();
 

tests/oracle/oracle_importer_test.cpp

@@ -488,6 +488,112 @@ TEST_F(OracleImporterTest, ApostropheNormalized)
     ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity"));
 }
 
+// ============================================================================
+// RawJson scanner tests
+// ============================================================================
+
+TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse)
+{
+    QJsonObject root;
+    QJsonObject data;
+    data["AAA"] = makeCard("Alpha Card");
+    data["BBB"] = makeCard("Beta Card");
+    root["data"] = data;
+
+    const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact);
+
+    RawJson::ScanError error;
+    const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(bytes, &error);
+    ASSERT_FALSE(error.isError()) << error.message.toStdString();
+    ASSERT_EQ(ranges.size(), 2);
+
+    const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject();
+    for (const RawJson::SetRange &range : ranges) {
+        QJsonParseError parseError;
+        const QJsonDocument sliceDoc =
+            QJsonDocument::fromJson(QByteArray(bytes.constData() + range.start, range.length), &parseError);
+        ASSERT_EQ(parseError.error, QJsonParseError::NoError)
+            << range.code.toStdString() << ": " << parseError.errorString().toStdString();
+        ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString();
+    }
+}
+
+TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards)
+{
+    const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\","
+                            "\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\","
+                            "\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}";
+
+    RawJson::ScanError error;
+    const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(json, &error);
+    ASSERT_FALSE(error.isError());
+    ASSERT_EQ(ranges.size(), 1);
+
+    const RawJson::SetRange &range = ranges.first();
+    ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key
+    const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00);
+    ASSERT_EQ(range.name, expectedName);
+    ASSERT_EQ(range.type, "expansion");
+    ASSERT_EQ(range.releaseDate, "2024-01-05");
+    ASSERT_EQ(range.cardCount, 3);
+
+    QJsonParseError parseError;
+    const QJsonDocument sliceDoc =
+        QJsonDocument::fromJson(QByteArray(json.constData() + range.start, range.length), &parseError);
+    ASSERT_EQ(parseError.error, QJsonParseError::NoError);
+    ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName);
+    ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3);
+}
+
+TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson)
+{
+    const QList<QByteArray> invalid = {"not json",
+                                       "[]",
+                                       "{\"data\":[]}",
+                                       "{\"data\":{}}",
+                                       "{\"other\":{}}",
+                                       "{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\","
+                                       "\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing",
+                                       "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}",
+                                       "{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}",
+                                       "{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"};
+
+    for (const QByteArray &json : invalid) {
+        RawJson::ScanError error;
+        RawJson::scanSetRanges(json, &error);
+        EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData();
+    }
+}
+
+// ============================================================================
+// Lazy per-set parsing tests
+// ============================================================================
+
+TEST_F(OracleImporterTest, StartImportParsesSetsLazily)
+{
+    QJsonObject setObj = makeCard("Lazy Import Card");
+    QJsonArray cards;
+    cards.append(setObj);
+    QJsonObject dataSet;
+    dataSet["code"] = "tst";
+    dataSet["name"] = "Test Set";
+    dataSet["type"] = "expansion";
+    dataSet["releaseDate"] = "2024-01-01";
+    dataSet["cards"] = cards;
+
+    QJsonObject root;
+    root["data"] = QJsonObject{{"TST", dataSet}};
+
+    const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact);
+    ASSERT_TRUE(importer->readSetsFromByteArray(data));
+    ASSERT_FALSE(importer->getRawSetsData().isEmpty());
+
+    const int importedSets = importer->startImport();
+    ASSERT_EQ(importedSets, 1);
+    ASSERT_EQ(importer->getCardList().size(), 1);
+    ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull());
+}
+
 int main(int argc, char **argv)
 {
     ::testing::InitGoogleTest(&argc, argv);