mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-09-23 10:05:10 -07:00
[Oracle] Parse sets lazily to slash importer peak memory
- Add a raw JSON scanner that splits the document into per-set byte ranges without materializing the JSON tree - Keep only the raw document bytes and parse one set at a time in startImport - Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied - Clear the retained raw data in releaseSetData()/clear() - Cover the scanner and lazy parsing with tests Took 2 minutes
This commit is contained in:
Lukas Brübach
parent
21c84a2e1a
commit
68d67ff3e5
8 changed files with 811 additions and 31 deletions
|
|
@ -23,6 +23,7 @@ set(oracle_SOURCES
|
|||
src/pages.cpp
|
||||
src/pagetemplates.cpp
|
||||
src/parsehelpers.cpp
|
||||
src/raw_json_scanner.cpp
|
||||
../cockatrice/src/client/settings/cache_settings.cpp
|
||||
../cockatrice/src/client/settings/card_counter_settings.cpp
|
||||
../cockatrice/src/client/settings/shortcuts_settings.cpp
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include <QDebug>
|
||||
#include <QJsonDocument>
|
||||
#include <QJsonObject>
|
||||
#include <QJsonParseError>
|
||||
#include <QRegularExpression>
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
|
|
@ -43,26 +44,23 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s
|
|||
return priority;
|
||||
}
|
||||
|
||||
bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
||||
bool OracleImporter::readSetsFromByteArray(QByteArray data)
|
||||
{
|
||||
QJsonParseError error;
|
||||
auto doc = QJsonDocument::fromJson(data, &error);
|
||||
if (error.error != QJsonParseError::NoError) {
|
||||
qDebug() << "error: QJsonDocument::fromJson():" << error.errorString();
|
||||
RawJson::ScanError error;
|
||||
const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error);
|
||||
if (error.isError()) {
|
||||
qDebug() << "error: RawJson::scanSetRanges():" << error.message;
|
||||
return false;
|
||||
}
|
||||
|
||||
auto setsObj = doc.object().value("data").toObject();
|
||||
|
||||
QList<SetToDownload> newSetList;
|
||||
newSetList.reserve(ranges.size());
|
||||
|
||||
for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) {
|
||||
QJsonObject setObj = it.value().toObject();
|
||||
QString shortName = setObj.value("code").toString().toUpper();
|
||||
QString longName = setObj.value("name").toString();
|
||||
QJsonArray setCards = setObj.value("cards").toArray();
|
||||
QString setType = setObj.value("type").toString();
|
||||
QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate);
|
||||
for (const RawJson::SetRange &range : ranges) {
|
||||
QString shortName = range.code.toUpper();
|
||||
QString longName = range.name;
|
||||
QString setType = range.type;
|
||||
QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate);
|
||||
CardSet::Priority priority = getSetPriority(setType, shortName);
|
||||
// capitalize set type
|
||||
if (setType.length() > 0) {
|
||||
|
|
@ -82,7 +80,9 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
|||
}
|
||||
setType = setType.trimmed();
|
||||
}
|
||||
newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate));
|
||||
SetToDownload set(shortName, longName, priority, setType, releaseDate);
|
||||
set.setRawRange(range);
|
||||
newSetList.append(set);
|
||||
}
|
||||
|
||||
std::sort(newSetList.begin(), newSetList.end());
|
||||
|
|
@ -91,6 +91,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
|||
return false;
|
||||
}
|
||||
allSets = newSetList;
|
||||
rawSetsData = data;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -541,7 +542,7 @@ int OracleImporter::startImport()
|
|||
// Pre-allocate cards hash to avoid rehashing during import
|
||||
int estimatedCards = 0;
|
||||
for (const SetToDownload &curSetToParse : allSets) {
|
||||
estimatedCards += curSetToParse.getCards().size();
|
||||
estimatedCards += curSetToParse.getRawRange().cardCount;
|
||||
}
|
||||
cards.reserve(estimatedCards);
|
||||
|
||||
|
|
@ -560,7 +561,21 @@ int OracleImporter::startImport()
|
|||
sets.insert(newSet->getShortName(), newSet);
|
||||
}
|
||||
|
||||
int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards());
|
||||
// parse only this set's slice of the raw document so the whole JSON tree is
|
||||
// never kept in memory at once
|
||||
const RawJson::SetRange &rawRange = curSetToParse.getRawRange();
|
||||
const QByteArray setBytes(rawSetsData.constData() + rawRange.start, rawRange.length);
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError);
|
||||
if (parseError.error != QJsonParseError::NoError) {
|
||||
qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":"
|
||||
<< parseError.errorString();
|
||||
++setIndex;
|
||||
continue;
|
||||
}
|
||||
|
||||
const QJsonArray setCards = setDoc.object().value("cards").toArray();
|
||||
int numCardsInSet = importCardsFromSet(newSet, setCards);
|
||||
|
||||
++setIndex;
|
||||
|
||||
|
|
@ -583,6 +598,7 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr
|
|||
void OracleImporter::releaseSetData()
|
||||
{
|
||||
allSets.clear();
|
||||
rawSetsData.clear();
|
||||
}
|
||||
|
||||
void OracleImporter::clear()
|
||||
|
|
@ -590,4 +606,5 @@ void OracleImporter::clear()
|
|||
sets.clear();
|
||||
cards.clear();
|
||||
allSets.clear();
|
||||
rawSetsData.clear();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
#ifndef ORACLEIMPORTER_H
|
||||
#define ORACLEIMPORTER_H
|
||||
|
||||
#include "raw_json_scanner.h"
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QJsonArray>
|
||||
#include <QJsonObject>
|
||||
#include <QMap>
|
||||
|
|
@ -46,10 +49,12 @@ class SetToDownload
|
|||
{
|
||||
private:
|
||||
QString shortName, longName;
|
||||
QJsonArray cards;
|
||||
QDate releaseDate;
|
||||
QString setType;
|
||||
CardSet::Priority priority;
|
||||
// Byte range of this set's object within the importer's raw JSON text. Parsing
|
||||
// one set at a time keeps peak memory low instead of holding the whole document.
|
||||
RawJson::SetRange rawRange;
|
||||
|
||||
public:
|
||||
const QString &getShortName() const
|
||||
|
|
@ -60,10 +65,6 @@ public:
|
|||
{
|
||||
return longName;
|
||||
}
|
||||
const QJsonArray &getCards() const
|
||||
{
|
||||
return cards;
|
||||
}
|
||||
const QString &getSetType() const
|
||||
{
|
||||
return setType;
|
||||
|
|
@ -76,16 +77,23 @@ public:
|
|||
{
|
||||
return priority;
|
||||
}
|
||||
const RawJson::SetRange &getRawRange() const
|
||||
{
|
||||
return rawRange;
|
||||
}
|
||||
SetToDownload(QString _shortName,
|
||||
QString _longName,
|
||||
QJsonArray _cards,
|
||||
CardSet::Priority _priority,
|
||||
QString _setType = QString(),
|
||||
const QDate &_releaseDate = QDate())
|
||||
: shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)),
|
||||
releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority)
|
||||
: shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate),
|
||||
setType(std::move(_setType)), priority(_priority)
|
||||
{
|
||||
}
|
||||
void setRawRange(const RawJson::SetRange &_rawRange)
|
||||
{
|
||||
rawRange = _rawRange;
|
||||
}
|
||||
bool operator<(const SetToDownload &set) const
|
||||
{
|
||||
return longName.compare(set.longName, Qt::CaseInsensitive) < 0;
|
||||
|
|
@ -141,6 +149,12 @@ private:
|
|||
|
||||
QList<SetToDownload> allSets;
|
||||
|
||||
/**
|
||||
* The raw JSON text of the source document, retained for lazy per-set
|
||||
* parsing during startImport(). Frees the card data as each set is imported.
|
||||
*/
|
||||
QByteArray rawSetsData;
|
||||
|
||||
CardInfoPtr addCard(QString name,
|
||||
const QString &text,
|
||||
bool isToken,
|
||||
|
|
@ -153,7 +167,11 @@ signals:
|
|||
|
||||
public:
|
||||
explicit OracleImporter(QObject *parent = nullptr);
|
||||
bool readSetsFromByteArray(const QByteArray &data);
|
||||
/**
|
||||
* Scans the given JSON document for set metadata. Takes the data by value so
|
||||
* the wizard can hand over its decompressed buffer without copying it.
|
||||
*/
|
||||
bool readSetsFromByteArray(QByteArray data);
|
||||
int startImport();
|
||||
bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion);
|
||||
int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList);
|
||||
|
|
@ -166,6 +184,10 @@ public:
|
|||
{
|
||||
return allSets;
|
||||
}
|
||||
const QByteArray &getRawSetsData() const
|
||||
{
|
||||
return rawSetsData;
|
||||
}
|
||||
void releaseSetData();
|
||||
void clear();
|
||||
};
|
||||
|
|
|
|||
573
oracle/src/raw_json_scanner.cpp
Normal file
573
oracle/src/raw_json_scanner.cpp
Normal file
|
|
@ -0,0 +1,573 @@
|
|||
#include "raw_json_scanner.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
inline bool isWhitespace(char c)
|
||||
{
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
||||
}
|
||||
|
||||
const char *skipWhitespace(const char *p, const char *end)
|
||||
{
|
||||
while (p < end && isWhitespace(*p)) {
|
||||
++p;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
inline bool isHexDigit(char c)
|
||||
{
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
inline quint8 hexValue(char c)
|
||||
{
|
||||
if (c >= '0' && c <= '9') {
|
||||
return c - '0';
|
||||
}
|
||||
if (c >= 'a' && c <= 'f') {
|
||||
return c - 'a' + 10;
|
||||
}
|
||||
return c - 'A' + 10;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Skips past a JSON string without decoding it, validating escapes.
|
||||
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
|
||||
*/
|
||||
bool skipString(const char *&p, const char *end)
|
||||
{
|
||||
++p; // opening quote
|
||||
for (;;) {
|
||||
const void *quote = memchr(p, '"', static_cast<size_t>(end - p));
|
||||
if (!quote) {
|
||||
return false; // unterminated string
|
||||
}
|
||||
// Backslash escapes can only appear before the closing quote, so bound
|
||||
// the scan to the string extent instead of the rest of the document.
|
||||
const void *backslash = memchr(p, '\\', static_cast<size_t>(static_cast<const char *>(quote) - p));
|
||||
if (!backslash) {
|
||||
p = static_cast<const char *>(quote) + 1;
|
||||
return true;
|
||||
}
|
||||
const char *b = static_cast<const char *>(backslash);
|
||||
if (end - b < 2) {
|
||||
return false;
|
||||
}
|
||||
const char escaped = b[1];
|
||||
if (escaped == 'u') {
|
||||
if (end - b < 6) {
|
||||
return false;
|
||||
}
|
||||
quint32 codepoint = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(b[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
codepoint = codepoint * 16 + hexValue(b[2 + i]);
|
||||
}
|
||||
p = b + 6;
|
||||
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
|
||||
// expect the low-surrogate escape for the second half
|
||||
if (end - p < 6 || p[0] != '\\' || p[1] != 'u') {
|
||||
return false; // unpaired high surrogate
|
||||
}
|
||||
quint32 low = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
low = low * 16 + hexValue(p[2 + i]);
|
||||
}
|
||||
if (low < 0xDC00 || low > 0xDFFF) {
|
||||
return false;
|
||||
}
|
||||
p += 6;
|
||||
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
|
||||
return false; // unpaired low surrogate
|
||||
}
|
||||
continue;
|
||||
}
|
||||
switch (escaped) {
|
||||
case '"':
|
||||
case '\\':
|
||||
case '/':
|
||||
case 'b':
|
||||
case 'f':
|
||||
case 'n':
|
||||
case 'r':
|
||||
case 't':
|
||||
p = b + 2;
|
||||
continue;
|
||||
default:
|
||||
return false; // invalid escape
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Decodes a JSON string into @p out, validating it as it goes.
|
||||
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
|
||||
*/
|
||||
bool decodeString(const char *&p, const char *end, QString &out)
|
||||
{
|
||||
out.clear();
|
||||
QByteArray utf8;
|
||||
auto flush = [&out, &utf8]() {
|
||||
if (!utf8.isEmpty()) {
|
||||
out += QString::fromUtf8(utf8);
|
||||
utf8.clear();
|
||||
}
|
||||
};
|
||||
|
||||
++p; // opening quote
|
||||
while (p < end) {
|
||||
const char c = *p;
|
||||
if (c == '\\') {
|
||||
flush();
|
||||
++p; // escaped character
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char escaped = *p;
|
||||
if (escaped == 'u') {
|
||||
++p; // first hex digit
|
||||
if (p + 4 > end) {
|
||||
return false;
|
||||
}
|
||||
quint32 codepoint = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[i])) {
|
||||
return false;
|
||||
}
|
||||
codepoint = codepoint * 16 + hexValue(p[i]);
|
||||
}
|
||||
p += 4;
|
||||
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
|
||||
// expect a low-surrogate escape for the second half
|
||||
if (p + 6 > end || p[0] != '\\' || p[1] != 'u') {
|
||||
return false; // unpaired high surrogate
|
||||
}
|
||||
quint32 low = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
low = low * 16 + hexValue(p[2 + i]);
|
||||
}
|
||||
if (low < 0xDC00 || low > 0xDFFF) {
|
||||
return false;
|
||||
}
|
||||
out += QChar(codepoint);
|
||||
out += QChar(low);
|
||||
p += 6;
|
||||
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
|
||||
return false; // unpaired low surrogate
|
||||
} else {
|
||||
out += QChar(codepoint);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
switch (escaped) {
|
||||
case '"':
|
||||
out += '"';
|
||||
break;
|
||||
case '\\':
|
||||
out += '\\';
|
||||
break;
|
||||
case '/':
|
||||
out += '/';
|
||||
break;
|
||||
case 'b':
|
||||
out += '\b';
|
||||
break;
|
||||
case 'f':
|
||||
out += '\f';
|
||||
break;
|
||||
case 'n':
|
||||
out += '\n';
|
||||
break;
|
||||
case 'r':
|
||||
out += '\r';
|
||||
break;
|
||||
case 't':
|
||||
out += '\t';
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (c == '"') {
|
||||
++p;
|
||||
flush();
|
||||
return true;
|
||||
}
|
||||
if (static_cast<unsigned char>(c) < 0x20) {
|
||||
return false; // unescaped control character is invalid JSON
|
||||
}
|
||||
utf8 += c;
|
||||
++p;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool matchLiteral(const char *&p, const char *end, const char *literal, int length)
|
||||
{
|
||||
if (end - p < length || memcmp(p, literal, static_cast<size_t>(length)) != 0) {
|
||||
return false;
|
||||
}
|
||||
const char *after = p + length;
|
||||
if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) {
|
||||
return false;
|
||||
}
|
||||
p = after;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool skipNumber(const char *&p, const char *end)
|
||||
{
|
||||
// JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
|
||||
if (p < end && *p == '-') {
|
||||
++p;
|
||||
}
|
||||
if (p < end && *p == '0') {
|
||||
++p;
|
||||
} else if (p < end && *p >= '1' && *p <= '9') {
|
||||
++p;
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
if (p < end && *p == '.') {
|
||||
++p;
|
||||
if (p >= end || !QChar::isDigit(*p)) {
|
||||
return false;
|
||||
}
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
}
|
||||
if (p < end && (*p == 'e' || *p == 'E')) {
|
||||
++p;
|
||||
if (p < end && (*p == '+' || *p == '-')) {
|
||||
++p;
|
||||
}
|
||||
if (p >= end || !QChar::isDigit(*p)) {
|
||||
return false;
|
||||
}
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool skipValue(const char *&p, const char *end);
|
||||
bool skipObject(const char *&p, const char *end);
|
||||
bool skipArray(const char *&p, const char *end);
|
||||
|
||||
bool skipPrimitive(const char *&p, const char *end)
|
||||
{
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char c = *p;
|
||||
if (c == '"') {
|
||||
return skipString(p, end);
|
||||
}
|
||||
if (c == 't') {
|
||||
return matchLiteral(p, end, "true", 4);
|
||||
}
|
||||
if (c == 'f') {
|
||||
return matchLiteral(p, end, "false", 5);
|
||||
}
|
||||
if (c == 'n') {
|
||||
return matchLiteral(p, end, "null", 4);
|
||||
}
|
||||
if (c == '-' || (c >= '0' && c <= '9')) {
|
||||
return skipNumber(p, end);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool skipObject(const char *&p, const char *end)
|
||||
{
|
||||
++p; // '{'
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != '"') {
|
||||
return false;
|
||||
}
|
||||
if (!skipString(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != ':') {
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
if (!skipValue(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool skipArray(const char *&p, const char *end)
|
||||
{
|
||||
++p; // '['
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == ']') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
if (!skipValue(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == ']') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool skipValue(const char *&p, const char *end)
|
||||
{
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char c = *p;
|
||||
if (c == '{') {
|
||||
return skipObject(p, end);
|
||||
}
|
||||
if (c == '[') {
|
||||
return skipArray(p, end);
|
||||
}
|
||||
return skipPrimitive(p, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Iterates the members of the object starting at @p p.
|
||||
*
|
||||
* For each member invokes @p memberCallback with the key and the byte range of
|
||||
* its value. Advancing @p p is unaffected by the callback.
|
||||
*/
|
||||
template <typename F> bool forEachObjectMember(const char *&p, const char *end, F &&memberCallback)
|
||||
{
|
||||
++p; // '{'
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != '"') {
|
||||
return false;
|
||||
}
|
||||
QString key;
|
||||
if (!decodeString(p, end, key)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != ':') {
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
const char *valueStart = skipWhitespace(p, end);
|
||||
const char *valueEnd = valueStart;
|
||||
if (!skipValue(valueEnd, end)) {
|
||||
return false;
|
||||
}
|
||||
if (!memberCallback(key, valueStart, valueEnd)) {
|
||||
return false;
|
||||
}
|
||||
p = valueEnd;
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Counts the direct elements of an array value; returns -1 if the array is malformed.
|
||||
int countArrayElements(const char *p, const char *end)
|
||||
{
|
||||
++p; // '['
|
||||
p = skipWhitespace(p, end);
|
||||
int count = 0;
|
||||
if (p < end && *p == ']') {
|
||||
return 0;
|
||||
}
|
||||
for (;;) {
|
||||
if (!skipValue(p, end)) {
|
||||
return -1;
|
||||
}
|
||||
++count;
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return -1;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == ']') {
|
||||
return count;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace RawJson
|
||||
{
|
||||
|
||||
QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error)
|
||||
{
|
||||
QList<SetRange> ranges;
|
||||
if (error) {
|
||||
*error = ScanError{};
|
||||
}
|
||||
|
||||
const auto fail = [&](const QString &message) -> QList<SetRange> {
|
||||
if (error) {
|
||||
error->message = message;
|
||||
}
|
||||
return {};
|
||||
};
|
||||
|
||||
const char *begin = json.constData();
|
||||
const char *end = begin + json.size();
|
||||
if (begin >= end) {
|
||||
return fail(QStringLiteral("empty JSON document"));
|
||||
}
|
||||
|
||||
const char *p = skipWhitespace(begin, end);
|
||||
if (p >= end || *p != '{') {
|
||||
return fail(QStringLiteral("top-level JSON must be an object"));
|
||||
}
|
||||
|
||||
bool foundData = false;
|
||||
bool malformedSetData = false;
|
||||
|
||||
const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) {
|
||||
if (key == QStringLiteral("data")) {
|
||||
foundData = true;
|
||||
if (valueStart >= valueEnd || *valueStart != '{') {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
const char *setP = valueStart;
|
||||
const bool ok = forEachObjectMember(
|
||||
setP, valueEnd, [&](const QString &setCode, const char *setStart, const char *setEnd) {
|
||||
if (setStart >= setEnd || *setStart != '{') {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
SetRange range;
|
||||
range.start = setStart - begin;
|
||||
range.length = setEnd - setStart;
|
||||
range.code = setCode;
|
||||
|
||||
const char *memberP = setStart;
|
||||
const bool metaOk =
|
||||
forEachObjectMember(memberP, setEnd, [&](const QString &field, const char *fs, const char *fe) {
|
||||
if (field == QStringLiteral("code")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.code);
|
||||
}
|
||||
if (field == QStringLiteral("name")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.name);
|
||||
}
|
||||
if (field == QStringLiteral("type")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.type);
|
||||
}
|
||||
if (field == QStringLiteral("releaseDate")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.releaseDate);
|
||||
}
|
||||
if (field == QStringLiteral("cards")) {
|
||||
if (fs >= fe || *fs != '[') {
|
||||
return false;
|
||||
}
|
||||
range.cardCount = countArrayElements(fs, fe);
|
||||
return range.cardCount >= 0;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (!metaOk) {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
ranges.append(range);
|
||||
return true;
|
||||
});
|
||||
if (!ok) {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
if (!forEachObjectMember(p, end, topLevelCallback)) {
|
||||
return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON"));
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p != end) {
|
||||
return fail(QStringLiteral("trailing content after top-level JSON object"));
|
||||
}
|
||||
if (!foundData) {
|
||||
return fail(QStringLiteral("missing \"data\" object"));
|
||||
}
|
||||
if (ranges.isEmpty()) {
|
||||
return fail(QStringLiteral("no sets found in \"data\""));
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
} // namespace RawJson
|
||||
60
oracle/src/raw_json_scanner.h
Normal file
60
oracle/src/raw_json_scanner.h
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
#ifndef RAW_JSON_SCANNER_H
|
||||
#define RAW_JSON_SCANNER_H
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QList>
|
||||
#include <QString>
|
||||
|
||||
namespace RawJson
|
||||
{
|
||||
|
||||
struct SetRange
|
||||
{
|
||||
/** @brief Byte offset of the set's object within the scanned buffer. */
|
||||
qsizetype start = -1;
|
||||
/** @brief Byte length of the set's object, including the surrounding braces. */
|
||||
qsizetype length = 0;
|
||||
/** @brief Number of entries in the set's "cards" array. */
|
||||
int cardCount = 0;
|
||||
QString code;
|
||||
QString name;
|
||||
QString type;
|
||||
QString releaseDate;
|
||||
};
|
||||
|
||||
struct ScanError
|
||||
{
|
||||
bool isError() const
|
||||
{
|
||||
return !message.isEmpty();
|
||||
}
|
||||
QString message;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Scans a full MTGJSON document without materializing the JSON tree.
|
||||
*
|
||||
* Splits the top-level "data" object into per-set byte ranges and reads each
|
||||
* set's metadata directly from the raw bytes. The oracle importer can then
|
||||
* parse one set at a time during import, keeping peak memory far below a single
|
||||
* QJsonDocument::fromJson() over the whole file.
|
||||
*
|
||||
* The whole document is structurally validated while scanning (strings,
|
||||
* escapes, braces, and a trailing-content check), so malformed input is
|
||||
* rejected just like QJsonDocument::fromJson would.
|
||||
*
|
||||
* Following QJsonDocument::fromJson's convention, the parsed ranges are
|
||||
* returned by value and any failure is reported through the @p error out
|
||||
* parameter.
|
||||
*
|
||||
* @param json The raw MTGJSON document bytes.
|
||||
* @param error Out parameter. Set to an error ScanError when the document
|
||||
* cannot be parsed, otherwise left empty. Passing a null
|
||||
* pointer disables error reporting.
|
||||
* @return The detected per-set ranges, or an empty list on failure.
|
||||
*/
|
||||
QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error = nullptr);
|
||||
|
||||
} // namespace RawJson
|
||||
|
||||
#endif // RAW_JSON_SCANNER_H
|
||||
|
|
@ -11,7 +11,7 @@ add_test(NAME parse_cipt_test COMMAND parse_cipt_test)
|
|||
# Oracle importer unit tests
|
||||
add_executable(
|
||||
oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
|
||||
oracle_importer_test.cpp
|
||||
../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp
|
||||
)
|
||||
|
||||
if(NOT GTEST_FOUND)
|
||||
|
|
@ -51,7 +51,7 @@ endif()
|
|||
add_executable(
|
||||
oracle_importer_benchmark_test
|
||||
${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
|
||||
oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
|
||||
../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
|
||||
)
|
||||
|
||||
if(NOT GTEST_FOUND)
|
||||
|
|
|
|||
|
|
@ -146,9 +146,10 @@ TEST(OracleBenchmark, ParseJsonThroughput)
|
|||
|
||||
for (int i = 0; i < iterations; ++i) {
|
||||
OracleImporter importer;
|
||||
QByteArray source = data;
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
bool ok = importer.readSetsFromByteArray(data);
|
||||
bool ok = importer.readSetsFromByteArray(std::move(source));
|
||||
ASSERT_TRUE(ok);
|
||||
totalMs += timer.elapsed();
|
||||
}
|
||||
|
|
@ -404,7 +405,7 @@ TEST(OracleBenchmark, ImportRamUsage)
|
|||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(data));
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data)));
|
||||
const qint64 parseMs = timer.elapsed();
|
||||
const MemorySnapshot afterParse = MemorySnapshot::current();
|
||||
|
||||
|
|
@ -468,7 +469,7 @@ TEST(OracleBenchmark, ImportRamUsageAllPrintings)
|
|||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(setsData));
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData)));
|
||||
const qint64 parseMs = timer.elapsed();
|
||||
const MemorySnapshot afterParse = MemorySnapshot::current();
|
||||
|
||||
|
|
|
|||
|
|
@ -488,6 +488,112 @@ TEST_F(OracleImporterTest, ApostropheNormalized)
|
|||
ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity"));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RawJson scanner tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse)
|
||||
{
|
||||
QJsonObject root;
|
||||
QJsonObject data;
|
||||
data["AAA"] = makeCard("Alpha Card");
|
||||
data["BBB"] = makeCard("Beta Card");
|
||||
root["data"] = data;
|
||||
|
||||
const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact);
|
||||
|
||||
RawJson::ScanError error;
|
||||
const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(bytes, &error);
|
||||
ASSERT_FALSE(error.isError()) << error.message.toStdString();
|
||||
ASSERT_EQ(ranges.size(), 2);
|
||||
|
||||
const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject();
|
||||
for (const RawJson::SetRange &range : ranges) {
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument sliceDoc =
|
||||
QJsonDocument::fromJson(QByteArray(bytes.constData() + range.start, range.length), &parseError);
|
||||
ASSERT_EQ(parseError.error, QJsonParseError::NoError)
|
||||
<< range.code.toStdString() << ": " << parseError.errorString().toStdString();
|
||||
ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards)
|
||||
{
|
||||
const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\","
|
||||
"\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\","
|
||||
"\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}";
|
||||
|
||||
RawJson::ScanError error;
|
||||
const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(json, &error);
|
||||
ASSERT_FALSE(error.isError());
|
||||
ASSERT_EQ(ranges.size(), 1);
|
||||
|
||||
const RawJson::SetRange &range = ranges.first();
|
||||
ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key
|
||||
const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00);
|
||||
ASSERT_EQ(range.name, expectedName);
|
||||
ASSERT_EQ(range.type, "expansion");
|
||||
ASSERT_EQ(range.releaseDate, "2024-01-05");
|
||||
ASSERT_EQ(range.cardCount, 3);
|
||||
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument sliceDoc =
|
||||
QJsonDocument::fromJson(QByteArray(json.constData() + range.start, range.length), &parseError);
|
||||
ASSERT_EQ(parseError.error, QJsonParseError::NoError);
|
||||
ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName);
|
||||
ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3);
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson)
|
||||
{
|
||||
const QList<QByteArray> invalid = {"not json",
|
||||
"[]",
|
||||
"{\"data\":[]}",
|
||||
"{\"data\":{}}",
|
||||
"{\"other\":{}}",
|
||||
"{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\","
|
||||
"\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"};
|
||||
|
||||
for (const QByteArray &json : invalid) {
|
||||
RawJson::ScanError error;
|
||||
RawJson::scanSetRanges(json, &error);
|
||||
EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Lazy per-set parsing tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, StartImportParsesSetsLazily)
|
||||
{
|
||||
QJsonObject setObj = makeCard("Lazy Import Card");
|
||||
QJsonArray cards;
|
||||
cards.append(setObj);
|
||||
QJsonObject dataSet;
|
||||
dataSet["code"] = "tst";
|
||||
dataSet["name"] = "Test Set";
|
||||
dataSet["type"] = "expansion";
|
||||
dataSet["releaseDate"] = "2024-01-01";
|
||||
dataSet["cards"] = cards;
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject{{"TST", dataSet}};
|
||||
|
||||
const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact);
|
||||
ASSERT_TRUE(importer->readSetsFromByteArray(data));
|
||||
ASSERT_FALSE(importer->getRawSetsData().isEmpty());
|
||||
|
||||
const int importedSets = importer->startImport();
|
||||
ASSERT_EQ(importedSets, 1);
|
||||
ASSERT_EQ(importer->getCardList().size(), 1);
|
||||
ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull());
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue