mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-09-24 18:33:03 -07:00
Compare commits
4 commits
68d67ff3e5
...
276fb9f7ee
| Author | SHA1 | Date | |
|---|---|---|---|
|
276fb9f7ee | ||
|
|
60fda3a5a2 | ||
|
|
a1b4c4da4d | ||
|
|
ef39b4967f |
11 changed files with 1943 additions and 843 deletions
|
|
@ -23,7 +23,7 @@ set(oracle_SOURCES
|
|||
src/pages.cpp
|
||||
src/pagetemplates.cpp
|
||||
src/parsehelpers.cpp
|
||||
src/qt-json/json.cpp
|
||||
src/raw_json_scanner.cpp
|
||||
../cockatrice/src/client/settings/cache_settings.cpp
|
||||
../cockatrice/src/client/settings/card_counter_settings.cpp
|
||||
../cockatrice/src/client/settings/shortcuts_settings.cpp
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
#include "libcockatrice/interfaces/noop_card_preference_provider.h"
|
||||
#include "libcockatrice/interfaces/noop_card_set_priority_controller.h"
|
||||
#include "parsehelpers.h"
|
||||
#include "qt-json/json.h"
|
||||
|
||||
#include <QDebug>
|
||||
#include <QJsonDocument>
|
||||
#include <QJsonObject>
|
||||
#include <QJsonParseError>
|
||||
#include <QRegularExpression>
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
|
|
@ -42,31 +44,29 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s
|
|||
return priority;
|
||||
}
|
||||
|
||||
bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
||||
bool OracleImporter::readSetsFromByteArray(QByteArray data)
|
||||
{
|
||||
bool ok;
|
||||
auto setsMap = QtJson::Json::parse(QString(data), ok).toMap().value("data").toMap();
|
||||
if (!ok) {
|
||||
qDebug() << "error: QtJson::Json::parse()";
|
||||
QList<RawJson::SetRange> ranges;
|
||||
const RawJson::ScanError scanError = RawJson::scanSetRanges(data, ranges);
|
||||
if (scanError.isError()) {
|
||||
qDebug() << "error: RawJson::scanSetRanges():" << scanError.message;
|
||||
return false;
|
||||
}
|
||||
|
||||
QList<SetToDownload> newSetList;
|
||||
newSetList.reserve(ranges.size());
|
||||
|
||||
QListIterator it(setsMap.values());
|
||||
|
||||
while (it.hasNext()) {
|
||||
QVariantMap map = it.next().toMap();
|
||||
QString shortName = map.value("code").toString().toUpper();
|
||||
QString longName = map.value("name").toString();
|
||||
QList<QVariant> setCards = map.value("cards").toList();
|
||||
QString setType = map.value("type").toString();
|
||||
QDate releaseDate = map.value("releaseDate").toDate();
|
||||
for (const RawJson::SetRange &range : ranges) {
|
||||
QString shortName = range.code.toUpper();
|
||||
QString longName = range.name;
|
||||
QString setType = range.type;
|
||||
QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate);
|
||||
CardSet::Priority priority = getSetPriority(setType, shortName);
|
||||
// capitalize set type
|
||||
if (setType.length() > 0) {
|
||||
// basic grammar for words that aren't capitalized, like in "From the Vault"
|
||||
const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", "of", "in", "and", "with", "or"};
|
||||
static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for",
|
||||
"of", "in", "and", "with", "or"};
|
||||
QStringList words = setType.split("_");
|
||||
setType.clear();
|
||||
bool first = false;
|
||||
|
|
@ -74,13 +74,15 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
|||
if (first && noCapitalize.contains(item)) {
|
||||
setType += item + QString(" ");
|
||||
} else {
|
||||
setType += item[0].toUpper() + item.mid(1, -1) + QString(" ");
|
||||
setType += item[0].toUpper() + item.mid(1) + QString(" ");
|
||||
first = true;
|
||||
}
|
||||
}
|
||||
setType = setType.trimmed();
|
||||
}
|
||||
newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate));
|
||||
SetToDownload set(shortName, longName, priority, setType, releaseDate);
|
||||
set.setRawRange(range.start, range.length, range.cardCount);
|
||||
newSetList.append(set);
|
||||
}
|
||||
|
||||
std::sort(newSetList.begin(), newSetList.end());
|
||||
|
|
@ -89,6 +91,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
|
|||
return false;
|
||||
}
|
||||
allSets = newSetList;
|
||||
rawSetsData = std::move(data);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -122,14 +125,8 @@ static void sortAndReduceColors(QString &colors)
|
|||
std::sort(colors.begin(), colors.end(),
|
||||
[](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); });
|
||||
// reduce
|
||||
QChar lastChar = '\0';
|
||||
for (int i = 0; i < colors.size(); ++i) {
|
||||
if (colors.at(i) == lastChar) {
|
||||
colors.remove(i, 1);
|
||||
} else {
|
||||
lastChar = colors.at(i);
|
||||
}
|
||||
}
|
||||
auto last = std::unique(colors.begin(), colors.end());
|
||||
colors.erase(last, colors.end());
|
||||
}
|
||||
|
||||
CardInfoPtr OracleImporter::addCard(QString name,
|
||||
|
|
@ -142,8 +139,9 @@ CardInfoPtr OracleImporter::addCard(QString name,
|
|||
// Workaround for card name weirdness
|
||||
name = name.replace("Æ", "AE");
|
||||
name = name.replace("’", "'");
|
||||
if (cards.contains(name)) {
|
||||
CardInfoPtr card = cards.value(name);
|
||||
auto existingIt = cards.constFind(name);
|
||||
if (existingIt != cards.constEnd()) {
|
||||
CardInfoPtr card = existingIt.value();
|
||||
card->addToSet(printingInfo.getSet(), printingInfo);
|
||||
if (card->getProperties().filter(formatRegex).empty()) {
|
||||
card->combineLegalities(properties);
|
||||
|
|
@ -182,8 +180,9 @@ CardInfoPtr OracleImporter::addCard(QString name,
|
|||
|
||||
// DETECT CARD POSITIONING INFO
|
||||
|
||||
bool landscapeOrientation = properties.value("maintype") == "Battle" || properties.value("layout") == "split" ||
|
||||
properties.value("layout") == "planar";
|
||||
QString layoutVal = properties.value("layout");
|
||||
bool landscapeOrientation =
|
||||
properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar";
|
||||
|
||||
// cards that enter the field tapped
|
||||
bool cipt = parseCipt(name, text) || landscapeOrientation;
|
||||
|
|
@ -222,12 +221,12 @@ CardInfoPtr OracleImporter::addCard(QString name,
|
|||
return newCard;
|
||||
}
|
||||
|
||||
static QString getStringPropertyFromMap(const QVariantMap &card, const QString &propertyName)
|
||||
static inline QString getJsonString(const QJsonObject &obj, const QString &key)
|
||||
{
|
||||
return card.contains(propertyName) ? card.value(propertyName).toString() : QString("");
|
||||
return obj.value(key).toString();
|
||||
}
|
||||
|
||||
int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList<QVariant> &cardsList)
|
||||
int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList)
|
||||
{
|
||||
// mtgjson name => xml name
|
||||
static const QMap<QString, QString> cardProperties{
|
||||
|
|
@ -248,7 +247,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
|
||||
static const QString ptSeparator = "/";
|
||||
static constexpr bool isToken = false;
|
||||
static const QList<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
|
||||
static const QSet<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
|
||||
|
||||
int numCards = 0;
|
||||
|
||||
|
|
@ -256,16 +255,16 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
QMap<QString, QPair<QList<SplitCardPart>, QString>> splitCards;
|
||||
|
||||
// Keeps track of all names encountered so far
|
||||
QList<QString> allNameProps;
|
||||
QSet<QString> allNameProps;
|
||||
|
||||
for (const QVariant &cardVar : cardsList) {
|
||||
QVariantMap card = cardVar.toMap();
|
||||
for (const QJsonValue &cardVal : cardsList) {
|
||||
QJsonObject card = cardVal.toObject();
|
||||
|
||||
/* Currently used layouts are:
|
||||
* augment, double_faced_token, flip, host, leveler, meld, normal, planar,
|
||||
* saga, scheme, split, token, transform, vanguard
|
||||
*/
|
||||
QString layout = getStringPropertyFromMap(card, "layout");
|
||||
QString layout = getJsonString(card, "layout");
|
||||
|
||||
// don't import tokens from the json file
|
||||
if (layout == "token") {
|
||||
|
|
@ -273,9 +272,9 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
}
|
||||
|
||||
// normal cards handling
|
||||
QString name = getStringPropertyFromMap(card, "name");
|
||||
QString text = getStringPropertyFromMap(card, "text");
|
||||
QString faceName = getStringPropertyFromMap(card, "faceName");
|
||||
QString name = getJsonString(card, "name");
|
||||
QString text = getJsonString(card, "text");
|
||||
QString faceName = getJsonString(card, "faceName");
|
||||
if (faceName.isEmpty()) {
|
||||
faceName = name;
|
||||
}
|
||||
|
|
@ -283,39 +282,34 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
// card properties
|
||||
QHash<QString, QString> properties;
|
||||
for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) {
|
||||
QString mtgjsonProperty = i.key();
|
||||
QString xmlPropertyName = i.value();
|
||||
QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty);
|
||||
QString propertyValue = getJsonString(card, i.key());
|
||||
if (!propertyValue.isEmpty()) {
|
||||
properties.insert(xmlPropertyName, propertyValue);
|
||||
properties.insert(i.value(), propertyValue);
|
||||
}
|
||||
}
|
||||
|
||||
// per-set properties
|
||||
QHash<QString, QString> printingProps;
|
||||
for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) {
|
||||
QString mtgjsonProperty = i.key();
|
||||
QString xmlPropertyName = i.value();
|
||||
QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty);
|
||||
QString propertyValue = getJsonString(card, i.key());
|
||||
if (!propertyValue.isEmpty()) {
|
||||
printingProps.insert(xmlPropertyName, propertyValue);
|
||||
printingProps.insert(i.value(), propertyValue);
|
||||
}
|
||||
}
|
||||
|
||||
// handle flavorNames specially due to double-faced cards
|
||||
QString faceFlavorName = getStringPropertyFromMap(card, "faceFlavorName");
|
||||
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getStringPropertyFromMap(card, "flavorName");
|
||||
QString faceFlavorName = getJsonString(card, "faceFlavorName");
|
||||
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName");
|
||||
if (!flavorName.isEmpty()) {
|
||||
printingProps.insert("flavorName", flavorName);
|
||||
}
|
||||
|
||||
// Identifiers
|
||||
QJsonObject identifiers = card.value("identifiers").toObject();
|
||||
for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) {
|
||||
QString mtgjsonProperty = i.key();
|
||||
QString xmlPropertyName = i.value();
|
||||
QString propertyValue = getStringPropertyFromMap(card.value("identifiers").toMap(), mtgjsonProperty);
|
||||
QString propertyValue = getJsonString(identifiers, i.key());
|
||||
if (!propertyValue.isEmpty()) {
|
||||
printingProps.insert(xmlPropertyName, propertyValue);
|
||||
printingProps.insert(i.value(), propertyValue);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -331,21 +325,20 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) {
|
||||
numComponent = " (" + QString(lastChar).toLower() + ")";
|
||||
}
|
||||
allNameProps.append(faceName);
|
||||
allNameProps.insert(faceName);
|
||||
|
||||
// special handling properties
|
||||
QString colors = card.value("colors").toStringList().join("");
|
||||
QString colors = card.value("colors").toVariant().toStringList().join("");
|
||||
if (!colors.isEmpty()) {
|
||||
properties.insert("colors", colors);
|
||||
}
|
||||
|
||||
// special handling properties
|
||||
QString colorIdentity = card.value("colorIdentity").toStringList().join("");
|
||||
QString colorIdentity = card.value("colorIdentity").toVariant().toStringList().join("");
|
||||
if (!colorIdentity.isEmpty()) {
|
||||
properties.insert("coloridentity", colorIdentity);
|
||||
}
|
||||
|
||||
const auto &mainCardType = getMainCardType(card.value("types").toStringList());
|
||||
const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList());
|
||||
if (mainCardType.isEmpty()) {
|
||||
qDebug() << "warning: no mainCardType for card:" << name;
|
||||
} else {
|
||||
|
|
@ -354,22 +347,22 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
|
||||
// Depending on whether power and/or toughness are present, the format
|
||||
// is either P/T (most common), P (no toughness), or /T (no power).
|
||||
QString power = getStringPropertyFromMap(card, "power");
|
||||
QString toughness = getStringPropertyFromMap(card, "toughness");
|
||||
QString power = getJsonString(card, "power");
|
||||
QString toughness = getJsonString(card, "toughness");
|
||||
if (toughness.isEmpty() && !power.isEmpty()) {
|
||||
properties.insert("pt", power);
|
||||
} else if (!toughness.isEmpty()) {
|
||||
properties.insert("pt", power + ptSeparator + toughness);
|
||||
}
|
||||
|
||||
auto legalities = card.value("legalities").toMap();
|
||||
for (auto i = legalities.cbegin(), end = legalities.cend(); i != end; ++i) {
|
||||
auto legalities = card.value("legalities").toObject();
|
||||
for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) {
|
||||
properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower());
|
||||
}
|
||||
|
||||
// split cards are considered a single card, enqueue for later merging
|
||||
if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") {
|
||||
auto _faceName = getStringPropertyFromMap(card, "faceName");
|
||||
auto _faceName = getJsonString(card, "faceName");
|
||||
SplitCardPart split(_faceName, text, properties, printingInfo);
|
||||
auto found_iter = splitCards.find(name + numProperty);
|
||||
if (found_iter == splitCards.end()) {
|
||||
|
|
@ -382,11 +375,11 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
QList<CardRelation *> relatedCards;
|
||||
|
||||
// add other face for split cards as card relation
|
||||
if (!getStringPropertyFromMap(card, "side").isEmpty()) {
|
||||
auto faceManaValue = getStringPropertyFromMap(card, "faceManaValue");
|
||||
if (!getJsonString(card, "side").isEmpty()) {
|
||||
auto faceManaValue = getJsonString(card, "faceManaValue");
|
||||
if (faceManaValue.isEmpty()) {
|
||||
// check the old name for the property, for backwards compatibility purposes
|
||||
faceManaValue = getStringPropertyFromMap(card, "faceConvertedManaCost");
|
||||
faceManaValue = getJsonString(card, "faceConvertedManaCost");
|
||||
}
|
||||
properties["cmc"] = faceManaValue;
|
||||
|
||||
|
|
@ -406,20 +399,21 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
name = faceName;
|
||||
}
|
||||
|
||||
// mtgjon related cards
|
||||
if (card.contains("relatedCards")) {
|
||||
QVariantMap givenRelated = card.value("relatedCards").toMap();
|
||||
// mtgjson related cards
|
||||
QJsonObject givenRelated = card.value("relatedCards").toObject();
|
||||
if (!givenRelated.isEmpty()) {
|
||||
// conjured cards from a spellbook
|
||||
if (givenRelated.contains("spellbook")) {
|
||||
auto spbk = givenRelated.value("spellbook").toStringList();
|
||||
for (const QString &spbkName : spbk) {
|
||||
relatedCards.append(
|
||||
new CardRelation(spbkName, CardRelationType::DoesNotAttach, false, false, 1, true));
|
||||
QJsonArray spellbook = givenRelated.value("spellbook").toArray();
|
||||
if (!spellbook.isEmpty()) {
|
||||
for (const QJsonValue &spbkVal : spellbook) {
|
||||
relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false,
|
||||
false, 1, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CardInfoPtr newCard = addCard(name + numComponent, text, isToken, properties, relatedCards, printingInfo);
|
||||
CardInfoPtr newCard =
|
||||
addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo);
|
||||
numCards++;
|
||||
}
|
||||
}
|
||||
|
|
@ -427,7 +421,6 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
// split cards handling
|
||||
static const QString splitCardPropSeparator = QString(" // ");
|
||||
static const QString splitCardTextSeparator = QString("\n\n---\n\n");
|
||||
static const QList<CardRelation *> noRelatedCards = {};
|
||||
|
||||
QList<QPair<QList<SplitCardPart>, QString>> partsAndNames = splitCards.values();
|
||||
for (auto [splitCardParts, name] : partsAndNames) {
|
||||
|
|
@ -453,7 +446,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) {
|
||||
if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty
|
||||
properties.insert(prop, thisCardPropertyValue);
|
||||
} else if (prop == "colors") { // the card is both colors
|
||||
} else if (prop == "colors" || prop == "coloridentity") { // the card is both colors
|
||||
properties.insert(prop, originalPropertyValue + thisCardPropertyValue);
|
||||
} else if (prop == "maintype") { // don't create maintypes with //es in them
|
||||
continue;
|
||||
|
|
@ -465,7 +458,7 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
}
|
||||
}
|
||||
}
|
||||
CardInfoPtr newCard = addCard(name, text, isToken, properties, noRelatedCards, printingInfo);
|
||||
CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo);
|
||||
numCards++;
|
||||
}
|
||||
|
||||
|
|
@ -474,11 +467,16 @@ int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QList
|
|||
|
||||
FormatRulesNameMap OracleImporter::createDefaultMagicFormats()
|
||||
{
|
||||
static FormatRulesNameMap cached;
|
||||
if (!cached.isEmpty()) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Predefined common exceptions
|
||||
CardCondition superTypeIsBasic;
|
||||
superTypeIsBasic.field = "type";
|
||||
superTypeIsBasic.matchType = "regex";
|
||||
superTypeIsBasic.value = "\bBasic\b[^—]+\bLand\b";
|
||||
superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)";
|
||||
|
||||
ExceptionRule basicLands;
|
||||
basicLands.conditions.append(superTypeIsBasic);
|
||||
|
|
@ -492,7 +490,7 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats()
|
|||
mayContainAnyNumber.conditions.append(anyNumberAllowed);
|
||||
|
||||
// Map to store default rules
|
||||
FormatRulesNameMap defaultFormatRulesNameMap;
|
||||
FormatRulesNameMap &defaultFormatRulesNameMap = cached;
|
||||
|
||||
// ----------------- Helper lambda to create format -----------------
|
||||
auto makeFormat = [&](const QString &name, int minDeck = 60, int maxDeck = -1, int maxSideboardSize = 15,
|
||||
|
|
@ -541,6 +539,13 @@ int OracleImporter::startImport()
|
|||
{
|
||||
static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController();
|
||||
|
||||
// Pre-allocate cards hash to avoid rehashing during import
|
||||
int estimatedCards = 0;
|
||||
for (const SetToDownload &curSetToParse : allSets) {
|
||||
estimatedCards += curSetToParse.getCardCount();
|
||||
}
|
||||
cards.reserve(estimatedCards);
|
||||
|
||||
// add an empty set for tokens
|
||||
CardSetPtr tokenSet =
|
||||
CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens");
|
||||
|
|
@ -556,7 +561,20 @@ int OracleImporter::startImport()
|
|||
sets.insert(newSet->getShortName(), newSet);
|
||||
}
|
||||
|
||||
int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards());
|
||||
// parse only this set's slice of the raw document so the whole JSON tree is
|
||||
// never kept in memory at once
|
||||
const QByteArray setBytes(rawSetsData.constData() + curSetToParse.getRawStart(), curSetToParse.getRawLength());
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError);
|
||||
if (parseError.error != QJsonParseError::NoError) {
|
||||
qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":"
|
||||
<< parseError.errorString();
|
||||
++setIndex;
|
||||
continue;
|
||||
}
|
||||
|
||||
const QJsonArray setCards = setDoc.object().value("cards").toArray();
|
||||
int numCardsInSet = importCardsFromSet(newSet, setCards);
|
||||
|
||||
++setIndex;
|
||||
|
||||
|
|
@ -576,9 +594,18 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr
|
|||
return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion);
|
||||
}
|
||||
|
||||
void OracleImporter::releaseSetData()
|
||||
{
|
||||
allSets.clear();
|
||||
rawSetsData.clear();
|
||||
}
|
||||
|
||||
void OracleImporter::clear()
|
||||
{
|
||||
sets.clear();
|
||||
cards.clear();
|
||||
allSets.clear();
|
||||
rawSetsData.clear();
|
||||
// Note: createDefaultMagicFormats() uses a function-local static cache that is
|
||||
// intentionally not cleared here since format rules are hardcoded constants.
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
#ifndef ORACLEIMPORTER_H
|
||||
#define ORACLEIMPORTER_H
|
||||
|
||||
#include "raw_json_scanner.h"
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QJsonArray>
|
||||
#include <QJsonObject>
|
||||
#include <QMap>
|
||||
#include <QRegularExpression>
|
||||
#include <QVariant>
|
||||
|
|
@ -44,10 +49,14 @@ class SetToDownload
|
|||
{
|
||||
private:
|
||||
QString shortName, longName;
|
||||
QList<QVariant> cards;
|
||||
QDate releaseDate;
|
||||
QString setType;
|
||||
CardSet::Priority priority;
|
||||
// Byte range of this set's object within the importer's raw JSON text. Parsing
|
||||
// one set at a time keeps peak memory low instead of holding the whole document.
|
||||
qsizetype rawStart = -1;
|
||||
qsizetype rawLength = 0;
|
||||
int cardCount = 0;
|
||||
|
||||
public:
|
||||
const QString &getShortName() const
|
||||
|
|
@ -58,10 +67,6 @@ public:
|
|||
{
|
||||
return longName;
|
||||
}
|
||||
const QList<QVariant> &getCards() const
|
||||
{
|
||||
return cards;
|
||||
}
|
||||
const QString &getSetType() const
|
||||
{
|
||||
return setType;
|
||||
|
|
@ -74,16 +79,33 @@ public:
|
|||
{
|
||||
return priority;
|
||||
}
|
||||
qsizetype getRawStart() const
|
||||
{
|
||||
return rawStart;
|
||||
}
|
||||
qsizetype getRawLength() const
|
||||
{
|
||||
return rawLength;
|
||||
}
|
||||
int getCardCount() const
|
||||
{
|
||||
return cardCount;
|
||||
}
|
||||
SetToDownload(QString _shortName,
|
||||
QString _longName,
|
||||
QList<QVariant> _cards,
|
||||
CardSet::Priority _priority,
|
||||
QString _setType = QString(),
|
||||
const QDate &_releaseDate = QDate())
|
||||
: shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)),
|
||||
releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority)
|
||||
: shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate),
|
||||
setType(std::move(_setType)), priority(_priority)
|
||||
{
|
||||
}
|
||||
void setRawRange(qsizetype _rawStart, qsizetype _rawLength, int _cardCount)
|
||||
{
|
||||
rawStart = _rawStart;
|
||||
rawLength = _rawLength;
|
||||
cardCount = _cardCount;
|
||||
}
|
||||
bool operator<(const SetToDownload &set) const
|
||||
{
|
||||
return longName.compare(set.longName, Qt::CaseInsensitive) < 0;
|
||||
|
|
@ -139,6 +161,12 @@ private:
|
|||
|
||||
QList<SetToDownload> allSets;
|
||||
|
||||
/**
|
||||
* The raw JSON text of the source document, retained for lazy per-set
|
||||
* parsing during startImport(). Frees the card data as each set is imported.
|
||||
*/
|
||||
QByteArray rawSetsData;
|
||||
|
||||
CardInfoPtr addCard(QString name,
|
||||
const QString &text,
|
||||
bool isToken,
|
||||
|
|
@ -151,10 +179,14 @@ signals:
|
|||
|
||||
public:
|
||||
explicit OracleImporter(QObject *parent = nullptr);
|
||||
bool readSetsFromByteArray(const QByteArray &data);
|
||||
/**
|
||||
* Scans the given JSON document for set metadata. Takes the data by value so
|
||||
* the wizard can hand over its decompressed buffer without copying it.
|
||||
*/
|
||||
bool readSetsFromByteArray(QByteArray data);
|
||||
int startImport();
|
||||
bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion);
|
||||
int importCardsFromSet(const CardSetPtr ¤tSet, const QList<QVariant> &cardsList);
|
||||
int importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList);
|
||||
FormatRulesNameMap createDefaultMagicFormats();
|
||||
const CardNameMap &getCardList() const
|
||||
{
|
||||
|
|
@ -164,6 +196,11 @@ public:
|
|||
{
|
||||
return allSets;
|
||||
}
|
||||
const QByteArray &getRawSetsData() const
|
||||
{
|
||||
return rawSetsData;
|
||||
}
|
||||
void releaseSetData();
|
||||
void clear();
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -560,6 +560,9 @@ void SaveSetsPage::initializePage()
|
|||
|
||||
int setsImported = wizard()->importer->startImport();
|
||||
|
||||
// JSON data no longer needed after CardInfo objects are built
|
||||
wizard()->importer->releaseSetData();
|
||||
|
||||
if (setsImported == 0) {
|
||||
QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,545 +0,0 @@
|
|||
/* Copyright 2011 Eeli Reilin. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ''AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
* EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
|
||||
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* The views and conclusions contained in the software and documentation
|
||||
* are those of the authors and should not be interpreted as representing
|
||||
* official policies, either expressed or implied, of Eeli Reilin.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file json.cpp
|
||||
*/
|
||||
|
||||
#include "json.h"
|
||||
|
||||
#include <QMetaType>
|
||||
#include <iostream>
|
||||
|
||||
namespace QtJson
|
||||
{
|
||||
|
||||
static QString sanitizeString(QString str)
|
||||
{
|
||||
str.replace(QLatin1String("\\"), QLatin1String("\\\\"));
|
||||
str.replace(QLatin1String("\""), QLatin1String("\\\""));
|
||||
str.replace(QLatin1String("\b"), QLatin1String("\\b"));
|
||||
str.replace(QLatin1String("\f"), QLatin1String("\\f"));
|
||||
str.replace(QLatin1String("\n"), QLatin1String("\\n"));
|
||||
str.replace(QLatin1String("\r"), QLatin1String("\\r"));
|
||||
str.replace(QLatin1String("\t"), QLatin1String("\\t"));
|
||||
return QString(QLatin1String("\"%1\"")).arg(str);
|
||||
}
|
||||
|
||||
static QByteArray join(const QList<QByteArray> &list, const QByteArray &sep)
|
||||
{
|
||||
QByteArray res;
|
||||
for (const QByteArray &i : list) {
|
||||
if (!res.isEmpty()) {
|
||||
res += sep;
|
||||
}
|
||||
res += i;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* parse
|
||||
*/
|
||||
QVariant Json::parse(const QString &json)
|
||||
{
|
||||
bool success = true;
|
||||
return Json::parse(json, success);
|
||||
}
|
||||
|
||||
/**
|
||||
* parse
|
||||
*/
|
||||
QVariant Json::parse(const QString &json, bool &success)
|
||||
{
|
||||
success = true;
|
||||
|
||||
// Return an empty QVariant if the JSON data is either null or empty
|
||||
if (!json.isNull() || !json.isEmpty()) {
|
||||
// We'll start from index 0
|
||||
int index = 0;
|
||||
|
||||
// Parse the first value
|
||||
QVariant value = Json::parseValue(json, index, success);
|
||||
|
||||
// Return the parsed value
|
||||
return value;
|
||||
} else {
|
||||
// Return the empty QVariant
|
||||
return QVariant();
|
||||
}
|
||||
}
|
||||
|
||||
QByteArray Json::serialize(const QVariant &data)
|
||||
{
|
||||
bool success = true;
|
||||
return Json::serialize(data, success);
|
||||
}
|
||||
|
||||
QByteArray Json::serialize(const QVariant &data, bool &success)
|
||||
{
|
||||
QByteArray str;
|
||||
success = true;
|
||||
|
||||
if (!data.isValid()) // invalid or null?
|
||||
{
|
||||
str = "null";
|
||||
}
|
||||
else if ((data.typeId() == QMetaType::Type::QVariantList) ||
|
||||
(data.typeId() == QMetaType::Type::QStringList)) // variant is a list?
|
||||
{
|
||||
QList<QByteArray> values;
|
||||
const QVariantList list = data.toList();
|
||||
for (const QVariant &v : list) {
|
||||
QByteArray serializedValue = serialize(v);
|
||||
if (serializedValue.isNull()) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
values << serializedValue;
|
||||
}
|
||||
|
||||
str = "[ " + join(values, ", ") + " ]";
|
||||
}
|
||||
else if ((data.typeId() == QMetaType::Type::QVariantHash)) // variant is a hash?
|
||||
{
|
||||
const QVariantHash vhash = data.toHash();
|
||||
QHashIterator<QString, QVariant> it(vhash);
|
||||
str = "{ ";
|
||||
QList<QByteArray> pairs;
|
||||
|
||||
while (it.hasNext()) {
|
||||
it.next();
|
||||
QByteArray serializedValue = serialize(it.value());
|
||||
|
||||
if (serializedValue.isNull()) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue;
|
||||
}
|
||||
|
||||
str += join(pairs, ", ");
|
||||
str += " }";
|
||||
}
|
||||
else if ((data.typeId() == QMetaType::Type::QVariantMap)) // variant is a map?
|
||||
{
|
||||
const QVariantMap vmap = data.toMap();
|
||||
QMapIterator<QString, QVariant> it(vmap);
|
||||
str = "{ ";
|
||||
QList<QByteArray> pairs;
|
||||
while (it.hasNext()) {
|
||||
it.next();
|
||||
QByteArray serializedValue = serialize(it.value());
|
||||
if (serializedValue.isNull()) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue;
|
||||
}
|
||||
str += join(pairs, ", ");
|
||||
str += " }";
|
||||
}
|
||||
else if ((data.typeId() == QMetaType::Type::QString) ||
|
||||
(data.typeId() == QMetaType::Type::QByteArray)) // a string or a byte array?
|
||||
{
|
||||
str = sanitizeString(data.toString()).toUtf8();
|
||||
}
|
||||
else if (data.typeId() == QMetaType::Type::Double) // double?
|
||||
{
|
||||
str = QByteArray::number(data.toDouble(), 'g', 20);
|
||||
if (!str.contains(".") && !str.contains("e")) {
|
||||
str += ".0";
|
||||
}
|
||||
}
|
||||
else if (data.typeId() == QMetaType::Type::Bool) // boolean value?
|
||||
{
|
||||
str = data.toBool() ? "true" : "false";
|
||||
}
|
||||
else if (data.typeId() == QMetaType::Type::ULongLong) // large unsigned number?
|
||||
{
|
||||
str = QByteArray::number(data.value<qulonglong>());
|
||||
} else if (data.canConvert<qlonglong>()) // any signed number?
|
||||
{
|
||||
str = QByteArray::number(data.value<qlonglong>());
|
||||
} else if (data.canConvert<long>()) {
|
||||
str = QString::number(data.value<long>()).toUtf8();
|
||||
} else if (data.canConvert<QString>()) // can value be converted to string?
|
||||
{
|
||||
// this will catch QDate, QDateTime, QUrl, ...
|
||||
str = sanitizeString(data.toString()).toUtf8();
|
||||
} else {
|
||||
success = false;
|
||||
}
|
||||
if (success) {
|
||||
return str;
|
||||
} else {
|
||||
return QByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* parseValue
|
||||
*/
|
||||
QVariant Json::parseValue(const QString &json, int &index, bool &success)
|
||||
{
|
||||
// Determine what kind of data we should parse by
|
||||
// checking out the upcoming token
|
||||
switch (Json::lookAhead(json, index)) {
|
||||
case JsonTokenString:
|
||||
return Json::parseString(json, index, success);
|
||||
case JsonTokenNumber:
|
||||
return Json::parseNumber(json, index);
|
||||
case JsonTokenCurlyOpen:
|
||||
return Json::parseObject(json, index, success);
|
||||
case JsonTokenSquaredOpen:
|
||||
return Json::parseArray(json, index, success);
|
||||
case JsonTokenTrue:
|
||||
Json::nextToken(json, index);
|
||||
return QVariant(true);
|
||||
case JsonTokenFalse:
|
||||
Json::nextToken(json, index);
|
||||
return QVariant(false);
|
||||
case JsonTokenNull:
|
||||
Json::nextToken(json, index);
|
||||
return QVariant();
|
||||
case JsonTokenNone:
|
||||
break;
|
||||
}
|
||||
|
||||
// If there were no tokens, flag the failure and return an empty QVariant
|
||||
success = false;
|
||||
return QVariant();
|
||||
}
|
||||
|
||||
/**
|
||||
* parseObject
|
||||
*/
|
||||
QVariant Json::parseObject(const QString &json, int &index, bool &success)
|
||||
{
|
||||
QVariantMap map;
|
||||
int token;
|
||||
|
||||
// Get rid of the whitespace and increment index
|
||||
Json::nextToken(json, index);
|
||||
|
||||
// Loop through all of the key/value pairs of the object
|
||||
bool done = false;
|
||||
while (!done) {
|
||||
// Get the upcoming token
|
||||
token = Json::lookAhead(json, index);
|
||||
|
||||
if (token == JsonTokenNone) {
|
||||
success = false;
|
||||
return QVariantMap();
|
||||
} else if (token == JsonTokenComma) {
|
||||
Json::nextToken(json, index);
|
||||
} else if (token == JsonTokenCurlyClose) {
|
||||
Json::nextToken(json, index);
|
||||
return map;
|
||||
} else {
|
||||
// Parse the key/value pair's name
|
||||
QString name = Json::parseString(json, index, success).toString();
|
||||
|
||||
if (!success) {
|
||||
return QVariantMap();
|
||||
}
|
||||
|
||||
// Get the next token
|
||||
token = Json::nextToken(json, index);
|
||||
|
||||
// If the next token is not a colon, flag the failure
|
||||
// return an empty QVariant
|
||||
if (token != JsonTokenColon) {
|
||||
success = false;
|
||||
return QVariant(QVariantMap());
|
||||
}
|
||||
|
||||
// Parse the key/value pair's value
|
||||
QVariant value = Json::parseValue(json, index, success);
|
||||
|
||||
if (!success) {
|
||||
return QVariantMap();
|
||||
}
|
||||
|
||||
// Assign the value to the key in the map
|
||||
map[name] = value;
|
||||
}
|
||||
}
|
||||
|
||||
// Return the map successfully
|
||||
return QVariant(map);
|
||||
}
|
||||
|
||||
/**
|
||||
* parseArray
|
||||
*/
|
||||
QVariant Json::parseArray(const QString &json, int &index, bool &success)
|
||||
{
|
||||
QVariantList list;
|
||||
|
||||
Json::nextToken(json, index);
|
||||
|
||||
bool done = false;
|
||||
while (!done) {
|
||||
int token = Json::lookAhead(json, index);
|
||||
|
||||
if (token == JsonTokenNone) {
|
||||
success = false;
|
||||
return QVariantList();
|
||||
} else if (token == JsonTokenComma) {
|
||||
Json::nextToken(json, index);
|
||||
} else if (token == JsonTokenSquaredClose) {
|
||||
Json::nextToken(json, index);
|
||||
break;
|
||||
} else {
|
||||
QVariant value = Json::parseValue(json, index, success);
|
||||
|
||||
if (!success) {
|
||||
return QVariantList();
|
||||
}
|
||||
|
||||
list.push_back(value);
|
||||
}
|
||||
}
|
||||
|
||||
return QVariant(list);
|
||||
}
|
||||
|
||||
/**
|
||||
* parseString
|
||||
*/
|
||||
QVariant Json::parseString(const QString &json, int &index, bool &success)
|
||||
{
|
||||
QString s;
|
||||
QChar c;
|
||||
|
||||
Json::eatWhitespace(json, index);
|
||||
|
||||
c = json[index++];
|
||||
|
||||
bool complete = false;
|
||||
while (!complete) {
|
||||
if (index == json.size()) {
|
||||
break;
|
||||
}
|
||||
|
||||
c = json[index++];
|
||||
|
||||
if (c == '\"') {
|
||||
complete = true;
|
||||
break;
|
||||
} else if (c == '\\') {
|
||||
if (index == json.size()) {
|
||||
break;
|
||||
}
|
||||
|
||||
c = json[index++];
|
||||
|
||||
if (c == '\"') {
|
||||
s.append('\"');
|
||||
} else if (c == '\\') {
|
||||
s.append('\\');
|
||||
} else if (c == '/') {
|
||||
s.append('/');
|
||||
} else if (c == 'b') {
|
||||
s.append('\b');
|
||||
} else if (c == 'f') {
|
||||
s.append('\f');
|
||||
} else if (c == 'n') {
|
||||
s.append('\n');
|
||||
} else if (c == 'r') {
|
||||
s.append('\r');
|
||||
} else if (c == 't') {
|
||||
s.append('\t');
|
||||
} else if (c == 'u') {
|
||||
int remainingLength = json.size() - index;
|
||||
|
||||
if (remainingLength >= 4) {
|
||||
QString unicodeStr = json.mid(index, 4);
|
||||
|
||||
int symbol = unicodeStr.toInt(0, 16);
|
||||
|
||||
s.append(QChar(symbol));
|
||||
|
||||
index += 4;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.append(c);
|
||||
}
|
||||
}
|
||||
|
||||
if (!complete) {
|
||||
success = false;
|
||||
return QVariant();
|
||||
}
|
||||
|
||||
return QVariant(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* parseNumber
|
||||
*/
|
||||
QVariant Json::parseNumber(const QString &json, int &index)
|
||||
{
|
||||
Json::eatWhitespace(json, index);
|
||||
|
||||
int lastIndex = Json::lastIndexOfNumber(json, index);
|
||||
int charLength = (lastIndex - index) + 1;
|
||||
QString numberStr;
|
||||
|
||||
numberStr = json.mid(index, charLength);
|
||||
|
||||
index = lastIndex + 1;
|
||||
|
||||
if (numberStr.contains('.')) {
|
||||
return QVariant(numberStr.toDouble(NULL));
|
||||
} else if (numberStr.startsWith('-')) {
|
||||
return QVariant(numberStr.toLongLong(NULL));
|
||||
} else {
|
||||
return QVariant(numberStr.toULongLong(NULL));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* lastIndexOfNumber
|
||||
*/
|
||||
int Json::lastIndexOfNumber(const QString &json, int index)
|
||||
{
|
||||
static const QString numericCharacters("0123456789+-.eE");
|
||||
int lastIndex;
|
||||
|
||||
for (lastIndex = index; lastIndex < json.size(); lastIndex++) {
|
||||
if (numericCharacters.indexOf(json[lastIndex]) == -1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return lastIndex - 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* eatWhitespace
|
||||
*/
|
||||
void Json::eatWhitespace(const QString &json, int &index)
|
||||
{
|
||||
static const QString whitespaceChars(" \t\n\r");
|
||||
for (; index < json.size(); index++) {
|
||||
if (whitespaceChars.indexOf(json[index]) == -1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* lookAhead
|
||||
*/
|
||||
int Json::lookAhead(const QString &json, int index)
|
||||
{
|
||||
int saveIndex = index;
|
||||
return Json::nextToken(json, saveIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* nextToken
|
||||
*/
|
||||
int Json::nextToken(const QString &json, int &index)
|
||||
{
|
||||
Json::eatWhitespace(json, index);
|
||||
|
||||
if (index == json.size()) {
|
||||
return JsonTokenNone;
|
||||
}
|
||||
|
||||
QChar c = json[index];
|
||||
index++;
|
||||
switch (c.toLatin1()) {
|
||||
case '{':
|
||||
return JsonTokenCurlyOpen;
|
||||
case '}':
|
||||
return JsonTokenCurlyClose;
|
||||
case '[':
|
||||
return JsonTokenSquaredOpen;
|
||||
case ']':
|
||||
return JsonTokenSquaredClose;
|
||||
case ',':
|
||||
return JsonTokenComma;
|
||||
case '"':
|
||||
return JsonTokenString;
|
||||
case '0':
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9':
|
||||
case '-':
|
||||
return JsonTokenNumber;
|
||||
case ':':
|
||||
return JsonTokenColon;
|
||||
}
|
||||
|
||||
index--;
|
||||
|
||||
int remainingLength = json.size() - index;
|
||||
|
||||
// True
|
||||
if (remainingLength >= 4) {
|
||||
if (json[index] == 't' && json[index + 1] == 'r' && json[index + 2] == 'u' && json[index + 3] == 'e') {
|
||||
index += 4;
|
||||
return JsonTokenTrue;
|
||||
}
|
||||
}
|
||||
|
||||
// False
|
||||
if (remainingLength >= 5) {
|
||||
if (json[index] == 'f' && json[index + 1] == 'a' && json[index + 2] == 'l' && json[index + 3] == 's' &&
|
||||
json[index + 4] == 'e') {
|
||||
index += 5;
|
||||
return JsonTokenFalse;
|
||||
}
|
||||
}
|
||||
|
||||
// Null
|
||||
if (remainingLength >= 4) {
|
||||
if (json[index] == 'n' && json[index + 1] == 'u' && json[index + 2] == 'l' && json[index + 3] == 'l') {
|
||||
index += 4;
|
||||
return JsonTokenNull;
|
||||
}
|
||||
}
|
||||
|
||||
return JsonTokenNone;
|
||||
}
|
||||
|
||||
} // namespace QtJson
|
||||
|
|
@ -1,204 +0,0 @@
|
|||
/* Copyright 2011 Eeli Reilin. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ''AS IS'' AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
* EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
|
||||
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* The views and conclusions contained in the software and documentation
|
||||
* are those of the authors and should not be interpreted as representing
|
||||
* official policies, either expressed or implied, of Eeli Reilin.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file json.h
|
||||
*/
|
||||
|
||||
#ifndef JSON_H
|
||||
#define JSON_H
|
||||
|
||||
#include <QVariant>
|
||||
#include <QString>
|
||||
|
||||
namespace QtJson
|
||||
{
|
||||
|
||||
/**
|
||||
* \enum JsonToken
|
||||
*/
|
||||
enum JsonToken
|
||||
{
|
||||
JsonTokenNone = 0,
|
||||
JsonTokenCurlyOpen = 1,
|
||||
JsonTokenCurlyClose = 2,
|
||||
JsonTokenSquaredOpen = 3,
|
||||
JsonTokenSquaredClose = 4,
|
||||
JsonTokenColon = 5,
|
||||
JsonTokenComma = 6,
|
||||
JsonTokenString = 7,
|
||||
JsonTokenNumber = 8,
|
||||
JsonTokenTrue = 9,
|
||||
JsonTokenFalse = 10,
|
||||
JsonTokenNull = 11
|
||||
};
|
||||
|
||||
/**
|
||||
* \class Json
|
||||
* \brief A JSON data parser
|
||||
*
|
||||
* Json parses a JSON data into a QVariant hierarchy.
|
||||
*/
|
||||
class Json
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Parse a JSON string
|
||||
*
|
||||
* \param json The JSON data
|
||||
*/
|
||||
static QVariant parse(const QString &json);
|
||||
|
||||
/**
|
||||
* Parse a JSON string
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param success The success of the parsing
|
||||
*/
|
||||
static QVariant parse(const QString &json, bool &success);
|
||||
|
||||
/**
|
||||
* This method generates a textual JSON representation
|
||||
*
|
||||
* \param data The JSON data generated by the parser.
|
||||
* \param success The success of the serialization
|
||||
*/
|
||||
static QByteArray serialize(const QVariant &data);
|
||||
|
||||
/**
|
||||
* This method generates a textual JSON representation
|
||||
*
|
||||
* \param data The JSON data generated by the parser.
|
||||
* \param success The success of the serialization
|
||||
*
|
||||
* \return QByteArray Textual JSON representation
|
||||
*/
|
||||
static QByteArray serialize(const QVariant &data, bool &success);
|
||||
|
||||
private:
|
||||
/**
|
||||
* Parses a value starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The start index
|
||||
* \param success The success of the parse process
|
||||
*
|
||||
* \return QVariant The parsed value
|
||||
*/
|
||||
static QVariant parseValue(const QString &json, int &index,
|
||||
bool &success);
|
||||
|
||||
/**
|
||||
* Parses an object starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The start index
|
||||
* \param success The success of the object parse
|
||||
*
|
||||
* \return QVariant The parsed object map
|
||||
*/
|
||||
static QVariant parseObject(const QString &json, int &index,
|
||||
bool &success);
|
||||
|
||||
/**
|
||||
* Parses an array starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
* \param success The success of the array parse
|
||||
*
|
||||
* \return QVariant The parsed variant array
|
||||
*/
|
||||
static QVariant parseArray(const QString &json, int &index,
|
||||
bool &success);
|
||||
|
||||
/**
|
||||
* Parses a string starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
* \param success The success of the string parse
|
||||
*
|
||||
* \return QVariant The parsed string
|
||||
*/
|
||||
static QVariant parseString(const QString &json, int &index,
|
||||
bool &success);
|
||||
|
||||
/**
|
||||
* Parses a number starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
*
|
||||
* \return QVariant The parsed number
|
||||
*/
|
||||
static QVariant parseNumber(const QString &json, int &index);
|
||||
|
||||
/**
|
||||
* Get the last index of a number starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
*
|
||||
* \return The last index of the number
|
||||
*/
|
||||
static int lastIndexOfNumber(const QString &json, int index);
|
||||
|
||||
/**
|
||||
* Skip unwanted whitespace symbols starting from index
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The start index
|
||||
*/
|
||||
static void eatWhitespace(const QString &json, int &index);
|
||||
|
||||
/**
|
||||
* Check what token lies ahead
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
*
|
||||
* \return int The upcoming token
|
||||
*/
|
||||
static int lookAhead(const QString &json, int index);
|
||||
|
||||
/**
|
||||
* Get the next JSON token
|
||||
*
|
||||
* \param json The JSON data
|
||||
* \param index The starting index
|
||||
*
|
||||
* \return int The next JSON token
|
||||
*/
|
||||
static int nextToken(const QString &json, int &index);
|
||||
};
|
||||
|
||||
|
||||
} //end namespace
|
||||
|
||||
#endif //JSON_H
|
||||
563
oracle/src/raw_json_scanner.cpp
Normal file
563
oracle/src/raw_json_scanner.cpp
Normal file
|
|
@ -0,0 +1,563 @@
|
|||
#include "raw_json_scanner.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
inline bool isWhitespace(char c)
|
||||
{
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
||||
}
|
||||
|
||||
const char *skipWhitespace(const char *p, const char *end)
|
||||
{
|
||||
while (p < end && isWhitespace(*p)) {
|
||||
++p;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
inline bool isHexDigit(char c)
|
||||
{
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
inline quint8 hexValue(char c)
|
||||
{
|
||||
if (c >= '0' && c <= '9') {
|
||||
return c - '0';
|
||||
}
|
||||
if (c >= 'a' && c <= 'f') {
|
||||
return c - 'a' + 10;
|
||||
}
|
||||
return c - 'A' + 10;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Skips past a JSON string without decoding it, validating escapes.
|
||||
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
|
||||
*/
|
||||
bool skipString(const char *&p, const char *end)
|
||||
{
|
||||
++p; // opening quote
|
||||
for (;;) {
|
||||
const void *quote = memchr(p, '"', static_cast<size_t>(end - p));
|
||||
if (!quote) {
|
||||
return false; // unterminated string
|
||||
}
|
||||
// Backslash escapes can only appear before the closing quote, so bound
|
||||
// the scan to the string extent instead of the rest of the document.
|
||||
const void *backslash = memchr(p, '\\', static_cast<size_t>(static_cast<const char *>(quote) - p));
|
||||
if (!backslash) {
|
||||
p = static_cast<const char *>(quote) + 1;
|
||||
return true;
|
||||
}
|
||||
const char *b = static_cast<const char *>(backslash);
|
||||
if (end - b < 2) {
|
||||
return false;
|
||||
}
|
||||
const char escaped = b[1];
|
||||
if (escaped == 'u') {
|
||||
if (end - b < 6) {
|
||||
return false;
|
||||
}
|
||||
quint32 codepoint = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(b[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
codepoint = codepoint * 16 + hexValue(b[2 + i]);
|
||||
}
|
||||
p = b + 6;
|
||||
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
|
||||
// expect the low-surrogate escape for the second half
|
||||
if (end - p < 6 || p[0] != '\\' || p[1] != 'u') {
|
||||
return false; // unpaired high surrogate
|
||||
}
|
||||
quint32 low = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
low = low * 16 + hexValue(p[2 + i]);
|
||||
}
|
||||
if (low < 0xDC00 || low > 0xDFFF) {
|
||||
return false;
|
||||
}
|
||||
p += 6;
|
||||
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
|
||||
return false; // unpaired low surrogate
|
||||
}
|
||||
continue;
|
||||
}
|
||||
switch (escaped) {
|
||||
case '"':
|
||||
case '\\':
|
||||
case '/':
|
||||
case 'b':
|
||||
case 'f':
|
||||
case 'n':
|
||||
case 'r':
|
||||
case 't':
|
||||
p = b + 2;
|
||||
continue;
|
||||
default:
|
||||
return false; // invalid escape
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Decodes a JSON string into @p out, validating it as it goes.
|
||||
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
|
||||
*/
|
||||
bool decodeString(const char *&p, const char *end, QString &out)
|
||||
{
|
||||
out.clear();
|
||||
QByteArray utf8;
|
||||
auto flush = [&out, &utf8]() {
|
||||
if (!utf8.isEmpty()) {
|
||||
out += QString::fromUtf8(utf8);
|
||||
utf8.clear();
|
||||
}
|
||||
};
|
||||
|
||||
++p; // opening quote
|
||||
while (p < end) {
|
||||
const char c = *p;
|
||||
if (c == '\\') {
|
||||
flush();
|
||||
++p; // escaped character
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char escaped = *p;
|
||||
if (escaped == 'u') {
|
||||
++p; // first hex digit
|
||||
if (p + 4 > end) {
|
||||
return false;
|
||||
}
|
||||
quint32 codepoint = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[i])) {
|
||||
return false;
|
||||
}
|
||||
codepoint = codepoint * 16 + hexValue(p[i]);
|
||||
}
|
||||
p += 4;
|
||||
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
|
||||
// expect a low-surrogate escape for the second half
|
||||
if (p + 6 > end || p[0] != '\\' || p[1] != 'u') {
|
||||
return false; // unpaired high surrogate
|
||||
}
|
||||
quint32 low = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (!isHexDigit(p[2 + i])) {
|
||||
return false;
|
||||
}
|
||||
low = low * 16 + hexValue(p[2 + i]);
|
||||
}
|
||||
if (low < 0xDC00 || low > 0xDFFF) {
|
||||
return false;
|
||||
}
|
||||
out += QChar(codepoint);
|
||||
out += QChar(low);
|
||||
p += 6;
|
||||
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
|
||||
return false; // unpaired low surrogate
|
||||
} else {
|
||||
out += QChar(codepoint);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
switch (escaped) {
|
||||
case '"':
|
||||
out += '"';
|
||||
break;
|
||||
case '\\':
|
||||
out += '\\';
|
||||
break;
|
||||
case '/':
|
||||
out += '/';
|
||||
break;
|
||||
case 'b':
|
||||
out += '\b';
|
||||
break;
|
||||
case 'f':
|
||||
out += '\f';
|
||||
break;
|
||||
case 'n':
|
||||
out += '\n';
|
||||
break;
|
||||
case 'r':
|
||||
out += '\r';
|
||||
break;
|
||||
case 't':
|
||||
out += '\t';
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (c == '"') {
|
||||
++p;
|
||||
flush();
|
||||
return true;
|
||||
}
|
||||
if (static_cast<unsigned char>(c) < 0x20) {
|
||||
return false; // unescaped control character is invalid JSON
|
||||
}
|
||||
utf8 += c;
|
||||
++p;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool matchLiteral(const char *&p, const char *end, const char *literal, int length)
|
||||
{
|
||||
if (end - p < length || memcmp(p, literal, static_cast<size_t>(length)) != 0) {
|
||||
return false;
|
||||
}
|
||||
const char *after = p + length;
|
||||
if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) {
|
||||
return false;
|
||||
}
|
||||
p = after;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool skipNumber(const char *&p, const char *end)
|
||||
{
|
||||
// JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
|
||||
if (p < end && *p == '-') {
|
||||
++p;
|
||||
}
|
||||
if (p < end && *p == '0') {
|
||||
++p;
|
||||
} else if (p < end && *p >= '1' && *p <= '9') {
|
||||
++p;
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
if (p < end && *p == '.') {
|
||||
++p;
|
||||
if (p >= end || !QChar::isDigit(*p)) {
|
||||
return false;
|
||||
}
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
}
|
||||
if (p < end && (*p == 'e' || *p == 'E')) {
|
||||
++p;
|
||||
if (p < end && (*p == '+' || *p == '-')) {
|
||||
++p;
|
||||
}
|
||||
if (p >= end || !QChar::isDigit(*p)) {
|
||||
return false;
|
||||
}
|
||||
while (p < end && QChar::isDigit(*p)) {
|
||||
++p;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool skipValue(const char *&p, const char *end);
|
||||
bool skipObject(const char *&p, const char *end);
|
||||
bool skipArray(const char *&p, const char *end);
|
||||
|
||||
bool skipPrimitive(const char *&p, const char *end)
|
||||
{
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char c = *p;
|
||||
if (c == '"') {
|
||||
return skipString(p, end);
|
||||
}
|
||||
if (c == 't') {
|
||||
return matchLiteral(p, end, "true", 4);
|
||||
}
|
||||
if (c == 'f') {
|
||||
return matchLiteral(p, end, "false", 5);
|
||||
}
|
||||
if (c == 'n') {
|
||||
return matchLiteral(p, end, "null", 4);
|
||||
}
|
||||
if (c == '-' || (c >= '0' && c <= '9')) {
|
||||
return skipNumber(p, end);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool skipObject(const char *&p, const char *end)
|
||||
{
|
||||
++p; // '{'
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != '"') {
|
||||
return false;
|
||||
}
|
||||
if (!skipString(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != ':') {
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
if (!skipValue(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool skipArray(const char *&p, const char *end)
|
||||
{
|
||||
++p; // '['
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == ']') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
if (!skipValue(p, end)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == ']') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool skipValue(const char *&p, const char *end)
|
||||
{
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
const char c = *p;
|
||||
if (c == '{') {
|
||||
return skipObject(p, end);
|
||||
}
|
||||
if (c == '[') {
|
||||
return skipArray(p, end);
|
||||
}
|
||||
return skipPrimitive(p, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Iterates the members of the object starting at @p p.
|
||||
*
|
||||
* For each member invokes @p memberCallback with the key and the byte range of
|
||||
* its value. Advancing @p p is unaffected by the callback.
|
||||
*/
|
||||
template <typename F> bool forEachObjectMember(const char *&p, const char *end, F &&memberCallback)
|
||||
{
|
||||
++p; // '{'
|
||||
p = skipWhitespace(p, end);
|
||||
if (p < end && *p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
for (;;) {
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != '"') {
|
||||
return false;
|
||||
}
|
||||
QString key;
|
||||
if (!decodeString(p, end, key)) {
|
||||
return false;
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end || *p != ':') {
|
||||
return false;
|
||||
}
|
||||
++p;
|
||||
const char *valueStart = skipWhitespace(p, end);
|
||||
const char *valueEnd = valueStart;
|
||||
if (!skipValue(valueEnd, end)) {
|
||||
return false;
|
||||
}
|
||||
if (!memberCallback(key, valueStart, valueEnd)) {
|
||||
return false;
|
||||
}
|
||||
p = valueEnd;
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return false;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == '}') {
|
||||
++p;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Counts the direct elements of an array value; returns -1 if the array is malformed.
|
||||
int countArrayElements(const char *p, const char *end)
|
||||
{
|
||||
++p; // '['
|
||||
p = skipWhitespace(p, end);
|
||||
int count = 0;
|
||||
if (p < end && *p == ']') {
|
||||
return 0;
|
||||
}
|
||||
for (;;) {
|
||||
if (!skipValue(p, end)) {
|
||||
return -1;
|
||||
}
|
||||
++count;
|
||||
p = skipWhitespace(p, end);
|
||||
if (p >= end) {
|
||||
return -1;
|
||||
}
|
||||
if (*p == ',') {
|
||||
++p;
|
||||
continue;
|
||||
}
|
||||
if (*p == ']') {
|
||||
return count;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace RawJson
|
||||
{
|
||||
|
||||
ScanError scanSetRanges(const QByteArray &json, QList<SetRange> &ranges)
|
||||
{
|
||||
ranges.clear();
|
||||
|
||||
const char *begin = json.constData();
|
||||
const char *end = begin + json.size();
|
||||
if (begin >= end) {
|
||||
return ScanError{"empty JSON document"};
|
||||
}
|
||||
|
||||
const char *p = skipWhitespace(begin, end);
|
||||
if (p >= end || *p != '{') {
|
||||
return ScanError{"top-level JSON must be an object"};
|
||||
}
|
||||
|
||||
bool foundData = false;
|
||||
bool malformedSetData = false;
|
||||
|
||||
const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) {
|
||||
if (key == QStringLiteral("data")) {
|
||||
foundData = true;
|
||||
if (valueStart >= valueEnd || *valueStart != '{') {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
const char *setP = valueStart;
|
||||
const bool ok = forEachObjectMember(
|
||||
setP, valueEnd, [&](const QString &setCode, const char *setStart, const char *setEnd) {
|
||||
if (setStart >= setEnd || *setStart != '{') {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
SetRange range;
|
||||
range.start = setStart - begin;
|
||||
range.length = setEnd - setStart;
|
||||
range.code = setCode;
|
||||
|
||||
const char *memberP = setStart;
|
||||
const bool metaOk =
|
||||
forEachObjectMember(memberP, setEnd, [&](const QString &field, const char *fs, const char *fe) {
|
||||
if (field == QStringLiteral("code")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.code);
|
||||
}
|
||||
if (field == QStringLiteral("name")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.name);
|
||||
}
|
||||
if (field == QStringLiteral("type")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.type);
|
||||
}
|
||||
if (field == QStringLiteral("releaseDate")) {
|
||||
return fs < fe && *fs == '"' && decodeString(fs, fe, range.releaseDate);
|
||||
}
|
||||
if (field == QStringLiteral("cards")) {
|
||||
if (fs >= fe || *fs != '[') {
|
||||
return false;
|
||||
}
|
||||
range.cardCount = countArrayElements(fs, fe);
|
||||
return range.cardCount >= 0;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (!metaOk) {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
ranges.append(range);
|
||||
return true;
|
||||
});
|
||||
if (!ok) {
|
||||
malformedSetData = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
if (!forEachObjectMember(p, end, topLevelCallback)) {
|
||||
return ScanError{malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON")};
|
||||
}
|
||||
p = skipWhitespace(p, end);
|
||||
if (p != end) {
|
||||
return ScanError{"trailing content after top-level JSON object"};
|
||||
}
|
||||
if (!foundData) {
|
||||
return ScanError{"missing \"data\" object"};
|
||||
}
|
||||
if (ranges.isEmpty()) {
|
||||
return ScanError{"no sets found in \"data\""};
|
||||
}
|
||||
return ScanError{};
|
||||
}
|
||||
|
||||
} // namespace RawJson
|
||||
50
oracle/src/raw_json_scanner.h
Normal file
50
oracle/src/raw_json_scanner.h
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#ifndef RAW_JSON_SCANNER_H
|
||||
#define RAW_JSON_SCANNER_H
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QList>
|
||||
#include <QString>
|
||||
|
||||
namespace RawJson
|
||||
{
|
||||
|
||||
struct SetRange
|
||||
{
|
||||
/** @brief Byte offset of the set's object within the scanned buffer. */
|
||||
qsizetype start = -1;
|
||||
/** @brief Byte length of the set's object, including the surrounding braces. */
|
||||
qsizetype length = 0;
|
||||
/** @brief Number of entries in the set's "cards" array. */
|
||||
int cardCount = 0;
|
||||
QString code;
|
||||
QString name;
|
||||
QString type;
|
||||
QString releaseDate;
|
||||
};
|
||||
|
||||
struct ScanError
|
||||
{
|
||||
bool isError() const
|
||||
{
|
||||
return !message.isEmpty();
|
||||
}
|
||||
QString message;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Scans a full MTGJSON document without materializing the JSON tree.
|
||||
*
|
||||
* Splits the top-level "data" object into per-set byte ranges and reads each
|
||||
* set's metadata directly from the raw bytes. The oracle importer can then
|
||||
* parse one set at a time during import, keeping peak memory far below a single
|
||||
* QJsonDocument::fromJson() over the whole file.
|
||||
*
|
||||
* The whole document is structurally validated while scanning (strings,
|
||||
* escapes, braces, and a trailing-content check), so malformed input is
|
||||
* rejected just like QJsonDocument::fromJson would.
|
||||
*/
|
||||
ScanError scanSetRanges(const QByteArray &json, QList<SetRange> &ranges);
|
||||
|
||||
} // namespace RawJson
|
||||
|
||||
#endif // RAW_JSON_SCANNER_H
|
||||
|
|
@ -7,3 +7,66 @@ endif()
|
|||
target_link_libraries(parse_cipt_test Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES})
|
||||
|
||||
add_test(NAME parse_cipt_test COMMAND parse_cipt_test)
|
||||
|
||||
# Oracle importer unit tests
|
||||
add_executable(
|
||||
oracle_importer_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
|
||||
../../oracle/src/raw_json_scanner.cpp oracle_importer_test.cpp
|
||||
)
|
||||
|
||||
if(NOT GTEST_FOUND)
|
||||
add_dependencies(oracle_importer_test gtest)
|
||||
endif()
|
||||
|
||||
target_link_libraries(
|
||||
oracle_importer_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES}
|
||||
${TEST_QT_MODULES}
|
||||
)
|
||||
|
||||
add_test(NAME oracle_importer_test COMMAND oracle_importer_test)
|
||||
|
||||
# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark
|
||||
# can download and decompress whatever AllPrintings format the default URL selects.
|
||||
find_package(ZLIB)
|
||||
if(ZLIB_FOUND)
|
||||
add_definitions("-DHAS_ZLIB")
|
||||
set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp)
|
||||
set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES})
|
||||
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||
else()
|
||||
message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled")
|
||||
endif()
|
||||
|
||||
find_package(LibLZMA)
|
||||
if(LIBLZMA_FOUND)
|
||||
add_definitions("-DHAS_LZMA")
|
||||
list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp)
|
||||
list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES})
|
||||
include_directories(${LIBLZMA_INCLUDE_DIRS})
|
||||
else()
|
||||
message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled")
|
||||
endif()
|
||||
|
||||
# Oracle importer benchmark tests
|
||||
add_executable(
|
||||
oracle_importer_benchmark_test
|
||||
${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
|
||||
../../oracle/src/raw_json_scanner.cpp oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
|
||||
)
|
||||
|
||||
if(NOT GTEST_FOUND)
|
||||
add_dependencies(oracle_importer_benchmark_test gtest)
|
||||
endif()
|
||||
|
||||
target_link_libraries(
|
||||
oracle_importer_benchmark_test
|
||||
libcockatrice_card
|
||||
libcockatrice_interfaces
|
||||
Threads::Threads
|
||||
${GTEST_BOTH_LIBRARIES}
|
||||
${TEST_QT_MODULES}
|
||||
${_ORACLE_BENCH_EXTRA_LIBRARIES}
|
||||
)
|
||||
|
||||
add_test(NAME oracle_importer_benchmark_test COMMAND oracle_importer_benchmark_test)
|
||||
set_tests_properties(oracle_importer_benchmark_test PROPERTIES TIMEOUT 120)
|
||||
|
|
|
|||
506
tests/oracle/oracle_importer_benchmark_test.cpp
Normal file
506
tests/oracle/oracle_importer_benchmark_test.cpp
Normal file
|
|
@ -0,0 +1,506 @@
|
|||
#include "../../oracle/src/oracleimporter.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include <QBuffer>
|
||||
#include <QCoreApplication>
|
||||
#include <QDebug>
|
||||
#include <QElapsedTimer>
|
||||
#include <QEventLoop>
|
||||
#include <QFile>
|
||||
#include <QJsonArray>
|
||||
#include <QJsonDocument>
|
||||
#include <QJsonObject>
|
||||
#include <QNetworkAccessManager>
|
||||
#include <QNetworkReply>
|
||||
#include <QNetworkRequest>
|
||||
#include <QTimer>
|
||||
#include <QUrl>
|
||||
#include <libcockatrice/interfaces/noop_card_set_priority_controller.h>
|
||||
|
||||
#if defined(HAS_LZMA)
|
||||
#include "../../oracle/src/lzma/decompress.h"
|
||||
#endif
|
||||
#if defined(HAS_ZLIB)
|
||||
#include "../../oracle/src/zip/unzip.h"
|
||||
#endif
|
||||
#if defined(Q_OS_MACOS)
|
||||
#include <sys/resource.h>
|
||||
#endif
|
||||
|
||||
// Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set
|
||||
static QByteArray buildSyntheticData(int numSets, int cardsPerSet)
|
||||
{
|
||||
QJsonObject dataObj;
|
||||
for (int s = 0; s < numSets; ++s) {
|
||||
QJsonArray cardsArray;
|
||||
for (int c = 0; c < cardsPerSet; ++c) {
|
||||
QJsonObject card;
|
||||
card["name"] = QString("Card %1").arg(s * cardsPerSet + c);
|
||||
card["text"] = "This is a test card with some rules text.";
|
||||
card["layout"] = "normal";
|
||||
card["manaCost"] = "{W}";
|
||||
card["type"] = "Creature — Human";
|
||||
card["power"] = "2";
|
||||
card["toughness"] = "2";
|
||||
card["colors"] = QJsonArray{"W"};
|
||||
card["colorIdentity"] = QJsonArray{"W"};
|
||||
card["types"] = QJsonArray{"Creature"};
|
||||
card["convertedManaCost"] = "1";
|
||||
|
||||
QJsonObject legalities;
|
||||
legalities["standard"] = "legal";
|
||||
legalities["modern"] = "legal";
|
||||
legalities["legacy"] = "legal";
|
||||
legalities["vintage"] = "legal";
|
||||
legalities["commander"] = "legal";
|
||||
card["legalities"] = legalities;
|
||||
|
||||
QJsonObject identifiers;
|
||||
identifiers["scryfallId"] = QString("id-%1-%2").arg(s).arg(c);
|
||||
card["identifiers"] = identifiers;
|
||||
|
||||
QJsonObject numObj;
|
||||
numObj["number"] = QString::number(c + 1);
|
||||
numObj["rarity"] = "common";
|
||||
// Add per-set properties via nested object (mtgjson format for AllPrintings)
|
||||
card["number"] = QString::number(c + 1);
|
||||
card["rarity"] = "common";
|
||||
|
||||
cardsArray.append(card);
|
||||
}
|
||||
|
||||
QJsonObject setObj;
|
||||
setObj["code"] = QString("T%1").arg(s, 2, 10, QChar('0'));
|
||||
setObj["name"] = QString("Test Set %1").arg(s);
|
||||
setObj["type"] = "expansion";
|
||||
setObj["releaseDate"] = "2024-01-01";
|
||||
setObj["cards"] = cardsArray;
|
||||
|
||||
dataObj[QString("T%1").arg(s, 2, 10, QChar('0'))] = setObj;
|
||||
}
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = dataObj;
|
||||
return QJsonDocument(root).toJson(QJsonDocument::Compact);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Import throughput benchmark
|
||||
// ============================================================================
|
||||
|
||||
TEST(OracleBenchmark, ImportThroughput)
|
||||
{
|
||||
static constexpr int numSets = 10;
|
||||
static constexpr int cardsPerSet = 500;
|
||||
|
||||
QByteArray data = buildSyntheticData(numSets, cardsPerSet);
|
||||
|
||||
NoopCardSetPriorityController controller;
|
||||
OracleImporter importer;
|
||||
|
||||
// Phase 1: Parse JSON
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
bool ok = importer.readSetsFromByteArray(data);
|
||||
ASSERT_TRUE(ok);
|
||||
qint64 parseMs = timer.elapsed();
|
||||
|
||||
// Phase 2: Import cards
|
||||
timer.restart();
|
||||
int importedSets = importer.startImport();
|
||||
qint64 importMs = timer.elapsed();
|
||||
|
||||
int totalImported = 0;
|
||||
for (const auto &card : importer.getCardList()) {
|
||||
Q_UNUSED(card);
|
||||
totalImported++;
|
||||
}
|
||||
|
||||
qDebug().noquote()
|
||||
<< QString("Oracle Import Benchmark: %1 sets, %2 unique cards").arg(importedSets).arg(totalImported);
|
||||
qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs);
|
||||
qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs);
|
||||
qDebug().noquote() << QString(" Total: %1 ms").arg(parseMs + importMs);
|
||||
if (importMs > 0) {
|
||||
qDebug().noquote() << QString(" Throughput: %1 cards/sec")
|
||||
.arg(static_cast<double>(totalImported) / importMs * 1000.0, 0, 'f', 0);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// readSetsFromByteArray benchmark
|
||||
// ============================================================================
|
||||
|
||||
TEST(OracleBenchmark, ParseJsonThroughput)
|
||||
{
|
||||
static constexpr int numSets = 20;
|
||||
static constexpr int cardsPerSet = 1000;
|
||||
|
||||
QByteArray data = buildSyntheticData(numSets, cardsPerSet);
|
||||
|
||||
NoopCardSetPriorityController controller;
|
||||
|
||||
// Run 5 iterations and report average
|
||||
static constexpr int iterations = 5;
|
||||
qint64 totalMs = 0;
|
||||
|
||||
for (int i = 0; i < iterations; ++i) {
|
||||
OracleImporter importer;
|
||||
QByteArray source = data;
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
bool ok = importer.readSetsFromByteArray(std::move(source));
|
||||
ASSERT_TRUE(ok);
|
||||
totalMs += timer.elapsed();
|
||||
}
|
||||
|
||||
qint64 avgMs = totalMs / iterations;
|
||||
qDebug().noquote() << QString("Parse Benchmark (%1 iterations): avg %2 ms for %3 sets x %4 cards")
|
||||
.arg(iterations)
|
||||
.arg(avgMs)
|
||||
.arg(numSets)
|
||||
.arg(cardsPerSet);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Split card merging benchmark
|
||||
// ============================================================================
|
||||
|
||||
TEST(OracleBenchmark, SplitCardMerging)
|
||||
{
|
||||
static constexpr int numSplitCards = 1000;
|
||||
|
||||
QJsonArray cardsList;
|
||||
for (int i = 0; i < numSplitCards; ++i) {
|
||||
QJsonObject face1;
|
||||
face1["name"] = QString("Fire %1 // Ice %1").arg(i);
|
||||
face1["text"] = "Fire side text.";
|
||||
face1["layout"] = "split";
|
||||
face1["side"] = "a";
|
||||
face1["faceName"] = QString("Fire %1").arg(i);
|
||||
face1["colors"] = QJsonArray{"R"};
|
||||
face1["colorIdentity"] = QJsonArray{"R"};
|
||||
face1["types"] = QJsonArray{"Instant"};
|
||||
face1["manaCost"] = "{R}";
|
||||
face1["legalities"] = QJsonObject{{"standard", "not_legal"}};
|
||||
face1["identifiers"] = QJsonObject{{"scryfallId", QString("f-%1").arg(i)}};
|
||||
face1["number"] = QString::number(i + 1);
|
||||
face1["rarity"] = "uncommon";
|
||||
|
||||
QJsonObject face2;
|
||||
face2["name"] = QString("Fire %1 // Ice %1").arg(i);
|
||||
face2["text"] = "Ice side text.";
|
||||
face2["layout"] = "split";
|
||||
face2["side"] = "b";
|
||||
face2["faceName"] = QString("Ice %1").arg(i);
|
||||
face2["colors"] = QJsonArray{"U"};
|
||||
face2["colorIdentity"] = QJsonArray{"U"};
|
||||
face2["types"] = QJsonArray{"Instant"};
|
||||
face2["manaCost"] = "{U}";
|
||||
face2["legalities"] = QJsonObject{{"standard", "not_legal"}};
|
||||
face2["identifiers"] = QJsonObject{{"scryfallId", QString("i-%1").arg(i)}};
|
||||
face2["number"] = QString::number(i + 1);
|
||||
face2["rarity"] = "uncommon";
|
||||
|
||||
cardsList.append(face1);
|
||||
cardsList.append(face2);
|
||||
}
|
||||
|
||||
NoopCardSetPriorityController controller;
|
||||
OracleImporter importer;
|
||||
CardSetPtr set = CardSet::newInstance(&controller, "TST", "Split Test");
|
||||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
int count = importer.importCardsFromSet(set, cardsList);
|
||||
qint64 ms = timer.elapsed();
|
||||
|
||||
ASSERT_EQ(count, numSplitCards);
|
||||
qDebug().noquote() << QString("Split Card Merge Benchmark: %1 cards in %2 ms (%3 cards/sec)")
|
||||
.arg(count)
|
||||
.arg(ms)
|
||||
.arg(ms > 0 ? static_cast<double>(count) / ms * 1000.0 : 0.0, 0, 'f', 0);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// sortAndReduceColors microbenchmark
|
||||
// ============================================================================
|
||||
|
||||
// We can't call sortAndReduceColors directly (it's static), so we benchmark
|
||||
// through importCardsFromSet with color properties.
|
||||
|
||||
TEST(OracleBenchmark, ImportCardsWithColors)
|
||||
{
|
||||
static constexpr int numCards = 10000;
|
||||
|
||||
NoopCardSetPriorityController controller;
|
||||
OracleImporter importer;
|
||||
CardSetPtr set = CardSet::newInstance(&controller, "TST", "Color Test");
|
||||
|
||||
QJsonArray cardsList;
|
||||
for (int i = 0; i < numCards; ++i) {
|
||||
QJsonObject card;
|
||||
card["name"] = QString("Color Card %1").arg(i);
|
||||
card["text"] = "Rules text.";
|
||||
card["layout"] = "normal";
|
||||
card["manaCost"] = "{W}";
|
||||
card["type"] = "Creature — Human";
|
||||
card["types"] = QJsonArray{"Creature"};
|
||||
card["colors"] = QJsonArray{"B", "R", "G", "W", "U"};
|
||||
card["colorIdentity"] = QJsonArray{"B", "R", "G", "W", "U"};
|
||||
card["number"] = QString::number(i + 1);
|
||||
card["rarity"] = "common";
|
||||
card["legalities"] = QJsonObject{{"standard", "legal"}};
|
||||
card["identifiers"] = QJsonObject{{"scryfallId", QString("c-%1").arg(i)}};
|
||||
cardsList.append(card);
|
||||
}
|
||||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
int count = importer.importCardsFromSet(set, cardsList);
|
||||
qint64 ms = timer.elapsed();
|
||||
|
||||
ASSERT_EQ(count, numCards);
|
||||
qDebug().noquote() << QString("Import with Colors Benchmark: %1 cards in %2 ms (%3 cards/sec)")
|
||||
.arg(count)
|
||||
.arg(ms)
|
||||
.arg(ms > 0 ? static_cast<double>(count) / ms * 1000.0 : 0.0, 0, 'f', 0);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RAM usage measurement
|
||||
// ============================================================================
|
||||
|
||||
// Mirrors the default AllPrintings URL selection in oracle/src/pages.cpp.
|
||||
#if defined(HAS_LZMA)
|
||||
static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.xz");
|
||||
#elif defined(HAS_ZLIB)
|
||||
static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.zip");
|
||||
#else
|
||||
static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json");
|
||||
#endif
|
||||
|
||||
// Magic bytes also from oracle/src/pages.cpp
|
||||
static const QByteArray kXzSignature("\xFD\x37\x7A\x58\x5A", 6);
|
||||
static const QByteArray kZipSignature("PK");
|
||||
|
||||
struct MemorySnapshot
|
||||
{
|
||||
qint64 peakRssKb = -1; // process high-water mark (VmHWM on Linux, ru_maxrss on macOS)
|
||||
qint64 rssKb = -1; // current resident set size
|
||||
bool available = false;
|
||||
|
||||
static MemorySnapshot current()
|
||||
{
|
||||
MemorySnapshot snap;
|
||||
#if defined(Q_OS_LINUX)
|
||||
QFile statusFile("/proc/self/status");
|
||||
if (statusFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
|
||||
// /proc files report size() == 0, so atEnd() is immediately true: read everything first.
|
||||
const QList<QByteArray> lines = statusFile.readAll().split('\n');
|
||||
for (const QByteArray &line : lines) {
|
||||
if (line.startsWith("VmHWM:")) {
|
||||
snap.peakRssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong();
|
||||
} else if (line.startsWith("VmRSS:")) {
|
||||
snap.rssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong();
|
||||
}
|
||||
}
|
||||
snap.available = snap.peakRssKb >= 0;
|
||||
}
|
||||
#elif defined(Q_OS_MACOS)
|
||||
struct rusage usage;
|
||||
if (getrusage(RUSAGE_SELF, &usage) == 0) {
|
||||
snap.peakRssKb = usage.ru_maxrss / 1024; // bytes -> kB
|
||||
snap.available = snap.peakRssKb >= 0;
|
||||
}
|
||||
#endif
|
||||
return snap;
|
||||
}
|
||||
};
|
||||
|
||||
static QString formatKb(qint64 kb)
|
||||
{
|
||||
if (kb < 0) {
|
||||
return "N/A";
|
||||
}
|
||||
return QString("%1 MB").arg(kb / 1024.0, 0, 'f', 1);
|
||||
}
|
||||
|
||||
static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, const MemorySnapshot ¤t)
|
||||
{
|
||||
if (!baseline.available || !current.available) {
|
||||
qDebug().noquote() << QString(" %1: memory stats unavailable on this platform").arg(phase);
|
||||
return;
|
||||
}
|
||||
qDebug().noquote() << QString(" %1: current RSS %2 | peak added %3 | process peak %4")
|
||||
.arg(phase)
|
||||
.arg(formatKb(current.rssKb))
|
||||
.arg(formatKb(current.peakRssKb - baseline.peakRssKb))
|
||||
.arg(formatKb(current.peakRssKb));
|
||||
}
|
||||
|
||||
// Decompresses the download payload when the default URL is a compressed build,
|
||||
// mirroring the wizard's magic-byte handling in oracle/src/pages.cpp.
|
||||
static QByteArray decompressSetsData(const QByteArray &payload)
|
||||
{
|
||||
if (payload.startsWith(kXzSignature)) {
|
||||
#if defined(HAS_LZMA)
|
||||
QBuffer inBuffer(const_cast<QByteArray *>(&payload));
|
||||
QByteArray out;
|
||||
QBuffer outBuffer(&out);
|
||||
inBuffer.open(QIODevice::ReadOnly);
|
||||
outBuffer.open(QIODevice::WriteOnly);
|
||||
XzDecompressor xz;
|
||||
if (!xz.decompress(&inBuffer, &outBuffer)) {
|
||||
qDebug() << "RAM benchmark: xz decompression failed";
|
||||
return {};
|
||||
}
|
||||
return out;
|
||||
#else
|
||||
qDebug() << "RAM benchmark: download is xz-compressed but this build has no LZMA support";
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
if (payload.startsWith(kZipSignature)) {
|
||||
#if defined(HAS_ZLIB)
|
||||
QBuffer inBuffer(const_cast<QByteArray *>(&payload));
|
||||
inBuffer.open(QIODevice::ReadOnly);
|
||||
UnZip unzip;
|
||||
if (unzip.openArchive(&inBuffer) != UnZip::Ok) {
|
||||
qDebug() << "RAM benchmark: zip archive open failed";
|
||||
return {};
|
||||
}
|
||||
if (unzip.fileList().size() != 1) {
|
||||
qDebug() << "RAM benchmark: zip archive doesn't contain exactly one file";
|
||||
return {};
|
||||
}
|
||||
QByteArray out;
|
||||
QBuffer outBuffer(&out);
|
||||
outBuffer.open(QIODevice::WriteOnly);
|
||||
const auto errorCode = unzip.extractFile(unzip.fileList().value(0), &outBuffer);
|
||||
unzip.closeArchive();
|
||||
if (errorCode != UnZip::Ok) {
|
||||
qDebug() << "RAM benchmark: zip extraction failed";
|
||||
return {};
|
||||
}
|
||||
return out;
|
||||
#else
|
||||
qDebug() << "RAM benchmark: download is zip-compressed but this build has no zlib support";
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
TEST(OracleBenchmark, ImportRamUsage)
|
||||
{
|
||||
static constexpr int numSets = 30;
|
||||
static constexpr int cardsPerSet = 2000; // ~60k cards, roughly AllPrintings scale
|
||||
|
||||
const QByteArray data = buildSyntheticData(numSets, cardsPerSet);
|
||||
|
||||
const MemorySnapshot baseline = MemorySnapshot::current();
|
||||
NoopCardSetPriorityController controller;
|
||||
OracleImporter importer;
|
||||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(std::move(data)));
|
||||
const qint64 parseMs = timer.elapsed();
|
||||
const MemorySnapshot afterParse = MemorySnapshot::current();
|
||||
|
||||
timer.restart();
|
||||
const int importedSets = importer.startImport();
|
||||
const qint64 importMs = timer.elapsed();
|
||||
const MemorySnapshot afterImport = MemorySnapshot::current();
|
||||
|
||||
importer.releaseSetData();
|
||||
const MemorySnapshot afterRelease = MemorySnapshot::current();
|
||||
|
||||
const int totalCards = importer.getCardList().size();
|
||||
qDebug().noquote() << QString("Oracle RAM Benchmark (synthetic): %1 sets, %2 cards, %3 MB JSON")
|
||||
.arg(importedSets)
|
||||
.arg(totalCards)
|
||||
.arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1);
|
||||
qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs);
|
||||
qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs);
|
||||
logRamPhase("parse", baseline, afterParse);
|
||||
logRamPhase("import", afterParse, afterImport);
|
||||
logRamPhase("after releaseSetData()", afterImport, afterRelease);
|
||||
}
|
||||
|
||||
TEST(OracleBenchmark, ImportRamUsageAllPrintings)
|
||||
{
|
||||
if (qEnvironmentVariableIsEmpty("COCKATRICE_ORACLE_RAM_BENCHMARK")) {
|
||||
GTEST_SKIP() << "Set COCKATRICE_ORACLE_RAM_BENCHMARK=1 to download the real AllPrintings dataset for this "
|
||||
"RAM benchmark. Default URL: "
|
||||
<< kDefaultAllPrintingsUrl.toDisplayString().toStdString();
|
||||
}
|
||||
|
||||
QNetworkAccessManager nam;
|
||||
QNetworkRequest request(kDefaultAllPrintingsUrl);
|
||||
request.setHeader(QNetworkRequest::UserAgentHeader, "Cockatrice Oracle RAM benchmark");
|
||||
QNetworkReply *reply = nam.get(request);
|
||||
|
||||
QEventLoop loop;
|
||||
QTimer timeoutTimer;
|
||||
timeoutTimer.setSingleShot(true);
|
||||
QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
|
||||
QObject::connect(&timeoutTimer, &QTimer::timeout, &loop, &QEventLoop::quit);
|
||||
timeoutTimer.start(10 * 60 * 1000);
|
||||
loop.exec();
|
||||
timeoutTimer.stop();
|
||||
|
||||
if (reply->error() != QNetworkReply::NoError) {
|
||||
GTEST_SKIP() << "Download failed: " << reply->errorString().toStdString();
|
||||
}
|
||||
const QByteArray payload = reply->readAll();
|
||||
reply->deleteLater();
|
||||
|
||||
const MemorySnapshot baseline = MemorySnapshot::current();
|
||||
const QByteArray setsData = decompressSetsData(payload);
|
||||
const MemorySnapshot afterDownload = MemorySnapshot::current();
|
||||
if (setsData.isEmpty()) {
|
||||
GTEST_SKIP() << "No data to import (download or decompression failed)";
|
||||
}
|
||||
|
||||
NoopCardSetPriorityController controller;
|
||||
OracleImporter importer;
|
||||
|
||||
QElapsedTimer timer;
|
||||
timer.start();
|
||||
ASSERT_TRUE(importer.readSetsFromByteArray(std::move(setsData)));
|
||||
const qint64 parseMs = timer.elapsed();
|
||||
const MemorySnapshot afterParse = MemorySnapshot::current();
|
||||
|
||||
timer.restart();
|
||||
const int importedSets = importer.startImport();
|
||||
const qint64 importMs = timer.elapsed();
|
||||
const MemorySnapshot afterImport = MemorySnapshot::current();
|
||||
|
||||
importer.releaseSetData();
|
||||
const MemorySnapshot afterRelease = MemorySnapshot::current();
|
||||
|
||||
const int totalCards = importer.getCardList().size();
|
||||
qDebug().noquote() << QString("Oracle RAM Benchmark (real AllPrintings): %1 sets, %2 unique cards")
|
||||
.arg(importedSets)
|
||||
.arg(totalCards);
|
||||
qDebug().noquote() << QString(" URL: %1").arg(kDefaultAllPrintingsUrl.toDisplayString());
|
||||
qDebug().noquote() << QString(" Downloaded: %1 MB, decompressed: %2 MB")
|
||||
.arg(payload.size() / (1024.0 * 1024.0), 0, 'f', 1)
|
||||
.arg(setsData.size() / (1024.0 * 1024.0), 0, 'f', 1);
|
||||
qDebug().noquote() << QString(" JSON parse: %1 ms").arg(parseMs);
|
||||
qDebug().noquote() << QString(" Card import: %1 ms").arg(importMs);
|
||||
logRamPhase("download+decompress", baseline, afterDownload);
|
||||
logRamPhase("parse", afterDownload, afterParse);
|
||||
logRamPhase("import", afterParse, afterImport);
|
||||
logRamPhase("after releaseSetData()", afterImport, afterRelease);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
// Required for the event loop used by the real-AllPrintings download benchmark
|
||||
QCoreApplication app(argc, argv);
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
600
tests/oracle/oracle_importer_test.cpp
Normal file
600
tests/oracle/oracle_importer_test.cpp
Normal file
|
|
@ -0,0 +1,600 @@
|
|||
#include "../../oracle/src/oracleimporter.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include <QJsonArray>
|
||||
#include <QJsonDocument>
|
||||
#include <QJsonObject>
|
||||
#include <QSet>
|
||||
#include <libcockatrice/card/format/format_legality_rules.h>
|
||||
#include <libcockatrice/card/set/card_set.h>
|
||||
#include <libcockatrice/interfaces/noop_card_set_priority_controller.h>
|
||||
|
||||
class OracleImporterTest : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
void SetUp() override
|
||||
{
|
||||
controller = new NoopCardSetPriorityController();
|
||||
importer = new OracleImporter();
|
||||
set = CardSet::newInstance(controller, "TST", "Test Set");
|
||||
}
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
delete importer;
|
||||
delete controller;
|
||||
}
|
||||
|
||||
// Helper: build a minimal card JSON object
|
||||
QJsonObject makeCard(const QString &name,
|
||||
const QString &colors = "",
|
||||
const QString &colorIdentity = "",
|
||||
const QVariantMap &legalities = {})
|
||||
{
|
||||
QJsonObject card;
|
||||
card["name"] = name;
|
||||
card["text"] = "Rules text.";
|
||||
card["layout"] = "normal";
|
||||
card["manaCost"] = "{W}";
|
||||
card["type"] = "Creature — Human";
|
||||
card["types"] = QJsonArray{"Creature"};
|
||||
card["number"] = "1";
|
||||
card["rarity"] = "common";
|
||||
|
||||
if (!colors.isEmpty()) {
|
||||
QJsonArray arr;
|
||||
for (const QChar &c : colors) {
|
||||
arr.append(QString(c));
|
||||
}
|
||||
card["colors"] = arr;
|
||||
}
|
||||
if (!colorIdentity.isEmpty()) {
|
||||
QJsonArray arr;
|
||||
for (const QChar &c : colorIdentity) {
|
||||
arr.append(QString(c));
|
||||
}
|
||||
card["colorIdentity"] = arr;
|
||||
}
|
||||
if (!legalities.isEmpty()) {
|
||||
QJsonObject legalObj;
|
||||
for (auto it = legalities.constBegin(); it != legalities.constEnd(); ++it) {
|
||||
legalObj[it.key()] = it.value().toString();
|
||||
}
|
||||
card["legalities"] = legalObj;
|
||||
}
|
||||
|
||||
QJsonObject identifiers;
|
||||
identifiers["scryfallId"] = QUuid::createUuid().toString(QUuid::WithoutBraces);
|
||||
card["identifiers"] = identifiers;
|
||||
|
||||
return card;
|
||||
}
|
||||
|
||||
NoopCardSetPriorityController *controller;
|
||||
OracleImporter *importer;
|
||||
CardSetPtr set;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// sortAndReduceColors tests (tested via importCardsFromSet)
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, SortAndReduceColorsSingleColor)
|
||||
{
|
||||
QJsonArray cards{makeCard("Red Card", "R", "R")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Red Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("colors"), "R");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SortAndReduceColorsDeduplicates)
|
||||
{
|
||||
QJsonArray cards{makeCard("Dedup Card", "WWUUB", "WU")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Dedup Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("colors"), "WUB");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SortAndReduceColorsSortsWUBRG)
|
||||
{
|
||||
QJsonArray cards{makeCard("Sort Card", "RGW", "RGW")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Sort Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("colors"), "WRG");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SortAndReduceColorsAllFive)
|
||||
{
|
||||
QJsonArray cards{makeCard("Five Color", "BRGWU", "BRGWU")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Five Color");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("colors"), "WUBRG");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SortAndReduceColorIdentity)
|
||||
{
|
||||
QJsonArray cards{makeCard("Color Id Card", "W", "GWR")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Color Id Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("coloridentity"), "WRG");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SingleColorNotSorted)
|
||||
{
|
||||
QJsonArray cards{makeCard("Single Card", "B", "B")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
auto card = importer->getCardList().value("Single Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("colors"), "B");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Legality guard tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, LegalityGuardCombinesForNewCard)
|
||||
{
|
||||
QVariantMap leg;
|
||||
leg["standard"] = "legal";
|
||||
leg["modern"] = "legal";
|
||||
QJsonArray cards{makeCard("Legal Card", "", "", leg)};
|
||||
|
||||
importer->importCardsFromSet(set, cards);
|
||||
auto card = importer->getCardList().value("Legal Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
ASSERT_EQ(card->getProperty("format-standard"), "legal");
|
||||
ASSERT_EQ(card->getProperty("format-modern"), "legal");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, LegalityGuardPreservesFirstPrinting)
|
||||
{
|
||||
// First printing: standard=legal, modern=legal
|
||||
QVariantMap leg1;
|
||||
leg1["standard"] = "legal";
|
||||
leg1["modern"] = "legal";
|
||||
QJsonArray cards1{makeCard("Guarded Card", "", "", leg1)};
|
||||
importer->importCardsFromSet(set, cards1);
|
||||
|
||||
// Second printing: standard=banned, modern=not_legal
|
||||
CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set");
|
||||
QVariantMap leg2;
|
||||
leg2["standard"] = "banned";
|
||||
leg2["modern"] = "not_legal";
|
||||
QJsonArray cards2{makeCard("Guarded Card", "", "", leg2)};
|
||||
importer->importCardsFromSet(set2, cards2);
|
||||
|
||||
auto card = importer->getCardList().value("Guarded Card");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
// Guard should preserve first printing's legalities
|
||||
ASSERT_EQ(card->getProperty("format-standard"), "legal");
|
||||
ASSERT_EQ(card->getProperty("format-modern"), "legal");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// createDefaultMagicFormats tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, CreateDefaultMagicFormatsContainsExpectedFormats)
|
||||
{
|
||||
auto formats = importer->createDefaultMagicFormats();
|
||||
ASSERT_TRUE(formats.contains("standard"));
|
||||
ASSERT_TRUE(formats.contains("modern"));
|
||||
ASSERT_TRUE(formats.contains("legacy"));
|
||||
ASSERT_TRUE(formats.contains("vintage"));
|
||||
ASSERT_TRUE(formats.contains("commander"));
|
||||
ASSERT_TRUE(formats.contains("pauper"));
|
||||
ASSERT_TRUE(formats.contains("pioneer"));
|
||||
ASSERT_TRUE(formats.contains("brawl"));
|
||||
ASSERT_TRUE(formats.contains("historic"));
|
||||
ASSERT_TRUE(formats.contains("timeless"));
|
||||
ASSERT_TRUE(formats.contains("duel"));
|
||||
ASSERT_TRUE(formats.contains("oathbreaker"));
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, CreateDefaultMagicFormatsSingletonDeckSizes)
|
||||
{
|
||||
auto formats = importer->createDefaultMagicFormats();
|
||||
auto commander = formats.value("commander");
|
||||
ASSERT_EQ(commander->minDeckSize, 100);
|
||||
ASSERT_EQ(commander->maxDeckSize, 100);
|
||||
ASSERT_EQ(commander->maxSideboardSize, 15);
|
||||
|
||||
auto brawl = formats.value("brawl");
|
||||
ASSERT_EQ(brawl->minDeckSize, 60);
|
||||
ASSERT_EQ(brawl->maxDeckSize, 60);
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, CreateDefaultMagicFormatsVintageHasRestricted)
|
||||
{
|
||||
auto formats = importer->createDefaultMagicFormats();
|
||||
auto vintage = formats.value("vintage");
|
||||
bool hasRestricted = false;
|
||||
for (const auto &ac : vintage->allowedCounts) {
|
||||
if (ac.label == "restricted") {
|
||||
hasRestricted = true;
|
||||
ASSERT_EQ(ac.max, 1);
|
||||
}
|
||||
}
|
||||
ASSERT_TRUE(hasRestricted);
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, CreateDefaultMagicFormatsRegexMatchesBasicLands)
|
||||
{
|
||||
auto formats = importer->createDefaultMagicFormats();
|
||||
auto standard = formats.value("standard");
|
||||
ASSERT_FALSE(standard->exceptions.isEmpty());
|
||||
|
||||
auto &basicLandsException = standard->exceptions.first();
|
||||
ASSERT_FALSE(basicLandsException.conditions.isEmpty());
|
||||
|
||||
auto &condition = basicLandsException.conditions.first();
|
||||
ASSERT_EQ(condition.field, "type");
|
||||
ASSERT_EQ(condition.matchType, "regex");
|
||||
|
||||
// Verify the regex actually works (was broken before: \b = backspace, not word boundary)
|
||||
QRegularExpression regex(condition.value);
|
||||
ASSERT_TRUE(regex.isValid());
|
||||
ASSERT_TRUE(regex.match("Basic Land — Forest").hasMatch());
|
||||
ASSERT_TRUE(regex.match("Basic Snow Land — Mountain").hasMatch());
|
||||
ASSERT_FALSE(regex.match("Creature — Elf Warrior").hasMatch());
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, CreateDefaultMagicFormatsCaching)
|
||||
{
|
||||
auto first = importer->createDefaultMagicFormats();
|
||||
auto second = importer->createDefaultMagicFormats();
|
||||
ASSERT_EQ(first.size(), second.size());
|
||||
ASSERT_EQ(first.keys(), second.keys());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// readSetsFromByteArray tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, ReadSetsFromByteArrayValidJson)
|
||||
{
|
||||
QJsonObject setObj;
|
||||
setObj["code"] = "tst";
|
||||
setObj["name"] = "Test Set";
|
||||
setObj["type"] = "expansion";
|
||||
setObj["releaseDate"] = "2024-01-01";
|
||||
setObj["cards"] = QJsonArray();
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject{{"TST", setObj}};
|
||||
|
||||
QByteArray data = QJsonDocument(root).toJson();
|
||||
ASSERT_TRUE(importer->readSetsFromByteArray(data));
|
||||
ASSERT_EQ(importer->getSets().size(), 1);
|
||||
ASSERT_EQ(importer->getSets().first().getShortName(), "TST");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ReadSetsFromByteArrayInvalidJson)
|
||||
{
|
||||
QByteArray data = "not valid json";
|
||||
ASSERT_FALSE(importer->readSetsFromByteArray(data));
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ReadSetsFromByteArrayEmptyData)
|
||||
{
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject();
|
||||
|
||||
QByteArray data = QJsonDocument(root).toJson();
|
||||
ASSERT_FALSE(importer->readSetsFromByteArray(data));
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ReadSetsFromByteArrayCapitalizesSetType)
|
||||
{
|
||||
QJsonObject setObj;
|
||||
setObj["code"] = "ftv";
|
||||
setObj["name"] = "From The Vault";
|
||||
setObj["type"] = "from_the_vault";
|
||||
setObj["releaseDate"] = "2024-01-01";
|
||||
setObj["cards"] = QJsonArray();
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject{{"FTV", setObj}};
|
||||
|
||||
QByteArray data = QJsonDocument(root).toJson();
|
||||
ASSERT_TRUE(importer->readSetsFromByteArray(data));
|
||||
ASSERT_EQ(importer->getSets().first().getSetType(), "From the Vault");
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ReadSetsFromByteArraySortsSetsByName)
|
||||
{
|
||||
QJsonObject setA;
|
||||
setA["code"] = "zzz";
|
||||
setA["name"] = "Zeta Set";
|
||||
setA["type"] = "expansion";
|
||||
setA["releaseDate"] = "2024-01-01";
|
||||
setA["cards"] = QJsonArray();
|
||||
|
||||
QJsonObject setB;
|
||||
setB["code"] = "aaa";
|
||||
setB["name"] = "Alpha Set";
|
||||
setB["type"] = "expansion";
|
||||
setB["releaseDate"] = "2024-01-01";
|
||||
setB["cards"] = QJsonArray();
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject{{"ZZZ", setA}, {"AAA", setB}};
|
||||
|
||||
QByteArray data = QJsonDocument(root).toJson();
|
||||
ASSERT_TRUE(importer->readSetsFromByteArray(data));
|
||||
auto sets = importer->getSets();
|
||||
ASSERT_GE(sets.size(), 2);
|
||||
ASSERT_EQ(sets.first().getShortName(), "AAA");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Split card coloridentity tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, SplitCardColorIdentityConcatenated)
|
||||
{
|
||||
QJsonObject leg{{"standard", "not_legal"}};
|
||||
|
||||
QJsonObject face1;
|
||||
face1["name"] = "Fire // Ice";
|
||||
face1["text"] = "Fire deals 2 damage.";
|
||||
face1["layout"] = "split";
|
||||
face1["side"] = "a";
|
||||
face1["faceName"] = "Fire";
|
||||
face1["colors"] = QJsonArray{"R"};
|
||||
face1["colorIdentity"] = QJsonArray{"R"};
|
||||
face1["types"] = QJsonArray{"Instant"};
|
||||
face1["manaCost"] = "{R}";
|
||||
face1["legalities"] = leg;
|
||||
face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}};
|
||||
face1["number"] = "1";
|
||||
face1["rarity"] = "uncommon";
|
||||
|
||||
QJsonObject face2;
|
||||
face2["name"] = "Fire // Ice";
|
||||
face2["text"] = "Ice taps target artifact.";
|
||||
face2["layout"] = "split";
|
||||
face2["side"] = "b";
|
||||
face2["faceName"] = "Ice";
|
||||
face2["colors"] = QJsonArray{"U"};
|
||||
face2["colorIdentity"] = QJsonArray{"U"};
|
||||
face2["types"] = QJsonArray{"Instant"};
|
||||
face2["manaCost"] = "{U}";
|
||||
face2["legalities"] = leg;
|
||||
face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}};
|
||||
face2["number"] = "1";
|
||||
face2["rarity"] = "uncommon";
|
||||
|
||||
QJsonArray cardsList{face1, face2};
|
||||
int count = importer->importCardsFromSet(set, cardsList);
|
||||
ASSERT_EQ(count, 1);
|
||||
|
||||
auto card = importer->getCardList().value("Fire // Ice");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
|
||||
// coloridentity should be "RU" (concatenated), not "R // U"
|
||||
QString ci = card->getProperty("coloridentity");
|
||||
ASSERT_FALSE(ci.contains("//")) << "coloridentity should not contain '//', got: " << ci.toStdString();
|
||||
ASSERT_TRUE(ci.contains("R"));
|
||||
ASSERT_TRUE(ci.contains("U"));
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, SplitCardColorsConcatenated)
|
||||
{
|
||||
QJsonObject leg{{"standard", "not_legal"}};
|
||||
|
||||
QJsonObject face1;
|
||||
face1["name"] = "Fire // Ice";
|
||||
face1["text"] = "Fire deals 2 damage.";
|
||||
face1["layout"] = "split";
|
||||
face1["side"] = "a";
|
||||
face1["faceName"] = "Fire";
|
||||
face1["colors"] = QJsonArray{"R"};
|
||||
face1["colorIdentity"] = QJsonArray{"R"};
|
||||
face1["types"] = QJsonArray{"Instant"};
|
||||
face1["manaCost"] = "{R}";
|
||||
face1["legalities"] = leg;
|
||||
face1["identifiers"] = QJsonObject{{"scryfallId", "aaa"}};
|
||||
face1["number"] = "1";
|
||||
face1["rarity"] = "uncommon";
|
||||
|
||||
QJsonObject face2;
|
||||
face2["name"] = "Fire // Ice";
|
||||
face2["text"] = "Ice taps target artifact.";
|
||||
face2["layout"] = "split";
|
||||
face2["side"] = "b";
|
||||
face2["faceName"] = "Ice";
|
||||
face2["colors"] = QJsonArray{"U"};
|
||||
face2["colorIdentity"] = QJsonArray{"U"};
|
||||
face2["types"] = QJsonArray{"Instant"};
|
||||
face2["manaCost"] = "{U}";
|
||||
face2["legalities"] = leg;
|
||||
face2["identifiers"] = QJsonObject{{"scryfallId", "bbb"}};
|
||||
face2["number"] = "1";
|
||||
face2["rarity"] = "uncommon";
|
||||
|
||||
QJsonArray cardsList{face1, face2};
|
||||
importer->importCardsFromSet(set, cardsList);
|
||||
|
||||
auto card = importer->getCardList().value("Fire // Ice");
|
||||
ASSERT_FALSE(card.isNull());
|
||||
|
||||
QString colors = card->getProperty("colors");
|
||||
ASSERT_FALSE(colors.contains("//")) << "colors should not contain '//', got: " << colors.toStdString();
|
||||
ASSERT_TRUE(colors.contains("R"));
|
||||
ASSERT_TRUE(colors.contains("U"));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Mana cost formatting tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, ManaCostStripsBraces)
|
||||
{
|
||||
QJsonObject card = makeCard("Mana Card");
|
||||
card["manaCost"] = "{2}{W}{B}";
|
||||
QJsonArray cards{card};
|
||||
|
||||
importer->importCardsFromSet(set, cards);
|
||||
auto result = importer->getCardList().value("Mana Card");
|
||||
ASSERT_FALSE(result.isNull());
|
||||
ASSERT_EQ(result->getProperty("manacost"), "2WB");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Card deduplication tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, DuplicateCardNameReturnsExisting)
|
||||
{
|
||||
QJsonArray cards{makeCard("Dupe Card")};
|
||||
importer->importCardsFromSet(set, cards);
|
||||
|
||||
CardSetPtr set2 = CardSet::newInstance(controller, "TS2", "Second Set");
|
||||
QJsonArray cards2{makeCard("Dupe Card")};
|
||||
importer->importCardsFromSet(set2, cards2);
|
||||
|
||||
ASSERT_EQ(importer->getCardList().size(), 1);
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, AELigatureReplaced)
|
||||
{
|
||||
QJsonObject card = makeCard(QString::fromUtf8("\xC3\x86ther Vial")); // Æther Vial
|
||||
QJsonArray cards{card};
|
||||
|
||||
importer->importCardsFromSet(set, cards);
|
||||
// Æ is replaced with AE, resulting in "AEther Vial"
|
||||
ASSERT_FALSE(importer->getCardList().contains(QString::fromUtf8("\xC3\x86ther Vial")));
|
||||
ASSERT_TRUE(importer->getCardList().contains("AEther Vial"));
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ApostropheNormalized)
|
||||
{
|
||||
QJsonObject card = makeCard(QString::fromUtf8("Jace\u2019s Ingenuity"));
|
||||
QJsonArray cards{card};
|
||||
|
||||
importer->importCardsFromSet(set, cards);
|
||||
ASSERT_TRUE(importer->getCardList().contains("Jace's Ingenuity"));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RawJson scanner tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesMatchFullJsonParse)
|
||||
{
|
||||
QJsonObject root;
|
||||
QJsonObject data;
|
||||
data["AAA"] = makeCard("Alpha Card");
|
||||
data["BBB"] = makeCard("Beta Card");
|
||||
root["data"] = data;
|
||||
|
||||
const QByteArray bytes = QJsonDocument(root).toJson(QJsonDocument::Compact);
|
||||
|
||||
QList<RawJson::SetRange> ranges;
|
||||
const RawJson::ScanError error = RawJson::scanSetRanges(bytes, ranges);
|
||||
ASSERT_FALSE(error.isError()) << error.message.toStdString();
|
||||
ASSERT_EQ(ranges.size(), 2);
|
||||
|
||||
const QJsonObject wholeData = QJsonDocument::fromJson(bytes).object().value("data").toObject();
|
||||
for (const RawJson::SetRange &range : ranges) {
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument sliceDoc =
|
||||
QJsonDocument::fromJson(QByteArray(bytes.constData() + range.start, range.length), &parseError);
|
||||
ASSERT_EQ(parseError.error, QJsonParseError::NoError)
|
||||
<< range.code.toStdString() << ": " << parseError.errorString().toStdString();
|
||||
ASSERT_EQ(sliceDoc.object(), wholeData.value(range.code).toObject()) << "set " << range.code.toStdString();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesDecodesEscapesAndCountsCards)
|
||||
{
|
||||
const QByteArray json = "{\"data\":{\"KEY\":{\"code\":\"zzz\",\"name\":\"\\u00c9tude \\ud83d\\ude00\","
|
||||
"\"type\":\"expansion\",\"releaseDate\":\"2024-01-05\","
|
||||
"\"cards\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}}}";
|
||||
|
||||
QList<RawJson::SetRange> ranges;
|
||||
ASSERT_FALSE(RawJson::scanSetRanges(json, ranges).isError());
|
||||
ASSERT_EQ(ranges.size(), 1);
|
||||
|
||||
const RawJson::SetRange &range = ranges.first();
|
||||
ASSERT_EQ(range.code, "zzz"); // inner "code" wins over the object key
|
||||
const QString expectedName = QString::fromUtf8("\xC3\x89tude ") + QChar(0xD83D) + QChar(0xDE00);
|
||||
ASSERT_EQ(range.name, expectedName);
|
||||
ASSERT_EQ(range.type, "expansion");
|
||||
ASSERT_EQ(range.releaseDate, "2024-01-05");
|
||||
ASSERT_EQ(range.cardCount, 3);
|
||||
|
||||
QJsonParseError parseError;
|
||||
const QJsonDocument sliceDoc =
|
||||
QJsonDocument::fromJson(QByteArray(json.constData() + range.start, range.length), &parseError);
|
||||
ASSERT_EQ(parseError.error, QJsonParseError::NoError);
|
||||
ASSERT_EQ(sliceDoc.object().value("name").toString(), expectedName);
|
||||
ASSERT_EQ(sliceDoc.object().value("cards").toArray().size(), 3);
|
||||
}
|
||||
|
||||
TEST_F(OracleImporterTest, ScanSetRangesRejectsInvalidJson)
|
||||
{
|
||||
const QList<QByteArray> invalid = {"not json",
|
||||
"[]",
|
||||
"{\"data\":[]}",
|
||||
"{\"data\":{}}",
|
||||
"{\"other\":{}}",
|
||||
"{\"data\":{\"A\":{\"code\":\"a\",\"name\":\"ok\",\"type\":\"x\","
|
||||
"\"releaseDate\":\"2024-01-01\",\"cards\":[]}}} trailing",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\uZZZZ\"}]}}}",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"bad \\q escape\"}]}}}",
|
||||
"{\"data\":{\"A\":{\"cards\":[{\"name\":\"\\ud800\"}]}}}"};
|
||||
|
||||
for (const QByteArray &json : invalid) {
|
||||
QList<RawJson::SetRange> ranges;
|
||||
const RawJson::ScanError error = RawJson::scanSetRanges(json, ranges);
|
||||
EXPECT_TRUE(error.isError()) << "expected failure for: " << json.constData();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Lazy per-set parsing tests
|
||||
// ============================================================================
|
||||
|
||||
TEST_F(OracleImporterTest, StartImportParsesSetsLazily)
|
||||
{
|
||||
QJsonObject setObj = makeCard("Lazy Import Card");
|
||||
QJsonArray cards;
|
||||
cards.append(setObj);
|
||||
QJsonObject dataSet;
|
||||
dataSet["code"] = "tst";
|
||||
dataSet["name"] = "Test Set";
|
||||
dataSet["type"] = "expansion";
|
||||
dataSet["releaseDate"] = "2024-01-01";
|
||||
dataSet["cards"] = cards;
|
||||
|
||||
QJsonObject root;
|
||||
root["data"] = QJsonObject{{"TST", dataSet}};
|
||||
|
||||
const QByteArray data = QJsonDocument(root).toJson(QJsonDocument::Compact);
|
||||
ASSERT_TRUE(importer->readSetsFromByteArray(data));
|
||||
ASSERT_FALSE(importer->getRawSetsData().isEmpty());
|
||||
|
||||
const int importedSets = importer->startImport();
|
||||
ASSERT_EQ(importedSets, 1);
|
||||
ASSERT_EQ(importer->getCardList().size(), 1);
|
||||
ASSERT_FALSE(importer->getCardList().value("Lazy Import Card").isNull());
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue