mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-09-22 01:25:10 -07:00
c011ea7ceb
* [Oracle] Parse sets lazily to slash importer peak memory - Add a raw JSON scanner that splits the document into per-set byte ranges without materializing the JSON tree - Keep only the raw document bytes and parse one set at a time in startImport - Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied - Clear the retained raw data in releaseSetData()/clear() - Cover the scanner and lazy parsing with tests Took 2 minutes * [Oracle] Fix nesting-depth cap, tolerate unescaped control chars, lazy-parse review fixes --------- Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>
645 lines
26 KiB
C++
645 lines
26 KiB
C++
#include "oracleimporter.h"
|
||
|
||
#include "libcockatrice/interfaces/noop_card_preference_provider.h"
|
||
#include "libcockatrice/interfaces/noop_card_set_priority_controller.h"
|
||
#include "parsehelpers.h"
|
||
|
||
#include <QDebug>
|
||
#include <QJsonDocument>
|
||
#include <QJsonObject>
|
||
#include <QJsonParseError>
|
||
#include <QRegularExpression>
|
||
#include <QSet>
|
||
#include <algorithm>
|
||
#include <climits>
|
||
#include <libcockatrice/card/database/parser/cockatrice_xml_4.h>
|
||
#include <libcockatrice/card/relation/card_relation.h>
|
||
|
||
static const QList<AllowedCount> kConstructedCounts = {{4, "legal"}, {0, "banned"}};
|
||
|
||
static const QList<AllowedCount> kSingletonCounts = {{1, "legal"}, {0, "banned"}};
|
||
|
||
SplitCardPart::SplitCardPart(const QString &_name,
|
||
const QString &_text,
|
||
const QHash<QString, QString> &_properties,
|
||
const PrintingInfo &_printingInfo)
|
||
: name(_name), text(_text), properties(_properties), printingInfo(_printingInfo)
|
||
{
|
||
}
|
||
|
||
const QRegularExpression OracleImporter::formatRegex = QRegularExpression("^format-");
|
||
|
||
OracleImporter::OracleImporter(QObject *parent) : QObject(parent)
|
||
{
|
||
}
|
||
|
||
static CardSet::Priority getSetPriority(const QString &setType, const QString &shortName)
|
||
{
|
||
if (!setTypePriorities.contains(setType.toLower())) {
|
||
qDebug() << "warning: Set type" << setType << "unrecognized for prioritization";
|
||
}
|
||
CardSet::Priority priority = setTypePriorities.value(setType.toLower(), CardSet::PriorityOther);
|
||
if (nonEnglishSets.contains(shortName)) {
|
||
priority = CardSet::PriorityLowest;
|
||
}
|
||
return priority;
|
||
}
|
||
|
||
bool OracleImporter::readSetsFromByteArray(QByteArray data)
|
||
{
|
||
RawJson::ScanError error;
|
||
const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error);
|
||
if (error.isError()) {
|
||
qDebug() << "error: RawJson::scanSetRanges():" << error.message;
|
||
return false;
|
||
}
|
||
|
||
QList<SetToDownload> newSetList;
|
||
newSetList.reserve(ranges.size());
|
||
|
||
for (const RawJson::SetRange &range : ranges) {
|
||
QString shortName = range.code.toUpper();
|
||
QString longName = range.name;
|
||
QString setType = range.type;
|
||
QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate);
|
||
CardSet::Priority priority = getSetPriority(setType, shortName);
|
||
// capitalize set type
|
||
if (setType.length() > 0) {
|
||
// basic grammar for words that aren't capitalized, like in "From the Vault"
|
||
static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for",
|
||
"of", "in", "and", "with", "or"};
|
||
QStringList words = setType.split("_");
|
||
setType.clear();
|
||
bool first = false;
|
||
for (auto &item : words) {
|
||
if (first && noCapitalize.contains(item)) {
|
||
setType += item + QString(" ");
|
||
} else {
|
||
setType += item[0].toUpper() + item.mid(1) + QString(" ");
|
||
first = true;
|
||
}
|
||
}
|
||
setType = setType.trimmed();
|
||
}
|
||
SetToDownload set(shortName, longName, priority, setType, releaseDate);
|
||
set.setRawRange(range.dataRange);
|
||
newSetList.append(set);
|
||
}
|
||
|
||
std::sort(newSetList.begin(), newSetList.end());
|
||
|
||
if (newSetList.isEmpty()) {
|
||
return false;
|
||
}
|
||
allSets = newSetList;
|
||
rawSetsData = std::move(data);
|
||
return true;
|
||
}
|
||
|
||
static QString getMainCardType(const QStringList &typeList)
|
||
{
|
||
if (typeList.isEmpty()) {
|
||
return {};
|
||
}
|
||
|
||
static const QStringList typePriority = {"Planeswalker", "Creature", "Land", "Sorcery",
|
||
"Instant", "Artifact", "Enchantment"};
|
||
|
||
for (const auto &type : typePriority) {
|
||
if (typeList.contains(type)) {
|
||
return type;
|
||
}
|
||
}
|
||
|
||
return typeList.first();
|
||
}
|
||
|
||
/**
|
||
* Sorts and deduplicates the color chars in the string by WUBRG order.
|
||
*
|
||
* @param colors The string containing the color chars. Will be modified in-place
|
||
*/
|
||
static void sortAndReduceColors(QString &colors)
|
||
{
|
||
// sort
|
||
static const QHash<QChar, unsigned int> colorOrder{{'W', 0}, {'U', 1}, {'B', 2}, {'R', 3}, {'G', 4}};
|
||
std::sort(colors.begin(), colors.end(),
|
||
[](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); });
|
||
// reduce
|
||
auto last = std::unique(colors.begin(), colors.end());
|
||
colors.erase(last, colors.end());
|
||
}
|
||
|
||
CardInfoPtr OracleImporter::addCard(QString name,
|
||
const QString &text,
|
||
bool isToken,
|
||
QHash<QString, QString> properties,
|
||
const QList<CardRelation *> &relatedCards,
|
||
const PrintingInfo &printingInfo)
|
||
{
|
||
// Workaround for card name weirdness
|
||
name = name.replace("Æ", "AE");
|
||
name = name.replace("’", "'");
|
||
auto existingIt = cards.constFind(name);
|
||
if (existingIt != cards.constEnd()) {
|
||
CardInfoPtr card = existingIt.value();
|
||
card->addToSet(printingInfo.getSet(), printingInfo);
|
||
// Only merge legalities when the card has none yet, so multi-format
|
||
// printings don't overwrite each other's legality lists.
|
||
if (card->getProperties().filter(formatRegex).empty()) {
|
||
card->combineLegalities(properties);
|
||
}
|
||
return card;
|
||
}
|
||
|
||
// Remove {} around mana costs, except if it's split cost
|
||
QString manacost = properties.value("manacost");
|
||
if (!manacost.isEmpty()) {
|
||
QStringList symbols = manacost.split("}");
|
||
QString formattedCardCost;
|
||
for (QString symbol : symbols) {
|
||
static const auto manaCostPattern = QRegularExpression("[0-9WUBGRP]/[0-9WUBGRP]");
|
||
if (symbol.contains(manaCostPattern)) {
|
||
symbol.append("}");
|
||
} else {
|
||
symbol.remove(QChar('{'));
|
||
}
|
||
formattedCardCost.append(symbol);
|
||
}
|
||
properties.insert("manacost", formattedCardCost);
|
||
}
|
||
|
||
// fix colors
|
||
QString allColors = properties.value("colors");
|
||
if (allColors.size() > 1) {
|
||
sortAndReduceColors(allColors);
|
||
properties.insert("colors", allColors);
|
||
}
|
||
QString allColorIdent = properties.value("coloridentity");
|
||
if (allColorIdent.size() > 1) {
|
||
sortAndReduceColors(allColorIdent);
|
||
properties.insert("coloridentity", allColorIdent);
|
||
}
|
||
|
||
// DETECT CARD POSITIONING INFO
|
||
|
||
QString layoutVal = properties.value("layout");
|
||
bool landscapeOrientation =
|
||
properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar";
|
||
|
||
// cards that enter the field tapped
|
||
bool cipt = parseCipt(name, text) || landscapeOrientation;
|
||
|
||
// table row
|
||
int tableRow = 1;
|
||
QString mainCardType = properties.value("maintype");
|
||
if (mainCardType == "Land") {
|
||
tableRow = 0;
|
||
} else if (mainCardType == "Sorcery" || mainCardType == "Instant") {
|
||
tableRow = 3;
|
||
} else if (mainCardType == "Creature") {
|
||
tableRow = 2;
|
||
}
|
||
|
||
// card side
|
||
QString side = properties.value("side") == "b" ? "back" : "front";
|
||
properties.insert("side", side);
|
||
|
||
// upsideDown (flip cards)
|
||
QString layout = properties.value("layout");
|
||
bool upsideDown = layout == "flip" && side == "back";
|
||
|
||
// insert the card and its properties
|
||
SetToPrintingsMap setsInfo;
|
||
setsInfo[printingInfo.getSet()->getShortName()].append(printingInfo);
|
||
CardInfo::UiAttributes attributes = {cipt, landscapeOrientation, tableRow, upsideDown};
|
||
CardInfoPtr newCard =
|
||
CardInfo::newInstance(name, text, isToken, properties, relatedCards, {}, setsInfo, attributes);
|
||
|
||
if (name.isEmpty()) {
|
||
qDebug() << "warning: an empty card was added to set" << printingInfo.getSet()->getShortName();
|
||
}
|
||
cards.insert(name, newCard);
|
||
|
||
return newCard;
|
||
}
|
||
|
||
static QString getJsonString(const QJsonObject &obj, const QString &key)
|
||
{
|
||
// QVariant coerces numbers and booleans to text, while QJsonValue::toString()
|
||
// returns a null string for them — some MTGJSON fields (manaValue,
|
||
// convertedManaCost, isOnlineOnly, isRebalanced) carry those types.
|
||
return obj.value(key).toVariant().toString();
|
||
}
|
||
|
||
int OracleImporter::importCardsFromSet(const CardSetPtr ¤tSet, const QJsonArray &cardsList)
|
||
{
|
||
// mtgjson name => xml name
|
||
static const QMap<QString, QString> cardProperties{
|
||
{"manaCost", "manacost"}, {"manaValue", "cmc"}, {"type", "type"},
|
||
{"loyalty", "loyalty"}, {"layout", "layout"}, {"side", "side"},
|
||
{"convertedManaCost", "cmc"}, // old name for manaValue, for backwards compatibility
|
||
};
|
||
|
||
// mtgjson name => xml name
|
||
static const QMap<QString, QString> setInfoProperties{{"number", "num"},
|
||
{"rarity", "rarity"},
|
||
{"isOnlineOnly", "isOnlineOnly"},
|
||
{"isRebalanced", "isRebalanced"},
|
||
{"artist", "artist"}};
|
||
|
||
// mtgjson name => xml name
|
||
static const QMap<QString, QString> identifierProperties{{"multiverseId", "muid"}, {"scryfallId", "uuid"}};
|
||
|
||
static const QString ptSeparator = "/";
|
||
static constexpr bool isToken = false;
|
||
static const QSet<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
|
||
|
||
int numCards = 0;
|
||
|
||
// Keeps track of any split card faces encountered so far
|
||
QMap<QString, QPair<QList<SplitCardPart>, QString>> splitCards;
|
||
|
||
// Keeps track of all names encountered so far
|
||
QSet<QString> allNameProps;
|
||
|
||
for (const QJsonValue &cardVal : cardsList) {
|
||
QJsonObject card = cardVal.toObject();
|
||
|
||
/* Currently used layouts are:
|
||
* augment, double_faced_token, flip, host, leveler, meld, normal, planar,
|
||
* saga, scheme, split, token, transform, vanguard
|
||
*/
|
||
QString layout = getJsonString(card, "layout");
|
||
|
||
// don't import tokens from the json file
|
||
if (layout == "token") {
|
||
continue;
|
||
}
|
||
|
||
// normal cards handling
|
||
QString name = getJsonString(card, "name");
|
||
QString text = getJsonString(card, "text");
|
||
QString faceName = getJsonString(card, "faceName");
|
||
if (faceName.isEmpty()) {
|
||
faceName = name;
|
||
}
|
||
|
||
// card properties
|
||
QHash<QString, QString> properties;
|
||
for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) {
|
||
QString propertyValue = getJsonString(card, i.key());
|
||
if (!propertyValue.isEmpty()) {
|
||
properties.insert(i.value(), propertyValue);
|
||
}
|
||
}
|
||
|
||
// per-set properties
|
||
QHash<QString, QString> printingProps;
|
||
for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) {
|
||
QString propertyValue = getJsonString(card, i.key());
|
||
if (!propertyValue.isEmpty()) {
|
||
printingProps.insert(i.value(), propertyValue);
|
||
}
|
||
}
|
||
|
||
// handle flavorNames specially due to double-faced cards
|
||
QString faceFlavorName = getJsonString(card, "faceFlavorName");
|
||
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName");
|
||
if (!flavorName.isEmpty()) {
|
||
printingProps.insert("flavorName", flavorName);
|
||
}
|
||
|
||
// Identifiers
|
||
QJsonObject identifiers = card.value("identifiers").toObject();
|
||
for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) {
|
||
QString propertyValue = getJsonString(identifiers, i.key());
|
||
if (!propertyValue.isEmpty()) {
|
||
printingProps.insert(i.value(), propertyValue);
|
||
}
|
||
}
|
||
|
||
PrintingInfo printingInfo(currentSet, LazyPropertiesHash(printingProps));
|
||
|
||
QString numComponent;
|
||
const QString numProperty = printingInfo.getProperty("num");
|
||
const QChar lastChar = numProperty.isEmpty() ? QChar() : numProperty.back();
|
||
|
||
// Un-Sets do some wonky stuff. Split up these cards as individual entries.
|
||
// these cards will have a num with a letter (abc) behind it, put that letter into the name
|
||
if (setsWithCardsWithSameNameButDifferentText.contains(currentSet->getShortName()) &&
|
||
allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) {
|
||
numComponent = " (" + QString(lastChar).toLower() + ")";
|
||
}
|
||
allNameProps.insert(faceName);
|
||
|
||
// special handling properties
|
||
QString colors;
|
||
for (const QJsonValue &color : card.value("colors").toArray()) {
|
||
colors += color.toString();
|
||
}
|
||
if (!colors.isEmpty()) {
|
||
properties.insert("colors", colors);
|
||
}
|
||
|
||
QString colorIdentity;
|
||
for (const QJsonValue &color : card.value("colorIdentity").toArray()) {
|
||
colorIdentity += color.toString();
|
||
}
|
||
if (!colorIdentity.isEmpty()) {
|
||
properties.insert("coloridentity", colorIdentity);
|
||
}
|
||
|
||
const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList());
|
||
if (mainCardType.isEmpty()) {
|
||
qDebug() << "warning: no mainCardType for card:" << name;
|
||
} else {
|
||
properties.insert("maintype", mainCardType);
|
||
}
|
||
|
||
// Depending on whether power and/or toughness are present, the format
|
||
// is either P/T (most common), P (no toughness), or /T (no power).
|
||
QString power = getJsonString(card, "power");
|
||
QString toughness = getJsonString(card, "toughness");
|
||
if (toughness.isEmpty() && !power.isEmpty()) {
|
||
properties.insert("pt", power);
|
||
} else if (!toughness.isEmpty()) {
|
||
properties.insert("pt", power + ptSeparator + toughness);
|
||
}
|
||
|
||
auto legalities = card.value("legalities").toObject();
|
||
for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) {
|
||
properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower());
|
||
}
|
||
|
||
// split cards are considered a single card, enqueue for later merging
|
||
if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") {
|
||
auto _faceName = getJsonString(card, "faceName");
|
||
SplitCardPart split(_faceName, text, properties, printingInfo);
|
||
auto found_iter = splitCards.find(name + numProperty);
|
||
if (found_iter == splitCards.end()) {
|
||
splitCards.insert(name + numProperty, {{split}, name});
|
||
} else {
|
||
found_iter->first.append(split);
|
||
}
|
||
} else {
|
||
// relations
|
||
QList<CardRelation *> relatedCards;
|
||
|
||
// add other face for split cards as card relation
|
||
if (!getJsonString(card, "side").isEmpty()) {
|
||
auto faceManaValue = getJsonString(card, "faceManaValue");
|
||
if (faceManaValue.isEmpty()) {
|
||
// check the old name for the property, for backwards compatibility purposes
|
||
faceManaValue = getJsonString(card, "faceConvertedManaCost");
|
||
}
|
||
properties["cmc"] = faceManaValue;
|
||
|
||
if (layout == "meld") { // meld cards don't work
|
||
static const QRegularExpression meldNameRegex{"then meld them into ([^\\.]*)"};
|
||
QString additionalName = meldNameRegex.match(text).captured(1);
|
||
if (!additionalName.isNull()) {
|
||
relatedCards.append(new CardRelation(additionalName, CardRelationType::TransformInto));
|
||
}
|
||
} else {
|
||
for (const QString &additionalName : name.split(" // ")) {
|
||
if (additionalName != faceName) {
|
||
relatedCards.append(new CardRelation(additionalName, CardRelationType::TransformInto));
|
||
}
|
||
}
|
||
}
|
||
name = faceName;
|
||
}
|
||
|
||
// mtgjson related cards
|
||
QJsonObject givenRelated = card.value("relatedCards").toObject();
|
||
if (!givenRelated.isEmpty()) {
|
||
// conjured cards from a spellbook
|
||
QJsonArray spellbook = givenRelated.value("spellbook").toArray();
|
||
if (!spellbook.isEmpty()) {
|
||
for (const QJsonValue &spbkVal : spellbook) {
|
||
relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false,
|
||
false, 1, true));
|
||
}
|
||
}
|
||
}
|
||
|
||
CardInfoPtr newCard =
|
||
addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo);
|
||
numCards++;
|
||
}
|
||
}
|
||
|
||
// split cards handling
|
||
static const QString splitCardPropSeparator = QString(" // ");
|
||
static const QString splitCardTextSeparator = QString("\n\n---\n\n");
|
||
|
||
QList<QPair<QList<SplitCardPart>, QString>> partsAndNames = splitCards.values();
|
||
for (auto [splitCardParts, name] : partsAndNames) {
|
||
QString text;
|
||
QHash<QString, QString> properties;
|
||
PrintingInfo printingInfo;
|
||
|
||
for (const SplitCardPart &tmp : splitCardParts) {
|
||
if (!text.isEmpty()) {
|
||
text.append(splitCardTextSeparator);
|
||
}
|
||
text.append(tmp.getText());
|
||
|
||
if (properties.isEmpty()) {
|
||
properties = tmp.getProperties();
|
||
printingInfo = tmp.getPrintingInfo();
|
||
} else {
|
||
const QHash<QString, QString> &tmpProps = tmp.getProperties();
|
||
for (auto i = tmpProps.cbegin(), end = tmpProps.cend(); i != end; ++i) {
|
||
QString prop = i.key();
|
||
QString originalPropertyValue = properties.value(prop);
|
||
QString thisCardPropertyValue = i.value();
|
||
if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) {
|
||
if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty
|
||
properties.insert(prop, thisCardPropertyValue);
|
||
} else if (prop == "colors" || prop == "coloridentity") { // the card is both colors
|
||
properties.insert(prop, originalPropertyValue + thisCardPropertyValue);
|
||
} else if (prop == "maintype") { // don't create maintypes with //es in them
|
||
continue;
|
||
} else {
|
||
properties.insert(prop,
|
||
originalPropertyValue + splitCardPropSeparator + thisCardPropertyValue);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo);
|
||
numCards++;
|
||
}
|
||
|
||
return numCards;
|
||
}
|
||
|
||
static FormatRulesNameMap buildDefaultMagicFormats()
|
||
{
|
||
// Predefined common exceptions
|
||
CardCondition superTypeIsBasic;
|
||
superTypeIsBasic.field = "type";
|
||
superTypeIsBasic.matchType = "regex";
|
||
superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)";
|
||
|
||
ExceptionRule basicLands;
|
||
basicLands.conditions.append(superTypeIsBasic);
|
||
|
||
CardCondition anyNumberAllowed;
|
||
anyNumberAllowed.field = "text";
|
||
anyNumberAllowed.matchType = "contains";
|
||
anyNumberAllowed.value = "A deck can have any number of";
|
||
|
||
ExceptionRule mayContainAnyNumber;
|
||
mayContainAnyNumber.conditions.append(anyNumberAllowed);
|
||
|
||
FormatRulesNameMap defaultFormatRulesNameMap;
|
||
|
||
// ----------------- Helper lambda to create format -----------------
|
||
auto makeFormat = [&](const QString &name, int minDeck = 60, int maxDeck = -1, int maxSideboardSize = 15,
|
||
const QList<AllowedCount> &allowedCounts = kConstructedCounts) -> FormatRulesPtr {
|
||
FormatRulesPtr f(new FormatRules);
|
||
f->formatName = name;
|
||
f->allowedCounts = allowedCounts;
|
||
f->minDeckSize = minDeck;
|
||
f->maxDeckSize = maxDeck;
|
||
f->maxSideboardSize = maxSideboardSize;
|
||
f->exceptions.append(basicLands);
|
||
f->exceptions.append(mayContainAnyNumber);
|
||
defaultFormatRulesNameMap.insert(name.toLower(), f);
|
||
return f;
|
||
};
|
||
|
||
// ----------------- Standard formats -----------------
|
||
makeFormat("Standard");
|
||
makeFormat("Modern");
|
||
makeFormat("Legacy");
|
||
makeFormat("Pioneer");
|
||
makeFormat("Historic");
|
||
makeFormat("Timeless");
|
||
makeFormat("Future");
|
||
makeFormat("OldSchool");
|
||
makeFormat("Premodern");
|
||
makeFormat("Pauper");
|
||
makeFormat("Penny");
|
||
|
||
// ----------------- Singleton formats -----------------
|
||
makeFormat("Commander", 100, 100, 15, kSingletonCounts);
|
||
makeFormat("Duel", 100, 100, 15, kSingletonCounts);
|
||
makeFormat("Brawl", 60, 60, 15, kSingletonCounts);
|
||
makeFormat("StandardBrawl", 60, 60, 15, kSingletonCounts);
|
||
makeFormat("Oathbreaker", 60, 60, 15, kSingletonCounts);
|
||
makeFormat("PauperCommander", 100, 100, 15, kSingletonCounts);
|
||
makeFormat("Predh", 100, 100, 15, kSingletonCounts);
|
||
|
||
// ----------------- Restricted formats -----------------
|
||
makeFormat("Vintage", 60, -1, 15, {{4, "legal"}, {1, "restricted"}, {0, "banned"}});
|
||
|
||
return defaultFormatRulesNameMap;
|
||
}
|
||
|
||
const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats()
|
||
{
|
||
static const FormatRulesNameMap cached = buildDefaultMagicFormats();
|
||
return cached;
|
||
}
|
||
|
||
int OracleImporter::startImport()
|
||
{
|
||
static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController();
|
||
|
||
// Pre-allocate the cards hash to avoid rehashing during import. Keys are
|
||
// distinct card names while raw ranges only count printings (AllPrintings
|
||
// ~100k printings vs ~35k names), so this over-reserves somewhat; an exact
|
||
// distinct-name count would require eagerly parsing, which the lazy reader
|
||
// deliberately avoids. It's a capacity hint, so the overshoot is harmless.
|
||
int estimatedCards = 0;
|
||
for (const SetToDownload &curSetToParse : allSets) {
|
||
estimatedCards += curSetToParse.getRawRange().cardCount;
|
||
}
|
||
cards.reserve(estimatedCards);
|
||
|
||
// add an empty set for tokens
|
||
CardSetPtr tokenSet =
|
||
CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens");
|
||
sets.insert(CardSet::TOKENS_SETNAME, tokenSet);
|
||
|
||
int setIndex = 0;
|
||
|
||
for (const SetToDownload &curSetToParse : allSets) {
|
||
CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(),
|
||
curSetToParse.getLongName(), curSetToParse.getSetType(),
|
||
curSetToParse.getReleaseDate(), curSetToParse.getPriority());
|
||
|
||
// parse only this set's slice of the raw document so the whole JSON tree is
|
||
// never kept in memory at once
|
||
const RawJson::SetDataRange &rawRange = curSetToParse.getRawRange();
|
||
const qsizetype rangeEnd = rawRange.start + rawRange.length;
|
||
if (rawRange.start < 0 || rawRange.length <= 0 || rangeEnd > rawSetsData.size()) {
|
||
// rawSetsData is cleared by releaseSetData() while SetToDownload copies
|
||
// taken from getSets() keep their ranges, and nothing else enforces the
|
||
// pairing — so never index past the buffer on stale/mismatched ranges.
|
||
qWarning() << "error: out-of-bounds raw range for set" << curSetToParse.getShortName() << "skipping";
|
||
++setIndex;
|
||
emit setIndexChanged(0, setIndex, curSetToParse.getLongName());
|
||
continue;
|
||
}
|
||
// sliced() shares the buffer instead of deep-copying the slice; the largest
|
||
// sets in AllPrintings are tens of MB, so the copy is worth avoiding here.
|
||
const QByteArray setBytes = rawSetsData.sliced(rawRange.start, rawRange.length);
|
||
QJsonParseError parseError;
|
||
const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError);
|
||
if (parseError.error != QJsonParseError::NoError) {
|
||
qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":"
|
||
<< parseError.errorString();
|
||
++setIndex;
|
||
// Keep the progress accounting honest: a set that failed to parse
|
||
// still advanced the index, so report it (with zero imported cards)
|
||
// rather than letting SaveSetsPage's bar stall per failed set.
|
||
emit setIndexChanged(0, setIndex, curSetToParse.getLongName());
|
||
continue;
|
||
}
|
||
|
||
// Only add the set to the database once its slice parsed cleanly;
|
||
// a set that fails here must not persist as an empty set in cards.xml.
|
||
if (!sets.contains(newSet->getShortName())) {
|
||
sets.insert(newSet->getShortName(), newSet);
|
||
}
|
||
|
||
const QJsonArray setCards = setDoc.object().value("cards").toArray();
|
||
int numCardsInSet = importCardsFromSet(newSet, setCards);
|
||
|
||
++setIndex;
|
||
|
||
emit setIndexChanged(numCardsInSet, setIndex, curSetToParse.getLongName());
|
||
}
|
||
|
||
emit setIndexChanged(0, setIndex, QString());
|
||
|
||
// total number of sets
|
||
return setIndex;
|
||
}
|
||
|
||
bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion)
|
||
{
|
||
CockatriceXml4Parser parser(new NoopCardPreferenceProvider(), new NoopCardSetPriorityController());
|
||
|
||
return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion);
|
||
}
|
||
|
||
void OracleImporter::releaseSetData()
|
||
{
|
||
allSets.clear();
|
||
rawSetsData.clear();
|
||
}
|
||
|
||
void OracleImporter::clear()
|
||
{
|
||
sets.clear();
|
||
cards.clear();
|
||
allSets.clear();
|
||
rawSetsData.clear();
|
||
}
|