Cockatrice/oracle/src/oracleimporter.cpp
BruebachL 61e6a9913e
Some checks are pending
CodeQL / Analyze (cpp) (push) Waiting to run
CodeQL / Analyze (actions) (push) Waiting to run
Build Desktop / Configure (push) Waiting to run
Build Desktop / Debian 13 (push) Blocked by required conditions
Build Desktop / Debian 12 (push) Blocked by required conditions
Build Desktop / Fedora 44 (push) Blocked by required conditions
Build Desktop / Fedora 43 (push) Blocked by required conditions
Build Desktop / Servatrice_Debian 12 (push) Blocked by required conditions
Build Desktop / Ubuntu 26.04 (push) Blocked by required conditions
Build Desktop / Ubuntu 24.04 (push) Blocked by required conditions
Build Desktop / Arch (push) Blocked by required conditions
Build Desktop / macOS 13 Intel (push) Blocked by required conditions
Build Desktop / macOS 14 (push) Blocked by required conditions
Build Desktop / macOS 15 (push) Blocked by required conditions
Build Desktop / macOS 26 Debug (push) Blocked by required conditions
Build Desktop / Windows 10 (push) Blocked by required conditions
Build Docker / Servatrice (arm) (push) Waiting to run
Build Docker / Servatrice (x86) (push) Waiting to run
Build Docker / Publish multi-platform Servatrice image (push) Blocked by required conditions
[Oracle] Add oracle importer tests and fix set parsing details (#7215)
* [Oracle] Add oracle importer tests and fix set parsing details

- Add oracle_importer_test and oracle_importer_benchmark_test targets
- Preserve the first printing's legalities when an existing card is reused
- Concatenate split-card coloridentity and sort/dedupe card colors
- Use a raw string for the Basic Land format regex
- Pre-allocate the card hash and micro-optimize string handling

Took 2 minutes

* [Oracle/Tests] Pin cmc coercion in CI run; scope the reserve pass

The #7214 coercion assertion lived only in oracle_importer_benchmark_test,
which gets no add_test and so never runs under ctest. Add NumericManaValueCoercedToCmc
and LegacyConvertedManaCostCoercedToCmc to oracle_importer_test (a CI-ran
binary): manaValue/convertedManaCost are JSON numbers in AllPrintings, and
QJsonValue::toString() would drop them to an empty cmc without the
#7214 coercion fix.

Wrap the distinct-name reserve pass in a bare block so the ~35k name
QStrings are handed back before the memory-heavy import loop starts.

---------

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>
2026-09-04 22:14:16 +02:00

615 lines
24 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#include "oracleimporter.h"
#include "libcockatrice/interfaces/noop_card_preference_provider.h"
#include "libcockatrice/interfaces/noop_card_set_priority_controller.h"
#include "parsehelpers.h"
#include <QDebug>
#include <QJsonDocument>
#include <QJsonObject>
#include <QRegularExpression>
#include <QSet>
#include <algorithm>
#include <climits>
#include <libcockatrice/card/database/parser/cockatrice_xml_4.h>
#include <libcockatrice/card/relation/card_relation.h>
static const QList<AllowedCount> kConstructedCounts = {{4, "legal"}, {0, "banned"}};
static const QList<AllowedCount> kSingletonCounts = {{1, "legal"}, {0, "banned"}};
SplitCardPart::SplitCardPart(const QString &_name,
const QString &_text,
const QHash<QString, QString> &_properties,
const PrintingInfo &_printingInfo)
: name(_name), text(_text), properties(_properties), printingInfo(_printingInfo)
{
}
const QRegularExpression OracleImporter::formatRegex = QRegularExpression("^format-");
OracleImporter::OracleImporter(QObject *parent) : QObject(parent)
{
}
static CardSet::Priority getSetPriority(const QString &setType, const QString &shortName)
{
if (!setTypePriorities.contains(setType.toLower())) {
qDebug() << "warning: Set type" << setType << "unrecognized for prioritization";
}
CardSet::Priority priority = setTypePriorities.value(setType.toLower(), CardSet::PriorityOther);
if (nonEnglishSets.contains(shortName)) {
priority = CardSet::PriorityLowest;
}
return priority;
}
bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
{
QJsonParseError error;
auto doc = QJsonDocument::fromJson(data, &error);
if (error.error != QJsonParseError::NoError) {
qDebug() << "error: QJsonDocument::fromJson():" << error.errorString();
return false;
}
auto setsObj = doc.object().value("data").toObject();
QList<SetToDownload> newSetList;
for (auto it = setsObj.constBegin(); it != setsObj.constEnd(); ++it) {
QJsonObject setObj = it.value().toObject();
QString shortName = setObj.value("code").toString().toUpper();
QString longName = setObj.value("name").toString();
QJsonArray setCards = setObj.value("cards").toArray();
QString setType = setObj.value("type").toString();
QDate releaseDate = QDate::fromString(setObj.value("releaseDate").toString(), Qt::ISODate);
CardSet::Priority priority = getSetPriority(setType, shortName);
// capitalize set type
if (setType.length() > 0) {
// basic grammar for words that aren't capitalized, like in "From the Vault"
static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for",
"of", "in", "and", "with", "or"};
QStringList words = setType.split("_");
setType.clear();
bool first = false;
for (auto &item : words) {
if (first && noCapitalize.contains(item)) {
setType += item + QString(" ");
} else {
setType += item[0].toUpper() + item.mid(1) + QString(" ");
first = true;
}
}
setType = setType.trimmed();
}
newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate));
}
std::sort(newSetList.begin(), newSetList.end());
if (newSetList.isEmpty()) {
return false;
}
allSets = newSetList;
return true;
}
static QString getMainCardType(const QStringList &typeList)
{
if (typeList.isEmpty()) {
return {};
}
static const QStringList typePriority = {"Planeswalker", "Creature", "Land", "Sorcery",
"Instant", "Artifact", "Enchantment"};
for (const auto &type : typePriority) {
if (typeList.contains(type)) {
return type;
}
}
return typeList.first();
}
/**
* Sorts and deduplicates the color chars in the string by WUBRG order.
*
* @param colors The string containing the color chars. Will be modified in-place
*/
static void sortAndReduceColors(QString &colors)
{
// sort
static const QHash<QChar, unsigned int> colorOrder{{'W', 0}, {'U', 1}, {'B', 2}, {'R', 3}, {'G', 4}};
std::sort(colors.begin(), colors.end(),
[](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); });
// reduce
auto last = std::unique(colors.begin(), colors.end());
colors.erase(last, colors.end());
}
CardInfoPtr OracleImporter::addCard(QString name,
const QString &text,
bool isToken,
QHash<QString, QString> properties,
const QList<CardRelation *> &relatedCards,
const PrintingInfo &printingInfo)
{
// Workaround for card name weirdness
name = name.replace("Æ", "AE");
name = name.replace("", "'");
auto existingIt = cards.constFind(name);
if (existingIt != cards.constEnd()) {
CardInfoPtr card = existingIt.value();
card->addToSet(printingInfo.getSet(), printingInfo);
// Only merge legalities when the card has none yet, so multi-format
// printings don't overwrite each other's legality lists.
if (card->getProperties().filter(formatRegex).empty()) {
card->combineLegalities(properties);
}
return card;
}
// Remove {} around mana costs, except if it's split cost
QString manacost = properties.value("manacost");
if (!manacost.isEmpty()) {
QStringList symbols = manacost.split("}");
QString formattedCardCost;
for (QString symbol : symbols) {
static const auto manaCostPattern = QRegularExpression("[0-9WUBGRP]/[0-9WUBGRP]");
if (symbol.contains(manaCostPattern)) {
symbol.append("}");
} else {
symbol.remove(QChar('{'));
}
formattedCardCost.append(symbol);
}
properties.insert("manacost", formattedCardCost);
}
// fix colors
QString allColors = properties.value("colors");
if (allColors.size() > 1) {
sortAndReduceColors(allColors);
properties.insert("colors", allColors);
}
QString allColorIdent = properties.value("coloridentity");
if (allColorIdent.size() > 1) {
sortAndReduceColors(allColorIdent);
properties.insert("coloridentity", allColorIdent);
}
// DETECT CARD POSITIONING INFO
QString layoutVal = properties.value("layout");
bool landscapeOrientation =
properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar";
// cards that enter the field tapped
bool cipt = parseCipt(name, text) || landscapeOrientation;
// table row
int tableRow = 1;
QString mainCardType = properties.value("maintype");
if (mainCardType == "Land") {
tableRow = 0;
} else if (mainCardType == "Sorcery" || mainCardType == "Instant") {
tableRow = 3;
} else if (mainCardType == "Creature") {
tableRow = 2;
}
// card side
QString side = properties.value("side") == "b" ? "back" : "front";
properties.insert("side", side);
// upsideDown (flip cards)
QString layout = properties.value("layout");
bool upsideDown = layout == "flip" && side == "back";
// insert the card and its properties
SetToPrintingsMap setsInfo;
setsInfo[printingInfo.getSet()->getShortName()].append(printingInfo);
CardInfo::UiAttributes attributes = {cipt, landscapeOrientation, tableRow, upsideDown};
CardInfoPtr newCard =
CardInfo::newInstance(name, text, isToken, properties, relatedCards, {}, setsInfo, attributes);
if (name.isEmpty()) {
qDebug() << "warning: an empty card was added to set" << printingInfo.getSet()->getShortName();
}
cards.insert(name, newCard);
return newCard;
}
static QString getJsonString(const QJsonObject &obj, const QString &key)
{
// QVariant coerces numbers and booleans to text, while QJsonValue::toString()
// returns a null string for them — some MTGJSON fields (manaValue,
// convertedManaCost, isOnlineOnly, isRebalanced) carry those types.
return obj.value(key).toVariant().toString();
}
int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QJsonArray &cardsList)
{
// mtgjson name => xml name
static const QMap<QString, QString> cardProperties{
{"manaCost", "manacost"}, {"manaValue", "cmc"}, {"type", "type"},
{"loyalty", "loyalty"}, {"layout", "layout"}, {"side", "side"},
{"convertedManaCost", "cmc"}, // old name for manaValue, for backwards compatibility
};
// mtgjson name => xml name
static const QMap<QString, QString> setInfoProperties{{"number", "num"},
{"rarity", "rarity"},
{"isOnlineOnly", "isOnlineOnly"},
{"isRebalanced", "isRebalanced"},
{"artist", "artist"}};
// mtgjson name => xml name
static const QMap<QString, QString> identifierProperties{{"multiverseId", "muid"}, {"scryfallId", "uuid"}};
static const QString ptSeparator = "/";
static constexpr bool isToken = false;
static const QSet<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
int numCards = 0;
// Keeps track of any split card faces encountered so far
QMap<QString, QPair<QList<SplitCardPart>, QString>> splitCards;
// Keeps track of all names encountered so far
QSet<QString> allNameProps;
for (const QJsonValue &cardVal : cardsList) {
QJsonObject card = cardVal.toObject();
/* Currently used layouts are:
* augment, double_faced_token, flip, host, leveler, meld, normal, planar,
* saga, scheme, split, token, transform, vanguard
*/
QString layout = getJsonString(card, "layout");
// don't import tokens from the json file
if (layout == "token") {
continue;
}
// normal cards handling
QString name = getJsonString(card, "name");
QString text = getJsonString(card, "text");
QString faceName = getJsonString(card, "faceName");
if (faceName.isEmpty()) {
faceName = name;
}
// card properties
QHash<QString, QString> properties;
for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) {
QString propertyValue = getJsonString(card, i.key());
if (!propertyValue.isEmpty()) {
properties.insert(i.value(), propertyValue);
}
}
// per-set properties
QHash<QString, QString> printingProps;
for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) {
QString propertyValue = getJsonString(card, i.key());
if (!propertyValue.isEmpty()) {
printingProps.insert(i.value(), propertyValue);
}
}
// handle flavorNames specially due to double-faced cards
QString faceFlavorName = getJsonString(card, "faceFlavorName");
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName");
if (!flavorName.isEmpty()) {
printingProps.insert("flavorName", flavorName);
}
// Identifiers
QJsonObject identifiers = card.value("identifiers").toObject();
for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) {
QString propertyValue = getJsonString(identifiers, i.key());
if (!propertyValue.isEmpty()) {
printingProps.insert(i.value(), propertyValue);
}
}
PrintingInfo printingInfo(currentSet, LazyPropertiesHash(printingProps));
QString numComponent;
const QString numProperty = printingInfo.getProperty("num");
const QChar lastChar = numProperty.isEmpty() ? QChar() : numProperty.back();
// Un-Sets do some wonky stuff. Split up these cards as individual entries.
// these cards will have a num with a letter (abc) behind it, put that letter into the name
if (setsWithCardsWithSameNameButDifferentText.contains(currentSet->getShortName()) &&
allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) {
numComponent = " (" + QString(lastChar).toLower() + ")";
}
allNameProps.insert(faceName);
// special handling properties
QString colors;
for (const QJsonValue &color : card.value("colors").toArray()) {
colors += color.toString();
}
if (!colors.isEmpty()) {
properties.insert("colors", colors);
}
QString colorIdentity;
for (const QJsonValue &color : card.value("colorIdentity").toArray()) {
colorIdentity += color.toString();
}
if (!colorIdentity.isEmpty()) {
properties.insert("coloridentity", colorIdentity);
}
const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList());
if (mainCardType.isEmpty()) {
qDebug() << "warning: no mainCardType for card:" << name;
} else {
properties.insert("maintype", mainCardType);
}
// Depending on whether power and/or toughness are present, the format
// is either P/T (most common), P (no toughness), or /T (no power).
QString power = getJsonString(card, "power");
QString toughness = getJsonString(card, "toughness");
if (toughness.isEmpty() && !power.isEmpty()) {
properties.insert("pt", power);
} else if (!toughness.isEmpty()) {
properties.insert("pt", power + ptSeparator + toughness);
}
auto legalities = card.value("legalities").toObject();
for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) {
properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower());
}
// split cards are considered a single card, enqueue for later merging
if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") {
auto _faceName = getJsonString(card, "faceName");
SplitCardPart split(_faceName, text, properties, printingInfo);
auto found_iter = splitCards.find(name + numProperty);
if (found_iter == splitCards.end()) {
splitCards.insert(name + numProperty, {{split}, name});
} else {
found_iter->first.append(split);
}
} else {
// relations
QList<CardRelation *> relatedCards;
// add other face for split cards as card relation
if (!getJsonString(card, "side").isEmpty()) {
auto faceManaValue = getJsonString(card, "faceManaValue");
if (faceManaValue.isEmpty()) {
// check the old name for the property, for backwards compatibility purposes
faceManaValue = getJsonString(card, "faceConvertedManaCost");
}
properties["cmc"] = faceManaValue;
if (layout == "meld") { // meld cards don't work
static const QRegularExpression meldNameRegex{"then meld them into ([^\\.]*)"};
QString additionalName = meldNameRegex.match(text).captured(1);
if (!additionalName.isNull()) {
relatedCards.append(new CardRelation(additionalName, CardRelationType::TransformInto));
}
} else {
for (const QString &additionalName : name.split(" // ")) {
if (additionalName != faceName) {
relatedCards.append(new CardRelation(additionalName, CardRelationType::TransformInto));
}
}
}
name = faceName;
}
// mtgjson related cards
QJsonObject givenRelated = card.value("relatedCards").toObject();
if (!givenRelated.isEmpty()) {
// conjured cards from a spellbook
QJsonArray spellbook = givenRelated.value("spellbook").toArray();
if (!spellbook.isEmpty()) {
for (const QJsonValue &spbkVal : spellbook) {
relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false,
false, 1, true));
}
}
}
CardInfoPtr newCard =
addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo);
numCards++;
}
}
// split cards handling
static const QString splitCardPropSeparator = QString(" // ");
static const QString splitCardTextSeparator = QString("\n\n---\n\n");
QList<QPair<QList<SplitCardPart>, QString>> partsAndNames = splitCards.values();
for (auto [splitCardParts, name] : partsAndNames) {
QString text;
QHash<QString, QString> properties;
PrintingInfo printingInfo;
for (const SplitCardPart &tmp : splitCardParts) {
if (!text.isEmpty()) {
text.append(splitCardTextSeparator);
}
text.append(tmp.getText());
if (properties.isEmpty()) {
properties = tmp.getProperties();
printingInfo = tmp.getPrintingInfo();
} else {
const QHash<QString, QString> &tmpProps = tmp.getProperties();
for (auto i = tmpProps.cbegin(), end = tmpProps.cend(); i != end; ++i) {
QString prop = i.key();
QString originalPropertyValue = properties.value(prop);
QString thisCardPropertyValue = i.value();
if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) {
if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty
properties.insert(prop, thisCardPropertyValue);
} else if (prop == "colors" || prop == "coloridentity") { // the card is both colors
properties.insert(prop, originalPropertyValue + thisCardPropertyValue);
} else if (prop == "maintype") { // don't create maintypes with //es in them
continue;
} else {
properties.insert(prop,
originalPropertyValue + splitCardPropSeparator + thisCardPropertyValue);
}
}
}
}
}
CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo);
numCards++;
}
return numCards;
}
static FormatRulesNameMap buildDefaultMagicFormats()
{
// Predefined common exceptions
CardCondition superTypeIsBasic;
superTypeIsBasic.field = "type";
superTypeIsBasic.matchType = "regex";
superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)";
ExceptionRule basicLands;
basicLands.conditions.append(superTypeIsBasic);
CardCondition anyNumberAllowed;
anyNumberAllowed.field = "text";
anyNumberAllowed.matchType = "contains";
anyNumberAllowed.value = "A deck can have any number of";
ExceptionRule mayContainAnyNumber;
mayContainAnyNumber.conditions.append(anyNumberAllowed);
FormatRulesNameMap defaultFormatRulesNameMap;
// ----------------- Helper lambda to create format -----------------
auto makeFormat = [&](const QString &name, int minDeck = 60, int maxDeck = -1, int maxSideboardSize = 15,
const QList<AllowedCount> &allowedCounts = kConstructedCounts) -> FormatRulesPtr {
FormatRulesPtr f(new FormatRules);
f->formatName = name;
f->allowedCounts = allowedCounts;
f->minDeckSize = minDeck;
f->maxDeckSize = maxDeck;
f->maxSideboardSize = maxSideboardSize;
f->exceptions.append(basicLands);
f->exceptions.append(mayContainAnyNumber);
defaultFormatRulesNameMap.insert(name.toLower(), f);
return f;
};
// ----------------- Standard formats -----------------
makeFormat("Standard");
makeFormat("Modern");
makeFormat("Legacy");
makeFormat("Pioneer");
makeFormat("Historic");
makeFormat("Timeless");
makeFormat("Future");
makeFormat("OldSchool");
makeFormat("Premodern");
makeFormat("Pauper");
makeFormat("Penny");
// ----------------- Singleton formats -----------------
makeFormat("Commander", 100, 100, 15, kSingletonCounts);
makeFormat("Duel", 100, 100, 15, kSingletonCounts);
makeFormat("Brawl", 60, 60, 15, kSingletonCounts);
makeFormat("StandardBrawl", 60, 60, 15, kSingletonCounts);
makeFormat("Oathbreaker", 60, 60, 15, kSingletonCounts);
makeFormat("PauperCommander", 100, 100, 15, kSingletonCounts);
makeFormat("Predh", 100, 100, 15, kSingletonCounts);
// ----------------- Restricted formats -----------------
makeFormat("Vintage", 60, -1, 15, {{4, "legal"}, {1, "restricted"}, {0, "banned"}});
return defaultFormatRulesNameMap;
}
const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats()
{
static const FormatRulesNameMap cached = buildDefaultMagicFormats();
return cached;
}
int OracleImporter::startImport()
{
static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController();
// Pre-allocate the cards hash to avoid rehashing during import. The hash
// is keyed by distinct card name rather than by printings: AllPrintings
// ships ~100k printings for ~35k names, so reserving the printing count
// would overallocate ~3x (against this stack's RAM goal). Collecting
// distinct names is cheap — one pass over the already-parsed name fields.
{
QSet<QString> distinctNames;
for (const SetToDownload &curSetToParse : allSets) {
for (const QJsonValue &cardValue : curSetToParse.getCards()) {
distinctNames.insert(cardValue.toObject().value("name").toString());
}
}
cards.reserve(distinctNames.size());
// The set goes out of scope here, handing the ~35k name QStrings back
// to the allocator before the (memory-heavy) import loop starts.
}
// add an empty set for tokens
CardSetPtr tokenSet =
CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens");
sets.insert(CardSet::TOKENS_SETNAME, tokenSet);
int setIndex = 0;
for (const SetToDownload &curSetToParse : allSets) {
CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(),
curSetToParse.getLongName(), curSetToParse.getSetType(),
curSetToParse.getReleaseDate(), curSetToParse.getPriority());
if (!sets.contains(newSet->getShortName())) {
sets.insert(newSet->getShortName(), newSet);
}
int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards());
++setIndex;
emit setIndexChanged(numCardsInSet, setIndex, curSetToParse.getLongName());
}
emit setIndexChanged(0, setIndex, QString());
// total number of sets
return setIndex;
}
bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion)
{
CockatriceXml4Parser parser(new NoopCardPreferenceProvider(), new NoopCardSetPriorityController());
return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion);
}
void OracleImporter::releaseSetData()
{
allSets.clear();
}
void OracleImporter::clear()
{
sets.clear();
cards.clear();
allSets.clear();
}