jeirmeister/Cockatrice
1
0
Fork 0
mirror of https://github.com/Cockatrice/Cockatrice.git synced 2026-09-24 10:23:02 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'master' into tooomm_ninja-win

Browse source
This commit is contained in:
tooomm 2026-09-20 12:46:35 +02:00 • committed by GitHub
commit 0467031c6e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
810 changed files with 52151 additions and 9604 deletions
Download patch file Download diff file Expand all files Collapse all files

58
oracle/CMakeLists.txt
View file

@ -23,11 +23,12 @@ set(oracle_SOURCES
src/pages.cpp
src/pagetemplates.cpp
src/parsehelpers.cpp
src/qt-json/json.cpp
src/raw_json_scanner.cpp
../cockatrice/src/client/settings/cache_settings.cpp
../cockatrice/src/client/settings/card_counter_settings.cpp
../cockatrice/src/client/settings/shortcuts_settings.cpp
../cockatrice/src/client/network/update/client/release_channel.cpp
../cockatrice/src/interface/pixel_map_generator.cpp
../cockatrice/src/interface/theme_config.cpp
../cockatrice/src/interface/theme_manager.cpp
../cockatrice/src/interface/widgets/quick_settings/settings_button_widget.cpp
@ -64,11 +65,7 @@ set(oracle_RESOURCES oracle.qrc)
# ------------------------
# Qt resources
# ------------------------
if(Qt6_FOUND)
qt6_add_resources(oracle_RESOURCES_RCC ${oracle_RESOURCES})
elseif(Qt5_FOUND)
qt5_add_resources(oracle_RESOURCES_RCC ${oracle_RESOURCES})
endif()
qt6_add_resources(oracle_RESOURCES_RCC ${oracle_RESOURCES})
# ------------------------
# Include directories
@ -106,37 +103,18 @@ set(ORACLE_MAC_QM_INSTALL_DIR "oracle.app/Contents/Resources/translations")
set(ORACLE_UNIX_QM_INSTALL_DIR "share/oracle/translations")
set(ORACLE_WIN32_QM_INSTALL_DIR "translations")
if(Qt6_FOUND)
# Qt6 Translations are linked after the executable is created in manual mode
qt6_add_executable(
oracle
WIN32
MACOSX_BUNDLE
${oracle_SOURCES}
${oracle_RESOURCES_RCC}
${oracle_MOC_SRCS}
MANUAL_FINALIZATION
)
elseif(Qt5_FOUND)
# Qt5 Translations need to be linked at executable creation time
if(Qt5LinguistTools_FOUND)
if(UPDATE_TRANSLATIONS)
qt5_create_translation(oracle_QM ${translate_SRCS} ${oracle_TS})
else()
qt5_add_translation(oracle_QM ${oracle_TS})
endif()
endif()
add_executable(oracle WIN32 MACOSX_BUNDLE ${oracle_MOC_SRCS} ${oracle_QM} ${oracle_RESOURCES_RCC} ${oracle_SOURCES})
if(UNIX)
if(APPLE)
install(FILES ${oracle_QM} DESTINATION ${ORACLE_MAC_QM_INSTALL_DIR})
else()
install(FILES ${oracle_QM} DESTINATION ${ORACLE_UNIX_QM_INSTALL_DIR})
endif()
elseif(WIN32)
install(FILES ${oracle_QM} DESTINATION ${ORACLE_WIN32_QM_INSTALL_DIR})
endif()
endif()
# Qt6 Translations are linked after the executable is created in manual mode
qt6_add_executable(
oracle
WIN32
MACOSX_BUNDLE
${oracle_SOURCES}
${oracle_RESOURCES_RCC}
${oracle_MOC_SRCS}
MANUAL_FINALIZATION
)
target_precompile_headers(oracle PRIVATE "${CMAKE_SOURCE_DIR}/cmake/pch/qtwidgets_pch.h")
# ------------------------
# Link libraries
@ -286,7 +264,7 @@ endif()
# ------------------------
# Qt translations
# ------------------------
if(Qt6_FOUND AND Qt6LinguistTools_FOUND)
if(Qt6LinguistTools_FOUND)
#Qt6 Translations happen after the executable is built up
if(UPDATE_TRANSLATIONS)
qt6_add_translations(
@ -313,6 +291,4 @@ if(Qt6_FOUND AND Qt6LinguistTools_FOUND)
endif()
endif()
if(Qt6_FOUND)
qt6_finalize_target(oracle)
endif()
qt6_finalize_target(oracle)

13
oracle/src/main.cpp
View file

@ -10,6 +10,7 @@
#include <QLibraryInfo>
#include <QTimer>
#include <QTranslator>
#include <libcockatrice/settings/personal_settings.h>
QTranslator *translator, *qtTranslator;
ThemeManager *themeManager;
@ -21,14 +22,10 @@ bool isBackgrounded;
void installNewTranslator()
{
QString lang = SettingsCache::instance().getLang();
QString lang = SettingsCache::instance().personal().getLang();
QString qtNameHint = "qt_" + lang;
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
QString qtTranslationPath = QLibraryInfo::path(QLibraryInfo::TranslationsPath);
#else
QString qtTranslationPath = QLibraryInfo::location(QLibraryInfo::TranslationsPath);
#endif
bool qtTranslationLoaded = qtTranslator->load(qtNameHint, qtTranslationPath);
if (!qtTranslationLoaded) {
@ -53,8 +50,8 @@ int main(int argc, char *argv[])
QApplication app(argc, argv);
QCoreApplication::setOrganizationName("Cockatrice");
QCoreApplication::setOrganizationDomain("cockatrice");
// this can't be changed, as it influences the default save path for cards.xml
QCoreApplication::setOrganizationDomain("Cockatrice");
// This can't be changed, as it influences the default save path for cards.xml
QCoreApplication::setApplicationName("Cockatrice");
// If the program is opened with the -s flag, it will only do spoilers. Otherwise it will do MTGJSON/Tokens
@ -86,7 +83,7 @@ int main(int argc, char *argv[])
QIcon icon("theme:appicon.svg");
wizard.setWindowIcon(icon);
// set name of the app desktop file; used by wayland to load the window icon
QGuiApplication::setDesktopFileName("oracle");
QGuiApplication::setDesktopFileName("Oracle");
wizard.show();

500
oracle/src/oracleimporter.cpp
View file

@ -3,12 +3,16 @@
#include "libcockatrice/interfaces/noop_card_preference_provider.h"
#include "libcockatrice/interfaces/noop_card_set_priority_controller.h"
#include "parsehelpers.h"
#include "qt-json/json.h"
#include <QDebug>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonParseError>
#include <QRegularExpression>
#include <QSet>
#include <algorithm>
#include <climits>
#include <libcockatrice/card/card_localization.h>
#include <libcockatrice/card/database/parser/cockatrice_xml_4.h>
#include <libcockatrice/card/relation/card_relation.h>
@ -18,9 +22,10 @@ static const QList<AllowedCount> kSingletonCounts = {{1, "legal"}, {0, "banned"}
SplitCardPart::SplitCardPart(const QString &_name,
const QString &_text,
const QVariantHash &_properties,
const PrintingInfo &_printingInfo)
: name(_name), text(_text), properties(_properties), printingInfo(_printingInfo)
const QHash<QString, QString> &_properties,
const PrintingInfo &_printingInfo,
const QString &_localizedText)
: name(_name), text(_text), localizedText(_localizedText), properties(_properties), printingInfo(_printingInfo)
{
}
@ -30,6 +35,42 @@ OracleImporter::OracleImporter(QObject *parent) : QObject(parent)
{
}
void OracleImporter::setCardLang(const QString &lang)
{
cardLang = lang.trimmed().toLower();
localizationEnabled = cardLang != "en" && CardLocalization::supportedLanguages().contains(cardLang);
}
/**
* @brief Maps the MTGJSON foreignData language names to the short codes used by
* Scryfall and stored in cards.xml (e.g. "German" -> "de").
* @param language The language name found in the MTGJSON foreignData entries.
* @return The short language code, or an empty string if unknown.
*/
static QString mtgjsonLanguageToCode(const QString &language)
{
static const QHash<QString, QString> map = {
{"Chinese Simplified", "zhs"},
{"Chinese Traditional", "zht"},
{"English", "en"},
{"French", "fr"},
{"German", "de"},
{"Greek", "grc"},
{"Ancient Greek", "grc"},
{"Hebrew", "he"},
{"Italian", "it"},
{"Japanese", "ja"},
{"Korean", "ko"},
{"Latin", "la"},
{"Phyrexian", "ph"},
{"Portuguese (Brazil)", "pt"},
{"Russian", "ru"},
{"Sanskrit", "sa"},
{"Spanish", "es"},
};
return map.value(language);
}
static CardSet::Priority getSetPriority(const QString &setType, const QString &shortName)
{
if (!setTypePriorities.contains(setType.toLower())) {
@ -42,31 +83,37 @@ static CardSet::Priority getSetPriority(const QString &setType, const QString &s
return priority;
}
bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
bool OracleImporter::readSetsFromByteArray(QByteArray data)
{
bool ok;
auto setsMap = QtJson::Json::parse(QString(data), ok).toMap().value("data").toMap();
if (!ok) {
qDebug() << "error: QtJson::Json::parse()";
const RawJson::ScanProgressCallback progress =
progressReporting
? [this](
qsizetype bytesRead,
qsizetype
totalBytes) { emit dataReadProgress(static_cast<int>(bytesRead), static_cast<int>(totalBytes)); }
: RawJson::ScanProgressCallback{};
RawJson::ScanError error;
const QList<RawJson::SetRange> ranges = RawJson::scanSetRanges(data, &error, progress);
if (error.isError()) {
qDebug() << "error: RawJson::scanSetRanges():" << error.message;
return false;
}
QList<SetToDownload> newSetList;
newSetList.reserve(ranges.size());
QListIterator it(setsMap.values());
while (it.hasNext()) {
QVariantMap map = it.next().toMap();
QString shortName = map.value("code").toString().toUpper();
QString longName = map.value("name").toString();
QList<QVariant> setCards = map.value("cards").toList();
QString setType = map.value("type").toString();
QDate releaseDate = map.value("releaseDate").toDate();
for (const RawJson::SetRange &range : ranges) {
QString shortName = range.code.toUpper();
QString longName = range.name;
QString setType = range.type;
QDate releaseDate = QDate::fromString(range.releaseDate, Qt::ISODate);
CardSet::Priority priority = getSetPriority(setType, shortName);
// capitalize set type
if (setType.length() > 0) {
// basic grammar for words that aren't capitalized, like in "From the Vault"
const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for", "of", "in", "and", "with", "or"};
static const QStringList noCapitalize = {"the", "a", "an", "on", "to", "for",
"of", "in", "and", "with", "or"};
QStringList words = setType.split("_");
setType.clear();
bool first = false;
@ -74,13 +121,15 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
if (first && noCapitalize.contains(item)) {
setType += item + QString(" ");
} else {
setType += item[0].toUpper() + item.mid(1, -1) + QString(" ");
setType += item[0].toUpper() + item.mid(1) + QString(" ");
first = true;
}
}
setType = setType.trimmed();
}
newSetList.append(SetToDownload(shortName, longName, setCards, priority, setType, releaseDate));
SetToDownload set(shortName, longName, priority, setType, releaseDate);
set.setRawRange(range.dataRange);
newSetList.append(set);
}
std::sort(newSetList.begin(), newSetList.end());
@ -89,19 +138,34 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
return false;
}
allSets = newSetList;
rawSetsData = std::move(data);
return true;
}
/**
* The priority order used to pick a card's main type when a card has multiple
* types (e.g. "Artifact Creature") or multiple faces (e.g. split/adventure cards).
* A lower index means a higher priority.
*/
static const QStringList MAIN_CARD_TYPE_PRIORITY = {"Planeswalker", "Creature", "Land", "Sorcery",
"Instant", "Artifact", "Enchantment"};
/**
* Returns the priority (index) of the given main card type. Known types map to their
* position in {@link mainCardTypePriority()}, unknown types map to -1 (lowest priority).
*/
static int mainCardTypePriority(const QString &mainCardType)
{
return MAIN_CARD_TYPE_PRIORITY.indexOf(mainCardType);
}
static QString getMainCardType(const QStringList &typeList)
{
if (typeList.isEmpty()) {
return {};
}
static const QStringList typePriority = {"Planeswalker", "Creature", "Land", "Sorcery",
"Instant", "Artifact", "Enchantment"};
for (const auto &type : typePriority) {
for (const auto &type : MAIN_CARD_TYPE_PRIORITY) {
if (typeList.contains(type)) {
return type;
}
@ -122,29 +186,26 @@ static void sortAndReduceColors(QString &colors)
std::sort(colors.begin(), colors.end(),
[](const QChar a, const QChar b) { return colorOrder.value(a, INT_MAX) < colorOrder.value(b, INT_MAX); });
// reduce
QChar lastChar = '\0';
for (int i = 0; i < colors.size(); ++i) {
if (colors.at(i) == lastChar) {
colors.remove(i, 1);
} else {
lastChar = colors.at(i);
}
}
auto last = std::unique(colors.begin(), colors.end());
colors.erase(last, colors.end());
}
CardInfoPtr OracleImporter::addCard(QString name,
const QString &text,
bool isToken,
QVariantHash properties,
QHash<QString, QString> properties,
const QList<CardRelation *> &relatedCards,
const PrintingInfo &printingInfo)
{
// Workaround for card name weirdness
name = name.replace("Æ", "AE");
name = name.replace("’", "'");
if (cards.contains(name)) {
CardInfoPtr card = cards.value(name);
auto existingIt = cards.constFind(name);
if (existingIt != cards.constEnd()) {
CardInfoPtr card = existingIt.value();
card->addToSet(printingInfo.getSet(), printingInfo);
// Only merge legalities when the card has none yet, so multi-format
// printings don't overwrite each other's legality lists.
if (card->getProperties().filter(formatRegex).empty()) {
card->combineLegalities(properties);
}
@ -152,7 +213,7 @@ CardInfoPtr OracleImporter::addCard(QString name,
}
// Remove {} around mana costs, except if it's split cost
QString manacost = properties.value("manacost").toString();
QString manacost = properties.value("manacost");
if (!manacost.isEmpty()) {
QStringList symbols = manacost.split("}");
QString formattedCardCost;
@ -169,12 +230,12 @@ CardInfoPtr OracleImporter::addCard(QString name,
}
// fix colors
QString allColors = properties.value("colors").toString();
QString allColors = properties.value("colors");
if (allColors.size() > 1) {
sortAndReduceColors(allColors);
properties.insert("colors", allColors);
}
QString allColorIdent = properties.value("coloridentity").toString();
QString allColorIdent = properties.value("coloridentity");
if (allColorIdent.size() > 1) {
sortAndReduceColors(allColorIdent);
properties.insert("coloridentity", allColorIdent);
@ -182,16 +243,16 @@ CardInfoPtr OracleImporter::addCard(QString name,
// DETECT CARD POSITIONING INFO
bool landscapeOrientation = properties.value("maintype").toString() == "Battle" ||
properties.value("layout").toString() == "split" ||
properties.value("layout").toString() == "planar";
QString layoutVal = properties.value("layout");
bool landscapeOrientation =
properties.value("maintype") == "Battle" || layoutVal == "split" || layoutVal == "planar";
// cards that enter the field tapped
bool cipt = parseCipt(name, text) || landscapeOrientation;
// table row
int tableRow = 1;
QString mainCardType = properties.value("maintype").toString();
QString mainCardType = properties.value("maintype");
if (mainCardType == "Land") {
tableRow = 0;
} else if (mainCardType == "Sorcery" || mainCardType == "Instant") {
@ -201,11 +262,11 @@ CardInfoPtr OracleImporter::addCard(QString name,
}
// card side
QString side = properties.value("side").toString() == "b" ? "back" : "front";
QString side = properties.value("side") == "b" ? "back" : "front";
properties.insert("side", side);
// upsideDown (flip cards)
QString layout = properties.value("layout").toString();
QString layout = properties.value("layout");
bool upsideDown = layout == "flip" && side == "back";
// insert the card and its properties
@ -223,12 +284,110 @@ CardInfoPtr OracleImporter::addCard(QString name,
return newCard;
}
static QString getStringPropertyFromMap(const QVariantMap &card, const QString &propertyName)
static QString getJsonString(const QJsonObject &obj, const QString &key)
{
return card.contains(propertyName) ? card.value(propertyName).toString() : QString("");
// QVariant coerces numbers and booleans to text, while QJsonValue::toString()
// returns a null string for them — some MTGJSON fields (manaValue,
// convertedManaCost, isOnlineOnly, isRebalanced) carry those types.
return obj.value(key).toVariant().toString();
}
int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList<QVariant> &cardsList)
static QString normalizeCardName(QString name)
{
// Mirror of the name cleanup applied in addCard(), so collected localization
// keys line up with the card map keys (Æ → AE, curly apostrophe → straight).
name = name.replace("Æ", "AE");
name = name.replace("’", "'");
return name;
}
static QString matchingForeignEntryText(const QJsonObject &card, const QString &cardLang)
{
// Multi-face cards (split/aftermath/adventure/prepare) expose each face as a
// separate card object, each with its own foreignData entry carrying that
// face's rules text; single-face cards carry the full text in one entry.
const QJsonArray foreignData = card.value("foreignData").toArray();
for (const QJsonValue &entryValue : foreignData) {
const QJsonObject entry = entryValue.toObject();
// MTGJSON reports languages by long-form name ("German"); match the
// short code ("de") that Scryfall and cards.xml use.
if (mtgjsonLanguageToCode(getJsonString(entry, "language")) == cardLang) {
return getJsonString(entry, "text");
}
}
return QString();
}
void OracleImporter::collectForeignData(const QString &cardKey,
const CardSetPtr &currentSet,
const QJsonObject &card,
bool collectText)
{
if (!localizationEnabled) {
return;
}
LocalizedCardEntry incoming;
bool found = false;
const QJsonArray foreignData = card.value("foreignData").toArray();
for (const QJsonValue &entryValue : foreignData) {
const QJsonObject entry = entryValue.toObject();
// MTGJSON reports languages by long-form name ("German"); match the
// short code ("de") that Scryfall and cards.xml use.
if (mtgjsonLanguageToCode(getJsonString(entry, "language")) != cardLang) {
continue;
}
incoming.name = getJsonString(entry, "name");
incoming.text = collectText ? getJsonString(entry, "text") : QString();
found = true;
break;
}
if (!found) {
return;
}
// Prefer the entry from the highest-priority set (lower enum value = more
// authoritative); printings of equal priority keep the first one seen.
incoming.priority = currentSet->getPriority();
const auto existing = localizedEntries.constFind(cardKey);
if (existing == localizedEntries.constEnd() || incoming.priority < existing->priority) {
localizedEntries.insert(cardKey, incoming);
}
}
void OracleImporter::applyLocalizedData()
{
if (!localizationEnabled) {
return;
}
for (auto it = localizedEntries.constBegin(); it != localizedEntries.constEnd(); ++it) {
CardInfoPtr card = cards.value(it.key());
if (card.isNull()) {
continue;
}
const LocalizedCardEntry &entry = it.value();
if (!entry.name.isEmpty()) {
card->setLocalizedName(cardLang, entry.name);
}
if (!entry.text.isEmpty()) {
card->setLocalizedText(cardLang, entry.text);
}
}
localizedEntries.clear();
for (auto it = splitLocalizedTexts.constBegin(); it != splitLocalizedTexts.constEnd(); ++it) {
CardInfoPtr card = cards.value(it.key());
if (card.isNull()) {
continue;
}
if (!it.value().text.isEmpty()) {
card->setLocalizedText(cardLang, it.value().text);
}
}
splitLocalizedTexts.clear();
}
int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QJsonArray &cardsList)
{
// mtgjson name => xml name
static const QMap<QString, QString> cardProperties{
@ -238,15 +397,18 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
};
// mtgjson name => xml name
static const QMap<QString, QString> setInfoProperties{
{"number", "num"}, {"rarity", "rarity"}, {"isOnlineOnly", "isOnlineOnly"}, {"isRebalanced", "isRebalanced"}};
static const QMap<QString, QString> setInfoProperties{{"number", "num"},
{"rarity", "rarity"},
{"isOnlineOnly", "isOnlineOnly"},
{"isRebalanced", "isRebalanced"},
{"artist", "artist"}};
// mtgjson name => xml name
static const QMap<QString, QString> identifierProperties{{"multiverseId", "muid"}, {"scryfallId", "uuid"}};
static const QString ptSeparator = "/";
static constexpr bool isToken = false;
static const QList<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
static const QSet<QString> setsWithCardsWithSameNameButDifferentText = {"UST"};
int numCards = 0;
@ -254,16 +416,16 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
QMap<QString, QPair<QList<SplitCardPart>, QString>> splitCards;
// Keeps track of all names encountered so far
QList<QString> allNameProps;
QSet<QString> allNameProps;
for (const QVariant &cardVar : cardsList) {
QVariantMap card = cardVar.toMap();
for (const QJsonValue &cardVal : cardsList) {
QJsonObject card = cardVal.toObject();
/* Currently used layouts are:
* augment, double_faced_token, flip, host, leveler, meld, normal, planar,
* saga, scheme, split, token, transform, vanguard
*/
QString layout = getStringPropertyFromMap(card, "layout");
QString layout = getJsonString(card, "layout");
// don't import tokens from the json file
if (layout == "token") {
@ -271,52 +433,49 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
}
// normal cards handling
QString name = getStringPropertyFromMap(card, "name");
QString text = getStringPropertyFromMap(card, "text");
QString faceName = getStringPropertyFromMap(card, "faceName");
QString name = getJsonString(card, "name");
QString text = getJsonString(card, "text");
QString faceName = getJsonString(card, "faceName");
if (faceName.isEmpty()) {
faceName = name;
}
// card properties
QVariantHash properties;
QHash<QString, QString> properties;
for (auto i = cardProperties.cbegin(), end = cardProperties.cend(); i != end; ++i) {
QString mtgjsonProperty = i.key();
QString xmlPropertyName = i.value();
QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty);
QString propertyValue = getJsonString(card, i.key());
if (!propertyValue.isEmpty()) {
properties.insert(xmlPropertyName, propertyValue);
properties.insert(i.value(), propertyValue);
}
}
// per-set properties
PrintingInfo printingInfo = PrintingInfo(currentSet);
QHash<QString, QString> printingProps;
for (auto i = setInfoProperties.cbegin(), end = setInfoProperties.cend(); i != end; ++i) {
QString mtgjsonProperty = i.key();
QString xmlPropertyName = i.value();
QString propertyValue = getStringPropertyFromMap(card, mtgjsonProperty);
QString propertyValue = getJsonString(card, i.key());
if (!propertyValue.isEmpty()) {
printingInfo.setProperty(xmlPropertyName, propertyValue);
printingProps.insert(i.value(), propertyValue);
}
}
// handle flavorNames specially due to double-faced cards
QString faceFlavorName = getStringPropertyFromMap(card, "faceFlavorName");
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getStringPropertyFromMap(card, "flavorName");
QString faceFlavorName = getJsonString(card, "faceFlavorName");
QString flavorName = !faceFlavorName.isEmpty() ? faceFlavorName : getJsonString(card, "flavorName");
if (!flavorName.isEmpty()) {
printingInfo.setProperty("flavorName", flavorName);
printingProps.insert("flavorName", flavorName);
}
// Identifiers
QJsonObject identifiers = card.value("identifiers").toObject();
for (auto i = identifierProperties.cbegin(), end = identifierProperties.cend(); i != end; ++i) {
QString mtgjsonProperty = i.key();
QString xmlPropertyName = i.value();
QString propertyValue = getStringPropertyFromMap(card.value("identifiers").toMap(), mtgjsonProperty);
QString propertyValue = getJsonString(identifiers, i.key());
if (!propertyValue.isEmpty()) {
printingInfo.setProperty(xmlPropertyName, propertyValue);
printingProps.insert(i.value(), propertyValue);
}
}
PrintingInfo printingInfo(currentSet, LazyPropertiesHash(printingProps));
QString numComponent;
const QString numProperty = printingInfo.getProperty("num");
const QChar lastChar = numProperty.isEmpty() ? QChar() : numProperty.back();
@ -327,21 +486,26 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
allNameProps.contains(faceName) && layout == "normal" && lastChar.isLetter()) {
numComponent = " (" + QString(lastChar).toLower() + ")";
}
allNameProps.append(faceName);
allNameProps.insert(faceName);
// special handling properties
QString colors = card.value("colors").toStringList().join("");
QString colors;
for (const QJsonValue &color : card.value("colors").toArray()) {
colors += color.toString();
}
if (!colors.isEmpty()) {
properties.insert("colors", colors);
}
// special handling properties
QString colorIdentity = card.value("colorIdentity").toStringList().join("");
QString colorIdentity;
for (const QJsonValue &color : card.value("colorIdentity").toArray()) {
colorIdentity += color.toString();
}
if (!colorIdentity.isEmpty()) {
properties.insert("coloridentity", colorIdentity);
}
const auto &mainCardType = getMainCardType(card.value("types").toStringList());
const auto &mainCardType = getMainCardType(card.value("types").toVariant().toStringList());
if (mainCardType.isEmpty()) {
qDebug() << "warning: no mainCardType for card:" << name;
} else {
@ -350,39 +514,47 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
// Depending on whether power and/or toughness are present, the format
// is either P/T (most common), P (no toughness), or /T (no power).
QString power = getStringPropertyFromMap(card, "power");
QString toughness = getStringPropertyFromMap(card, "toughness");
QString power = getJsonString(card, "power");
QString toughness = getJsonString(card, "toughness");
if (toughness.isEmpty() && !power.isEmpty()) {
properties.insert("pt", power);
} else if (!toughness.isEmpty()) {
properties.insert("pt", power + ptSeparator + toughness);
}
auto legalities = card.value("legalities").toMap();
for (auto i = legalities.cbegin(), end = legalities.cend(); i != end; ++i) {
auto legalities = card.value("legalities").toObject();
for (auto i = legalities.constBegin(), end = legalities.constEnd(); i != end; ++i) {
properties.insert(QString("format-%1").arg(i.key()), i.value().toString().toLower());
}
// split cards are considered a single card, enqueue for later merging
if (layout == "split" || layout == "aftermath" || layout == "adventure" || layout == "prepare") {
auto _faceName = getStringPropertyFromMap(card, "faceName");
SplitCardPart split(_faceName, text, properties, printingInfo);
auto _faceName = getJsonString(card, "faceName");
// MTGJSON exposes each face as a separate card object, each with its
// own foreignData entry holding that face's rules text; collect it so
// the per-face texts can be joined the same way as the English text.
const QString faceLocalizedText =
localizationEnabled ? matchingForeignEntryText(card, cardLang) : QString();
SplitCardPart split(_faceName, text, properties, printingInfo, faceLocalizedText);
auto found_iter = splitCards.find(name + numProperty);
if (found_iter == splitCards.end()) {
splitCards.insert(name + numProperty, {{split}, name});
} else {
found_iter->first.append(split);
}
// MTGJSON's foreignData name is the joined name present on every
// face, so collect the name once.
collectForeignData(normalizeCardName(name), currentSet, card, false);
} else {
// relations
QList<CardRelation *> relatedCards;
// add other face for split cards as card relation
if (!getStringPropertyFromMap(card, "side").isEmpty()) {
auto faceManaValue = getStringPropertyFromMap(card, "faceManaValue");
if (!getJsonString(card, "side").isEmpty()) {
auto faceManaValue = getJsonString(card, "faceManaValue");
if (faceManaValue.isEmpty()) {
// check the old name for the property, for backwards compatibility purposes
faceManaValue = getStringPropertyFromMap(card, "faceConvertedManaCost");
faceManaValue = getJsonString(card, "faceConvertedManaCost");
}
properties["cmc"] = faceManaValue;
@ -402,20 +574,23 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
name = faceName;
}
// mtgjon related cards
if (card.contains("relatedCards")) {
QVariantMap givenRelated = card.value("relatedCards").toMap();
// mtgjson related cards
QJsonObject givenRelated = card.value("relatedCards").toObject();
if (!givenRelated.isEmpty()) {
// conjured cards from a spellbook
if (givenRelated.contains("spellbook")) {
auto spbk = givenRelated.value("spellbook").toStringList();
for (const QString &spbkName : spbk) {
relatedCards.append(
new CardRelation(spbkName, CardRelationType::DoesNotAttach, false, false, 1, true));
QJsonArray spellbook = givenRelated.value("spellbook").toArray();
if (!spellbook.isEmpty()) {
for (const QJsonValue &spbkVal : spellbook) {
relatedCards.append(new CardRelation(spbkVal.toString(), CardRelationType::DoesNotAttach, false,
false, 1, true));
}
}
}
CardInfoPtr newCard = addCard(name + numComponent, text, isToken, properties, relatedCards, printingInfo);
collectForeignData(normalizeCardName(name + numComponent), currentSet, card);
CardInfoPtr newCard =
addCard(name + numComponent, text, isToken, std::move(properties), relatedCards, printingInfo);
numCards++;
}
}
@ -423,12 +598,13 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
// split cards handling
static const QString splitCardPropSeparator = QString(" // ");
static const QString splitCardTextSeparator = QString("\n\n---\n\n");
static const QList<CardRelation *> noRelatedCards = {};
QList<QPair<QList<SplitCardPart>, QString>> partsAndNames = splitCards.values();
for (auto [splitCardParts, name] : partsAndNames) {
QString text;
QVariantHash properties;
QString localizedText;
bool localizedTextComplete = true;
QHash<QString, QString> properties;
PrintingInfo printingInfo;
for (const SplitCardPart &tmp : splitCardParts) {
@ -437,22 +613,44 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
}
text.append(tmp.getText());
// Build the cardLang text by joining each face's translated text with
// the same separator as the English text. Any face missing a complete
// translation abandons the whole join, falling back to the English text.
if (localizedTextComplete) {
const QString partLocalizedText = tmp.getLocalizedText();
if (partLocalizedText.isEmpty()) {
localizedTextComplete = false;
} else {
if (!localizedText.isEmpty()) {
localizedText.append(splitCardTextSeparator);
}
localizedText.append(partLocalizedText);
}
}
if (properties.isEmpty()) {
properties = tmp.getProperties();
printingInfo = tmp.getPrintingInfo();
} else {
const QVariantHash &tmpProps = tmp.getProperties();
const QHash<QString, QString> &tmpProps = tmp.getProperties();
for (auto i = tmpProps.cbegin(), end = tmpProps.cend(); i != end; ++i) {
QString prop = i.key();
QString originalPropertyValue = properties.value(prop).toString();
QString thisCardPropertyValue = i.value().toString();
QString originalPropertyValue = properties.value(prop);
QString thisCardPropertyValue = i.value();
if (!thisCardPropertyValue.isEmpty() && originalPropertyValue != thisCardPropertyValue) {
if (originalPropertyValue.isEmpty()) { // don't create //es if one field is empty
properties.insert(prop, thisCardPropertyValue);
} else if (prop == "colors") { // the card is both colors
} else if (prop == "colors" || prop == "coloridentity") { // the card is both colors
properties.insert(prop, originalPropertyValue + thisCardPropertyValue);
} else if (prop == "maintype") { // don't create maintypes with //es in them
continue;
} else if (prop == "maintype") {
// Use the same priority as getMainCardType() to pick the
// "best" type across faces — e.g. Creature over Instant
// for adventure cards like Bonecrusher Giant.
int currentPriority = mainCardTypePriority(originalPropertyValue);
int newPriority = mainCardTypePriority(thisCardPropertyValue);
if (newPriority >= 0 && (currentPriority < 0 || newPriority < currentPriority)) {
properties.insert(prop, thisCardPropertyValue);
}
} else {
properties.insert(prop,
originalPropertyValue + splitCardPropSeparator + thisCardPropertyValue);
@ -461,20 +659,32 @@ int OracleImporter::importCardsFromSet(const CardSetPtr &currentSet, const QList
}
}
}
CardInfoPtr newCard = addCard(name, text, isToken, properties, noRelatedCards, printingInfo);
CardInfoPtr newCard = addCard(name, text, isToken, std::move(properties), {}, printingInfo);
if (localizationEnabled && localizedTextComplete && !localizedText.isEmpty()) {
// Same priority policy as collectForeignData(): the joined text from the
// highest-priority set seen so far wins, applied once all printings are in.
LocalizedCardEntry entry;
entry.text = localizedText;
entry.priority = currentSet->getPriority();
const QString entryKey = normalizeCardName(name);
const auto existing = splitLocalizedTexts.constFind(entryKey);
if (existing == splitLocalizedTexts.constEnd() || entry.priority < existing->priority) {
splitLocalizedTexts.insert(entryKey, entry);
}
}
numCards++;
}
return numCards;
}
FormatRulesNameMap OracleImporter::createDefaultMagicFormats()
static FormatRulesNameMap buildDefaultMagicFormats()
{
// Predefined common exceptions
CardCondition superTypeIsBasic;
superTypeIsBasic.field = "type";
superTypeIsBasic.matchType = "regex";
superTypeIsBasic.value = "\bBasic\b[^—]+\bLand\b";
superTypeIsBasic.value = R"(\bBasic\b[^—]+\bLand\b)";
ExceptionRule basicLands;
basicLands.conditions.append(superTypeIsBasic);
@ -487,7 +697,6 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats()
ExceptionRule mayContainAnyNumber;
mayContainAnyNumber.conditions.append(anyNumberAllowed);
// Map to store default rules
FormatRulesNameMap defaultFormatRulesNameMap;
// ----------------- Helper lambda to create format -----------------
@ -533,10 +742,29 @@ FormatRulesNameMap OracleImporter::createDefaultMagicFormats()
return defaultFormatRulesNameMap;
}
const FormatRulesNameMap &OracleImporter::createDefaultMagicFormats()
{
static const FormatRulesNameMap cached = buildDefaultMagicFormats();
return cached;
}
int OracleImporter::startImport()
{
static ICardSetPriorityController *noOpController = new NoopCardSetPriorityController();
importCancelled.storeRelease(0);
// Pre-allocate the cards hash to avoid rehashing during import. Keys are
// distinct card names while raw ranges only count printings (AllPrintings
// ~100k printings vs ~35k names), so this over-reserves somewhat; an exact
// distinct-name count would require eagerly parsing, which the lazy reader
// deliberately avoids. It's a capacity hint, so the overshoot is harmless.
int estimatedCards = 0;
for (const SetToDownload &curSetToParse : allSets) {
estimatedCards += curSetToParse.getRawRange().cardCount;
}
cards.reserve(estimatedCards);
// add an empty set for tokens
CardSetPtr tokenSet =
CardSet::newInstance(noOpController, CardSet::TOKENS_SETNAME, tr("Dummy set containing tokens"), "Tokens");
@ -545,20 +773,61 @@ int OracleImporter::startImport()
int setIndex = 0;
for (const SetToDownload &curSetToParse : allSets) {
if (importCancelled.loadAcquire()) {
// The wizard was closed mid-import: stop at the next set boundary so
// the caller can wait for this future without processing every set.
break;
}
CardSetPtr newSet = CardSet::newInstance(noOpController, curSetToParse.getShortName(),
curSetToParse.getLongName(), curSetToParse.getSetType(),
curSetToParse.getReleaseDate(), curSetToParse.getPriority());
// parse only this set's slice of the raw document so the whole JSON tree is
// never kept in memory at once
const RawJson::SetDataRange &rawRange = curSetToParse.getRawRange();
const qsizetype rangeEnd = rawRange.start + rawRange.length;
if (rawRange.start < 0 || rawRange.length <= 0 || rangeEnd > rawSetsData.size()) {
// rawSetsData is cleared by releaseSetData() while SetToDownload copies
// taken from getSets() keep their ranges, and nothing else enforces the
// pairing — so never index past the buffer on stale/mismatched ranges.
qWarning() << "error: out-of-bounds raw range for set" << curSetToParse.getShortName() << "skipping";
++setIndex;
emit setIndexChanged(0, setIndex, curSetToParse.getLongName());
continue;
}
// sliced() shares the buffer instead of deep-copying the slice; the largest
// sets in AllPrintings are tens of MB, so the copy is worth avoiding here.
const QByteArray setBytes = rawSetsData.sliced(rawRange.start, rawRange.length);
QJsonParseError parseError;
const QJsonDocument setDoc = QJsonDocument::fromJson(setBytes, &parseError);
if (parseError.error != QJsonParseError::NoError) {
qWarning() << "error: parsing card data for set" << curSetToParse.getShortName() << ":"
<< parseError.errorString();
++setIndex;
// Keep the progress accounting honest: a set that failed to parse
// still advanced the index, so report it (with zero imported cards)
// rather than letting SaveSetsPage's bar stall per failed set.
emit setIndexChanged(0, setIndex, curSetToParse.getLongName());
continue;
}
// Only add the set to the database once its slice parsed cleanly;
// a set that fails here must not persist as an empty set in cards.xml.
if (!sets.contains(newSet->getShortName())) {
sets.insert(newSet->getShortName(), newSet);
}
int numCardsInSet = importCardsFromSet(newSet, curSetToParse.getCards());
const QJsonArray setCards = setDoc.object().value("cards").toArray();
int numCardsInSet = importCardsFromSet(newSet, setCards);
++setIndex;
emit setIndexChanged(numCardsInSet, setIndex, curSetToParse.getLongName());
}
applyLocalizedData();
emit setIndexChanged(0, setIndex, QString());
// total number of sets
@ -572,9 +841,18 @@ bool OracleImporter::saveToFile(const QString &fileName, const QString &sourceUr
return parser.saveToFile(createDefaultMagicFormats(), sets, cards, fileName, sourceUrl, sourceVersion);
}
void OracleImporter::releaseSetData()
{
allSets.clear();
rawSetsData.clear();
}
void OracleImporter::clear()
{
sets.clear();
cards.clear();
allSets.clear();
rawSetsData.clear();
localizedEntries.clear();
splitLocalizedTexts.clear();
}

171
oracle/src/oracleimporter.h
View file

@ -1,6 +1,12 @@
#ifndef ORACLEIMPORTER_H
#define ORACLEIMPORTER_H
#include "raw_json_scanner.h"
#include <QAtomicInt>
#include <QByteArray>
#include <QJsonArray>
#include <QJsonObject>
#include <QMap>
#include <QRegularExpression>
#include <QVariant>
@ -44,10 +50,12 @@ class SetToDownload
{
private:
QString shortName, longName;
QList<QVariant> cards;
QDate releaseDate;
QString setType;
CardSet::Priority priority;
// Byte range of this set's object within the importer's raw JSON text. Parsing
// one set at a time keeps peak memory low instead of holding the whole document.
RawJson::SetDataRange rawRange;
public:
const QString &getShortName() const
@ -58,10 +66,6 @@ public:
{
return longName;
}
const QList<QVariant> &getCards() const
{
return cards;
}
const QString &getSetType() const
{
return setType;
@ -74,16 +78,23 @@ public:
{
return priority;
}
const RawJson::SetDataRange &getRawRange() const
{
return rawRange;
}
SetToDownload(QString _shortName,
QString _longName,
QList<QVariant> _cards,
CardSet::Priority _priority,
QString _setType = QString(),
const QDate &_releaseDate = QDate())
: shortName(std::move(_shortName)), longName(std::move(_longName)), cards(std::move(_cards)),
releaseDate(_releaseDate), setType(std::move(_setType)), priority(_priority)
: shortName(std::move(_shortName)), longName(std::move(_longName)), releaseDate(_releaseDate),
setType(std::move(_setType)), priority(_priority)
{
}
void setRawRange(const RawJson::SetDataRange &_rawRange)
{
rawRange = _rawRange;
}
bool operator<(const SetToDownload &set) const
{
return longName.compare(set.longName, Qt::CaseInsensitive) < 0;
@ -95,8 +106,9 @@ class SplitCardPart
public:
SplitCardPart(const QString &_name,
const QString &_text,
const QVariantHash &_properties,
const PrintingInfo &_printingInfo);
const QHash<QString, QString> &_properties,
const PrintingInfo &_printingInfo,
const QString &_localizedText = QString());
inline const QString &getName() const
{
return name;
@ -105,7 +117,14 @@ public:
{
return text;
}
inline const QVariantHash &getProperties() const
/**
* @brief The cardLang rules text of this face's foreignData entry, if any.
*/
inline const QString &getLocalizedText() const
{
return localizedText;
}
inline const QHash<QString, QString> &getProperties() const
{
return properties;
}
@ -117,10 +136,18 @@ public:
private:
QString name;
QString text;
QVariantHash properties;
QString localizedText;
QHash<QString, QString> properties;
PrintingInfo printingInfo;
};
struct LocalizedCardEntry
{
QString name;
QString text;
CardSet::Priority priority = CardSet::PriorityLowest;
};
class OracleImporter : public QObject
{
Q_OBJECT
@ -139,23 +166,130 @@ private:
QList<SetToDownload> allSets;
/**
* The raw JSON text of the source document, retained for lazy per-set
* parsing during startImport(). Frees the card data as each set is imported.
*/
QByteArray rawSetsData;
/**
* Whether readSetsFromByteArray() should report scan progress via
* dataReadProgress. A background run routes that signal to stdout (for the
* hosting Cockatrice client to parse); the flag exists to skip the scanner
* instrumentation entirely when no consumer needs it.
*/
bool progressReporting = true;
/**
* Atomic "please stop importing" flag. startImport() checks it between sets
* so a wizard being closed mid-import can be torn down without waiting for
* the whole import (or racing it).
*/
QAtomicInt importCancelled;
/**
* The ISO-639 language code whose foreignData is imported; "en" by default.
*/
QString cardLang = "en";
/**
* Whether cardLang is a supported language other than English, so per-card
* foreignData scanning can be skipped entirely when disabled.
*/
bool localizationEnabled = false;
/**
* Localized name/text collected per imported card key while parsing sets,
* applied to the CardInfo objects by applyLocalizedData() once all
* printings have been seen so the best-priority one wins.
*/
QMap<QString, LocalizedCardEntry> localizedEntries;
/**
* cardLang rules text collected for split-card names while parsing sets,
* applied by applyLocalizedData(). Kept apart from localizedEntries because
* MTGJSON emits each split face as its own card object with the joined name
* on every foreignData entry: names and the per-face text join have different
* completeness and must not overwrite each other under the same key.
*/
QMap<QString, LocalizedCardEntry> splitLocalizedTexts;
CardInfoPtr addCard(QString name,
const QString &text,
bool isToken,
QVariantHash properties,
QHash<QString, QString> properties,
const QList<CardRelation *> &relatedCards,
const PrintingInfo &printingInfo);
/**
* Records the first foreignData entry matching cardLang for the given card
* key, keeping the entry from the highest-priority set seen so far.
*
* Multi-face cards (split, adventure, aftermath, prepare) pass collectText =
* false: MTGJSON emits one foreignData entry per face with the same joined
* name but only that face's text, so the name is collected here while the
* per-face texts are joined during the split-card merge.
*/
void collectForeignData(const QString &cardKey,
const CardSetPtr &currentSet,
const QJsonObject &card,
bool collectText = true);
signals:
void setIndexChanged(int cardsImported, int setIndex, const QString &setName);
void dataReadProgress(int bytesRead, int totalBytes);
public:
explicit OracleImporter(QObject *parent = nullptr);
bool readSetsFromByteArray(const QByteArray &data);
/**
* @brief Controls whether readSetsFromByteArray() instruments the raw scan.
*
* When enabled (the default) the raw scanner reports progress via
* dataReadProgress(), which an interactive wizard shows on its progress bar
* and a background run routes to stdout for the hosting client. Switch it
* off only when nothing will consume scan progress.
*/
void setProgressReporting(bool enabled)
{
progressReporting = enabled;
}
/**
* Selects the ISO-639 language code whose foreignData is imported.
* English (the default) and unsupported codes disable localization.
*/
void setCardLang(const QString &lang);
const QString &getCardLang() const
{
return cardLang;
}
/**
* Scans the given JSON document for set metadata. Takes the data by value so
* the wizard can hand over its decompressed buffer without copying it.
*/
bool readSetsFromByteArray(QByteArray data);
int startImport();
/**
* @brief Requests an in-flight startImport() to stop at the next set boundary.
*
* Works by setting an atomic flag that startImport() polls between sets, so
* cancelImport() followed by a short waitForFinished() on the running future is
* safe the moment the wizard is about to be destroyed.
*/
void cancelImport()
{
importCancelled.storeRelease(1);
}
/**
* Applies the collected localized names/texts to the imported cards.
* Called automatically at the end of startImport(); exposed separately so
* tests can drive it after importing sets directly.
*/
void applyLocalizedData();
bool saveToFile(const QString &fileName, const QString &sourceUrl, const QString &sourceVersion);
int importCardsFromSet(const CardSetPtr &currentSet, const QList<QVariant> &cardsList);
FormatRulesNameMap createDefaultMagicFormats();
int importCardsFromSet(const CardSetPtr &currentSet, const QJsonArray &cardsList);
/**
* @brief Returns the default format rules. The result is memoized on first use and must be treated as immutable.
*/
const FormatRulesNameMap &createDefaultMagicFormats();
const CardNameMap &getCardList() const
{
return cards;
@ -164,6 +298,11 @@ public:
{
return allSets;
}
const QByteArray &getRawSetsData() const
{
return rawSetsData;
}
void releaseSetData();
void clear();
};

25
oracle/src/oraclewizard.cpp
View file

@ -15,6 +15,8 @@
#include <QScrollBar>
#include <QtConcurrent>
#include <QtGui>
#include <libcockatrice/settings/cards_display_settings.h>
#include <libcockatrice/settings/personal_settings.h>
OracleWizard::OracleWizard(QWidget *parent) : QWizard(parent)
{
@ -33,9 +35,12 @@ OracleWizard::OracleWizard(QWidget *parent) : QWizard(parent)
migrateOracleSettings();
}
connect(&SettingsCache::instance(), &SettingsCache::langChanged, this, &OracleWizard::updateLanguage);
connect(&SettingsCache::instance().personal(), &PersonalSettings::langChanged, this, &OracleWizard::updateLanguage);
importer = new OracleImporter(this);
// Import card text in the language the client displays, if supported:
// foreignData for any other language is never imported.
importer->setCardLang(SettingsCache::instance().cardsDisplay().getCardLang());
nam = new QNetworkAccessManager(this);
@ -109,6 +114,24 @@ void OracleWizard::accept()
QDialog::accept();
}
void OracleWizard::reject()
{
// The wizard is being closed while a page may still run a worker on the
// importer. Ask it to stop before the wizard (and the importer child) is
// destroyed, so the worker thread never touches freed memory.
if (auto *active = dynamic_cast<OracleWizardPage *>(currentPage())) {
active->cancelWork();
}
QWizard::reject();
}
void OracleWizard::runInBackground()
{
backgroundMode = true;
hide();
currentPage()->initializePage();
}
void OracleWizard::enableButtons()
{
button(QWizard::NextButton)->setDisabled(false);

8
oracle/src/oraclewizard.h
View file

@ -23,6 +23,7 @@ class OracleWizard : public QWizard
public:
explicit OracleWizard(QWidget *parent = nullptr);
void accept() override;
void reject() override;
void enableButtons();
void disableButtons();
void retranslateUi();
@ -52,12 +53,7 @@ public:
}
bool saveTokensToFile(const QString &fileName);
void runInBackground()
{
backgroundMode = true;
hide();
currentPage()->initializePage();
}
void runInBackground();
public:
OracleImporter *importer;

484
oracle/src/pages.cpp
View file

@ -12,6 +12,7 @@
#include <QBuffer>
#include <QCheckBox>
#include <QComboBox>
#include <QDebug>
#include <QDir>
#include <QFileDialog>
#include <QGridLayout>
@ -20,14 +21,18 @@
#include <QMessageBox>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QPointer>
#include <QProgressBar>
#include <QPushButton>
#include <QRadioButton>
#include <QScrollBar>
#include <QStandardPaths>
#include <QTextEdit>
#include <QTextStream>
#include <QtConcurrent>
#include <QtGui>
#include <cstdio>
#include <libcockatrice/settings/personal_settings.h>
#ifdef HAS_LZMA
#include "lzma/decompress.h"
@ -52,6 +57,115 @@
#define ALLSETS_URL "https://www.mtgjson.com/api/v5/AllPrintings.json"
#endif
/**
* @brief Emits one machine-readable background-run progress line to stdout.
*
* Used only in background mode, so the hosting Cockatrice client can parse these
* lines to drive a determinate progress bar. stderr stays reserved for
* human-readable log output.
*/
static void emitBackgroundProgress(const char *stage, qint64 done, qint64 total)
{
QTextStream out(stdout);
out << "PROGRESS " << stage << ' ' << done << ' ' << total << '\n';
out.flush();
}
namespace
{
/**
* @brief Decompresses and dispatches a sets-file payload on a worker thread.
*
* Iteratively unwraps xz/zip compression, then either hands the JSON to the
* importer (which reports scan progress via dataReadProgress) or returns the raw
* XML for the plain-XML path. Must never touch the wizard or the page: the caller
* consumes the returned LoadSetsResult on the UI thread in importFinished().
*/
LoadSetsResult loadSetsData(const QPointer<OracleImporter> &importer, QByteArray data)
{
LoadSetsResult result;
while (true) {
if (data.startsWith(XZ_SIGNATURE)) {
#ifdef HAS_LZMA
QBuffer inBuffer(&data);
QByteArray decompressed;
QBuffer outBuffer(&decompressed);
inBuffer.open(QBuffer::ReadOnly);
outBuffer.open(QBuffer::WriteOnly);
XzDecompressor xz;
if (!xz.decompress(&inBuffer, &outBuffer)) {
result.errorMessage = LoadSetsPage::tr("Xz extraction failed.");
result.offerUncompressedFallback = true;
return result;
}
data = decompressed;
continue;
#else
result.errorMessage =
LoadSetsPage::tr("Sorry, this version of Oracle does not support xz compressed files.");
result.offerUncompressedFallback = true;
return result;
#endif
}
if (data.startsWith(ZIP_SIGNATURE)) {
#ifdef HAS_ZLIB
QBuffer inBuffer(&data);
UnZip uz;
const UnZip::ErrorCode openEc = uz.openArchive(&inBuffer);
if (openEc != UnZip::Ok) {
result.errorMessage = LoadSetsPage::tr("Failed to open Zip archive: %1.").arg(uz.formatError(openEc));
result.offerUncompressedFallback = true;
return result;
}
if (uz.fileList().size() != 1) {
result.errorMessage =
LoadSetsPage::tr("Zip extraction failed: the Zip archive doesn't contain exactly one file.");
result.offerUncompressedFallback = true;
return result;
}
const QString fileName = uz.fileList().at(0);
QByteArray decompressed;
QBuffer outBuffer(&decompressed);
outBuffer.open(QBuffer::ReadWrite);
const UnZip::ErrorCode ec = uz.extractFile(fileName, &outBuffer);
uz.closeArchive();
if (ec != UnZip::Ok) {
result.errorMessage = LoadSetsPage::tr("Zip extraction failed: %1.").arg(uz.formatError(ec));
result.offerUncompressedFallback = true;
return result;
}
data = decompressed;
continue;
#else
result.errorMessage = LoadSetsPage::tr("Sorry, this version of Oracle does not support zipped files.");
result.offerUncompressedFallback = true;
return result;
#endif
}
break;
}
if (data.startsWith("<")) {
result.ok = true;
result.plainXml = true;
result.xmlData = std::move(data);
return result;
}
if (data.startsWith("{")) {
result.ok = importer && importer->readSetsFromByteArray(std::move(data));
return result;
}
result.errorMessage = LoadSetsPage::tr("Failed to interpret downloaded data.");
return result;
}
} // namespace
#define TOKENS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Token/master/tokens.xml"
#define SPOILERS_URL "https://raw.githubusercontent.com/Cockatrice/Magic-Spoiler/files/spoiler.xml"
@ -119,7 +233,7 @@ QString IntroPage::languageName(const QString &lang)
void IntroPage::languageBoxChanged(int index)
{
SettingsCache::instance().setLang(languageBox->itemData(index).toString());
SettingsCache::instance().personal().setLang(languageBox->itemData(index).toString());
}
void IntroPage::retranslateUi()
@ -181,6 +295,11 @@ LoadSetsPage::LoadSetsPage(QWidget *parent) : OracleWizardPage(parent)
setLayout(layout);
}
bool LoadSetsPage::isComplete() const
{
return !loadActive;
}
void LoadSetsPage::initializePage()
{
urlLineEdit->setText(wizard()->settings->value("allsetsurl", ALLSETS_URL).toString());
@ -259,7 +378,7 @@ bool LoadSetsPage::validatePage()
return false;
}
progressLabel->setText(tr("Downloading (0MB)"));
progressLabel->setText(tr("Downloading (0 MB)"));
// show an infinite progressbar
progressBar->setMaximum(0);
progressBar->setMinimum(0);
@ -278,18 +397,13 @@ bool LoadSetsPage::validatePage()
return false;
}
if (!setsFile.open(QIODevice::ReadOnly)) {
QMessageBox::critical(nullptr, tr("Error"), tr("Cannot open file '%1'.").arg(fileLineEdit->text()));
return false;
}
wizard()->disableButtons();
setEnabled(false);
wizard()->setCardSourceUrl(setsFile.fileName());
wizard()->setCardSourceVersion("unknown");
readSetsFromByteArray(setsFile.readAll());
readSetsFromFile(setsFile.fileName());
}
return false;
@ -338,11 +452,14 @@ void LoadSetsPage::downloadSetsFile(const QUrl &url)
void LoadSetsPage::actDownloadProgressSetsFile(qint64 received, qint64 total)
{
if (wizard()->backgroundMode) {
emitBackgroundProgress("download", received, total);
}
if (total > 0) {
progressBar->setMaximum(static_cast<int>(total));
progressBar->setValue(static_cast<int>(received));
}
progressLabel->setText(tr("Downloading (%1MB)").arg((int)received / (1024 * 1024)));
progressLabel->setText(tr("Downloading (%1 MB)").arg((int)received / (1024 * 1024)));
}
void LoadSetsPage::actDownloadFinishedSetsFile()
@ -383,106 +500,97 @@ void LoadSetsPage::actDownloadFinishedSetsFile()
reply->deleteLater();
}
void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
void LoadSetsPage::updateParsingProgress(int bytesRead, int totalBytes)
{
// show an infinite progressbar
if (totalBytes <= 0) {
return;
}
progressBar->setRange(0, totalBytes);
progressBar->setValue(bytesRead);
const int percent = static_cast<int>((100.0 * bytesRead) / totalBytes);
progressLabel->setText(tr("Parsing file (%1%)").arg(percent));
}
void LoadSetsPage::scanProgressToStdout(int bytesRead, int totalBytes)
{
emitBackgroundProgress("scan", bytesRead, totalBytes);
}
void LoadSetsPage::beginLoadSets(bool compressedFile)
{
// Show an infinite progressbar while the worker decompresses; the scan
// steals the label via dataReadProgress as soon as it starts.
progressBar->setMaximum(0);
progressBar->setMinimum(0);
progressBar->setValue(0);
progressLabel->setText(tr("Parsing file"));
progressLabel->setText(compressedFile ? tr("Extracting file...") : tr("Parsing file"));
progressLabel->show();
progressBar->show();
// Keep Next disabled (via completeChanged) until the worker reports in;
// updateButtonStates() re-evaluates button state whenever we re-enable.
loadActive = true;
emit completeChanged();
wizard()->downloadedPlainXml = false;
wizard()->xmlData.clear();
readSetsFromByteArrayRef(_data);
if (wizard()->backgroundMode) {
connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::scanProgressToStdout,
Qt::UniqueConnection);
} else {
connect(wizard()->importer, &OracleImporter::dataReadProgress, this, &LoadSetsPage::updateParsingProgress,
Qt::UniqueConnection);
}
}
void LoadSetsPage::readSetsFromByteArrayRef(QByteArray &_data)
void LoadSetsPage::readSetsFromByteArray(QByteArray _data)
{
// unzip the file if needed
if (_data.startsWith(XZ_SIGNATURE)) {
#ifdef HAS_LZMA
// zipped file
auto *inBuffer = new QBuffer(&_data);
auto newData = QByteArray();
auto *outBuffer = new QBuffer(&newData);
inBuffer->open(QBuffer::ReadOnly);
outBuffer->open(QBuffer::WriteOnly);
XzDecompressor xz;
if (!xz.decompress(inBuffer, outBuffer)) {
zipDownloadFailed(tr("Xz extraction failed."));
return;
const bool compressed = _data.startsWith(XZ_SIGNATURE) || _data.startsWith(ZIP_SIGNATURE);
beginLoadSets(compressed);
// Decompress and scan off the UI thread so a large download can't freeze the window.
const QPointer<OracleImporter> importer = wizard()->importer;
future = QtConcurrent::run(
[importer, data = std::move(_data)]() mutable { return loadSetsData(importer, std::move(data)); });
watcher.setFuture(future);
}
void LoadSetsPage::readSetsFromFile(const QString &fileName)
{
// Peek at the header on the UI thread so the status text can distinguish
// "Extracting file..." from a plain JSON parse; the full read happens in the worker.
QFile headerFile(fileName);
bool compressed = false;
if (headerFile.open(QIODevice::ReadOnly)) {
const QByteArray header = headerFile.read(6);
compressed = header.startsWith(XZ_SIGNATURE) || header.startsWith(ZIP_SIGNATURE);
}
beginLoadSets(compressed);
// Read, decompress and scan off the UI thread (a plain JSON can be hundreds
// of MB, so even the read itself must not block the window).
const QPointer<OracleImporter> importer = wizard()->importer;
future = QtConcurrent::run([importer, fileName]() mutable -> LoadSetsResult {
QFile file(fileName);
if (!file.open(QIODevice::ReadOnly)) {
LoadSetsResult readError;
readError.errorMessage = LoadSetsPage::tr("Cannot open file '%1'.").arg(fileName);
return readError;
}
_data.clear();
readSetsFromByteArrayRef(newData);
return;
#else
zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files."));
return loadSetsData(importer, file.readAll());
});
watcher.setFuture(future);
}
wizard()->enableButtons();
setEnabled(true);
progressLabel->hide();
progressBar->hide();
return;
#endif
} else if (_data.startsWith(ZIP_SIGNATURE)) {
#ifdef HAS_ZLIB
// zipped file
auto *inBuffer = new QBuffer(&_data);
auto newData = QByteArray();
auto *outBuffer = new QBuffer(&newData);
QString fileName;
UnZip::ErrorCode ec;
UnZip uz;
ec = uz.openArchive(inBuffer);
if (ec != UnZip::Ok) {
zipDownloadFailed(tr("Failed to open Zip archive: %1.").arg(uz.formatError(ec)));
return;
}
if (uz.fileList().size() != 1) {
zipDownloadFailed(tr("Zip extraction failed: the Zip archive doesn't contain exactly one file."));
return;
}
fileName = uz.fileList().at(0);
outBuffer->open(QBuffer::ReadWrite);
ec = uz.extractFile(fileName, outBuffer);
if (ec != UnZip::Ok) {
zipDownloadFailed(tr("Zip extraction failed: %1.").arg(uz.formatError(ec)));
uz.closeArchive();
return;
}
_data.clear();
readSetsFromByteArrayRef(newData);
return;
#else
zipDownloadFailed(tr("Sorry, this version of Oracle does not support zipped files."));
wizard()->enableButtons();
setEnabled(true);
progressLabel->hide();
progressBar->hide();
return;
#endif
} else if (_data.startsWith("{")) {
// Start the computation.
jsonData = std::move(_data);
future = QtConcurrent::run([this] { return wizard()->importer->readSetsFromByteArray(std::move(jsonData)); });
watcher.setFuture(future);
} else if (_data.startsWith("<")) {
// save xml file and don't do any processing
wizard()->downloadedPlainXml = true;
wizard()->xmlData = std::move(_data);
importFinished();
} else {
wizard()->enableButtons();
setEnabled(true);
progressLabel->hide();
progressBar->hide();
QMessageBox::critical(this, tr("Error"), tr("Failed to interpret downloaded data."));
void LoadSetsPage::cancelWork()
{
// The scan is short-lived; just wait it out before the wizard (and its
// importer) can be torn down underneath the worker thread.
if (future.isRunning()) {
future.cancel();
watcher.cancel();
future.waitForFinished();
}
}
@ -508,17 +616,65 @@ void LoadSetsPage::zipDownloadFailed(const QString &message)
void LoadSetsPage::importFinished()
{
loadActive = false;
emit completeChanged();
wizard()->enableButtons();
setEnabled(true);
progressLabel->hide();
progressBar->hide();
if (wizard()->downloadedPlainXml || watcher.future().result()) {
wizard()->next();
} else {
QMessageBox::critical(this, tr("Error"),
tr("The file was retrieved successfully, but it does not contain any sets data."));
const LoadSetsResult result = watcher.result();
if (result.plainXml) {
wizard()->downloadedPlainXml = true;
wizard()->xmlData = result.xmlData;
}
if (wizard()->backgroundMode) {
progressLabel->hide();
progressBar->hide();
if (!result.errorMessage.isEmpty()) {
qWarning() << result.errorMessage;
} else if (!result.ok && !result.plainXml) {
qWarning() << tr("The file was retrieved successfully, but it does not contain any sets data.");
}
emit readyToContinue();
return;
}
const auto fail = [this](const QString &message) {
progressLabel->hide();
progressBar->hide();
QMessageBox::critical(this, tr("Error"), message);
};
if (!result.errorMessage.isEmpty()) {
if (result.offerUncompressedFallback) {
zipDownloadFailed(result.errorMessage);
return;
}
fail(result.errorMessage);
return;
}
if (!result.ok && !result.plainXml) {
fail(tr("The file was retrieved successfully, but it does not contain any sets data."));
return;
}
// Snap the bar to 100% and hold it there for a moment so the completed state
// is actually visible before the next page's own import progress takes over
// (a zero-length deferral can still fire before the repaint is delivered).
progressBar->setMaximum(1);
progressBar->setValue(1);
progressLabel->setText(tr("Parsing file (100%)"));
QTimer::singleShot(500, this, [this] {
if (wizard()->currentPage() == this) {
// Leave the page pristine: hide the completed load bar so a later
// Back from the save page doesn't show stale progress.
progressLabel->hide();
progressBar->hide();
wizard()->next();
}
});
}
SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
@ -526,46 +682,113 @@ SaveSetsPage::SaveSetsPage(QWidget *parent) : OracleWizardPage(parent)
pathLabel = new QLabel(this);
saveLabel = new QLabel(this);
progressBar = new QProgressBar(this);
progressBar->hide();
defaultPathCheckBox = new QCheckBox(this);
messageLog = new QTextEdit(this);
messageLog->setReadOnly(true);
auto *layout = new QGridLayout(this);
layout->addWidget(messageLog, 0, 0);
layout->addWidget(saveLabel, 1, 0);
layout->addWidget(pathLabel, 2, 0);
layout->addWidget(defaultPathCheckBox, 3, 0);
layout->addWidget(progressBar, 0, 0);
layout->addWidget(messageLog, 1, 0);
layout->addWidget(saveLabel, 2, 0);
layout->addWidget(pathLabel, 3, 0);
layout->addWidget(defaultPathCheckBox, 4, 0);
setLayout(layout);
}
bool SaveSetsPage::isComplete() const
{
return !importActive;
}
void SaveSetsPage::cleanupPage()
{
cancelWork();
disconnect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
disconnect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished);
wizard()->importer->clear();
disconnect(wizard()->importer, &OracleImporter::setIndexChanged, nullptr, nullptr);
}
void SaveSetsPage::initializePage()
{
messageLog->clear();
retranslateUi();
if (wizard()->downloadedPlainXml) {
messageLog->hide();
} else {
messageLog->show();
connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress);
int setsImported = wizard()->importer->startImport();
if (setsImported == 0) {
QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
progressBar->hide();
if (wizard()->backgroundMode) {
emit readyToContinue();
}
return;
}
messageLog->clear();
messageLog->show();
progressBar->show();
totalSets = wizard()->importer->getSets().size();
progressBar->setRange(0, totalSets);
progressBar->setValue(0);
connect(wizard()->importer, &OracleImporter::setIndexChanged, this, &SaveSetsPage::updateTotalProgress,
Qt::UniqueConnection);
connect(&importWatcher, &QFutureWatcher<int>::finished, this, &SaveSetsPage::importFinished, Qt::UniqueConnection);
wizard()->disableButtons();
importActive = true;
emit completeChanged();
const QPointer<OracleImporter> importer = wizard()->importer;
importFuture = QtConcurrent::run([importer] { return importer ? importer->startImport() : 0; });
importWatcher.setFuture(importFuture);
}
void SaveSetsPage::cancelWork()
{
if (!importActive) {
return;
}
// Ask the worker to stop at the next set boundary, then wait it out so the
// wizard (and the importer it owns) is never torn down under a running thread.
importActive = false;
emit completeChanged();
wizard()->importer->cancelImport();
importFuture.cancel();
importWatcher.cancel();
importFuture.waitForFinished();
}
void SaveSetsPage::importFinished()
{
if (!importActive) {
return;
}
importActive = false;
emit completeChanged();
wizard()->enableButtons();
const int setsImported = importWatcher.result();
const QPointer<OracleImporter> importer = wizard()->importer;
if (importer) {
importer->releaseSetData();
}
if (wizard()->backgroundMode) {
if (setsImported == 0) {
qWarning() << tr("No set has been imported.");
}
emit readyToContinue();
return;
}
progressBar->setValue(progressBar->maximum());
if (setsImported == 0) {
QMessageBox::critical(this, tr("Error"), tr("No set has been imported."));
}
}
@ -573,7 +796,7 @@ void SaveSetsPage::retranslateUi()
{
setTitle(tr("Sets imported"));
if (wizard()->downloadedPlainXml) {
setSubTitle(tr("A cockatrice database file of %1 MB has been downloaded.")
setSubTitle(tr("A Cockatrice card database file of %1 MB has been downloaded.")
.arg(qRound(wizard()->xmlData.size() / 1000000.0)));
} else {
setSubTitle(tr("The following sets have been found:"));
@ -587,13 +810,28 @@ void SaveSetsPage::retranslateUi()
setButtonText(QWizard::NextButton, tr("&Save"));
}
void SaveSetsPage::updateTotalProgress(int cardsImported, int /* setIndex */, const QString &setName)
void SaveSetsPage::updateTotalProgress(int cardsImported, int setIndex, const QString &setName)
{
if (!importActive) {
return;
}
if (setName.isEmpty()) {
messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(wizard()->importer->getCardList().size()) +
"</b>");
progressBar->setValue(progressBar->maximum());
const int cardCount = wizard()->importer->getCardList().size();
if (wizard()->backgroundMode) {
qInfo() << tr("Import finished: %1 cards.").arg(cardCount);
emitBackgroundProgress("import", totalSets, totalSets);
} else {
messageLog->append("<b>" + tr("Import finished: %1 cards.").arg(cardCount) + "</b>");
}
} else {
messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
progressBar->setValue(setIndex);
if (wizard()->backgroundMode) {
qInfo() << tr("%1: %2 cards imported").arg(setName).arg(cardsImported);
emitBackgroundProgress("import", setIndex, totalSets);
} else {
messageLog->append(tr("%1: %2 cards imported").arg(setName).arg(cardsImported));
}
}
messageLog->verticalScrollBar()->setValue(messageLog->verticalScrollBar()->maximum());
@ -739,4 +977,4 @@ void LoadSpoilersPage::retranslateUi()
pathLabel->setText(tr("The spoiler database will be saved at the following location:") + "<br>" +
SettingsCache::instance().getSpoilerCardDatabasePath());
defaultPathCheckBox->setText(tr("Save to a custom path (not recommended)"));
}
}

35
oracle/src/pages.h
View file

@ -3,8 +3,10 @@
#include "pagetemplates.h"
#include <QByteArray>
#include <QFuture>
#include <QFutureWatcher>
#include <QString>
#include <QTimer>
#include <QWizard>
#include <utility>
@ -57,19 +59,31 @@ protected:
void initializePage() override;
};
/** @brief Result of a worker-thread sets-file load (read + decompress + dispatch). */
struct LoadSetsResult
{
bool ok = false; ///< JSON scan produced set data (or plain XML was handled)
bool plainXml = false; ///< input was a plain Cockatrice XML database
QByteArray xmlData; ///< raw XML for the plain-XML path
QString errorMessage; ///< set when the input could not be processed
bool offerUncompressedFallback = false; ///< decompression-only failure: offer the uncompressed URL
};
class LoadSetsPage : public OracleWizardPage
{
Q_OBJECT
public:
explicit LoadSetsPage(QWidget *parent = nullptr);
void retranslateUi() override;
bool isComplete() const override;
protected:
void initializePage() override;
bool validatePage() override;
void readSetsFromByteArray(QByteArray _data);
void readSetsFromByteArrayRef(QByteArray &_data);
void readSetsFromFile(const QString &fileName);
void downloadSetsFile(const QUrl &url);
void cancelWork() override;
private:
QRadioButton *urlRadioButton;
@ -81,15 +95,19 @@ private:
QLabel *progressLabel;
QProgressBar *progressBar;
QFutureWatcher<bool> watcher;
QFuture<bool> future;
QByteArray jsonData;
QFutureWatcher<LoadSetsResult> watcher;
QFuture<LoadSetsResult> future;
bool loadActive = false;
void beginLoadSets(bool compressedFile = false);
private slots:
void actLoadSetsFile();
void actRestoreDefaultUrl();
void actDownloadProgressSetsFile(qint64 received, qint64 total);
void actDownloadFinishedSetsFile();
void updateParsingProgress(int bytesRead, int totalBytes);
void scanProgressToStdout(int bytesRead, int totalBytes);
void importFinished();
void zipDownloadFailed(const QString &message);
};
@ -100,19 +118,28 @@ class SaveSetsPage : public OracleWizardPage
public:
explicit SaveSetsPage(QWidget *parent = nullptr);
void retranslateUi() override;
bool isComplete() const override;
private:
QTextEdit *messageLog;
QProgressBar *progressBar;
QCheckBox *defaultPathCheckBox;
QLabel *pathLabel;
QLabel *saveLabel;
QFutureWatcher<int> importWatcher;
QFuture<int> importFuture;
int totalSets = 0;
bool importActive = false;
protected:
void initializePage() override;
void cleanupPage() override;
bool validatePage() override;
void cancelWork() override;
private slots:
void importFinished();
void updateTotalProgress(int cardsImported, int setIndex, const QString &setName);
};

2
oracle/src/pagetemplates.cpp
View file

@ -112,7 +112,7 @@ bool SimpleDownloadFilePage::validatePage()
return false;
}
progressLabel->setText(tr("Downloading (0MB)"));
progressLabel->setText(tr("Downloading (0 MB)"));
// show an infinite progressbar
progressBar->setMaximum(0);
progressBar->setMinimum(0);

8
oracle/src/pagetemplates.h
View file

@ -20,6 +20,14 @@ public:
}
virtual void retranslateUi() = 0;
/**
* @brief Asks an active page to stop any background worker before the wizard
* (and its importer) can be torn down underneath it. Default is a no-op.
*/
virtual void cancelWork()
{
}
signals:
void readyToContinue();

2
oracle/src/parsehelpers.cpp
View file

@ -20,7 +20,7 @@
* Note that "...enters tapped unless..." returns false.
*
* @param name The name of the card
* @param text The oracle text of the card
* @param text The Oracle text of the card
*/
bool parseCipt(const QString &name, const QString &text)
{

3
oracle/src/qt-json/AUTHORS
View file

@ -1,3 +0,0 @@
Eeli Reilin <eeli@emicode.fi>
Luis Gustavo S. Barreto <gustavosbarreto@gmail.com>
Stephen Kockentiedt <Stephen@Kockentiedt.name>

27
oracle/src/qt-json/LICENSE
View file

@ -1,27 +0,0 @@
Copyright 2011 Eeli Reilin. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ''AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The views and conclusions contained in the software and documentation
are those of the authors and should not be interpreted as representing
official policies, either expressed or implied, of Eeli Reilin.

96
oracle/src/qt-json/README
View file

@ -1,96 +0,0 @@
########################################################################
1. INTRODUCTION
The Json class is a simple class for parsing JSON data into a QVariant
hierarchies. Now, we can also reverse the process and serialize
QVariant hierarchies into valid JSON data.
########################################################################
2. HOW TO USE
The parser is really easy to use. Let's say we have the following
QString of JSON data:
------------------------------------------------------------------------
{
"encoding" : "UTF-8",
"plug-ins" : [
"python",
"c++",
"ruby"
],
"indent" : {
"length" : 3,
"use_space" : true
}
}
------------------------------------------------------------------------
We would first call the parse-method:
------------------------------------------------------------------------
//Say that we're using the QtJson namespace
using namespace QtJson;
bool ok;
//json is a QString containing the JSON data
QVariantMap result = Json::parse(json, ok).toMap();
if(!ok) {
qFatal("An error occurred during parsing");
exit(1);
}
------------------------------------------------------------------------
Assuming the parsing process completed without errors, we would then
go through the hierarchy:
------------------------------------------------------------------------
qDebug() << "encoding:" << result["encoding"].toString();
qDebug() << "plugins:";
foreach(QVariant plugin, result["plug-ins"].toList()) {
qDebug() << "\t-" << plugin.toString();
}
QVariantMap nestedMap = result["indent"].toMap();
qDebug() << "length:" << nestedMap["length"].toInt();
qDebug() << "use_space:" << nestedMap["use_space"].toBool();
------------------------------------------------------------------------
The previous code would print out the following:
------------------------------------------------------------------------
encoding: "UTF-8"
plugins:
- "python"
- "c++"
- "ruby"
length: 3
use_space: true
------------------------------------------------------------------------
To write JSON data from Qt object is as simple as parsing:
------------------------------------------------------------------------
QVariantMap map;
map["name"] = "Name";
map["age"] = 22;
QByteArray data = Json::serialize(map);
------------------------------------------------------------------------
The byte array 'data' contains valid JSON data:
------------------------------------------------------------------------
{
name: "Luis Gustavo",
age: 22,
}
------------------------------------------------------------------------
########################################################################
4. CONTRIBUTING
The code is available to download at GitHub. Contribute if you dare!

573
oracle/src/qt-json/json.cpp
View file

@ -1,573 +0,0 @@
/* Copyright 2011 Eeli Reilin. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ''AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation
* are those of the authors and should not be interpreted as representing
* official policies, either expressed or implied, of Eeli Reilin.
*/
/**
* \file json.cpp
*/
#include "json.h"
#include <QMetaType>
#include <iostream>
namespace QtJson
{
static QString sanitizeString(QString str)
{
str.replace(QLatin1String("\\"), QLatin1String("\\\\"));
str.replace(QLatin1String("\""), QLatin1String("\\\""));
str.replace(QLatin1String("\b"), QLatin1String("\\b"));
str.replace(QLatin1String("\f"), QLatin1String("\\f"));
str.replace(QLatin1String("\n"), QLatin1String("\\n"));
str.replace(QLatin1String("\r"), QLatin1String("\\r"));
str.replace(QLatin1String("\t"), QLatin1String("\\t"));
return QString(QLatin1String("\"%1\"")).arg(str);
}
static QByteArray join(const QList<QByteArray> &list, const QByteArray &sep)
{
QByteArray res;
for (const QByteArray &i : list) {
if (!res.isEmpty()) {
res += sep;
}
res += i;
}
return res;
}
/**
* parse
*/
QVariant Json::parse(const QString &json)
{
bool success = true;
return Json::parse(json, success);
}
/**
* parse
*/
QVariant Json::parse(const QString &json, bool &success)
{
success = true;
// Return an empty QVariant if the JSON data is either null or empty
if (!json.isNull() || !json.isEmpty()) {
// We'll start from index 0
int index = 0;
// Parse the first value
QVariant value = Json::parseValue(json, index, success);
// Return the parsed value
return value;
} else {
// Return the empty QVariant
return QVariant();
}
}
QByteArray Json::serialize(const QVariant &data)
{
bool success = true;
return Json::serialize(data, success);
}
QByteArray Json::serialize(const QVariant &data, bool &success)
{
QByteArray str;
success = true;
if (!data.isValid()) // invalid or null?
{
str = "null";
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if ((data.typeId() == QMetaType::Type::QVariantList) ||
(data.typeId() == QMetaType::Type::QStringList)) // variant is a list?
#else
else if ((data.type() == QVariant::List) || (data.type() == QVariant::StringList)) // variant is a list?
#endif
{
QList<QByteArray> values;
const QVariantList list = data.toList();
for (const QVariant &v : list) {
QByteArray serializedValue = serialize(v);
if (serializedValue.isNull()) {
success = false;
break;
}
values << serializedValue;
}
str = "[ " + join(values, ", ") + " ]";
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if ((data.typeId() == QMetaType::Type::QVariantHash)) // variant is a list?
#else
else if (data.type() == QVariant::Hash) // variant is a hash?
#endif
{
const QVariantHash vhash = data.toHash();
QHashIterator<QString, QVariant> it(vhash);
str = "{ ";
QList<QByteArray> pairs;
while (it.hasNext()) {
it.next();
QByteArray serializedValue = serialize(it.value());
if (serializedValue.isNull()) {
success = false;
break;
}
pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue;
}
str += join(pairs, ", ");
str += " }";
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if ((data.typeId() == QMetaType::Type::QVariantMap)) // variant is a list?
#else
else if (data.type() == QVariant::Map) // variant is a map?
#endif
{
const QVariantMap vmap = data.toMap();
QMapIterator<QString, QVariant> it(vmap);
str = "{ ";
QList<QByteArray> pairs;
while (it.hasNext()) {
it.next();
QByteArray serializedValue = serialize(it.value());
if (serializedValue.isNull()) {
success = false;
break;
}
pairs << sanitizeString(it.key()).toUtf8() + " : " + serializedValue;
}
str += join(pairs, ", ");
str += " }";
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if ((data.typeId() == QMetaType::Type::QString) ||
(data.typeId() == QMetaType::Type::QByteArray)) // variant is a list?
#else
else if ((data.type() == QVariant::String) || (data.type() == QVariant::ByteArray)) // a string or a byte array?
#endif
{
str = sanitizeString(data.toString()).toUtf8();
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if (data.typeId() == QMetaType::Type::Double)
#else
else if (data.type() == QVariant::Double) // double?
#endif
{
str = QByteArray::number(data.toDouble(), 'g', 20);
if (!str.contains(".") && !str.contains("e")) {
str += ".0";
}
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if (data.typeId() == QMetaType::Type::Bool)
#else
else if (data.type() == QVariant::Bool) // boolean value?
#endif
{
str = data.toBool() ? "true" : "false";
}
#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
else if (data.typeId() == QMetaType::Type::ULongLong)
#else
else if (data.type() == QVariant::ULongLong) // large unsigned number?
#endif
{
str = QByteArray::number(data.value<qulonglong>());
} else if (data.canConvert<qlonglong>()) // any signed number?
{
str = QByteArray::number(data.value<qlonglong>());
} else if (data.canConvert<long>()) {
str = QString::number(data.value<long>()).toUtf8();
} else if (data.canConvert<QString>()) // can value be converted to string?
{
// this will catch QDate, QDateTime, QUrl, ...
str = sanitizeString(data.toString()).toUtf8();
} else {
success = false;
}
if (success) {
return str;
} else {
return QByteArray();
}
}
/**
* parseValue
*/
QVariant Json::parseValue(const QString &json, int &index, bool &success)
{
// Determine what kind of data we should parse by
// checking out the upcoming token
switch (Json::lookAhead(json, index)) {
case JsonTokenString:
return Json::parseString(json, index, success);
case JsonTokenNumber:
return Json::parseNumber(json, index);
case JsonTokenCurlyOpen:
return Json::parseObject(json, index, success);
case JsonTokenSquaredOpen:
return Json::parseArray(json, index, success);
case JsonTokenTrue:
Json::nextToken(json, index);
return QVariant(true);
case JsonTokenFalse:
Json::nextToken(json, index);
return QVariant(false);
case JsonTokenNull:
Json::nextToken(json, index);
return QVariant();
case JsonTokenNone:
break;
}
// If there were no tokens, flag the failure and return an empty QVariant
success = false;
return QVariant();
}
/**
* parseObject
*/
QVariant Json::parseObject(const QString &json, int &index, bool &success)
{
QVariantMap map;
int token;
// Get rid of the whitespace and increment index
Json::nextToken(json, index);
// Loop through all of the key/value pairs of the object
bool done = false;
while (!done) {
// Get the upcoming token
token = Json::lookAhead(json, index);
if (token == JsonTokenNone) {
success = false;
return QVariantMap();
} else if (token == JsonTokenComma) {
Json::nextToken(json, index);
} else if (token == JsonTokenCurlyClose) {
Json::nextToken(json, index);
return map;
} else {
// Parse the key/value pair's name
QString name = Json::parseString(json, index, success).toString();
if (!success) {
return QVariantMap();
}
// Get the next token
token = Json::nextToken(json, index);
// If the next token is not a colon, flag the failure
// return an empty QVariant
if (token != JsonTokenColon) {
success = false;
return QVariant(QVariantMap());
}
// Parse the key/value pair's value
QVariant value = Json::parseValue(json, index, success);
if (!success) {
return QVariantMap();
}
// Assign the value to the key in the map
map[name] = value;
}
}
// Return the map successfully
return QVariant(map);
}
/**
* parseArray
*/
QVariant Json::parseArray(const QString &json, int &index, bool &success)
{
QVariantList list;
Json::nextToken(json, index);
bool done = false;
while (!done) {
int token = Json::lookAhead(json, index);
if (token == JsonTokenNone) {
success = false;
return QVariantList();
} else if (token == JsonTokenComma) {
Json::nextToken(json, index);
} else if (token == JsonTokenSquaredClose) {
Json::nextToken(json, index);
break;
} else {
QVariant value = Json::parseValue(json, index, success);
if (!success) {
return QVariantList();
}
list.push_back(value);
}
}
return QVariant(list);
}
/**
* parseString
*/
QVariant Json::parseString(const QString &json, int &index, bool &success)
{
QString s;
QChar c;
Json::eatWhitespace(json, index);
c = json[index++];
bool complete = false;
while (!complete) {
if (index == json.size()) {
break;
}
c = json[index++];
if (c == '\"') {
complete = true;
break;
} else if (c == '\\') {
if (index == json.size()) {
break;
}
c = json[index++];
if (c == '\"') {
s.append('\"');
} else if (c == '\\') {
s.append('\\');
} else if (c == '/') {
s.append('/');
} else if (c == 'b') {
s.append('\b');
} else if (c == 'f') {
s.append('\f');
} else if (c == 'n') {
s.append('\n');
} else if (c == 'r') {
s.append('\r');
} else if (c == 't') {
s.append('\t');
} else if (c == 'u') {
int remainingLength = json.size() - index;
if (remainingLength >= 4) {
QString unicodeStr = json.mid(index, 4);
int symbol = unicodeStr.toInt(0, 16);
s.append(QChar(symbol));
index += 4;
} else {
break;
}
}
} else {
s.append(c);
}
}
if (!complete) {
success = false;
return QVariant();
}
return QVariant(s);
}
/**
* parseNumber
*/
QVariant Json::parseNumber(const QString &json, int &index)
{
Json::eatWhitespace(json, index);
int lastIndex = Json::lastIndexOfNumber(json, index);
int charLength = (lastIndex - index) + 1;
QString numberStr;
numberStr = json.mid(index, charLength);
index = lastIndex + 1;
if (numberStr.contains('.')) {
return QVariant(numberStr.toDouble(NULL));
} else if (numberStr.startsWith('-')) {
return QVariant(numberStr.toLongLong(NULL));
} else {
return QVariant(numberStr.toULongLong(NULL));
}
}
/**
* lastIndexOfNumber
*/
int Json::lastIndexOfNumber(const QString &json, int index)
{
static const QString numericCharacters("0123456789+-.eE");
int lastIndex;
for (lastIndex = index; lastIndex < json.size(); lastIndex++) {
if (numericCharacters.indexOf(json[lastIndex]) == -1) {
break;
}
}
return lastIndex - 1;
}
/**
* eatWhitespace
*/
void Json::eatWhitespace(const QString &json, int &index)
{
static const QString whitespaceChars(" \t\n\r");
for (; index < json.size(); index++) {
if (whitespaceChars.indexOf(json[index]) == -1) {
break;
}
}
}
/**
* lookAhead
*/
int Json::lookAhead(const QString &json, int index)
{
int saveIndex = index;
return Json::nextToken(json, saveIndex);
}
/**
* nextToken
*/
int Json::nextToken(const QString &json, int &index)
{
Json::eatWhitespace(json, index);
if (index == json.size()) {
return JsonTokenNone;
}
QChar c = json[index];
index++;
switch (c.toLatin1()) {
case '{':
return JsonTokenCurlyOpen;
case '}':
return JsonTokenCurlyClose;
case '[':
return JsonTokenSquaredOpen;
case ']':
return JsonTokenSquaredClose;
case ',':
return JsonTokenComma;
case '"':
return JsonTokenString;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '-':
return JsonTokenNumber;
case ':':
return JsonTokenColon;
}
index--;
int remainingLength = json.size() - index;
// True
if (remainingLength >= 4) {
if (json[index] == 't' && json[index + 1] == 'r' && json[index + 2] == 'u' && json[index + 3] == 'e') {
index += 4;
return JsonTokenTrue;
}
}
// False
if (remainingLength >= 5) {
if (json[index] == 'f' && json[index + 1] == 'a' && json[index + 2] == 'l' && json[index + 3] == 's' &&
json[index + 4] == 'e') {
index += 5;
return JsonTokenFalse;
}
}
// Null
if (remainingLength >= 4) {
if (json[index] == 'n' && json[index + 1] == 'u' && json[index + 2] == 'l' && json[index + 3] == 'l') {
index += 4;
return JsonTokenNull;
}
}
return JsonTokenNone;
}
} // namespace QtJson

204
oracle/src/qt-json/json.h
View file

@ -1,204 +0,0 @@
/* Copyright 2011 Eeli Reilin. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ''AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL EELI REILIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation
* are those of the authors and should not be interpreted as representing
* official policies, either expressed or implied, of Eeli Reilin.
*/
/**
* \file json.h
*/
#ifndef JSON_H
#define JSON_H
#include <QVariant>
#include <QString>
namespace QtJson
{
/**
* \enum JsonToken
*/
enum JsonToken
{
JsonTokenNone = 0,
JsonTokenCurlyOpen = 1,
JsonTokenCurlyClose = 2,
JsonTokenSquaredOpen = 3,
JsonTokenSquaredClose = 4,
JsonTokenColon = 5,
JsonTokenComma = 6,
JsonTokenString = 7,
JsonTokenNumber = 8,
JsonTokenTrue = 9,
JsonTokenFalse = 10,
JsonTokenNull = 11
};
/**
* \class Json
* \brief A JSON data parser
*
* Json parses a JSON data into a QVariant hierarchy.
*/
class Json
{
public:
/**
* Parse a JSON string
*
* \param json The JSON data
*/
static QVariant parse(const QString &json);
/**
* Parse a JSON string
*
* \param json The JSON data
* \param success The success of the parsing
*/
static QVariant parse(const QString &json, bool &success);
/**
* This method generates a textual JSON representation
*
* \param data The JSON data generated by the parser.
* \param success The success of the serialization
*/
static QByteArray serialize(const QVariant &data);
/**
* This method generates a textual JSON representation
*
* \param data The JSON data generated by the parser.
* \param success The success of the serialization
*
* \return QByteArray Textual JSON representation
*/
static QByteArray serialize(const QVariant &data, bool &success);
private:
/**
* Parses a value starting from index
*
* \param json The JSON data
* \param index The start index
* \param success The success of the parse process
*
* \return QVariant The parsed value
*/
static QVariant parseValue(const QString &json, int &index,
bool &success);
/**
* Parses an object starting from index
*
* \param json The JSON data
* \param index The start index
* \param success The success of the object parse
*
* \return QVariant The parsed object map
*/
static QVariant parseObject(const QString &json, int &index,
bool &success);
/**
* Parses an array starting from index
*
* \param json The JSON data
* \param index The starting index
* \param success The success of the array parse
*
* \return QVariant The parsed variant array
*/
static QVariant parseArray(const QString &json, int &index,
bool &success);
/**
* Parses a string starting from index
*
* \param json The JSON data
* \param index The starting index
* \param success The success of the string parse
*
* \return QVariant The parsed string
*/
static QVariant parseString(const QString &json, int &index,
bool &success);
/**
* Parses a number starting from index
*
* \param json The JSON data
* \param index The starting index
*
* \return QVariant The parsed number
*/
static QVariant parseNumber(const QString &json, int &index);
/**
* Get the last index of a number starting from index
*
* \param json The JSON data
* \param index The starting index
*
* \return The last index of the number
*/
static int lastIndexOfNumber(const QString &json, int index);
/**
* Skip unwanted whitespace symbols starting from index
*
* \param json The JSON data
* \param index The start index
*/
static void eatWhitespace(const QString &json, int &index);
/**
* Check what token lies ahead
*
* \param json The JSON data
* \param index The starting index
*
* \return int The upcoming token
*/
static int lookAhead(const QString &json, int index);
/**
* Get the next JSON token
*
* \param json The JSON data
* \param index The starting index
*
* \return int The next JSON token
*/
static int nextToken(const QString &json, int &index);
};
} //end namespace
#endif //JSON_H

677
oracle/src/raw_json_scanner.cpp Normal file
View file

@ -0,0 +1,677 @@
#include "raw_json_scanner.h"
#include <algorithm>
#include <cstring>
namespace
{
// Nesting cap matching QJsonDocument's limit, so a pathologically deep document
// fails shallowly instead of overflowing the stack through the recursive
// skipValue/skipArray/skipObject walk (Qt's parser caps at 1024 for the same
// reason and reports DeepNesting).
constexpr int kMaxNestingDepth = 1024;
/**
* @brief Throttled byte-position reporting for scanSetRanges().
*
* Threaded through the skip walk so progress can be reported without materializing
* the whole document. Reports are rate-limited so a GUI showing progress isn't
* flooded with interrupts: a callback is invoked at most ~100 times per scan
* regardless of element count. The closing stretch (the last ~1%) is reported
* more finely so a large document doesn't stall the progress bar on the final
* percent before the scan wraps up.
*/
struct ScanProgress
{
const char *begin = nullptr;
qsizetype size = 0;
RawJson::ScanProgressCallback callback;
qsizetype step = 1;
qsizetype lastReported = 0;
/**
* @brief Reports the scanner's absolute offset, unless within @p step bytes
* of the previous report and not yet at the end of the document.
*/
void report(const char *p)
{
if (!callback) {
return;
}
const qsizetype offset = p - begin;
if (offset == lastReported) {
return; // the final element often already sits exactly at the end
}
const qsizetype reportingStep = offset >= size - step ? std::max<qsizetype>(1, step / 16) : step;
if (offset - lastReported < reportingStep && offset < size) {
return;
}
lastReported = offset;
callback(offset, size);
}
};
inline bool isWhitespace(char c)
{
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
}
const char *skipWhitespace(const char *p, const char *end)
{
while (p < end && isWhitespace(*p)) {
++p;
}
return p;
}
inline bool isHexDigit(char c)
{
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
inline quint8 hexValue(char c)
{
if (c >= '0' && c <= '9') {
return c - '0';
}
if (c >= 'a' && c <= 'f') {
return c - 'a' + 10;
}
return c - 'A' + 10;
}
/**
* @brief Skips past a JSON string without decoding it, validating escapes.
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
*/
bool skipString(const char *&p, const char *end)
{
++p; // opening quote
for (;;) {
const void *quote = memchr(p, '"', static_cast<size_t>(end - p));
if (!quote) {
return false; // unterminated string
}
// Backslash escapes can only appear before the closing quote, so bound
// the scan to the string extent instead of the rest of the document.
const void *backslash = memchr(p, '\\', static_cast<size_t>(static_cast<const char *>(quote) - p));
if (!backslash) {
p = static_cast<const char *>(quote) + 1;
return true;
}
const char *b = static_cast<const char *>(backslash);
if (end - b < 2) {
return false;
}
const char escaped = b[1];
if (escaped == 'u') {
if (end - b < 6) {
return false;
}
quint32 codepoint = 0;
for (int i = 0; i < 4; ++i) {
if (!isHexDigit(b[2 + i])) {
return false;
}
codepoint = codepoint * 16 + hexValue(b[2 + i]);
}
p = b + 6;
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
// expect the low-surrogate escape for the second half
if (end - p < 6 || p[0] != '\\' || p[1] != 'u') {
return false; // unpaired high surrogate
}
quint32 low = 0;
for (int i = 0; i < 4; ++i) {
if (!isHexDigit(p[2 + i])) {
return false;
}
low = low * 16 + hexValue(p[2 + i]);
}
if (low < 0xDC00 || low > 0xDFFF) {
return false;
}
p += 6;
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
return false; // unpaired low surrogate
}
continue;
}
switch (escaped) {
case '"':
case '\\':
case '/':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
p = b + 2;
continue;
default:
return false; // invalid escape
}
}
}
/**
* @brief Decodes a JSON string into @p out, validating it as it goes.
* @param p In: pointing at the opening quote. Out: pointing past the closing quote.
*/
bool decodeString(const char *&p, const char *end, QString &out)
{
out.clear();
QByteArray utf8;
auto flush = [&out, &utf8]() {
if (!utf8.isEmpty()) {
out += QString::fromUtf8(utf8);
utf8.clear();
}
};
++p; // opening quote
while (p < end) {
const char c = *p;
if (c == '\\') {
flush();
++p; // escaped character
if (p >= end) {
return false;
}
const char escaped = *p;
if (escaped == 'u') {
++p; // first hex digit
if (p + 4 > end) {
return false;
}
quint32 codepoint = 0;
for (int i = 0; i < 4; ++i) {
if (!isHexDigit(p[i])) {
return false;
}
codepoint = codepoint * 16 + hexValue(p[i]);
}
p += 4;
if (codepoint >= 0xD800 && codepoint <= 0xDBFF) {
// expect a low-surrogate escape for the second half
if (p + 6 > end || p[0] != '\\' || p[1] != 'u') {
return false; // unpaired high surrogate
}
quint32 low = 0;
for (int i = 0; i < 4; ++i) {
if (!isHexDigit(p[2 + i])) {
return false;
}
low = low * 16 + hexValue(p[2 + i]);
}
if (low < 0xDC00 || low > 0xDFFF) {
return false;
}
out += QChar(codepoint);
out += QChar(low);
p += 6;
} else if (codepoint >= 0xDC00 && codepoint <= 0xDFFF) {
return false; // unpaired low surrogate
} else {
out += QChar(codepoint);
}
continue;
}
switch (escaped) {
case '"':
out += '"';
break;
case '\\':
out += '\\';
break;
case '/':
out += '/';
break;
case 'b':
out += '\b';
break;
case 'f':
out += '\f';
break;
case 'n':
out += '\n';
break;
case 'r':
out += '\r';
break;
case 't':
out += '\t';
break;
default:
return false;
}
++p;
continue;
}
if (c == '"') {
++p;
flush();
return true;
}
// Deliberately accept unescaped control characters (e.g. a tab inside
// a set name): QJsonDocument and skipString accept them too, so
// rejecting them here would fail the whole document on a byte that
// Qt is fine with — the very total-failure mode this scanner avoids.
utf8 += c;
++p;
}
return false;
}
/**
* @brief Reads a set-metadata field, tolerating null and non-string values.
*
* A set's metadata may carry null or non-string values in otherwise-valid
* payloads ("releaseDate": null, "type": 7). The token itself was already
* structurally validated by skipValue, so a non-string value is accepted and
* leaves @p out at its default (empty) — one bad set must not abort the
* import of every other set in the document.
*/
bool decodeStringMember(const char *&fs, const char *&fe, QString &out)
{
if (fs >= fe) {
return false;
}
if (*fs != '"') {
return true;
}
return decodeString(fs, fe, out);
}
bool matchLiteral(const char *&p, const char *end, const char *literal, int length)
{
if (end - p < length || memcmp(p, literal, static_cast<size_t>(length)) != 0) {
return false;
}
const char *after = p + length;
if (after < end && (QChar::isLetter(*after) || QChar::isDigit(*after) || *after == '_')) {
return false;
}
p = after;
return true;
}
bool skipNumber(const char *&p, const char *end)
{
// JSON number: -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
if (p < end && *p == '-') {
++p;
}
if (p < end && *p == '0') {
++p;
} else if (p < end && *p >= '1' && *p <= '9') {
++p;
while (p < end && QChar::isDigit(*p)) {
++p;
}
} else {
return false;
}
if (p < end && *p == '.') {
++p;
if (p >= end || !QChar::isDigit(*p)) {
return false;
}
while (p < end && QChar::isDigit(*p)) {
++p;
}
}
if (p < end && (*p == 'e' || *p == 'E')) {
++p;
if (p < end && (*p == '+' || *p == '-')) {
++p;
}
if (p >= end || !QChar::isDigit(*p)) {
return false;
}
while (p < end && QChar::isDigit(*p)) {
++p;
}
}
return true;
}
bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan);
bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan);
bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan);
bool skipPrimitive(const char *&p, const char *end)
{
if (p >= end) {
return false;
}
const char c = *p;
if (c == '"') {
return skipString(p, end);
}
if (c == 't') {
return matchLiteral(p, end, "true", 4);
}
if (c == 'f') {
return matchLiteral(p, end, "false", 5);
}
if (c == 'n') {
return matchLiteral(p, end, "null", 4);
}
if (c == '-' || (c >= '0' && c <= '9')) {
return skipNumber(p, end);
}
return false;
}
bool skipObject(const char *&p, const char *end, int depth, ScanProgress &scan)
{
if (depth <= 0) {
return false; // nest deeper than the cap
}
++p; // '{'
p = skipWhitespace(p, end);
if (p < end && *p == '}') {
++p;
return true;
}
for (;;) {
p = skipWhitespace(p, end);
if (p >= end || *p != '"') {
return false;
}
if (!skipString(p, end)) {
return false;
}
p = skipWhitespace(p, end);
if (p >= end || *p != ':') {
return false;
}
++p;
if (!skipValue(p, end, depth - 1, scan)) {
return false;
}
scan.report(p);
p = skipWhitespace(p, end);
if (p >= end) {
return false;
}
if (*p == ',') {
++p;
continue;
}
if (*p == '}') {
++p;
return true;
}
return false;
}
}
bool skipArray(const char *&p, const char *end, int depth, ScanProgress &scan)
{
if (depth <= 0) {
return false; // nest deeper than the cap
}
++p; // '['
p = skipWhitespace(p, end);
if (p < end && *p == ']') {
++p;
return true;
}
for (;;) {
if (!skipValue(p, end, depth - 1, scan)) {
return false;
}
scan.report(p);
p = skipWhitespace(p, end);
if (p >= end) {
return false;
}
if (*p == ',') {
++p;
continue;
}
if (*p == ']') {
++p;
return true;
}
return false;
}
}
bool skipValue(const char *&p, const char *end, int depth, ScanProgress &scan)
{
p = skipWhitespace(p, end);
if (p >= end) {
return false;
}
const char c = *p;
if (c == '{') {
// pass depth through: skipObject consumes the single decrement for this level
return skipObject(p, end, depth, scan);
}
if (c == '[') {
return skipArray(p, end, depth, scan);
}
// a primitive is a leaf, so it never wastes a nesting level
return skipPrimitive(p, end);
}
/**
* @brief Iterates the members of the object starting at @p p.
*
* For each member invokes @p memberCallback with the key and the byte range of
* its value. Advancing @p p is unaffected by the callback.
*/
template <typename F>
bool forEachObjectMember(const char *&p, const char *end, int depth, F &&memberCallback, ScanProgress &scan)
{
if (depth <= 0) {
return false; // nest deeper than the cap
}
++p; // '{'
p = skipWhitespace(p, end);
if (p < end && *p == '}') {
++p;
return true;
}
for (;;) {
p = skipWhitespace(p, end);
if (p >= end || *p != '"') {
return false;
}
QString key;
if (!decodeString(p, end, key)) {
return false;
}
p = skipWhitespace(p, end);
if (p >= end || *p != ':') {
return false;
}
++p;
const char *valueStart = skipWhitespace(p, end);
const char *valueEnd = valueStart;
if (!skipValue(valueEnd, end, depth - 1, scan)) {
return false;
}
if (!memberCallback(key, valueStart, valueEnd)) {
return false;
}
p = valueEnd;
p = skipWhitespace(p, end);
if (p >= end) {
return false;
}
if (*p == ',') {
++p;
continue;
}
if (*p == '}') {
++p;
return true;
}
return false;
}
}
// Counts the direct elements of an array value; returns -1 if the array is malformed.
int countArrayElements(const char *p, const char *end, int depth, ScanProgress &scan)
{
if (depth <= 0) {
return -1; // nest deeper than the cap
}
++p; // '['
p = skipWhitespace(p, end);
int count = 0;
if (p < end && *p == ']') {
return 0;
}
for (;;) {
if (!skipValue(p, end, depth - 1, scan)) {
return -1;
}
scan.report(p);
++count;
p = skipWhitespace(p, end);
if (p >= end) {
return -1;
}
if (*p == ',') {
++p;
continue;
}
if (*p == ']') {
return count;
}
return -1;
}
}
} // namespace
namespace RawJson
{
QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error, const ScanProgressCallback &progress)
{
QList<SetRange> ranges;
if (error) {
*error = ScanError{};
}
const auto fail = [&](const QString &message) -> QList<SetRange> {
if (error) {
error->message = message;
}
return {};
};
const char *begin = json.constData();
const char *end = begin + json.size();
if (begin >= end) {
return fail(QStringLiteral("empty JSON document"));
}
// Throttle reports to ~100 per scan so a GUI thread unthrottling them never
// drowns under per-card interrupts, whatever the document size.
ScanProgress scan;
scan.begin = begin;
scan.size = end - begin;
scan.step = std::max<qsizetype>(1, scan.size / 100);
scan.callback = progress;
const char *p = skipWhitespace(begin, end);
if (p >= end || *p != '{') {
return fail(QStringLiteral("top-level JSON must be an object"));
}
bool foundData = false;
bool malformedSetData = false;
const auto topLevelCallback = [&](const QString &key, const char *valueStart, const char *valueEnd) {
if (key == QStringLiteral("data")) {
foundData = true;
if (valueStart >= valueEnd || *valueStart != '{') {
malformedSetData = true;
return false;
}
const char *setP = valueStart;
const bool ok = forEachObjectMember(
setP, valueEnd, kMaxNestingDepth - 1,
[&](const QString &setCode, const char *setStart, const char *setEnd) {
if (setStart >= setEnd || *setStart != '{') {
malformedSetData = true;
return false;
}
SetRange range;
range.dataRange.start = setStart - begin;
range.dataRange.length = setEnd - setStart;
range.code = setCode;
const char *memberP = setStart;
const bool metaOk = forEachObjectMember(
memberP, setEnd, kMaxNestingDepth - 2,
[&](const QString &field, const char *fs, const char *fe) {
if (field == QStringLiteral("code")) {
return decodeStringMember(fs, fe, range.code);
}
if (field == QStringLiteral("name")) {
return decodeStringMember(fs, fe, range.name);
}
if (field == QStringLiteral("type")) {
return decodeStringMember(fs, fe, range.type);
}
if (field == QStringLiteral("releaseDate")) {
return decodeStringMember(fs, fe, range.releaseDate);
}
if (field == QStringLiteral("cards")) {
if (fs >= fe) {
return false;
}
if (*fs != '[') {
// e.g. "cards": null — treat as an empty array,
// matching Qt's tolerance.
return true;
}
range.dataRange.cardCount = countArrayElements(fs, fe, kMaxNestingDepth - 2, scan);
return range.dataRange.cardCount >= 0;
}
return true;
},
scan);
if (!metaOk) {
malformedSetData = true;
return false;
}
ranges.append(range);
return true;
},
scan);
if (!ok) {
malformedSetData = true;
return false;
}
}
return true;
};
if (!forEachObjectMember(p, end, kMaxNestingDepth, topLevelCallback, scan)) {
return fail(malformedSetData ? QStringLiteral("malformed set data") : QStringLiteral("malformed JSON"));
}
p = skipWhitespace(p, end);
if (p != end) {
return fail(QStringLiteral("trailing content after top-level JSON object"));
}
if (!foundData) {
return fail(QStringLiteral("missing \"data\" object"));
}
if (ranges.isEmpty()) {
return fail(QStringLiteral("no sets found in \"data\""));
}
scan.report(end);
return ranges;
}
} // namespace RawJson

89
oracle/src/raw_json_scanner.h Normal file
View file

@ -0,0 +1,89 @@
#ifndef RAW_JSON_SCANNER_H
#define RAW_JSON_SCANNER_H
#include <QByteArray>
#include <QList>
#include <QString>
#include <functional>
namespace RawJson
{
/**
* @brief The byte extent of a set's object inside the scanned document, plus
* the size of its cards array. This is the slice SetToDownload needs for lazy
* per-set parsing; the metadata strings live in SetRange alongside it.
*/
struct SetDataRange
{
/** @brief Byte offset of the set's object within the scanned buffer. */
qsizetype start = -1;
/** @brief Byte length of the set's object, including the surrounding braces. */
qsizetype length = 0;
/** @brief Number of entries in the set's "cards" array. */
int cardCount = 0;
};
struct SetRange
{
/** @brief The byte slice of this set within the document. */
SetDataRange dataRange;
QString code;
QString name;
QString type;
QString releaseDate;
};
struct ScanError
{
bool isError() const
{
return !message.isEmpty();
}
QString message;
};
/**
* @brief Optional progress callback receiving @c (bytesRead, totalBytes) while
* the document is walked. Invoked from the scanning thread; the caller decides
* how the throttled offsets are relayed to a GUI event loop.
*/
using ScanProgressCallback = std::function<void(qsizetype bytesRead, qsizetype totalBytes)>;
/**
* @brief Scans a full MTGJSON document without materializing the JSON tree.
*
* Splits the top-level "data" object into per-set byte ranges and reads each
* set's metadata directly from the raw bytes. The Oracle importer can then
* parse one set at a time during import, keeping peak memory far below a single
* QJsonDocument::fromJson() over the whole file.
*
* The whole document is structurally validated while scanning (strings,
* escapes, braces, and a trailing-content check) and nesting depth is capped at
* 1024 to match QJsonDocument, so pathologically deep documents fail shallowly
* instead of exhausting the stack. Verdicts agree with QJsonDocument::fromJson
* on structurally malformed input; unlike Qt, string metadata fields
* ("name", "type", "releaseDate", "code") tolerate null / non-string values by
* defaulting to empty rather than rejecting the whole document, so one broken
* set cannot abort the import of the rest.
*
* Following QJsonDocument::fromJson's convention, the parsed ranges are
* returned by value and any failure is reported through the @p error out
* parameter.
*
* @param json The raw MTGJSON document bytes.
* @param error Out parameter. Set to an error ScanError when the document
* cannot be parsed, otherwise left empty. Passing a null
* pointer disables error reporting.
* @param progress Optional progress callback. When non-empty it is invoked as
* the scanner advances through the document, throttled to a
* tiny fraction of the total size.
* @return The detected per-set ranges, or an empty list on failure.
*/
QList<SetRange> scanSetRanges(const QByteArray &json,
ScanError *error = nullptr,
const ScanProgressCallback &progress = ScanProgressCallback());
} // namespace RawJson
#endif // RAW_JSON_SCANNER_H