jeirmeister/Cockatrice
1
0
Fork 0
mirror of https://github.com/Cockatrice/Cockatrice.git synced 2026-07-29 04:10:23 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

[Card Database] Improve loading times through binary cache (#7051)

Browse source 
* [Card Database] Improve loading times through binary cache

Took 10 minutes


Took 9 minutes

Took 16 seconds

* [Card Database] Remove lib qt include

Took 18 minutes


Took 14 seconds

* Downgrade to 6.3 datastream

Took 5 minutes

* go up to 6.4 datastream

Took 1 minute

* Address comments

* Small bug fixes

Took 20 minutes


Took 10 seconds

* More fixes.

Took 4 minutes


Took 4 seconds

* Even more fixes.

Took 11 minutes


Took 4 seconds

* More fixes.

Took 6 minutes

Took 26 seconds

Took 8 minutes

* Namespace instead of class

Took 6 minutes

---------

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>
This commit is contained in:
BruebachL 2026-07-27 23:12:53 +02:00 • committed by GitHub
parent 4bb8831531
commit 749223c2dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 1604 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

62
libcockatrice_card/libcockatrice/card/database/card_database.cpp
View file

@ -20,6 +20,7 @@ CardDatabase::CardDatabase(QObject *parent,
{
qRegisterMetaType<CardInfoPtr>("CardInfoPtr");
qRegisterMetaType<CardInfoPtr>("CardSetPtr");
qRegisterMetaType<CardDatabaseData>("CardDatabaseData");
// create loader and wire it up
loader = new CardDatabaseLoader(this, this, pathProvider, prefs, setPriorityController);
@ -28,6 +29,10 @@ CardDatabase::CardDatabase(QObject *parent,
connect(loader, &CardDatabaseLoader::loadingFailed, this, &CardDatabase::cardDatabaseLoadingFailed);
connect(loader, &CardDatabaseLoader::newSetsFound, this, &CardDatabase::cardDatabaseNewSetsFound);
connect(loader, &CardDatabaseLoader::allNewSetsEnabled, this, &CardDatabase::cardDatabaseAllNewSetsEnabled);
// swap the finished snapshot into the live database. Uses AutoConnection so
// that cross-thread loads are delivered via the event loop while same-thread/
// synchronous callers get the swap immediately.
connect(loader, &CardDatabaseLoader::databaseDataReady, this, &CardDatabase::swapInDatabaseData);
querier = new CardDatabaseQuerier(this, this, prefs);
}
@ -66,7 +71,12 @@ void CardDatabase::reloadCardDatabasesAndNotify()
loadCardDatabases();
if (loadStatus == Ok) {
notifyEnabledSetsChanged();
checkUnknownSets();
// A reload reconstructs the exact same enabled-set state, so the cached
// set-name / alt-name data computed during construction is still valid.
// Skipping the per-card refresh avoids a full 36k-card recompute that
// would otherwise duplicate the work already done while building cards.
notifyEnabledSetsChanged(false);
}
}
@ -77,13 +87,18 @@ bool CardDatabase::saveCustomTokensToFile()
void CardDatabase::refreshCachedReverseRelatedCards()
{
for (const auto &card : cards) {
refreshCachedReverseRelatedCards(cards);
}
void CardDatabase::refreshCachedReverseRelatedCards(CardNameMap &cardMap)
{
for (const auto &card : cardMap) {
card->resetReverseRelatedCards2Me();
}
for (const auto &card : cards) {
for (const auto &card : cardMap) {
for (auto *rel : card->getReverseRelatedCards()) {
if (auto target = cards.value(rel->getName())) {
if (auto target = cardMap.value(rel->getName())) {
auto *newRel = new CardRelation(card->getName(), rel->getAttachType(), rel->getIsCreateAllExclusion(),
rel->getIsVariable(), rel->getDefaultCount(), rel->getIsPersistent(),
rel->getIsFaceDown());
@ -205,11 +220,15 @@ void CardDatabase::markAllSetsAsKnown()
_sets.markAllAsKnown();
}
void CardDatabase::notifyEnabledSetsChanged()
void CardDatabase::notifyEnabledSetsChanged(bool recomputeCachedSets)
{
// refresh the list of cached set names
for (const CardInfoPtr &card : cards) {
card->refreshCachedSets();
if (recomputeCachedSets) {
// refresh the list of cached set names / alt names for every card; this is
// only needed when the enabled-set state actually changed (e.g. the user
// toggled a set in the settings UI).
for (const CardInfoPtr &card : cards) {
card->refreshCachedSets();
}
}
// inform the carddatabasemodels that they need to re-check their list of cards
@ -220,3 +239,30 @@ void CardDatabase::addFormat(const FormatRulesPtr &format)
{
formats.insert(format->formatName.toLower(), format);
}
void CardDatabase::swapInDatabaseData(CardDatabaseData data)
{
for (const auto &card : cards) {
for (auto *rel : card->getRelatedCards()) {
rel->deleteLater();
}
for (auto *rel : card->getReverseRelatedCards()) {
rel->deleteLater();
}
for (auto *rel : card->getReverseRelatedCards2Me()) {
rel->deleteLater();
}
}
cards = std::move(data.cards);
simpleNameCards = std::move(data.simpleNameCards);
sets = std::move(data.sets);
formats = std::move(data.formats);
loadStatus = cards.isEmpty() ? NotLoaded : Ok;
// inform listeners that the whole database was replaced; they should
// rebuild from the live containers in a single batch instead of reacting
// to individual card additions.
emit cardDatabaseReset();
}

49
libcockatrice_card/libcockatrice/card/database/card_database.h
View file

@ -2,6 +2,7 @@
#define CARDDATABASE_H
#include "../set/card_set_list.h"
#include "card_database_data.h"
#include "card_database_loader.h"
#include "card_database_querier.h"
@ -54,16 +55,17 @@ protected:
CardDatabaseQuerier *querier;
private:
/**
* @brief Check for sets that are unknown and emit signals if needed.
*/
void checkUnknownSets();
/**
* @brief Refreshes the cached reverse-related cards for all cards.
*/
void refreshCachedReverseRelatedCards();
/**
* @brief Refreshes the cached reverse-related cards for the given card map.
* @param cardMap Map of cards to process.
*/
void refreshCachedReverseRelatedCards(CardNameMap &cardMap);
/** @brief Mutexes for thread safety. */
QBasicMutex *clearDatabaseMutex = new QBasicMutex(), *addCardMutex = new QBasicMutex(),
*removeCardMutex = new QBasicMutex();
@ -127,8 +129,25 @@ public:
/** @brief Marks all sets as known. */
void markAllSetsAsKnown();
/** @brief Notifies listeners that enabled sets changed. */
void notifyEnabledSetsChanged();
/**
* @brief Notifies listeners that enabled sets changed.
* @param recomputeCachedSets When true, every card's cached set-name / alt-name
* data is recomputed (needed only when enabled-set state actually
* changed). Pass false after a plain reload, where enabled-set state is
* unchanged and the cached data is still valid.
*/
void notifyEnabledSetsChanged(bool recomputeCachedSets = true);
/**
* @brief Check for sets that are unknown and emit signals if needed.
*
* Called from MainWindow::startupConfigCheck() after signal connections are
* live (the front-loaded parse in main() runs before MainWindow exists, so
* signals emitted there would have no receivers). May also be called by
* the loader via the friend declaration. Has side effects: enables all
* sets when none are enabled (first-run), marks all sets as known.
*/
void checkUnknownSets();
ICardSetPriorityController *getPriorityController()
{
@ -159,10 +178,26 @@ public slots:
*/
bool saveCustomTokensToFile();
/**
* @brief Replaces the entire contents of the database with the provided
* snapshot in a single operation. Intended to be called once a
* background load has finished building a CardDatabaseData, so that
* observers never see a partially-populated database.
* @param data The fully-built snapshot to adopt.
*/
void swapInDatabaseData(CardDatabaseData data);
signals:
/** @brief Emitted when the card database has finished loading successfully. */
void cardDatabaseLoadingFinished();
/**
* @brief Emitted once after the live database contents have been replaced
* by a freshly loaded snapshot. Models should rebuild from the
* database in a single batch rather than reacting to per-card adds.
*/
void cardDatabaseReset();
/** @brief Emitted when the card database fails to load. */
void cardDatabaseLoadingFailed();

485
libcockatrice_card/libcockatrice/card/database/card_database_cache.cpp Normal file
View file

@ -0,0 +1,485 @@
#include "card_database_cache.h"
#include "../card_info.h"
#include "../format/format_legality_rules.h"
#include "../printing/printing_info.h"
#include "../relation/card_relation.h"
#include "../relation/card_relation_type.h"
#include "../set/card_set.h"
#include "card_database_loader.h"
#include <QBuffer>
#include <QDataStream>
#include <QElapsedTimer>
#include <QFile>
#include <QSaveFile>
#include <QVariantHash>
namespace
{
constexpr quint32 CACHE_MAGIC = 0x43445243; // "CDRC"
constexpr quint32 CACHE_VERSION = 1;
// ---- Primitives -----------------------------------------------------------
void writeString(QDataStream &out, const QString &s)
{
out << s;
}
QString readString(QDataStream &in)
{
QString s;
in >> s;
return s;
}
// Stores a QVariantHash as a single pre-serialized blob. The reader keeps the
// blob as-is and materializes the QVariantHash lazily on first query, which is
// what removes the allocation storm from database load (see card_info.cpp /
// printing_info.cpp).
void writeHashBlob(QDataStream &out, const QVariantHash &h)
{
QByteArray blob;
QDataStream blobOut(&blob, QIODevice::WriteOnly);
blobOut.setVersion(QDataStream::Qt_6_4);
blobOut << h;
out << static_cast<quint32>(blob.size());
out.writeRawData(blob.constData(), blob.size());
}
QByteArray readHashBlob(QDataStream &in)
{
quint32 len = 0;
in >> len;
if (in.status() != QDataStream::Ok || static_cast<qint64>(len) > in.device()->bytesAvailable()) {
return {};
}
QByteArray blob(len, Qt::Uninitialized);
in.readRawData(blob.data(), static_cast<int>(len));
return blob;
}
void writeDate(QDataStream &out, const QDate &d)
{
out << d;
}
QDate readDate(QDataStream &in)
{
QDate d;
in >> d;
return d;
}
// ---- CardRelation ----------------------------------------------------------
void writeRelation(QDataStream &out, const CardRelation *rel)
{
writeString(out, rel->getName());
out << static_cast<quint32>(rel->getAttachType());
out << rel->getIsCreateAllExclusion();
out << rel->getIsVariable();
out << rel->getDefaultCount();
out << rel->getIsPersistent();
out << rel->getIsFaceDown();
}
CardRelation *readRelation(QDataStream &in)
{
QString name = readString(in);
quint32 attachType = 0;
bool isExclusion = false;
bool isVariable = false;
int defaultCount = 1;
bool isPersistent = false;
bool isFaceDown = false;
in >> attachType;
in >> isExclusion;
in >> isVariable;
in >> defaultCount;
in >> isPersistent;
in >> isFaceDown;
return new CardRelation(name, static_cast<CardRelationType>(attachType), isExclusion, isVariable, defaultCount,
isPersistent, isFaceDown);
}
// ---- PrintingInfo ----------------------------------------------------------
// A printing references its set by short name; the CardSetPtr is resolved after
// all sets have been reconstructed on read.
void writePrinting(QDataStream &out, const PrintingInfo &p)
{
writeString(out, p.getSet() ? p.getSet()->getShortName() : QString());
writeHashBlob(out, p.getPropertiesHash());
}
PrintingInfo readPrinting(QDataStream &in, const SetNameMap &sets)
{
QString setName = readString(in);
QByteArray propsBlob = readHashBlob(in);
PrintingInfo p;
if (auto set = sets.value(setName)) {
p = PrintingInfo(set);
}
p.setPropertiesBlob(propsBlob);
return p;
}
// ---- CardSet ---------------------------------------------------------------
void writeSet(QDataStream &out, const CardSetPtr &set)
{
writeString(out, set->getShortName());
writeString(out, set->getLongName());
writeString(out, set->getSetType());
writeDate(out, set->getReleaseDate());
out << static_cast<quint32>(set->getPriority());
}
CardSetPtr readSet(QDataStream &in, ICardSetPriorityController *priorityController)
{
QString shortName = readString(in);
QString longName = readString(in);
QString setType = readString(in);
QDate releaseDate = readDate(in);
quint32 priority = 0;
in >> priority;
return CardSet::newInstance(priorityController, shortName, longName, setType, releaseDate,
static_cast<CardSet::Priority>(priority));
}
// ---- CardInfo --------------------------------------------------------------
void writeCard(QDataStream &out, const CardInfoPtr &card)
{
writeString(out, card->getName());
writeString(out, card->getText());
out << card->getIsToken();
writeHashBlob(out, card->getPropertiesHash());
CardInfo::UiAttributes ui = card->getUiAttributes();
out << ui.cipt;
out << ui.landscapeOrientation;
out << ui.tableRow;
out << ui.upsideDownArt;
// Precomputed derived state, so the reader can skip simplifyName() and the
// per-printing alt-name scan entirely.
writeString(out, card->getSimpleName());
{
const QSet<QString> altNames = card->getAltNames();
out << static_cast<quint32>(altNames.size());
for (const QString &alt : altNames) {
writeString(out, alt);
}
}
// setsToPrintings
const SetToPrintingsMap sets = card->getSets();
out << static_cast<quint32>(sets.size());
for (auto it = sets.constBegin(); it != sets.constEnd(); ++it) {
writeString(out, it.key());
out << static_cast<quint32>(it.value().size());
for (const PrintingInfo &p : it.value()) {
writePrinting(out, p);
}
}
// related cards
const QList<CardRelation *> related = card->getRelatedCards();
out << static_cast<quint32>(related.size());
for (const CardRelation *rel : related) {
writeRelation(out, rel);
}
// reverse-related cards (the reverseRelatedCards list, not the computed 2Me)
const QList<CardRelation *> reverse = card->getReverseRelatedCards();
out << static_cast<quint32>(reverse.size());
for (const CardRelation *rel : reverse) {
writeRelation(out, rel);
}
}
CardInfoPtr readCard(QDataStream &in, const SetNameMap &sets)
{
QString name = readString(in);
QString text = readString(in);
bool isToken = false;
in >> isToken;
QByteArray propertiesBlob = readHashBlob(in);
CardInfo::UiAttributes ui;
in >> ui.cipt;
in >> ui.landscapeOrientation;
in >> ui.tableRow;
in >> ui.upsideDownArt;
// Precomputed derived state, restored directly to skip simplifyName() and the
// per-printing alt-name scan.
QString simpleName = readString(in);
QSet<QString> altNames;
quint32 altNameCount = 0;
in >> altNameCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
altNames.reserve(altNameCount);
for (quint32 i = 0; i < altNameCount; ++i) {
altNames.insert(readString(in));
}
SetToPrintingsMap cardSets;
quint32 setCount = 0;
in >> setCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
for (quint32 i = 0; i < setCount; ++i) {
QString setName = readString(in);
quint32 printingCount = 0;
in >> printingCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
QList<PrintingInfo> printings;
printings.reserve(printingCount);
for (quint32 j = 0; j < printingCount; ++j) {
printings.append(readPrinting(in, sets));
}
cardSets.insert(setName, printings);
}
QList<CardRelation *> related;
quint32 relatedCount = 0;
in >> relatedCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
related.reserve(relatedCount);
for (quint32 i = 0; i < relatedCount; ++i) {
related.append(readRelation(in));
}
QList<CardRelation *> reverse;
quint32 reverseCount = 0;
in >> reverseCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
reverse.reserve(reverseCount);
for (quint32 i = 0; i < reverseCount; ++i) {
reverse.append(readRelation(in));
}
return CardInfo::newInstance(name, text, isToken, propertiesBlob, related, reverse, cardSets, ui, simpleName,
altNames, false);
}
// ---- FormatRules -----------------------------------------------------------
void writeFormat(QDataStream &out, const FormatRulesPtr &format)
{
writeString(out, format->formatName);
out << format->minDeckSize;
out << format->maxDeckSize;
out << format->maxSideboardSize;
out << static_cast<quint32>(format->allowedCounts.size());
for (const AllowedCount &ac : format->allowedCounts) {
out << ac.max;
writeString(out, ac.label);
}
out << static_cast<quint32>(format->exceptions.size());
for (const ExceptionRule &ex : format->exceptions) {
out << static_cast<quint32>(ex.conditions.size());
for (const CardCondition &cond : ex.conditions) {
writeString(out, cond.field);
writeString(out, cond.matchType);
writeString(out, cond.value);
}
out << ex.maxCopies;
}
}
FormatRulesPtr readFormat(QDataStream &in)
{
FormatRulesPtr format(new FormatRules());
format->formatName = readString(in);
in >> format->minDeckSize;
in >> format->maxDeckSize;
in >> format->maxSideboardSize;
quint32 allowedCount = 0;
in >> allowedCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
for (quint32 i = 0; i < allowedCount; ++i) {
AllowedCount ac;
in >> ac.max;
ac.label = readString(in);
format->allowedCounts.append(ac);
}
quint32 exceptionCount = 0;
in >> exceptionCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
for (quint32 i = 0; i < exceptionCount; ++i) {
ExceptionRule ex;
quint32 condCount = 0;
in >> condCount;
if (in.status() != QDataStream::Ok) {
return nullptr;
}
for (quint32 j = 0; j < condCount; ++j) {
CardCondition cond;
cond.field = readString(in);
cond.matchType = readString(in);
cond.value = readString(in);
ex.conditions.append(cond);
}
in >> ex.maxCopies;
format->exceptions.append(ex);
}
return format;
}
} // namespace
bool CardDatabaseCache::write(const QString &cachePath, const CardDatabaseData &data, const QByteArray &sourceHash)
{
QSaveFile file(cachePath);
if (!file.open(QIODevice::WriteOnly)) {
return false;
}
QDataStream out(&file);
out.setVersion(QDataStream::Qt_6_4);
// Header
out << CACHE_MAGIC;
out << CACHE_VERSION;
out << sourceHash;
// Sets
out << static_cast<quint32>(data.sets.size());
for (const CardSetPtr &set : data.sets) {
writeSet(out, set);
}
// Cards
const quint32 cardCount = static_cast<quint32>(data.cards.size());
out << cardCount;
for (const CardInfoPtr &card : data.cards) {
writeCard(out, card);
}
// Formats
out << static_cast<quint32>(data.formats.size());
for (const FormatRulesPtr &format : data.formats) {
writeFormat(out, format);
}
return file.commit();
}
bool CardDatabaseCache::read(const QString &cachePath,
CardDatabaseData &data,
const QByteArray &sourceHash,
ICardSetPriorityController *priorityController)
{
QFile file(cachePath);
if (!file.open(QIODevice::ReadOnly)) {
return false;
}
// Read the whole cache into memory up front; deserialization then works on an
// in-memory buffer with no further disk I/O.
QByteArray raw = file.readAll();
if (raw.isEmpty()) {
return false;
}
QBuffer buffer(&raw);
buffer.open(QIODevice::ReadOnly);
QDataStream in(&buffer);
in.setVersion(QDataStream::Qt_6_4);
quint32 magic = 0;
quint32 version = 0;
QByteArray storedHash;
in >> magic >> version >> storedHash;
if (magic != CACHE_MAGIC || version != CACHE_VERSION || storedHash != sourceHash) {
return false;
}
QElapsedTimer deserializeTimer;
deserializeTimer.start();
// Sets
quint32 setCount = 0;
in >> setCount;
if (in.status() != QDataStream::Ok) {
return false;
}
for (quint32 i = 0; i < setCount; ++i) {
CardSetPtr set = readSet(in, priorityController);
data.sets.insert(set->getShortName(), set);
}
// Cards
quint32 cardCount = 0;
in >> cardCount;
if (in.status() != QDataStream::Ok) {
return false;
}
if (cardCount > 0) {
data.cards.reserve(static_cast<int>(cardCount));
for (quint32 i = 0; i < cardCount; ++i) {
CardInfoPtr card = readCard(in, data.sets);
if (card == nullptr) {
return false;
}
data.cards.insert(card->getName(), card);
data.simpleNameCards.insert(card->getSimpleName(), card);
}
for (const CardInfoPtr &card : data.cards) {
for (const auto &printings : card->getSets()) {
for (const PrintingInfo &printing : printings) {
if (auto set = printing.getSet()) {
set->append(card);
break;
}
}
}
}
}
// Formats
quint32 formatCount = 0;
in >> formatCount;
if (in.status() != QDataStream::Ok) {
return false;
}
for (quint32 i = 0; i < formatCount; ++i) {
FormatRulesPtr format = readFormat(in);
if (format == nullptr) {
return false;
}
data.formats.insert(format->formatName.toLower(), format);
}
qCInfo(CardDatabaseLoadingLog) << "[cache] read + deserialize" << deserializeTimer.elapsed() << "ms for"
<< cardCount << "cards";
if (in.status() != QDataStream::Ok) {
return false;
}
return file.error() == QFile::NoError;
}

34
libcockatrice_card/libcockatrice/card/database/card_database_cache.h Normal file
View file

@ -0,0 +1,34 @@
#ifndef CARDDATABASE_CACHE_H
#define CARDDATABASE_CACHE_H
#include "card_database_data.h"
#include <QByteArray>
#include <libcockatrice/interfaces/interface_card_set_priority_controller.h>
namespace CardDatabaseCache
{
/**
* @brief Writes a fully-built database snapshot to a binary cache file.
* @param cachePath Path of the cache file to write.
* @param data The snapshot to serialize.
* @param sourceHash Hash of the source XML used to validate the cache later.
* @return True if the cache was written successfully.
*/
bool write(const QString &cachePath, const CardDatabaseData &data, const QByteArray &sourceHash);
/**
* @brief Reads a database snapshot from a binary cache file.
* @param cachePath Path of the cache file to read.
* @param data Snapshot to populate.
* @param sourceHash Expected hash of the source XML; read fails if it differs.
* @param priorityController Controller used to reconstruct CardSet state.
* @return True if a valid, up-to-date cache was read.
*/
bool read(const QString &cachePath,
CardDatabaseData &data,
const QByteArray &sourceHash,
ICardSetPriorityController *priorityController);
} // namespace CardDatabaseCache
#endif // CARDDATABASE_CACHE_H

24
libcockatrice_card/libcockatrice/card/database/card_database_data.h Normal file
View file

@ -0,0 +1,24 @@
#ifndef CARDDATABASE_DATA_H
#define CARDDATABASE_DATA_H
#include <libcockatrice/card/card_info.h>
/**
* @struct CardDatabaseData
* @ingroup CardDatabase
* @brief Value type holding the full parsed state of a card database.
*
* Parsers fill a CardDatabaseData off the UI thread; the finished snapshot is
* then swapped into the live CardDatabase atomically. Keeping this separate from
* the live container avoids emitting per-card change notifications and prevents
* readers from observing a half-loaded database.
*/
struct CardDatabaseData
{
CardNameMap cards;
CardNameMap simpleNameCards;
SetNameMap sets;
FormatRulesNameMap formats;
};
#endif // CARDDATABASE_DATA_H

142
libcockatrice_card/libcockatrice/card/database/card_database_loader.cpp
View file

@ -1,12 +1,18 @@
#include "card_database_loader.h"
#include "card_database.h"
#include "card_database_cache.h"
#include "parser/card_database_parser.h"
#include "parser/cockatrice_xml_3.h"
#include "parser/cockatrice_xml_4.h"
#include <QByteArray>
#include <QCoreApplication>
#include <QCryptographicHash>
#include <QDebug>
#include <QDirIterator>
#include <QFile>
#include <QFileInfo>
#include <QTime>
CardDatabaseLoader::CardDatabaseLoader(QObject *parent,
@ -14,18 +20,14 @@ CardDatabaseLoader::CardDatabaseLoader(QObject *parent,
ICardDatabasePathProvider *_pathProvider,
ICardPreferenceProvider *_preferenceProvider,
ICardSetPriorityController *_priorityController)
: QObject(parent), database(db), pathProvider(_pathProvider)
: QObject(parent), database(db), pathProvider(_pathProvider), priorityController(_priorityController)
{
// instantiate available parsers here and connect them to the database
// instantiate available parsers here
availableParsers << new CockatriceXml4Parser(_preferenceProvider, _priorityController);
availableParsers << new CockatriceXml3Parser(_priorityController);
for (auto *p : availableParsers) {
// connect parser outputs to the database adders
connect(p, &ICardDatabaseParser::addCard, database, &CardDatabase::addCard, Qt::DirectConnection);
connect(p, &ICardDatabaseParser::addSet, database, &CardDatabase::addSet, Qt::DirectConnection);
connect(p, &ICardDatabaseParser::addFormat, database, &CardDatabase::addFormat, Qt::DirectConnection);
}
// The load path parses into a snapshot and never emits per-card signals;
// the finished snapshot is swapped into the live database on the GUI thread.
// when SettingsCache's path changes, trigger reloads
connect(pathProvider, &ICardDatabasePathProvider::cardDatabasePathChanged, this,
@ -38,7 +40,7 @@ CardDatabaseLoader::~CardDatabaseLoader()
availableParsers.clear();
}
LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName, CardDatabaseData &data)
{
QFile file(fileName);
if (!file.open(QIODevice::ReadOnly)) {
@ -49,7 +51,7 @@ LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
file.reset();
if (parser->getCanParseFile(fileName, file)) {
file.reset();
parser->parseFile(file);
parser->parseFileInto(file, data);
return Ok;
}
}
@ -57,24 +59,29 @@ LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
return Invalid;
}
LoadStatus CardDatabaseLoader::loadCardDatabase(const QString &path)
LoadStatus CardDatabaseLoader::loadCardDatabase(const QString &path, CardDatabaseData &data)
{
auto startTime = QTime::currentTime();
LoadStatus tempLoadStatus = NotLoaded;
if (!path.isEmpty()) {
QMutexLocker locker(loadFromFileMutex);
tempLoadStatus = loadFromFile(path);
tempLoadStatus = loadFromFile(path, data);
}
int msecs = startTime.msecsTo(QTime::currentTime());
qCInfo(CardDatabaseLoadingLog) << "Loaded card database: Path =" << path << "Status =" << tempLoadStatus
<< "Cards =" << (database ? database->cards.size() : 0)
<< "Sets =" << (database ? database->sets.size() : 0) << QString("%1ms").arg(msecs);
<< "Cards =" << data.cards.size() << "Sets =" << data.sets.size()
<< QString("%1ms").arg(msecs);
return tempLoadStatus;
}
LoadStatus CardDatabaseLoader::loadCardDatabases()
{
return doLoadCardDatabases();
}
LoadStatus CardDatabaseLoader::doLoadCardDatabases()
{
QMutexLocker locker(reloadDatabaseMutex);
@ -86,31 +93,55 @@ LoadStatus CardDatabaseLoader::loadCardDatabases()
emit loadingStarted();
qCInfo(CardDatabaseLoadingLog) << "Card Database Loading Started";
database->clear(); // remove old db
ICardDatabaseParser::clearSetlist();
LoadStatus loadStatus = loadCardDatabase(pathProvider->getCardDatabasePath()); // load main card database
loadCardDatabase(pathProvider->getTokenDatabasePath()); // load tokens database
loadCardDatabase(pathProvider->getSpoilerCardDatabasePath()); // load spoilers database
CardDatabaseData data;
LoadStatus loadStatus = NotLoaded;
// find all custom card databases, recursively & following symlinks
// then load them alphabetically
// Try the binary cache first: a cache hit avoids re-parsing the (large) XML.
const QStringList customPaths = collectCustomDatabasePaths();
for (int i = 0; i < customPaths.size(); ++i) {
const auto &p = customPaths.at(i);
qCInfo(CardDatabaseLoadingLog) << "Loading Custom Set" << i << "(" << p << ")";
loadCardDatabase(p);
const QByteArray sourceHash = computeSourceHash(customPaths);
if (loadFromCache(data, sourceHash)) {
qCInfo(CardDatabaseLoadingLog) << "Loaded card database from binary cache";
loadStatus = Ok;
} else {
loadStatus = loadCardDatabase(pathProvider->getCardDatabasePath(), data); // main card database
if (loadStatus == Ok) {
loadCardDatabase(pathProvider->getTokenDatabasePath(), data); // tokens database
loadCardDatabase(pathProvider->getSpoilerCardDatabasePath(), data); // spoilers database
// find all custom card databases, recursively & following symlinks
// then load them alphabetically
for (int i = 0; i < customPaths.size(); ++i) {
const auto &p = customPaths.at(i);
qCInfo(CardDatabaseLoadingLog) << "Loading Custom Set" << i << "(" << p << ")";
loadCardDatabase(p, data);
}
if (!saveToCache(data, sourceHash)) {
qCWarning(CardDatabaseLoadingLog) << "Failed to write binary cache to" << cachePath();
}
}
}
// AFTER all the cards have been loaded
// resolve the reverse-related tags
database->refreshCachedReverseRelatedCards();
// AFTER all the cards have been loaded: resolve the reverse-related tags
// against the fully-built snapshot.
if (loadStatus == Ok) {
database->checkUnknownSets(); // update deck editors, etc
database->refreshCachedReverseRelatedCards(data.cards);
qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Success";
emit databaseDataReady(std::move(data));
emit loadingFinished();
// During the front-loaded parse in main() this runs before MainWindow
// exists, so cardDatabaseNewSetsFound / cardDatabaseAllNewSetsEnabled
// would be emitted with no receivers. Skip the check on the first load;
// MainWindow::startupConfigCheck() calls checkUnknownSets() once its
// signal connections are live. On subsequent reloads (e.g. path change)
// MainWindow exists and signals have live receivers.
if (initialLoadComplete) {
database->checkUnknownSets();
}
initialLoadComplete = true;
} else {
qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Failed";
emit loadingFailed(); // bring up the settings dialog
@ -119,6 +150,55 @@ LoadStatus CardDatabaseLoader::loadCardDatabases()
return loadStatus;
}
QString CardDatabaseLoader::cachePath() const
{
return pathProvider->getCardDatabasePath() + ".cache";
}
QByteArray CardDatabaseLoader::computeSourceHash(const QStringList &customPaths) const
{
// Hash over the paths, sizes and modification times of every input file so
// the cache invalidates when any source changes. Cheap (no content read).
QCryptographicHash hash(QCryptographicHash::Sha256);
// Include the application version so parser changes automatically invalidate
// old caches even when the XML files are byte-identical.
hash.addData(QCoreApplication::applicationVersion().toUtf8());
hash.addData(QByteArray(1, '\0'));
const QStringList inputs = QStringList()
<< pathProvider->getCardDatabasePath() << pathProvider->getTokenDatabasePath()
<< pathProvider->getSpoilerCardDatabasePath() << customPaths;
for (const QString &path : inputs) {
QFileInfo info(path);
if (info.exists()) {
hash.addData(path.toUtf8());
hash.addData(QByteArray(1, '\0'));
hash.addData(QByteArray::number(info.size()));
hash.addData(QByteArray(1, '\0'));
hash.addData(QByteArray::number(info.lastModified().toSecsSinceEpoch()));
hash.addData(QByteArray(1, '\0'));
}
}
return hash.result();
}
bool CardDatabaseLoader::loadFromCache(CardDatabaseData &data, const QByteArray &sourceHash)
{
if (sourceHash.isEmpty()) {
return false;
}
return CardDatabaseCache::read(cachePath(), data, sourceHash, priorityController);
}
bool CardDatabaseLoader::saveToCache(const CardDatabaseData &data, const QByteArray &sourceHash)
{
if (sourceHash.isEmpty()) {
return false;
}
return CardDatabaseCache::write(cachePath(), data, sourceHash);
}
QStringList CardDatabaseLoader::collectCustomDatabasePaths() const
{
QDirIterator it(pathProvider->getCustomCardDatabasePath(), {"*.xml"}, QDir::Files,

62
libcockatrice_card/libcockatrice/card/database/card_database_loader.h
View file

@ -4,6 +4,7 @@
#include <QBasicMutex>
#include <QList>
#include <QLoggingCategory>
#include <libcockatrice/card/database/card_database_data.h>
#include <libcockatrice/interfaces/interface_card_database_path_provider.h>
#include <libcockatrice/interfaces/interface_card_preference_provider.h>
#include <libcockatrice/interfaces/interface_card_set_priority_controller.h>
@ -62,16 +63,20 @@ public:
public slots:
/**
* @brief Loads all configured card databases.
*
* Runs synchronously on the calling thread. The caller should ensure
* that any signal receivers are already connected before invoking this.
* @return Status of the main database load.
*/
LoadStatus loadCardDatabases();
/**
* @brief Loads a single card database file.
* @brief Loads a single card database file into the given snapshot.
* @param path Path to the database file.
* @param data Snapshot to populate.
* @return LoadStatus indicating success or failure.
*/
LoadStatus loadCardDatabase(const QString &path);
LoadStatus loadCardDatabase(const QString &path, CardDatabaseData &data);
/**
* @brief Saves custom tokens to the user-defined custom database path.
@ -89,6 +94,12 @@ signals:
/** @brief Emitted when loading fails. */
void loadingFailed();
/**
* @brief Emitted with the fully-built snapshot once background loading
* completes. The receiver swaps it into the live database.
*/
void databaseDataReady(CardDatabaseData data);
/**
* @brief Emitted when new sets are discovered during loading.
* @param numSets Number of new sets.
@ -101,11 +112,18 @@ signals:
private:
/**
* @brief Loads a database from a single file using the available parsers.
* @brief Parses a single file into the given snapshot using the available parsers.
* @param fileName Path to the database file.
* @param data Snapshot to populate.
* @return LoadStatus indicating success or failure.
*/
LoadStatus loadFromFile(const QString &fileName);
LoadStatus loadFromFile(const QString &fileName, CardDatabaseData &data);
/**
* @brief Performs the actual load work synchronously on the calling thread.
* @return Status of the main database load.
*/
LoadStatus doLoadCardDatabases();
/**
* @brief Collects custom card database paths recursively.
@ -113,13 +131,43 @@ private:
*/
[[nodiscard]] QStringList collectCustomDatabasePaths() const;
/**
* @brief Computes a hash identifying the current set of input database files.
* @return Hash over the paths, sizes and modification times of all inputs.
*/
[[nodiscard]] QByteArray computeSourceHash(const QStringList &customPaths) const;
/**
* @brief Path of the binary cache file derived from the main card database path.
* @return Cache file path.
*/
[[nodiscard]] QString cachePath() const;
/**
* @brief Attempts to populate the snapshot from the binary cache.
* @param data Snapshot to populate.
* @param sourceHash Pre-computed hash identifying the current input files.
* @return True if a valid, up-to-date cache was read.
*/
bool loadFromCache(CardDatabaseData &data, const QByteArray &sourceHash);
/**
* @brief Writes the snapshot to the binary cache for the current inputs.
* @param data Snapshot to serialize.
* @param sourceHash Pre-computed hash identifying the current input files.
* @return True if the cache was written successfully.
*/
bool saveToCache(const CardDatabaseData &data, const QByteArray &sourceHash);
private:
CardDatabase *database; /**< Non-owning pointer to the target CardDatabase. */
ICardDatabasePathProvider *pathProvider; /**< Pointer to the path provider. */
QList<ICardDatabaseParser *> availableParsers; /**< List of available parsers for different formats. */
CardDatabase *database; /**< Non-owning pointer to the target CardDatabase. */
ICardDatabasePathProvider *pathProvider; /**< Pointer to the path provider. */
ICardSetPriorityController *priorityController; /**< Controller for reconstructing set state. */
QList<ICardDatabaseParser *> availableParsers; /**< List of available parsers for different formats. */
QBasicMutex *loadFromFileMutex = new QBasicMutex(); /**< Mutex for single-file loading. */
QBasicMutex *reloadDatabaseMutex = new QBasicMutex(); /**< Mutex for reloading entire database. */
bool initialLoadComplete = false; /**< Set after the first successful load. */
};
#endif // COCKATRICE_CARD_DATABASE_LOADER_H

6
libcockatrice_card/libcockatrice/card/database/parser/card_database_parser.cpp
View file

@ -30,6 +30,10 @@ CardSetPtr ICardDatabaseParser::internalAddSet(const QString &setName,
newSet->setPriority(priority);
sets.insert(setName, newSet);
emit addSet(newSet);
if (targetData) {
targetData->sets.insert(setName, newSet);
} else {
emit addSet(newSet);
}
return newSet;
}

25
libcockatrice_card/libcockatrice/card/database/parser/card_database_parser.h
View file

@ -2,6 +2,7 @@
#define CARDDATABASE_PARSER_H
#include "../../card_info.h"
#include "../card_database_data.h"
#include <QIODevice>
#include <QString>
@ -37,6 +38,15 @@ public:
*/
virtual void parseFile(QIODevice &device) = 0;
/**
* @brief Parses a database file into the given snapshot, without emitting
* any signals. Used for background loads that swap the snapshot into
* the live database once complete.
* @param device QIODevice representing the file content.
* @param data Target snapshot to populate.
*/
virtual void parseFileInto(QIODevice &device, CardDatabaseData &data) = 0;
/**
* @brief Saves card and set data to a file.
* @param _formats
@ -62,6 +72,21 @@ protected:
static SetNameMap sets;
ICardSetPriorityController *cardSetPriorityController;
/**
* @brief Snapshot the current parse is filling, or nullptr when emitting signals.
*
* This is an implicit-inheritance-via-member-variable pattern: parseFileInto()
* sets this before delegating to parseFile(), and loadCardsFromXml() /
* loadFormats() check it to decide between direct insertion and signal
* emission. Safe under the current model because loadFromFileMutex serialises
* all parser access and parseFile() is never re-entered. If a future change
* adds parallel parsing or a second parseFileInto() call within parseFile(),
* this pointer would race -- at that point refactor to pass CardDatabaseData*
* through the call chain instead. However, we are mostly RAM, not CPU bound on
* DB startup at this point so there's not much point to parallel parsing.
*/
CardDatabaseData *targetData = nullptr;
/**
* @brief Internal helper to add a set to the global set cache.
* @param setName Short set name.

34
libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_3.cpp
View file

@ -79,6 +79,13 @@ void CockatriceXml3Parser::parseFile(QIODevice &device)
}
}
void CockatriceXml3Parser::parseFileInto(QIODevice &device, CardDatabaseData &data)
{
targetData = &data;
parseFile(device);
targetData = nullptr;
}
void CockatriceXml3Parser::loadSetsFromXml(QXmlStreamReader &xml)
{
while (!xml.atEnd()) {
@ -222,25 +229,27 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
// behaviour. Without this check, disabling a set has no effect on v3 databases.
if (set->getEnabled()) {
PrintingInfo setInfo(set);
QVariantHash printingProps;
if (attrs.hasAttribute("muId")) {
setInfo.setProperty("muid", attrs.value("muId").toString());
printingProps.insert("muid", attrs.value("muId").toString());
}
if (attrs.hasAttribute("uuId")) {
setInfo.setProperty("uuid", attrs.value("uuId").toString());
printingProps.insert("uuid", attrs.value("uuId").toString());
}
if (attrs.hasAttribute("picURL")) {
setInfo.setProperty("picurl", attrs.value("picURL").toString());
printingProps.insert("picurl", attrs.value("picURL").toString());
}
if (attrs.hasAttribute("num")) {
setInfo.setProperty("num", attrs.value("num").toString());
printingProps.insert("num", attrs.value("num").toString());
}
if (attrs.hasAttribute("rarity")) {
setInfo.setProperty("rarity", attrs.value("rarity").toString());
printingProps.insert("rarity", attrs.value("rarity").toString());
}
setInfo.setProperties(printingProps);
_sets[setName].append(setInfo);
}
// related cards
@ -299,7 +308,20 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
.upsideDownArt = upsideDown};
CardInfoPtr newCard = CardInfo::newInstance(name, text, isToken, properties, relatedCards,
reverseRelatedCards, _sets, attributes);
emit addCard(newCard);
if (targetData) {
if (auto existing = targetData->cards.value(name)) {
for (const auto &printings : newCard->getSets()) {
for (const auto &printing : printings) {
existing->addToSet(printing.getSet(), printing);
}
}
} else {
targetData->cards.insert(name, newCard);
targetData->simpleNameCards.insert(newCard->getSimpleName(), newCard);
}
} else {
emit addCard(newCard);
}
}
}
}

7
libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_3.h
View file

@ -43,6 +43,13 @@ public:
*/
void parseFile(QIODevice &device) override;
/**
* @brief Parse the XML database into the given snapshot.
* @param device Open QIODevice positioned at start of file.
* @param data Target snapshot to populate.
*/
void parseFileInto(QIODevice &device, CardDatabaseData &data) override;
/**
* @brief Save sets and cards back to an XML3 file.
*/

34
libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_4.cpp
View file

@ -82,6 +82,13 @@ void CockatriceXml4Parser::parseFile(QIODevice &device)
}
}
void CockatriceXml4Parser::parseFileInto(QIODevice &device, CardDatabaseData &data)
{
targetData = &data;
parseFile(device);
targetData = nullptr;
}
static QSharedPointer<FormatRules> parseFormat(QXmlStreamReader &xml)
{
auto rulesPtr = FormatRulesPtr(new FormatRules());
@ -187,7 +194,11 @@ void CockatriceXml4Parser::loadFormats(QXmlStreamReader &xml)
if (xml.name().toString() == "format") {
auto rulesPtr = parseFormat(xml);
emit addFormat(rulesPtr);
if (targetData) {
targetData->formats.insert(rulesPtr->formatName.toLower(), rulesPtr);
} else {
emit addFormat(rulesPtr);
}
}
}
}
@ -304,13 +315,15 @@ void CockatriceXml4Parser::loadCardsFromXml(QXmlStreamReader &xml)
auto set = internalAddSet(setName);
if (set->getEnabled()) {
PrintingInfo printingInfo(set);
QVariantHash printingProps;
for (QXmlStreamAttribute attr : attrs) {
QString attrName = attr.name().toString();
if (attrName == "picURL") {
attrName = "picurl";
}
printingInfo.setProperty(attrName, attr.value().toString());
printingProps.insert(attrName, attr.value().toString());
}
printingInfo.setProperties(printingProps);
// This is very much a hack and not the right place to
// put this check, as it requires a reload of Cockatrice
@ -389,7 +402,22 @@ void CockatriceXml4Parser::loadCardsFromXml(QXmlStreamReader &xml)
.upsideDownArt = upsideDown};
CardInfoPtr newCard = CardInfo::newInstance(name, text, isToken, properties, relatedCards,
reverseRelatedCards, _sets, attributes);
emit addCard(newCard);
if (targetData) {
// Mirror CardDatabase::addCard: if a card with this name already
// exists, merge the new printings into it instead of replacing.
if (auto existing = targetData->cards.value(name)) {
for (const auto &printings : newCard->getSets()) {
for (const auto &printing : printings) {
existing->addToSet(printing.getSet(), printing);
}
}
} else {
targetData->cards.insert(name, newCard);
targetData->simpleNameCards.insert(newCard->getSimpleName(), newCard);
}
} else {
emit addCard(newCard);
}
}
}
}

7
libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_4.h
View file

@ -47,6 +47,13 @@ public:
*/
void parseFile(QIODevice &device) override;
/**
* @brief Parse the XML database into the given snapshot.
* @param device Open QIODevice positioned at start of file.
* @param data Target snapshot to populate.
*/
void parseFileInto(QIODevice &device, CardDatabaseData &data) override;
/**
* @brief Save sets and cards back to an XML4 file.
*/