mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-07-29 04:10:23 -07:00
[Card Database] Improve loading times through binary cache (#7051)
* [Card Database] Improve loading times through binary cache Took 10 minutes Took 9 minutes Took 16 seconds * [Card Database] Remove lib qt include Took 18 minutes Took 14 seconds * Downgrade to 6.3 datastream Took 5 minutes * go up to 6.4 datastream Took 1 minute * Address comments * Small bug fixes Took 20 minutes Took 10 seconds * More fixes. Took 4 minutes Took 4 seconds * Even more fixes. Took 11 minutes Took 4 seconds * More fixes. Took 6 minutes Took 26 seconds Took 8 minutes * Namespace instead of class Took 6 minutes --------- Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>
This commit is contained in:
BruebachL
GitHub
parent
4bb8831531
commit
749223c2dc
31 changed files with 1604 additions and 106 deletions
|
|
@ -1,12 +1,18 @@
|
|||
#include "card_database_loader.h"
|
||||
|
||||
#include "card_database.h"
|
||||
#include "card_database_cache.h"
|
||||
#include "parser/card_database_parser.h"
|
||||
#include "parser/cockatrice_xml_3.h"
|
||||
#include "parser/cockatrice_xml_4.h"
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QCoreApplication>
|
||||
#include <QCryptographicHash>
|
||||
#include <QDebug>
|
||||
#include <QDirIterator>
|
||||
#include <QFile>
|
||||
#include <QFileInfo>
|
||||
#include <QTime>
|
||||
|
||||
CardDatabaseLoader::CardDatabaseLoader(QObject *parent,
|
||||
|
|
@ -14,18 +20,14 @@ CardDatabaseLoader::CardDatabaseLoader(QObject *parent,
|
|||
ICardDatabasePathProvider *_pathProvider,
|
||||
ICardPreferenceProvider *_preferenceProvider,
|
||||
ICardSetPriorityController *_priorityController)
|
||||
: QObject(parent), database(db), pathProvider(_pathProvider)
|
||||
: QObject(parent), database(db), pathProvider(_pathProvider), priorityController(_priorityController)
|
||||
{
|
||||
// instantiate available parsers here and connect them to the database
|
||||
// instantiate available parsers here
|
||||
availableParsers << new CockatriceXml4Parser(_preferenceProvider, _priorityController);
|
||||
availableParsers << new CockatriceXml3Parser(_priorityController);
|
||||
|
||||
for (auto *p : availableParsers) {
|
||||
// connect parser outputs to the database adders
|
||||
connect(p, &ICardDatabaseParser::addCard, database, &CardDatabase::addCard, Qt::DirectConnection);
|
||||
connect(p, &ICardDatabaseParser::addSet, database, &CardDatabase::addSet, Qt::DirectConnection);
|
||||
connect(p, &ICardDatabaseParser::addFormat, database, &CardDatabase::addFormat, Qt::DirectConnection);
|
||||
}
|
||||
// The load path parses into a snapshot and never emits per-card signals;
|
||||
// the finished snapshot is swapped into the live database on the GUI thread.
|
||||
|
||||
// when SettingsCache's path changes, trigger reloads
|
||||
connect(pathProvider, &ICardDatabasePathProvider::cardDatabasePathChanged, this,
|
||||
|
|
@ -38,7 +40,7 @@ CardDatabaseLoader::~CardDatabaseLoader()
|
|||
availableParsers.clear();
|
||||
}
|
||||
|
||||
LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
|
||||
LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName, CardDatabaseData &data)
|
||||
{
|
||||
QFile file(fileName);
|
||||
if (!file.open(QIODevice::ReadOnly)) {
|
||||
|
|
@ -49,7 +51,7 @@ LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
|
|||
file.reset();
|
||||
if (parser->getCanParseFile(fileName, file)) {
|
||||
file.reset();
|
||||
parser->parseFile(file);
|
||||
parser->parseFileInto(file, data);
|
||||
return Ok;
|
||||
}
|
||||
}
|
||||
|
|
@ -57,24 +59,29 @@ LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName)
|
|||
return Invalid;
|
||||
}
|
||||
|
||||
LoadStatus CardDatabaseLoader::loadCardDatabase(const QString &path)
|
||||
LoadStatus CardDatabaseLoader::loadCardDatabase(const QString &path, CardDatabaseData &data)
|
||||
{
|
||||
auto startTime = QTime::currentTime();
|
||||
LoadStatus tempLoadStatus = NotLoaded;
|
||||
if (!path.isEmpty()) {
|
||||
QMutexLocker locker(loadFromFileMutex);
|
||||
tempLoadStatus = loadFromFile(path);
|
||||
tempLoadStatus = loadFromFile(path, data);
|
||||
}
|
||||
|
||||
int msecs = startTime.msecsTo(QTime::currentTime());
|
||||
qCInfo(CardDatabaseLoadingLog) << "Loaded card database: Path =" << path << "Status =" << tempLoadStatus
|
||||
<< "Cards =" << (database ? database->cards.size() : 0)
|
||||
<< "Sets =" << (database ? database->sets.size() : 0) << QString("%1ms").arg(msecs);
|
||||
<< "Cards =" << data.cards.size() << "Sets =" << data.sets.size()
|
||||
<< QString("%1ms").arg(msecs);
|
||||
|
||||
return tempLoadStatus;
|
||||
}
|
||||
|
||||
LoadStatus CardDatabaseLoader::loadCardDatabases()
|
||||
{
|
||||
return doLoadCardDatabases();
|
||||
}
|
||||
|
||||
LoadStatus CardDatabaseLoader::doLoadCardDatabases()
|
||||
{
|
||||
QMutexLocker locker(reloadDatabaseMutex);
|
||||
|
||||
|
|
@ -86,31 +93,55 @@ LoadStatus CardDatabaseLoader::loadCardDatabases()
|
|||
emit loadingStarted();
|
||||
qCInfo(CardDatabaseLoadingLog) << "Card Database Loading Started";
|
||||
|
||||
database->clear(); // remove old db
|
||||
ICardDatabaseParser::clearSetlist();
|
||||
|
||||
LoadStatus loadStatus = loadCardDatabase(pathProvider->getCardDatabasePath()); // load main card database
|
||||
loadCardDatabase(pathProvider->getTokenDatabasePath()); // load tokens database
|
||||
loadCardDatabase(pathProvider->getSpoilerCardDatabasePath()); // load spoilers database
|
||||
CardDatabaseData data;
|
||||
LoadStatus loadStatus = NotLoaded;
|
||||
|
||||
// find all custom card databases, recursively & following symlinks
|
||||
// then load them alphabetically
|
||||
// Try the binary cache first: a cache hit avoids re-parsing the (large) XML.
|
||||
const QStringList customPaths = collectCustomDatabasePaths();
|
||||
for (int i = 0; i < customPaths.size(); ++i) {
|
||||
const auto &p = customPaths.at(i);
|
||||
qCInfo(CardDatabaseLoadingLog) << "Loading Custom Set" << i << "(" << p << ")";
|
||||
loadCardDatabase(p);
|
||||
const QByteArray sourceHash = computeSourceHash(customPaths);
|
||||
if (loadFromCache(data, sourceHash)) {
|
||||
qCInfo(CardDatabaseLoadingLog) << "Loaded card database from binary cache";
|
||||
loadStatus = Ok;
|
||||
} else {
|
||||
loadStatus = loadCardDatabase(pathProvider->getCardDatabasePath(), data); // main card database
|
||||
if (loadStatus == Ok) {
|
||||
loadCardDatabase(pathProvider->getTokenDatabasePath(), data); // tokens database
|
||||
loadCardDatabase(pathProvider->getSpoilerCardDatabasePath(), data); // spoilers database
|
||||
|
||||
// find all custom card databases, recursively & following symlinks
|
||||
// then load them alphabetically
|
||||
for (int i = 0; i < customPaths.size(); ++i) {
|
||||
const auto &p = customPaths.at(i);
|
||||
qCInfo(CardDatabaseLoadingLog) << "Loading Custom Set" << i << "(" << p << ")";
|
||||
loadCardDatabase(p, data);
|
||||
}
|
||||
|
||||
if (!saveToCache(data, sourceHash)) {
|
||||
qCWarning(CardDatabaseLoadingLog) << "Failed to write binary cache to" << cachePath();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AFTER all the cards have been loaded
|
||||
|
||||
// resolve the reverse-related tags
|
||||
|
||||
database->refreshCachedReverseRelatedCards();
|
||||
|
||||
// AFTER all the cards have been loaded: resolve the reverse-related tags
|
||||
// against the fully-built snapshot.
|
||||
if (loadStatus == Ok) {
|
||||
database->checkUnknownSets(); // update deck editors, etc
|
||||
database->refreshCachedReverseRelatedCards(data.cards);
|
||||
|
||||
qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Success";
|
||||
emit databaseDataReady(std::move(data));
|
||||
emit loadingFinished();
|
||||
// During the front-loaded parse in main() this runs before MainWindow
|
||||
// exists, so cardDatabaseNewSetsFound / cardDatabaseAllNewSetsEnabled
|
||||
// would be emitted with no receivers. Skip the check on the first load;
|
||||
// MainWindow::startupConfigCheck() calls checkUnknownSets() once its
|
||||
// signal connections are live. On subsequent reloads (e.g. path change)
|
||||
// MainWindow exists and signals have live receivers.
|
||||
if (initialLoadComplete) {
|
||||
database->checkUnknownSets();
|
||||
}
|
||||
initialLoadComplete = true;
|
||||
} else {
|
||||
qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Failed";
|
||||
emit loadingFailed(); // bring up the settings dialog
|
||||
|
|
@ -119,6 +150,55 @@ LoadStatus CardDatabaseLoader::loadCardDatabases()
|
|||
return loadStatus;
|
||||
}
|
||||
|
||||
QString CardDatabaseLoader::cachePath() const
|
||||
{
|
||||
return pathProvider->getCardDatabasePath() + ".cache";
|
||||
}
|
||||
|
||||
QByteArray CardDatabaseLoader::computeSourceHash(const QStringList &customPaths) const
|
||||
{
|
||||
// Hash over the paths, sizes and modification times of every input file so
|
||||
// the cache invalidates when any source changes. Cheap (no content read).
|
||||
QCryptographicHash hash(QCryptographicHash::Sha256);
|
||||
|
||||
// Include the application version so parser changes automatically invalidate
|
||||
// old caches even when the XML files are byte-identical.
|
||||
hash.addData(QCoreApplication::applicationVersion().toUtf8());
|
||||
hash.addData(QByteArray(1, '\0'));
|
||||
|
||||
const QStringList inputs = QStringList()
|
||||
<< pathProvider->getCardDatabasePath() << pathProvider->getTokenDatabasePath()
|
||||
<< pathProvider->getSpoilerCardDatabasePath() << customPaths;
|
||||
for (const QString &path : inputs) {
|
||||
QFileInfo info(path);
|
||||
if (info.exists()) {
|
||||
hash.addData(path.toUtf8());
|
||||
hash.addData(QByteArray(1, '\0'));
|
||||
hash.addData(QByteArray::number(info.size()));
|
||||
hash.addData(QByteArray(1, '\0'));
|
||||
hash.addData(QByteArray::number(info.lastModified().toSecsSinceEpoch()));
|
||||
hash.addData(QByteArray(1, '\0'));
|
||||
}
|
||||
}
|
||||
return hash.result();
|
||||
}
|
||||
|
||||
bool CardDatabaseLoader::loadFromCache(CardDatabaseData &data, const QByteArray &sourceHash)
|
||||
{
|
||||
if (sourceHash.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return CardDatabaseCache::read(cachePath(), data, sourceHash, priorityController);
|
||||
}
|
||||
|
||||
bool CardDatabaseLoader::saveToCache(const CardDatabaseData &data, const QByteArray &sourceHash)
|
||||
{
|
||||
if (sourceHash.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return CardDatabaseCache::write(cachePath(), data, sourceHash);
|
||||
}
|
||||
|
||||
QStringList CardDatabaseLoader::collectCustomDatabasePaths() const
|
||||
{
|
||||
QDirIterator it(pathProvider->getCustomCardDatabasePath(), {"*.xml"}, QDir::Files,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue