Cockatrice/libcockatrice_card/libcockatrice/card/database/card_database_loader.cpp

#include "card_database_loader.h"

#include "card_database.h"
#include "card_database_cache.h"
#include "libcockatrice/card/card_info.h"
#include "libcockatrice/card/database/card_database_data.h"
#include "libcockatrice/card/set/card_set.h"
#include "libcockatrice/interfaces/interface_card_database_path_provider.h"
#include "libcockatrice/interfaces/interface_card_set_priority_controller.h"
#include "parser/card_database_parser.h"
#include "parser/cockatrice_xml_3.h"
#include "parser/cockatrice_xml_4.h"

#include <QByteArray>
#include <QCoreApplication>
#include <QCryptographicHash>
#include <QDateTime>
#include <QDebug>
#include <QDir>
#include <QDirIterator>
#include <QFile>
#include <QFileInfo>
#include <QIODevice>
#include <QMessageLogger>
#include <QMutexLocker>
#include <QSharedPointer>
#include <QTime>
#include <qalgorithms.h>
#include <utility>

class ICardPreferenceProvider;

CardDatabaseLoader::CardDatabaseLoader(QObject *parent,
                                       CardDatabase *db,
                                       ICardDatabasePathProvider *_pathProvider,
                                       ICardSetPriorityController *_priorityController)
    : QObject(parent), database(db), pathProvider(_pathProvider), priorityController(_priorityController)
{
    // instantiate available parsers here
    availableParsers << new CockatriceXml4Parser(_priorityController);
    availableParsers << new CockatriceXml3Parser(_priorityController);

    // The load path parses into a snapshot and never emits per-card signals;
    // the finished snapshot is swapped into the live database on the GUI thread.

    // when SettingsCache's path changes, trigger reloads
    connect(pathProvider, &ICardDatabasePathProvider::cardDatabasePathChanged, this, [this] { loadCardDatabases(); });
}

CardDatabaseLoader::~CardDatabaseLoader()
{
    qDeleteAll(availableParsers);
    availableParsers.clear();
}

LoadStatus CardDatabaseLoader::loadFromFile(const QString &fileName, CardDatabaseData &data)
{
    QFile file(fileName);
    if (!file.open(QIODevice::ReadOnly)) {
        return FileError;
    }

    for (auto parser : availableParsers) {
        file.reset();
        if (parser->getCanParseFile(fileName, file)) {
            file.reset();
            parser->parseFileInto(file, data);
            return Ok;
        }
    }

    return Invalid;
}

LoadStatus CardDatabaseLoader::loadCardDatabase(const QString &path, CardDatabaseData &data)
{
    auto startTime = QTime::currentTime();
    LoadStatus tempLoadStatus = NotLoaded;
    if (!path.isEmpty()) {
        QMutexLocker locker(loadFromFileMutex);
        tempLoadStatus = loadFromFile(path, data);
    }

    int msecs = startTime.msecsTo(QTime::currentTime());
    qCInfo(CardDatabaseLoadingLog) << "Loaded card database: Path =" << path << "Status =" << tempLoadStatus
                                   << "Cards =" << data.cards.size() << "Sets =" << data.sets.size()
                                   << QString("%1ms").arg(msecs);

    return tempLoadStatus;
}

LoadStatus CardDatabaseLoader::loadCardDatabases(bool ignoreCache)
{
    return doLoadCardDatabases(ignoreCache);
}

LoadStatus CardDatabaseLoader::doLoadCardDatabases(bool ignoreCache)
{
    QMutexLocker locker(reloadDatabaseMutex);

    if (!database) {
        qCWarning(CardDatabaseLoadingLog) << "Loader has no database pointer";
        emit loadingFailed();
        return FileError;
    }
    emit loadingStarted();
    qCInfo(CardDatabaseLoadingLog) << "Card Database Loading Started";

    ICardDatabaseParser::clearSetlist();

    CardDatabaseData data;
    LoadStatus loadStatus = NotLoaded;

    // Try the binary cache first: a cache hit avoids re-parsing the (large) XML.
    // An explicit reload skips the lookup so the files on disk are always
    // re-read, even when their size/mtime hash still matches the cache.
    const QStringList customPaths = collectCustomDatabasePaths();
    const QByteArray sourceHash = computeSourceHash(customPaths);
    if (ignoreCache) {
        qCInfo(CardDatabaseLoadingLog) << "Ignoring binary cache, re-parsing source files";
    }
    if (!ignoreCache && loadFromCache(data, sourceHash)) {
        qCInfo(CardDatabaseLoadingLog) << "Loaded card database from binary cache";
        loadStatus = Ok;
    } else {
        loadStatus = loadCardDatabase(pathProvider->getCardDatabasePath(), data); // main card database
        if (loadStatus == Ok) {
            loadCardDatabase(pathProvider->getTokenDatabasePath(), data);       // tokens database
            loadCardDatabase(pathProvider->getSpoilerCardDatabasePath(), data); // spoilers database

            // find all custom card databases, recursively & following symlinks
            // then load them alphabetically
            for (int i = 0; i < customPaths.size(); ++i) {
                const auto &p = customPaths.at(i);
                qCInfo(CardDatabaseLoadingLog) << "Loading Custom Set" << i << "(" << p << ")";
                loadCardDatabase(p, data);
            }

            if (!saveToCache(data, sourceHash)) {
                qCWarning(CardDatabaseLoadingLog) << "Failed to write binary cache to" << cachePath();
            }
        }
    }

    // AFTER all the cards have been loaded: resolve the reverse-related tags
    // against the fully-built snapshot.
    if (loadStatus == Ok) {
        database->refreshCachedReverseRelatedCards(data.cards);

        qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Success";
        emit databaseDataReady(std::move(data));
        emit loadingFinished();
        // During the front-loaded parse in main() this runs before MainWindow
        // exists, so cardDatabaseNewSetsFound / cardDatabaseAllNewSetsEnabled
        // would be emitted with no receivers.  Skip the check on the first load;
        // MainWindow::startupConfigCheck() calls checkUnknownSets() once its
        // signal connections are live.  On subsequent reloads (e.g. path change)
        // MainWindow exists and signals have live receivers.
        if (initialLoadComplete) {
            database->checkUnknownSets();
        }
        initialLoadComplete = true;
    } else {
        qCInfo(CardDatabaseLoadingSuccessOrFailureLog) << "Card Database Loading Failed";
        emit loadingFailed(); // bring up the settings dialog
    }

    return loadStatus;
}

QString CardDatabaseLoader::cachePath() const
{
    return pathProvider->getCardDatabasePath() + ".cache";
}

QByteArray CardDatabaseLoader::computeSourceHash(const QStringList &customPaths) const
{
    // Hash over the paths, sizes and modification times of every input file so
    // the cache invalidates when any source changes. Cheap (no content read).
    QCryptographicHash hash(QCryptographicHash::Sha256);

    // Include the application version so parser changes automatically invalidate
    // old caches even when the XML files are byte-identical.
    hash.addData(QCoreApplication::applicationVersion().toUtf8());
    hash.addData(QByteArray(1, '\0'));

    // Include parser schema version to invalidate caches when parser behavior changes.
    // Increment this if the parser logic changes in a way that affects cached data.
    static const char SCHEMA_VERSION[] = "carddb-parse-schema-v1";
    hash.addData(QByteArray(SCHEMA_VERSION, sizeof(SCHEMA_VERSION) - 1));
    hash.addData(QByteArray(1, '\0'));

    const QStringList inputs = QStringList()
                               << pathProvider->getCardDatabasePath() << pathProvider->getTokenDatabasePath()
                               << pathProvider->getSpoilerCardDatabasePath() << customPaths;
    for (const QString &path : inputs) {
        QFileInfo info(path);
        if (info.exists()) {
            hash.addData(path.toUtf8());
            hash.addData(QByteArray(1, '\0'));
            hash.addData(QByteArray::number(info.size()));
            hash.addData(QByteArray(1, '\0'));
            hash.addData(QByteArray::number(info.lastModified().toSecsSinceEpoch()));
            hash.addData(QByteArray(1, '\0'));
        }
    }

    // The parsers drop printings of disabled sets at parse time, so the cached
    // snapshot depends on enablement: fold the enabled set names in so that
    // toggling a set invalidates the cache and forces a re-parse.
    if (priorityController) {
        const QStringList enabledSets = priorityController->getEnabledSetNames();
        for (const QString &setName : enabledSets) {
            hash.addData(setName.toUtf8());
            hash.addData(QByteArray(1, '\0'));
        }
    }
    return hash.result();
}

bool CardDatabaseLoader::loadFromCache(CardDatabaseData &data, const QByteArray &sourceHash)
{
    if (sourceHash.isEmpty()) {
        return false;
    }
    return CardDatabaseCache::read(cachePath(), data, sourceHash, priorityController);
}

bool CardDatabaseLoader::saveToCache(const CardDatabaseData &data, const QByteArray &sourceHash)
{
    if (sourceHash.isEmpty()) {
        return false;
    }
    return CardDatabaseCache::write(cachePath(), data, sourceHash);
}

QStringList CardDatabaseLoader::collectCustomDatabasePaths() const
{
    QDirIterator it(pathProvider->getCustomCardDatabasePath(), {"*.xml"}, QDir::Files,
                    QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);

    QStringList paths;
    while (it.hasNext()) {
        paths << it.next();
    }
    paths.sort();
    return paths;
}

bool CardDatabaseLoader::saveCustomTokensToFile()
{
    if (!database) {
        qCWarning(CardDatabaseLog) << "saveCustomTokensToFile: database pointer missing";
        return false;
    }

    QString fileName = pathProvider->getCustomCardDatabasePath() + "/" + CardSet::TOKENS_SETNAME + ".xml";

    SetNameMap tmpSets;
    CardSetPtr customTokensSet = database->getSet(CardSet::TOKENS_SETNAME);
    tmpSets.insert(CardSet::TOKENS_SETNAME, customTokensSet);

    CardNameMap tmpCards;
    for (const CardInfoPtr &card : database->cards) {
        if (card->getSets().contains(CardSet::TOKENS_SETNAME)) {
            tmpCards.insert(card->getName(), card);
        }
    }

    availableParsers.first()->saveToFile(FormatRulesNameMap(), tmpSets, tmpCards, fileName);
    return true;
}