[Card Database] Improve loading times through binary cache (#7051)

* [Card Database] Improve loading times through binary cache

Took 10 minutes


Took 9 minutes

Took 16 seconds

* [Card Database] Remove lib qt include

Took 18 minutes


Took 14 seconds

* Downgrade to 6.3 datastream

Took 5 minutes

* go up to 6.4 datastream

Took 1 minute

* Address comments

* Small bug fixes

Took 20 minutes


Took 10 seconds

* More fixes.

Took 4 minutes


Took 4 seconds

* Even more fixes.

Took 11 minutes


Took 4 seconds

* More fixes.

Took 6 minutes

Took 26 seconds

Took 8 minutes

* Namespace instead of class

Took 6 minutes

---------

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>

libcockatrice_card/libcockatrice/card/database/parser/card_database_parser.cpp

@@ -30,6 +30,10 @@ CardSetPtr ICardDatabaseParser::internalAddSet(const QString &setName,
     newSet->setPriority(priority);
 
     sets.insert(setName, newSet);
-    emit addSet(newSet);
+    if (targetData) {
+        targetData->sets.insert(setName, newSet);
+    } else {
+        emit addSet(newSet);
+    }
     return newSet;
 }

libcockatrice_card/libcockatrice/card/database/parser/card_database_parser.h

@@ -2,6 +2,7 @@
 #define CARDDATABASE_PARSER_H
 
 #include "../../card_info.h"
+#include "../card_database_data.h"
 
 #include <QIODevice>
 #include <QString>
@@ -37,6 +38,15 @@ public:
      */
     virtual void parseFile(QIODevice &device) = 0;
 
+    /**
+     * @brief Parses a database file into the given snapshot, without emitting
+     *        any signals. Used for background loads that swap the snapshot into
+     *        the live database once complete.
+     * @param device QIODevice representing the file content.
+     * @param data Target snapshot to populate.
+     */
+    virtual void parseFileInto(QIODevice &device, CardDatabaseData &data) = 0;
+
     /**
      * @brief Saves card and set data to a file.
      * @param _formats
@@ -62,6 +72,21 @@ protected:
     static SetNameMap sets;
     ICardSetPriorityController *cardSetPriorityController;
 
+    /**
+     * @brief Snapshot the current parse is filling, or nullptr when emitting signals.
+     *
+     * This is an implicit-inheritance-via-member-variable pattern: parseFileInto()
+     * sets this before delegating to parseFile(), and loadCardsFromXml() /
+     * loadFormats() check it to decide between direct insertion and signal
+     * emission.  Safe under the current model because loadFromFileMutex serialises
+     * all parser access and parseFile() is never re-entered.  If a future change
+     * adds parallel parsing or a second parseFileInto() call within parseFile(),
+     * this pointer would race -- at that point refactor to pass CardDatabaseData*
+     * through the call chain instead. However, we are mostly RAM, not CPU bound on
+     * DB startup at this point so there's not much point to parallel parsing.
+     */
+    CardDatabaseData *targetData = nullptr;
+
     /**
      * @brief Internal helper to add a set to the global set cache.
      * @param setName Short set name.

libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_3.cpp

@@ -79,6 +79,13 @@ void CockatriceXml3Parser::parseFile(QIODevice &device)
     }
 }
 
+void CockatriceXml3Parser::parseFileInto(QIODevice &device, CardDatabaseData &data)
+{
+    targetData = &data;
+    parseFile(device);
+    targetData = nullptr;
+}
+
 void CockatriceXml3Parser::loadSetsFromXml(QXmlStreamReader &xml)
 {
     while (!xml.atEnd()) {
@@ -222,25 +229,27 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
                     // behaviour. Without this check, disabling a set has no effect on v3 databases.
                     if (set->getEnabled()) {
                         PrintingInfo setInfo(set);
+                        QVariantHash printingProps;
                         if (attrs.hasAttribute("muId")) {
-                            setInfo.setProperty("muid", attrs.value("muId").toString());
+                            printingProps.insert("muid", attrs.value("muId").toString());
                         }
 
                         if (attrs.hasAttribute("uuId")) {
-                            setInfo.setProperty("uuid", attrs.value("uuId").toString());
+                            printingProps.insert("uuid", attrs.value("uuId").toString());
                         }
 
                         if (attrs.hasAttribute("picURL")) {
-                            setInfo.setProperty("picurl", attrs.value("picURL").toString());
+                            printingProps.insert("picurl", attrs.value("picURL").toString());
                         }
 
                         if (attrs.hasAttribute("num")) {
-                            setInfo.setProperty("num", attrs.value("num").toString());
+                            printingProps.insert("num", attrs.value("num").toString());
                         }
 
                         if (attrs.hasAttribute("rarity")) {
-                            setInfo.setProperty("rarity", attrs.value("rarity").toString());
+                            printingProps.insert("rarity", attrs.value("rarity").toString());
                         }
+                        setInfo.setProperties(printingProps);
                         _sets[setName].append(setInfo);
                     }
                     // related cards
@@ -299,7 +308,20 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
                                                  .upsideDownArt = upsideDown};
             CardInfoPtr newCard = CardInfo::newInstance(name, text, isToken, properties, relatedCards,
                                                         reverseRelatedCards, _sets, attributes);
-            emit addCard(newCard);
+            if (targetData) {
+                if (auto existing = targetData->cards.value(name)) {
+                    for (const auto &printings : newCard->getSets()) {
+                        for (const auto &printing : printings) {
+                            existing->addToSet(printing.getSet(), printing);
+                        }
+                    }
+                } else {
+                    targetData->cards.insert(name, newCard);
+                    targetData->simpleNameCards.insert(newCard->getSimpleName(), newCard);
+                }
+            } else {
+                emit addCard(newCard);
+            }
         }
     }
 }

libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_3.h

@@ -43,6 +43,13 @@ public:
      */
     void parseFile(QIODevice &device) override;
 
+    /**
+     * @brief Parse the XML database into the given snapshot.
+     * @param device Open QIODevice positioned at start of file.
+     * @param data Target snapshot to populate.
+     */
+    void parseFileInto(QIODevice &device, CardDatabaseData &data) override;
+
     /**
      * @brief Save sets and cards back to an XML3 file.
      */

libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_4.cpp

@@ -82,6 +82,13 @@ void CockatriceXml4Parser::parseFile(QIODevice &device)
     }
 }
 
+void CockatriceXml4Parser::parseFileInto(QIODevice &device, CardDatabaseData &data)
+{
+    targetData = &data;
+    parseFile(device);
+    targetData = nullptr;
+}
+
 static QSharedPointer<FormatRules> parseFormat(QXmlStreamReader &xml)
 {
     auto rulesPtr = FormatRulesPtr(new FormatRules());
@@ -187,7 +194,11 @@ void CockatriceXml4Parser::loadFormats(QXmlStreamReader &xml)
 
         if (xml.name().toString() == "format") {
             auto rulesPtr = parseFormat(xml);
-            emit addFormat(rulesPtr);
+            if (targetData) {
+                targetData->formats.insert(rulesPtr->formatName.toLower(), rulesPtr);
+            } else {
+                emit addFormat(rulesPtr);
+            }
         }
     }
 }
@@ -304,13 +315,15 @@ void CockatriceXml4Parser::loadCardsFromXml(QXmlStreamReader &xml)
                     auto set = internalAddSet(setName);
                     if (set->getEnabled()) {
                         PrintingInfo printingInfo(set);
+                        QVariantHash printingProps;
                         for (QXmlStreamAttribute attr : attrs) {
                             QString attrName = attr.name().toString();
                             if (attrName == "picURL") {
                                 attrName = "picurl";
                             }
-                            printingInfo.setProperty(attrName, attr.value().toString());
+                            printingProps.insert(attrName, attr.value().toString());
                         }
+                        printingInfo.setProperties(printingProps);
 
                         // This is very much a hack and not the right place to
                         // put this check, as it requires a reload of Cockatrice
@@ -389,7 +402,22 @@ void CockatriceXml4Parser::loadCardsFromXml(QXmlStreamReader &xml)
                                                  .upsideDownArt = upsideDown};
             CardInfoPtr newCard = CardInfo::newInstance(name, text, isToken, properties, relatedCards,
                                                         reverseRelatedCards, _sets, attributes);
-            emit addCard(newCard);
+            if (targetData) {
+                // Mirror CardDatabase::addCard: if a card with this name already
+                // exists, merge the new printings into it instead of replacing.
+                if (auto existing = targetData->cards.value(name)) {
+                    for (const auto &printings : newCard->getSets()) {
+                        for (const auto &printing : printings) {
+                            existing->addToSet(printing.getSet(), printing);
+                        }
+                    }
+                } else {
+                    targetData->cards.insert(name, newCard);
+                    targetData->simpleNameCards.insert(newCard->getSimpleName(), newCard);
+                }
+            } else {
+                emit addCard(newCard);
+            }
         }
     }
 }

libcockatrice_card/libcockatrice/card/database/parser/cockatrice_xml_4.h

@@ -47,6 +47,13 @@ public:
      */
     void parseFile(QIODevice &device) override;
 
+    /**
+     * @brief Parse the XML database into the given snapshot.
+     * @param device Open QIODevice positioned at start of file.
+     * @param data Target snapshot to populate.
+     */
+    void parseFileInto(QIODevice &device, CardDatabaseData &data) override;
+
     /**
      * @brief Save sets and cards back to an XML4 file.
      */