jeirmeister/Cockatrice
1
0
Fork 0
mirror of https://github.com/Cockatrice/Cockatrice.git synced 2026-11-11 12:33:25 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

[Oracle] Load card tags from a tag database during import

Browse source 
Add a TagDatabase loader that understands Scryfall's Oracle Tags bulk data and user-supplied tag files (tag objects, flat oracle-id or name maps, optional manifest, JSON or JSONL, gzip/xz/zip). The importer joins the slugs on identifiers.scryfallOracleId while the identifiers are still available, falls back to the card name, and unions them with any tags the source already provides. Split-card faces merge through the existing tag union path.

This replaces oracle/scripts/inject_scryfall_tags.py, which patched AllPrintings.json before import. Loading during import keeps tags on by default for every source and lets users pair Oracle with any tag database.
This commit is contained in:
Lukas Brübach 2026-11-08 19:59:04 +01:00
parent 0f45dfa12a
commit 77179f913e
9 changed files with 1145 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

82
tests/oracle/CMakeLists.txt
View file

@ -1,3 +1,35 @@
# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the tag database
# loader can decompress whatever format the tag source serves, and the benchmark can
# download and decompress whatever AllPrintings format the default URL selects.
# add_definitions() is directory-scoped and order-independent, so HAS_ZLIB/HAS_LZMA
# reach every test target below regardless of where they are declared.
set(_ORACLE_COMPRESSION_SOURCES)
set(_ORACLE_COMPRESSION_LIBRARIES)
find_package(ZLIB)
if(ZLIB_FOUND)
add_definitions("-DHAS_ZLIB")
list(APPEND _ORACLE_COMPRESSION_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp)
cockatrice_disable_sanitizers_for(${_ORACLE_COMPRESSION_SOURCES})
cockatrice_disable_coverage_for(${_ORACLE_COMPRESSION_SOURCES})
list(APPEND _ORACLE_COMPRESSION_LIBRARIES ${ZLIB_LIBRARIES})
include_directories(${ZLIB_INCLUDE_DIRS})
else()
message(STATUS "Oracle tests: zlib not found; gzip/zip tag databases and the zip download benchmark disabled")
endif()
find_package(LibLZMA)
if(LIBLZMA_FOUND)
add_definitions("-DHAS_LZMA")
list(APPEND _ORACLE_COMPRESSION_SOURCES ../../oracle/src/lzma/decompress.cpp)
cockatrice_disable_sanitizers_for(../../oracle/src/lzma/decompress.cpp)
cockatrice_disable_coverage_for(../../oracle/src/lzma/decompress.cpp)
list(APPEND _ORACLE_COMPRESSION_LIBRARIES ${LIBLZMA_LIBRARIES})
include_directories(${LIBLZMA_INCLUDE_DIRS})
else()
message(STATUS "Oracle tests: LibLZMA not found; xz tag databases and the xz download benchmark disabled")
endif()
cockatrice_add_test(NAME parse_cipt_test SOURCES ../../oracle/src/parsehelpers.cpp parse_cipt_test.cpp)
# Oracle importer unit tests
@ -9,42 +41,29 @@ cockatrice_add_test(
../../oracle/src/oracleimporter.cpp
../../oracle/src/parsehelpers.cpp
../../oracle/src/raw_json_scanner.cpp
../../oracle/src/tag_database.cpp
oracle_importer_test.cpp
${_ORACLE_COMPRESSION_SOURCES}
LIBS
libcockatrice_card
libcockatrice_interfaces
EXTRA_LIBS
${_ORACLE_COMPRESSION_LIBRARIES}
)
# Tag database format detection, parsing and parent expansion
cockatrice_add_test(
NAME
tag_database_test
SOURCES
../../oracle/src/tag_database.cpp
tag_database_test.cpp
${_ORACLE_COMPRESSION_SOURCES}
EXTRA_LIBS
${_ORACLE_COMPRESSION_LIBRARIES}
)
# Oracle importer benchmark tests (manual, not run in CI, incl. RAM benchmark)
# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark
# can download and decompress whatever AllPrintings format the default URL selects.
# add_definitions() is directory-scoped and order-independent, so HAS_ZLIB/HAS_LZMA also
# reach the two unit tests above. That is harmless: those macros gate code only in
# oracle/src/pages.cpp and oracle/src/pagetemplates.cpp, which no test target compiles.
find_package(ZLIB)
if(ZLIB_FOUND)
add_definitions("-DHAS_ZLIB")
set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp)
cockatrice_disable_sanitizers_for(${_ORACLE_BENCH_EXTRA_SOURCES})
cockatrice_disable_coverage_for(${_ORACLE_BENCH_EXTRA_SOURCES})
set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES})
include_directories(${ZLIB_INCLUDE_DIRS})
else()
message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled")
endif()
find_package(LibLZMA)
if(LIBLZMA_FOUND)
add_definitions("-DHAS_LZMA")
list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp)
cockatrice_disable_sanitizers_for(../../oracle/src/lzma/decompress.cpp)
cockatrice_disable_coverage_for(../../oracle/src/lzma/decompress.cpp)
list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES})
include_directories(${LIBLZMA_INCLUDE_DIRS})
else()
message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled")
endif()
cockatrice_add_manual_test(
NAME
oracle_importer_benchmark_test
@ -53,12 +72,13 @@ cockatrice_add_manual_test(
../../oracle/src/oracleimporter.cpp
../../oracle/src/parsehelpers.cpp
../../oracle/src/raw_json_scanner.cpp
../../oracle/src/tag_database.cpp
oracle_importer_benchmark_test.cpp
${_ORACLE_BENCH_EXTRA_SOURCES}
${_ORACLE_COMPRESSION_SOURCES}
LIBS
libcockatrice_card
libcockatrice_interfaces
EXTRA_LIBS
${_ORACLE_BENCH_EXTRA_LIBRARIES}
${_ORACLE_COMPRESSION_LIBRARIES}
MANUAL
)

95
tests/oracle/oracle_importer_test.cpp
View file

@ -1,5 +1,6 @@
#include "../../oracle/src/oracleimporter.h"
#include "../../oracle/src/raw_json_scanner.h"
#include "../../oracle/src/tag_database.h"
#include "libcockatrice/card/card_info.h"
#include "gtest/gtest.h"
@ -600,6 +601,100 @@ TEST_F(OracleImporterTest, SplitCardTagsAreUnioned)
ASSERT_EQ(result->getProperty("tags"), "card-advantage removal");
}
// ============================================================================
// Tag database join tests
// ============================================================================
TEST_F(OracleImporterTest, AppliesTagDatabaseByOracleId)
{
QJsonObject card = makeCard("Tagged By Id");
QJsonObject identifiers = card["identifiers"].toObject();
identifiers["scryfallOracleId"] = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
card["identifiers"] = identifiers;
QString error;
const auto database = TagDatabase::loadFromData(
R"json({"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa": ["removal", "ramp"]})json", TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
importer->setTagDatabase(database);
importer->importCardsFromSet(set, QJsonArray{card});
auto result = importer->getCardList().value("Tagged By Id");
ASSERT_FALSE(result.isNull());
ASSERT_EQ(result->getProperty("tags"), "ramp removal");
}
TEST_F(OracleImporterTest, AppliesTagDatabaseByNameWhenOracleIdIsMissing)
{
QJsonObject card = makeCard("Lightning Bolt");
QString error;
const auto database = TagDatabase::loadFromData(R"json({"Lightning Bolt": ["burn", "removal"]})json",
TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
importer->setTagDatabase(database);
importer->importCardsFromSet(set, QJsonArray{card});
auto result = importer->getCardList().value("Lightning Bolt");
ASSERT_FALSE(result.isNull());
ASSERT_EQ(result->getProperty("tags"), "burn removal");
}
TEST_F(OracleImporterTest, TagDatabaseUnionsWithSourceTags)
{
QJsonObject card = makeCard("Union Card");
QJsonObject identifiers = card["identifiers"].toObject();
identifiers["scryfallOracleId"] = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
card["identifiers"] = identifiers;
card["tags"] = QJsonArray{"ramp"};
QString error;
const auto database = TagDatabase::loadFromData(
R"json({"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa": ["removal", "ramp"]})json", TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
importer->setTagDatabase(database);
importer->importCardsFromSet(set, QJsonArray{card});
auto result = importer->getCardList().value("Union Card");
ASSERT_FALSE(result.isNull());
ASSERT_EQ(result->getProperty("tags"), "ramp removal");
}
TEST_F(OracleImporterTest, TagDatabaseSplitFacesUnionTheirTags)
{
QJsonObject face1 = makeCard("Fire // Ice");
face1["layout"] = "split";
face1["side"] = "a";
face1["faceName"] = "Fire";
QJsonObject identifiers1 = face1["identifiers"].toObject();
identifiers1["scryfallOracleId"] = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
face1["identifiers"] = identifiers1;
QJsonObject face2 = makeCard("Fire // Ice");
face2["layout"] = "split";
face2["side"] = "b";
face2["faceName"] = "Ice";
QJsonObject identifiers2 = face2["identifiers"].toObject();
identifiers2["scryfallOracleId"] = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
face2["identifiers"] = identifiers2;
QString error;
const auto database = TagDatabase::loadFromData(
R"json({"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa": ["removal"], "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb": ["card-advantage"]})json",
TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
importer->setTagDatabase(database);
importer->importCardsFromSet(set, QJsonArray{face1, face2});
auto result = importer->getCardList().value("Fire // Ice");
ASSERT_FALSE(result.isNull());
ASSERT_EQ(result->getProperty("tags"), "card-advantage removal");
}
// ============================================================================
// Card deduplication tests
// ============================================================================

254
tests/oracle/tag_database_test.cpp Normal file
View file

@ -0,0 +1,254 @@
#include "../../oracle/src/tag_database.h"
#include "gtest/gtest.h"
#include <QByteArray>
#include <QString>
#include <QStringList>
#include <memory>
#ifdef HAS_ZLIB
#include <zlib.h>
#endif
namespace
{
const QString ORACLE_A = QStringLiteral("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
const QString ORACLE_B = QStringLiteral("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb");
std::shared_ptr<TagDatabase> load(const QByteArray &data,
const TagDatabase::LoadOptions &options = TagDatabase::LoadOptions{},
QString *errorMessage = nullptr)
{
return TagDatabase::loadFromData(data, options, errorMessage);
}
QString keyedByOracleId(const QString &oracleId, const QString &slugs)
{
return QStringLiteral("{\"%1\": [%2]}").arg(oracleId, slugs);
}
#ifdef HAS_ZLIB
QByteArray gzipCompress(const QByteArray &input)
{
z_stream stream{};
if (deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
return {};
}
stream.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.constData()));
stream.avail_in = static_cast<uInt>(input.size());
QByteArray output;
char buffer[4096];
int result = Z_OK;
while (result == Z_OK) {
stream.next_out = reinterpret_cast<Bytef *>(buffer);
stream.avail_out = sizeof(buffer);
result = deflate(&stream, Z_FINISH);
output.append(buffer, static_cast<int>(sizeof(buffer) - stream.avail_out));
}
deflateEnd(&stream);
return result == Z_STREAM_END ? output : QByteArray();
}
#endif
} // namespace
TEST(TagDatabaseTest, ParsesScryfallJsonLines)
{
const QByteArray data = R"json(
{"object":"tag","id":"11111111-1111-1111-1111-111111111111","slug":"removal","type":"oracle","parent_ids":[],"taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","weight":"median"}]}
{"object":"tag","id":"22222222-2222-2222-2222-222222222222","slug":"ramp","type":"oracle","parent_ids":[],"taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","weight":"median"},{"oracle_id":"bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb","weight":"strong"}]}
)json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagCount(), 2);
EXPECT_EQ(database->taggedCardCount(), 2);
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"ramp", "removal"}));
EXPECT_EQ(database->tagsForCard(ORACLE_B, QString()), QStringList({"ramp"}));
}
TEST(TagDatabaseTest, ParsesScryfallJsonArray)
{
const QByteArray data = R"json([
{"object":"tag","id":"11111111-1111-1111-1111-111111111111","slug":"removal","type":"oracle","taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}]}
])json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"removal"}));
}
TEST(TagDatabaseTest, IgnoresNonOracleTagTypes)
{
const QByteArray data = R"json(
{"object":"tag","id":"11111111-1111-1111-1111-111111111111","slug":"removal","type":"oracle","taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}]}
{"object":"tag","id":"22222222-2222-2222-2222-222222222222","slug":"art-only","type":"art","taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}]}
)json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"removal"}));
}
TEST(TagDatabaseTest, ParsesFlatOracleIdMap)
{
const QByteArray data = keyedByOracleId(ORACLE_A, R"("Removal", "removal", " ramp ")").toUtf8();
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
// Mixed case, duplicates and surrounding whitespace are normalized.
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"ramp", "removal"}));
EXPECT_EQ(database->tagsForCard(ORACLE_A.toUpper(), QString()), QStringList({"ramp", "removal"}));
EXPECT_EQ(database->tagsForCard(ORACLE_B, QString()), QStringList());
}
TEST(TagDatabaseTest, ParsesFlatNameMap)
{
const QByteArray data = R"json({"Lightning Bolt": ["removal", "burn"]})json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
// Name lookups are case-insensitive; an absent oracle id does not block them.
EXPECT_EQ(database->tagsForCard(QString(), QStringLiteral("lightning bolt")), QStringList({"burn", "removal"}));
EXPECT_EQ(database->tagsForCard(ORACLE_A, QStringLiteral("Lightning Bolt")), QStringList({"burn", "removal"}));
}
TEST(TagDatabaseTest, ParsesManifestWithNameKey)
{
const QByteArray data =
R"json({"version": 2, "key": "name", "tags": {"Lightning Bolt": ["removal"], "zzz-not-a-uuid": ["burn"]}})json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(QString(), QStringLiteral("Lightning Bolt")), QStringList({"removal"}));
EXPECT_EQ(database->tagsForCard(QString(), QStringLiteral("ZZZ-not-a-uuid")), QStringList({"burn"}));
}
TEST(TagDatabaseTest, ParsesJsonLinesRows)
{
const QByteArray data = R"json(
{"oracle_id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", "tags": ["removal"]}
{"name": "Lightning Bolt", "tags": ["burn", "removal"]}
)json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"removal"}));
EXPECT_EQ(database->tagsForCard(QString(), QStringLiteral("Lightning Bolt")), QStringList({"burn", "removal"}));
}
TEST(TagDatabaseTest, MergesOracleIdAndNameMatches)
{
const QByteArray data =
R"json({"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa": ["removal"], "Lightning Bolt": ["burn", "removal"]})json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QStringLiteral("Lightning Bolt")), QStringList({"burn", "removal"}));
}
TEST(TagDatabaseTest, DropsSlugsContainingWhitespace)
{
const QByteArray data = keyedByOracleId(ORACLE_A, R"("two words", "valid")").toUtf8();
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
// A slug with a space would corrupt the space-separated `tags` property.
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"valid"}));
}
TEST(TagDatabaseTest, ParentTagsAreOffByDefault)
{
const QByteArray data = R"json(
{"object":"tag","id":"child","slug":"tutor-creature-giant","type":"oracle","parent_ids":["parent","missing"],"taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}]}
{"object":"tag","id":"parent","slug":"tutor-creature","type":"oracle","parent_ids":["grandparent"],"taggings":[]}
{"object":"tag","id":"grandparent","slug":"tutor","type":"oracle","parent_ids":[],"taggings":[]}
)json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"tutor-creature-giant"}));
}
TEST(TagDatabaseTest, ParentTagsExpandTransitivelyWhenEnabled)
{
const QByteArray data = R"json(
{"object":"tag","id":"child","slug":"tutor-creature-giant","type":"oracle","parent_ids":["parent","missing"],"taggings":[{"oracle_id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}]}
{"object":"tag","id":"parent","slug":"tutor-creature","type":"oracle","parent_ids":["grandparent"],"taggings":[]}
{"object":"tag","id":"grandparent","slug":"tutor","type":"oracle","parent_ids":[],"taggings":[]}
)json";
QString error;
const auto database = load(data, TagDatabase::LoadOptions{.includeParentTags = true}, &error);
ASSERT_TRUE(database) << error.toStdString();
// The child precedes its parents in the file; expansion must not depend on order.
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()),
QStringList({"tutor", "tutor-creature", "tutor-creature-giant"}));
}
TEST(TagDatabaseTest, RejectsEmptyData)
{
QString error;
EXPECT_FALSE(load(QByteArray(), TagDatabase::LoadOptions{}, &error));
EXPECT_FALSE(error.isEmpty());
}
TEST(TagDatabaseTest, RejectsMalformedJson)
{
QString error;
EXPECT_FALSE(load("not a tag database", TagDatabase::LoadOptions{}, &error));
EXPECT_FALSE(error.isEmpty());
}
TEST(TagDatabaseTest, RejectsTagListsThatAreNotArrays)
{
QString error;
EXPECT_FALSE(load(R"json({"tags": {"Lightning Bolt": "removal"}})json", TagDatabase::LoadOptions{}, &error));
EXPECT_FALSE(error.isEmpty());
}
TEST(TagDatabaseTest, RejectsDatabaseWithoutUsableTags)
{
QString error;
EXPECT_FALSE(load(R"json({"Lightning Bolt": []})json", TagDatabase::LoadOptions{}, &error));
EXPECT_FALSE(error.isEmpty());
}
#ifdef HAS_ZLIB
TEST(TagDatabaseTest, DecompressesGzipPayloads)
{
const QByteArray plain = keyedByOracleId(ORACLE_A, R"("removal")").toUtf8();
const QByteArray compressed = gzipCompress(plain);
ASSERT_FALSE(compressed.isEmpty());
ASSERT_EQ(compressed.left(2), QByteArray::fromHex("1f8b"));
QString error;
const auto database = load(compressed, TagDatabase::LoadOptions{}, &error);
ASSERT_TRUE(database) << error.toStdString();
EXPECT_EQ(database->tagsForCard(ORACLE_A, QString()), QStringList({"removal"}));
}
#endif