[Oracle] Add RAM usage benchmarks for the oracle importer

- Measure process peak/current RSS via procfs (Linux) or getrusage (macOS)
- Add a synthetic-scale RAM benchmark and an opt-in real AllPrintings
  run gated by COCKATRICE_ORACLE_RAM_BENCHMARK=1
- Mirror the wizard's magic-byte handling to decompress .xz/.zip payloads
- Wire optional ZLIB/LibLZMA into the benchmark target and raise its timeout

Took 2 minutes

tests/oracle/CMakeLists.txt

@@ -25,10 +25,33 @@ target_link_libraries(
 
 add_test(NAME oracle_importer_test COMMAND oracle_importer_test)
 
-# Oracle importer benchmark tests (manual, not run in CI)
+# Oracle importer benchmark tests (manual, not run in CI, incl. RAM benchmark)
+# Optional compression libs, mirrored from oracle/CMakeLists.txt, so the benchmark
+# can download and decompress whatever AllPrintings format the default URL selects.
+find_package(ZLIB)
+if(ZLIB_FOUND)
+  add_definitions("-DHAS_ZLIB")
+  set(_ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/zip/unzip.cpp ../../oracle/src/zip/zipglobal.cpp)
+  set(_ORACLE_BENCH_EXTRA_LIBRARIES ${ZLIB_LIBRARIES})
+  include_directories(${ZLIB_INCLUDE_DIRS})
+else()
+  message(STATUS "Oracle tests: zlib not found; zip download benchmark disabled")
+endif()
+
+find_package(LibLZMA)
+if(LIBLZMA_FOUND)
+  add_definitions("-DHAS_LZMA")
+  list(APPEND _ORACLE_BENCH_EXTRA_SOURCES ../../oracle/src/lzma/decompress.cpp)
+  list(APPEND _ORACLE_BENCH_EXTRA_LIBRARIES ${LIBLZMA_LIBRARIES})
+  include_directories(${LIBLZMA_INCLUDE_DIRS})
+else()
+  message(STATUS "Oracle tests: LibLZMA not found; xz download benchmark disabled")
+endif()
+
 add_executable(
-  oracle_importer_benchmark_test ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp
+  oracle_importer_benchmark_test
-                                 ../../oracle/src/parsehelpers.cpp oracle_importer_benchmark_test.cpp
+  ${VERSION_STRING_CPP} ../../oracle/src/oracleimporter.cpp ../../oracle/src/parsehelpers.cpp
+  oracle_importer_benchmark_test.cpp ${_ORACLE_BENCH_EXTRA_SOURCES}
 )
 
 if(NOT GTEST_FOUND)
@@ -36,6 +59,11 @@ if(NOT GTEST_FOUND)
 endif()
 
 target_link_libraries(
-  oracle_importer_benchmark_test libcockatrice_card libcockatrice_interfaces Threads::Threads ${GTEST_BOTH_LIBRARIES}
+  oracle_importer_benchmark_test
+  libcockatrice_card
+  libcockatrice_interfaces
+  Threads::Threads
+  ${GTEST_BOTH_LIBRARIES}
   ${TEST_QT_MODULES}
+  ${_ORACLE_BENCH_EXTRA_LIBRARIES}
 )

tests/oracle/oracle_importer_benchmark_test.cpp

@@ -1,13 +1,33 @@
 #include "../../oracle/src/oracleimporter.h"
 
 #include "gtest/gtest.h"
+#include <QBuffer>
+#include <QCoreApplication>
 #include <QDebug>
 #include <QElapsedTimer>
+#include <QEventLoop>
+#include <QFile>
 #include <QJsonArray>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QNetworkRequest>
+#include <QTimer>
+#include <QUrl>
 #include <libcockatrice/interfaces/noop_card_set_priority_controller.h>
 
+#if defined(HAS_LZMA)
+#include "../../oracle/src/lzma/decompress.h"
+#endif
+#if defined(HAS_ZLIB)
+#include "../../oracle/src/zip/unzip.h"
+#endif
+#if defined(Q_OS_MACOS)
+#include <mach/mach.h>
+#include <sys/resource.h>
+#endif
+
 // Helper: build a synthetic MTGJSON-style JSON with the given number of sets and cards per set
 static QByteArray buildSyntheticData(int numSets, int cardsPerSet)
 {
@@ -257,8 +277,288 @@ TEST(OracleBenchmark, ImportCardsWithColors)
                               .arg(ms > 0 ? static_cast<double>(count) / ms * 1000.0 : 0.0, 0, 'f', 0);
 }
 
+// ============================================================================
+// RAM usage measurement
+// ============================================================================
+
+// Mirrors the default AllPrintings URL selection in oracle/src/pages.cpp.
+#if defined(HAS_LZMA)
+static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.xz");
+#elif defined(HAS_ZLIB)
+static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json.zip");
+#else
+static const QUrl kDefaultAllPrintingsUrl("https://www.mtgjson.com/api/v5/AllPrintings.json");
+#endif
+
+// Magic bytes also from oracle/src/pages.cpp
+static const QByteArray kXzSignature("\xFD\x37\x7A\x58\x5A", 6);
+static const QByteArray kZipSignature("PK");
+
+struct MemorySnapshot
+{
+    qint64 peakRssKb = -1; // process high-water mark (VmHWM on Linux, ru_maxrss on macOS)
+    qint64 rssKb = -1;     // current resident set size
+    bool available = false;
+
+    static MemorySnapshot current()
+    {
+        MemorySnapshot snap;
+#if defined(Q_OS_LINUX)
+        QFile statusFile("/proc/self/status");
+        if (statusFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            // /proc files report size() == 0, so atEnd() is immediately true: read everything first.
+            const QList<QByteArray> lines = statusFile.readAll().split('\n');
+            for (const QByteArray &line : lines) {
+                if (line.startsWith("VmHWM:")) {
+                    snap.peakRssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong();
+                } else if (line.startsWith("VmRSS:")) {
+                    snap.rssKb = line.mid(6).trimmed().split(' ').value(0).toLongLong();
+                }
+            }
+            snap.available = snap.peakRssKb >= 0;
+        }
+#elif defined(Q_OS_MACOS)
+        struct rusage usage;
+        if (getrusage(RUSAGE_SELF, &usage) == 0) {
+            snap.peakRssKb = usage.ru_maxrss / 1024; // bytes -> kB
+            snap.available = snap.peakRssKb >= 0;
+        }
+        // getrusage has no current-RSS equivalent; task_info's resident_size
+        // is the closest macOS analog to Linux VmRSS.
+        mach_task_basic_info info = {};
+        mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
+        if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, reinterpret_cast<task_info_t>(&info), &count) ==
+            KERN_SUCCESS) {
+            snap.rssKb = info.resident_size / 1024;
+        }
+#endif
+        return snap;
+    }
+};
+
+static QString formatKb(qint64 kb)
+{
+    if (kb < 0) {
+        return "N/A";
+    }
+    return QString("%1 MB").arg(kb / 1024.0, 0, 'f', 1);
+}
+
+static void logRamPhase(const QString &phase, const MemorySnapshot &baseline, const MemorySnapshot &current)
+{
+    if (!baseline.available || !current.available) {
+        qDebug().noquote() << QString("  %1: memory stats unavailable on this platform").arg(phase);
+        return;
+    }
+    // VmHWM / ru_maxrss are monotonically non-decreasing high-water marks, so a
+    // peak-based delta between phases is ~0.0 MB by construction once the
+    // fixture build has set the process peak. The live signals are current RSS
+    // and the process peak; the delta is only meaningful where current RSS is
+    // a per-phase value (see "after releaseSetData()").
+    QString rssDelta = "N/A";
+    if (current.rssKb >= 0 && baseline.rssKb >= 0) {
+        rssDelta = formatKb(current.rssKb - baseline.rssKb);
+    }
+    qDebug().noquote() << QString("  %1: current RSS %2 | delta vs baseline %3 | process peak %4")
+                              .arg(phase)
+                              .arg(formatKb(current.rssKb))
+                              .arg(rssDelta)
+                              .arg(formatKb(current.peakRssKb));
+}
+
+// Decompresses the download payload when the default URL is a compressed build,
+// mirroring the wizard's magic-byte handling in oracle/src/pages.cpp.
+static QByteArray decompressSetsData(const QByteArray &payload)
+{
+    if (payload.startsWith(kXzSignature)) {
+#if defined(HAS_LZMA)
+        QBuffer inBuffer(const_cast<QByteArray *>(&payload));
+        QByteArray out;
+        QBuffer outBuffer(&out);
+        inBuffer.open(QIODevice::ReadOnly);
+        outBuffer.open(QIODevice::WriteOnly);
+        XzDecompressor xz;
+        if (!xz.decompress(&inBuffer, &outBuffer)) {
+            qDebug() << "RAM benchmark: xz decompression failed";
+            return {};
+        }
+        return out;
+#else
+        qDebug() << "RAM benchmark: download is xz-compressed but this build has no LZMA support";
+        return {};
+#endif
+    }
+    if (payload.startsWith(kZipSignature)) {
+#if defined(HAS_ZLIB)
+        QBuffer inBuffer(const_cast<QByteArray *>(&payload));
+        inBuffer.open(QIODevice::ReadOnly);
+        UnZip unzip;
+        if (unzip.openArchive(&inBuffer) != UnZip::Ok) {
+            qDebug() << "RAM benchmark: zip archive open failed";
+            return {};
+        }
+        if (unzip.fileList().size() != 1) {
+            qDebug() << "RAM benchmark: zip archive doesn't contain exactly one file";
+            return {};
+        }
+        QByteArray out;
+        QBuffer outBuffer(&out);
+        outBuffer.open(QIODevice::WriteOnly);
+        const auto errorCode = unzip.extractFile(unzip.fileList().value(0), &outBuffer);
+        unzip.closeArchive();
+        if (errorCode != UnZip::Ok) {
+            qDebug() << "RAM benchmark: zip extraction failed";
+            return {};
+        }
+        return out;
+#else
+        qDebug() << "RAM benchmark: download is zip-compressed but this build has no zlib support";
+        return {};
+#endif
+    }
+    return payload;
+}
+
+TEST(OracleBenchmark, ImportRamUsage)
+{
+    static constexpr int numSets = 30;
+    static constexpr int cardsPerSet = 2000; // ~60k cards, roughly AllPrintings scale
+
+    // Baseline must precede the fixture build: a high-water mark set while
+    // generating the synthetic JSON would otherwise mask the importer phases.
+    // Where memory stats are unavailable (Windows), skip before doing the
+    // 60k-card fixture build, which would otherwise be pure wasted work.
+    const MemorySnapshot baseline = MemorySnapshot::current();
+    if (!baseline.available) {
+        GTEST_SKIP() << "Memory stats unavailable on this platform";
+    }
+
+    const QByteArray data = buildSyntheticData(numSets, cardsPerSet);
+
+    NoopCardSetPriorityController controller;
+    OracleImporter importer;
+
+    QElapsedTimer timer;
+    timer.start();
+    ASSERT_TRUE(importer.readSetsFromByteArray(data));
+    const qint64 parseMs = timer.elapsed();
+    const MemorySnapshot afterParse = MemorySnapshot::current();
+
+    timer.restart();
+    const int importedSets = importer.startImport();
+    const qint64 importMs = timer.elapsed();
+    const MemorySnapshot afterImport = MemorySnapshot::current();
+
+    importer.releaseSetData();
+    const MemorySnapshot afterRelease = MemorySnapshot::current();
+
+    const int totalCards = importer.getCardList().size();
+    qDebug().noquote() << QString("Oracle RAM Benchmark (synthetic): %1 sets, %2 cards, %3 MB JSON")
+                              .arg(importedSets)
+                              .arg(totalCards)
+                              .arg(data.size() / (1024.0 * 1024.0), 0, 'f', 1);
+    qDebug().noquote() << QString("  JSON parse: %1 ms").arg(parseMs);
+    qDebug().noquote() << QString("  Card import: %1 ms").arg(importMs);
+    logRamPhase("parse", baseline, afterParse);
+    logRamPhase("import", afterParse, afterImport);
+    logRamPhase("after releaseSetData()", afterImport, afterRelease);
+}
+
+TEST(OracleBenchmark, ImportRamUsageAllPrintings)
+{
+    // Only "1" enables the download: unset (the default and the CI setup) and
+    // an explicit "0" both disable it.
+    bool envOk = false;
+    const int enabled = qEnvironmentVariableIntValue("COCKATRICE_ORACLE_RAM_BENCHMARK", &envOk);
+    if (!envOk || enabled == 0) {
+        GTEST_SKIP() << "Set COCKATRICE_ORACLE_RAM_BENCHMARK=1 to download the real AllPrintings dataset for this "
+                        "RAM benchmark. Default URL: "
+                     << kDefaultAllPrintingsUrl.toDisplayString().toStdString();
+    }
+
+    // Baseline must precede the request so the phase covers the download +
+    // decompress step, including the payload materialized by readAll().
+    const MemorySnapshot baseline = MemorySnapshot::current();
+    if (!baseline.available) {
+        GTEST_SKIP() << "Memory stats unavailable on this platform";
+    }
+
+    QNetworkAccessManager nam;
+    QNetworkRequest request(kDefaultAllPrintingsUrl);
+    request.setHeader(QNetworkRequest::UserAgentHeader, "Cockatrice Oracle RAM benchmark");
+    QNetworkReply *reply = nam.get(request);
+
+    QEventLoop loop;
+    QTimer timeoutTimer;
+    timeoutTimer.setSingleShot(true);
+    bool timedOut = false;
+    QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
+    QObject::connect(&timeoutTimer, &QTimer::timeout, &loop, [&] {
+        timedOut = true;
+        reply->abort();
+    });
+    timeoutTimer.start(10 * 60 * 1000);
+    loop.exec();
+    timeoutTimer.stop();
+
+    // abort() leaves reply->error() as OperationCanceledError, so a timed-out
+    // download takes the same GTEST_SKIP path as any other network error
+    // instead of reading a truncated body and failing the parse below.
+    if (timedOut || reply->error() != QNetworkReply::NoError) {
+        GTEST_SKIP() << "Download failed: " << reply->errorString().toStdString();
+    }
+    const QByteArray payload = reply->readAll();
+    reply->deleteLater();
+
+    // mtgjson can answer 200 with an HTML page (mirrors the wizard's '<' check
+    // in pages.cpp); reject it before trying to decompress/parse.
+    if (payload.startsWith("<")) {
+        GTEST_SKIP() << "Download returned a non-JSON body (HTML page instead of data), skipping";
+    }
+
+    const QByteArray setsData = decompressSetsData(payload);
+    const MemorySnapshot afterDownload = MemorySnapshot::current();
+    if (setsData.isEmpty()) {
+        GTEST_SKIP() << "No data to import (download or decompression failed)";
+    }
+
+    NoopCardSetPriorityController controller;
+    OracleImporter importer;
+
+    QElapsedTimer timer;
+    timer.start();
+    ASSERT_TRUE(importer.readSetsFromByteArray(setsData));
+    const qint64 parseMs = timer.elapsed();
+    const MemorySnapshot afterParse = MemorySnapshot::current();
+
+    timer.restart();
+    const int importedSets = importer.startImport();
+    const qint64 importMs = timer.elapsed();
+    const MemorySnapshot afterImport = MemorySnapshot::current();
+
+    importer.releaseSetData();
+    const MemorySnapshot afterRelease = MemorySnapshot::current();
+
+    const int totalCards = importer.getCardList().size();
+    qDebug().noquote() << QString("Oracle RAM Benchmark (real AllPrintings): %1 sets, %2 unique cards")
+                              .arg(importedSets)
+                              .arg(totalCards);
+    qDebug().noquote() << QString("  URL: %1").arg(kDefaultAllPrintingsUrl.toDisplayString());
+    qDebug().noquote() << QString("  Downloaded: %1 MB, decompressed: %2 MB")
+                              .arg(payload.size() / (1024.0 * 1024.0), 0, 'f', 1)
+                              .arg(setsData.size() / (1024.0 * 1024.0), 0, 'f', 1);
+    qDebug().noquote() << QString("  JSON parse: %1 ms").arg(parseMs);
+    qDebug().noquote() << QString("  Card import: %1 ms").arg(importMs);
+    logRamPhase("download+decompress", baseline, afterDownload);
+    logRamPhase("parse", afterDownload, afterParse);
+    logRamPhase("import", afterParse, afterImport);
+    logRamPhase("after releaseSetData()", afterImport, afterRelease);
+}
+
 int main(int argc, char **argv)
 {
+    // Required for the event loop used by the real-AllPrintings download benchmark
+    QCoreApplication app(argc, argv);
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
 }