mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-09-23 18:06:26 -07:00
[Oracle] Parse sets lazily to slash importer peak memory
- Add a raw JSON scanner that splits the document into per-set byte ranges without materializing the JSON tree - Keep only the raw document bytes and parse one set at a time in startImport - Take readSetsFromByteArray by value so the wizard's buffer is moved, not copied - Clear the retained raw data in releaseSetData()/clear() - Cover the scanner and lazy parsing with tests Took 2 minutes
This commit is contained in:
parent
21c84a2e1a
commit
68d67ff3e5
8 changed files with 811 additions and 31 deletions
60
oracle/src/raw_json_scanner.h
Normal file
60
oracle/src/raw_json_scanner.h
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
#ifndef RAW_JSON_SCANNER_H
|
||||
#define RAW_JSON_SCANNER_H
|
||||
|
||||
#include <QByteArray>
|
||||
#include <QList>
|
||||
#include <QString>
|
||||
|
||||
namespace RawJson
|
||||
{
|
||||
|
||||
struct SetRange
|
||||
{
|
||||
/** @brief Byte offset of the set's object within the scanned buffer. */
|
||||
qsizetype start = -1;
|
||||
/** @brief Byte length of the set's object, including the surrounding braces. */
|
||||
qsizetype length = 0;
|
||||
/** @brief Number of entries in the set's "cards" array. */
|
||||
int cardCount = 0;
|
||||
QString code;
|
||||
QString name;
|
||||
QString type;
|
||||
QString releaseDate;
|
||||
};
|
||||
|
||||
struct ScanError
|
||||
{
|
||||
bool isError() const
|
||||
{
|
||||
return !message.isEmpty();
|
||||
}
|
||||
QString message;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Scans a full MTGJSON document without materializing the JSON tree.
|
||||
*
|
||||
* Splits the top-level "data" object into per-set byte ranges and reads each
|
||||
* set's metadata directly from the raw bytes. The oracle importer can then
|
||||
* parse one set at a time during import, keeping peak memory far below a single
|
||||
* QJsonDocument::fromJson() over the whole file.
|
||||
*
|
||||
* The whole document is structurally validated while scanning (strings,
|
||||
* escapes, braces, and a trailing-content check), so malformed input is
|
||||
* rejected just like QJsonDocument::fromJson would.
|
||||
*
|
||||
* Following QJsonDocument::fromJson's convention, the parsed ranges are
|
||||
* returned by value and any failure is reported through the @p error out
|
||||
* parameter.
|
||||
*
|
||||
* @param json The raw MTGJSON document bytes.
|
||||
* @param error Out parameter. Set to an error ScanError when the document
|
||||
* cannot be parsed, otherwise left empty. Passing a null
|
||||
* pointer disables error reporting.
|
||||
* @return The detected per-set ranges, or an empty list on failure.
|
||||
*/
|
||||
QList<SetRange> scanSetRanges(const QByteArray &json, ScanError *error = nullptr);
|
||||
|
||||
} // namespace RawJson
|
||||
|
||||
#endif // RAW_JSON_SCANNER_H
|
||||
Loading…
Add table
Add a link
Reference in a new issue