jeirmeister/Cockatrice
1
0
Fork 0
mirror of https://github.com/Cockatrice/Cockatrice.git synced 2026-06-10 08:14:47 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

[WIP] Basic mtgjsonv4 support (#3458)

Browse source 
* Basic mtgjsonv4 support

* Fix set type

* [WIP] Oracle: use zx instead of zip

* clanfigy fixes

* Fix reading last block of xz

* Added back zip support

* [WIP] adding xz on ci + fixes

* typo

* resolve conflict

* Make gcc an happy puppy

* test appveyor build

* appveyor maybe

* Appveyor: add  xz bindir

* Update ssl version (the old one is not available anymore)

* Windows is a really shitty platform to code on.

* test vcpkg

* again

* gosh

* nowarn

* warning 2

* static

* Maybe

* cmake fix

* fsck this pain

* FindWin32SslRuntime: add vcpkg path

* Appveyor: cache support, force usable of openssl from vcpkg

* updated as suggested

* ouch

* Import card uuids and expose this property as !uuid! for card image download

* Minor style fixes

* address changed URL
This commit is contained in:
ctrlaltca 2018-12-21 00:05:03 +01:00 • committed by Zach H
parent cee69705d8
commit 65f41e520e
22 changed files with 419 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

250
oracle/src/lzma/decompress.cpp Normal file
View file

@ -0,0 +1,250 @@
/*
* Simple routing to extract a single file from a xz archive
* Heavily based from doc/examples/02_decompress.c obtained from
* the official xz git repository: git.tukaani.org/xz.git
* The license from the original file header follows
*
* Author: Lasse Collin
* This file has been put into the public domain.
* You can do whatever you want with this file.
*/
#include <lzma.h>
#include <QDebug>
#include "decompress.h"
XzDecompressor::XzDecompressor(QObject *parent)
: QObject(parent)
{
}
bool XzDecompressor::decompress(QBuffer *in, QBuffer *out)
{
lzma_stream strm = LZMA_STREAM_INIT;
bool success;
if (!init_decoder(&strm)) {
return false;
}
success = internal_decompress(&strm, in, out);
// Free the memory allocated for the decoder. This only needs to be
// done after the last file.
lzma_end(&strm);
return success;
}
bool XzDecompressor::init_decoder(lzma_stream *strm)
{
// Initialize a .xz decoder. The decoder supports a memory usage limit
// and a set of flags.
//
// The memory usage of the decompressor depends on the settings used
// to compress a .xz file. It can vary from less than a megabyte to
// a few gigabytes, but in practice (at least for now) it rarely
// exceeds 65 MiB because that's how much memory is required to
// decompress files created with "xz -9". Settings requiring more
// memory take extra effort to use and don't (at least for now)
// provide significantly better compression in most cases.
//
// Memory usage limit is useful if it is important that the
// decompressor won't consume gigabytes of memory. The need
// for limiting depends on the application. In this example,
// no memory usage limiting is used. This is done by setting
// the limit to UINT64_MAX.
//
// The .xz format allows concatenating compressed files as is:
//
// echo foo | xz > foobar.xz
// echo bar | xz >> foobar.xz
//
// When decompressing normal standalone .xz files, LZMA_CONCATENATED
// should always be used to support decompression of concatenated
// .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop
// after the first .xz stream. This can be useful when .xz data has
// been embedded inside another file format.
//
// Flags other than LZMA_CONCATENATED are supported too, and can
// be combined with bitwise-or. See lzma/container.h
// (src/liblzma/api/lzma/container.h in the source package or e.g.
// /usr/include/lzma/container.h depending on the install prefix)
// for details.
lzma_ret ret = lzma_stream_decoder(
strm, UINT64_MAX, LZMA_CONCATENATED);
// Return successfully if the initialization went fine.
if (ret == LZMA_OK)
return true;
// Something went wrong. The possible errors are documented in
// lzma/container.h (src/liblzma/api/lzma/container.h in the source
// package or e.g. /usr/include/lzma/container.h depending on the
// install prefix).
//
// Note that LZMA_MEMLIMIT_ERROR is never possible here. If you
// specify a very tiny limit, the error will be delayed until
// the first headers have been parsed by a call to lzma_code().
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR:
msg = "Memory allocation failed";
break;
case LZMA_OPTIONS_ERROR:
msg = "Unsupported decompressor flags";
break;
default:
// This is most likely LZMA_PROG_ERROR indicating a bug in
// this program or in liblzma. It is inconvenient to have a
// separate error message for errors that should be impossible
// to occur, but knowing the error code is important for
// debugging. That's why it is good to print the error code
// at least when there is no good error message to show.
msg = "Unknown error, possibly a bug";
break;
}
qDebug() << "Error initializing the decoder:" << msg << "(error code " << ret << ")";
return false;
}
bool XzDecompressor::internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out)
{
// When LZMA_CONCATENATED flag was used when initializing the decoder,
// we need to tell lzma_code() when there will be no more input.
// This is done by setting action to LZMA_FINISH instead of LZMA_RUN
// in the same way as it is done when encoding.
//
// When LZMA_CONCATENATED isn't used, there is no need to use
// LZMA_FINISH to tell when all the input has been read, but it
// is still OK to use it if you want. When LZMA_CONCATENATED isn't
// used, the decoder will stop after the first .xz stream. In that
// case some unused data may be left in strm->next_in.
lzma_action action = LZMA_RUN;
uint8_t inbuf[BUFSIZ];
uint8_t outbuf[BUFSIZ];
qint64 bytesAvailable;
strm->next_in = NULL;
strm->avail_in = 0;
strm->next_out = outbuf;
strm->avail_out = sizeof(outbuf);
while (true) {
if (strm->avail_in == 0) {
strm->next_in = inbuf;
bytesAvailable = in->bytesAvailable();
if(bytesAvailable == 0) {
// Once the end of the input file has been reached,
// we need to tell lzma_code() that no more input
// will be coming. As said before, this isn't required
// if the LZMA_CONCATENATED flag isn't used when
// initializing the decoder.
action = LZMA_FINISH;
} else if(bytesAvailable >= BUFSIZ) {
in->read((char*) inbuf, BUFSIZ);
strm->avail_in = BUFSIZ;
} else {
in->read((char*) inbuf, bytesAvailable);
strm->avail_in = bytesAvailable;
}
}
lzma_ret ret = lzma_code(strm, action);
if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
qint64 write_size = sizeof(outbuf) - strm->avail_out;
if (out->write((char *) outbuf, write_size) != write_size) {
qDebug() << "Write error";
return false;
}
strm->next_out = outbuf;
strm->avail_out = sizeof(outbuf);
}
if (ret != LZMA_OK) {
// Once everything has been decoded successfully, the
// return value of lzma_code() will be LZMA_STREAM_END.
//
// It is important to check for LZMA_STREAM_END. Do not
// assume that getting ret != LZMA_OK would mean that
// everything has gone well or that when you aren't
// getting more output it must have successfully
// decoded everything.
if (ret == LZMA_STREAM_END)
return true;
// It's not LZMA_OK nor LZMA_STREAM_END,
// so it must be an error code. See lzma/base.h
// (src/liblzma/api/lzma/base.h in the source package
// or e.g. /usr/include/lzma/base.h depending on the
// install prefix) for the list and documentation of
// possible values. Many values listen in lzma_ret
// enumeration aren't possible in this example, but
// can be made possible by enabling memory usage limit
// or adding flags to the decoder initialization.
const char *msg;
switch (ret) {
case LZMA_MEM_ERROR:
msg = "Memory allocation failed";
break;
case LZMA_FORMAT_ERROR:
// .xz magic bytes weren't found.
msg = "The input is not in the .xz format";
break;
case LZMA_OPTIONS_ERROR:
// For example, the headers specify a filter
// that isn't supported by this liblzma
// version (or it hasn't been enabled when
// building liblzma, but no-one sane does
// that unless building liblzma for an
// embedded system). Upgrading to a newer
// liblzma might help.
//
// Note that it is unlikely that the file has
// accidentally became corrupt if you get this
// error. The integrity of the .xz headers is
// always verified with a CRC32, so
// unintentionally corrupt files can be
// distinguished from unsupported files.
msg = "Unsupported compression options";
break;
case LZMA_DATA_ERROR:
msg = "Compressed file is corrupt";
break;
case LZMA_BUF_ERROR:
// Typically this error means that a valid
// file has got truncated, but it might also
// be a damaged part in the file that makes
// the decoder think the file is truncated.
// If you prefer, you can use the same error
// message for this as for LZMA_DATA_ERROR.
msg = "Compressed file is truncated or "
"otherwise corrupt";
break;
default:
// This is most likely LZMA_PROG_ERROR.
msg = "Unknown error, possibly a bug";
break;
}
qDebug() << "Decoder error:" << msg << "(error code " << ret << ")";
return false;
}
}
}

19
oracle/src/lzma/decompress.h Normal file
View file

@ -0,0 +1,19 @@
#ifndef XZ_DECOMPRESS_H
#define XZ_DECOMPRESS_H
#include <lzma.h>
#include <QBuffer>
class XzDecompressor : public QObject
{
Q_OBJECT
public:
XzDecompressor(QObject *parent = 0);
~XzDecompressor() { };
bool decompress(QBuffer *in, QBuffer *out);
private:
bool init_decoder(lzma_stream *strm);
bool internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out);
};
#endif

60
oracle/src/oracleimporter.cpp
View file

@ -19,7 +19,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
setsMap = QtJson::Json::parse(QString(data), ok).toMap();
if (!ok) {
qDebug() << "error: QtJson::Json::parse()";
return 0;
return false;
}
QListIterator<QVariant> it(setsMap.values());
@ -33,7 +33,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
while (it.hasNext()) {
map = it.next().toMap();
edition = map.value("code").toString();
edition = map.value("code").toString().toUpper();
editionLong = map.value("name").toString();
editionCards = map.value("cards");
setType = map.value("type").toString();
@ -57,6 +57,7 @@ CardInfoPtr OracleImporter::addCard(const QString &setName,
QString cardName,
bool isToken,
int cardId,
QString &cardUuId,
QString &setNumber,
QString &cardCost,
QString &cmc,
@ -124,30 +125,13 @@ CardInfoPtr OracleImporter::addCard(const QString &setName,
cards.insert(cardName, card);
}
card->setMuId(setName, cardId);
card->setUuId(setName, cardUuId);
card->setSetNumber(setName, setNumber);
card->setRarity(setName, rarity);
return card;
}
void OracleImporter::extractColors(const QStringList &in, QStringList &out)
{
foreach (QString c, in) {
if (c == "White")
out << "W";
else if (c == "Blue")
out << "U";
else if (c == "Black")
out << "B";
else if (c == "Red")
out << "R";
else if (c == "Green")
out << "G";
else
qDebug() << "error: unknown color:" << c;
}
}
int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
{
int cards = 0;
@ -164,6 +148,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
QList<CardRelation *> relatedCards;
QList<CardRelation *> reverseRelatedCards; // dummy
int cardId;
QString cardUuId;
QString setNumber;
QString rarity;
QString cardLoyalty;
@ -173,15 +158,20 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
while (it.hasNext()) {
map = it.next().toMap();
/* Currently used layouts are:
* augment, double_faced_token, flip, host, leveler, meld, normal, planar,
* saga, scheme, split, token, transform, vanguard
*/
QString layout = map.value("layout").toString();
// don't import tokens from the json file
if (layout == "token")
continue;
if (layout == "split" || layout == "aftermath") {
// Aftermath card layout seems to have been integrated in "split"
if (layout == "split") {
// Enqueue split card for later handling
cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0;
cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0;
if (cardId)
splitCards.insertMulti(cardId, map);
continue;
@ -190,16 +180,18 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
// normal cards handling
cardName = map.contains("name") ? map.value("name").toString() : QString("");
cardCost = map.contains("manaCost") ? map.value("manaCost").toString() : QString("");
cmc = map.contains("cmc") ? map.value("cmc").toString() : QString("0");
cmc = map.contains("convertedManaCost") ? map.value("convertedManaCost").toString() : QString("0");
cardType = map.contains("type") ? map.value("type").toString() : QString("");
cardPT = map.contains("power") || map.contains("toughness")
? map.value("power").toString() + QString('/') + map.value("toughness").toString()
: QString("");
cardText = map.contains("text") ? map.value("text").toString() : QString("");
cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0;
cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0;
cardUuId = map.contains("uuid") ? map.value("uuid").toString() : QString("");
setNumber = map.contains("number") ? map.value("number").toString() : QString("");
rarity = map.contains("rarity") ? map.value("rarity").toString() : QString("");
cardLoyalty = map.contains("loyalty") ? map.value("loyalty").toString() : QString("");
colors = map.contains("colors") ? map.value("colors").toStringList() : QStringList();
relatedCards = QList<CardRelation *>();
if (map.contains("names"))
foreach (const QString &name, map.value("names").toStringList()) {
@ -214,11 +206,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
upsideDown = false;
}
colors.clear();
extractColors(map.value("colors").toStringList(), colors);
CardInfoPtr card =
addCard(set->getShortName(), cardName, false, cardId, setNumber, cardCost, cmc, cardType, cardPT,
addCard(set->getShortName(), cardName, false, cardId, cardUuId, setNumber, cardCost, cmc, cardType, cardPT,
cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
if (!set->contains(card)) {
@ -250,6 +239,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
cardType = "";
cardPT = "";
cardText = "";
cardUuId = "";
setNumber = "";
rarity = "";
cardLoyalty = "";
@ -269,10 +259,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
cardCost += prefix;
cardCost += map.value("manaCost").toString();
}
if (map.contains("cmc")) {
if (map.contains("convertedManaCost")) {
if (!cmc.isEmpty())
cmc += prefix;
cmc += map.value("cmc").toString();
cmc += map.value("convertedManaCost").toString();
}
if (map.contains("type")) {
if (!cardType.isEmpty())
@ -289,6 +279,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
cardText += prefix2;
cardText += map.value("text").toString();
}
if (map.contains("uuid")) {
if (cardUuId.isEmpty())
cardUuId = map.value("uuid").toString();
}
if (map.contains("number")) {
if (setNumber.isEmpty())
setNumber = map.value("number").toString();
@ -298,7 +292,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
rarity = map.value("rarity").toString();
}
extractColors(map.value("colors").toStringList(), colors);
colors << map.value("colors").toStringList();
}
colors.removeDuplicates();
@ -313,8 +307,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
// add the card
CardInfoPtr card =
addCard(set->getShortName(), cardName, false, muid, setNumber, cardCost, cmc, cardType, cardPT, cardLoyalty,
cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
addCard(set->getShortName(), cardName, false, muid, cardUuId, setNumber, cardCost, cmc, cardType, cardPT,
cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
if (!set->contains(card)) {
card->addToSet(set);

2
oracle/src/oracleimporter.h
View file

@ -61,6 +61,7 @@ private:
QString cardName,
bool isToken,
int cardId,
QString &cardUuId,
QString &setNumber,
QString &cardCost,
QString &cmc,
@ -93,7 +94,6 @@ public:
}
protected:
void extractColors(const QStringList &in, QStringList &out);
void sortColors(QStringList &colors);
};

51
oracle/src/oraclewizard.cpp
View file

@ -26,11 +26,22 @@
#include "settingscache.h"
#include "version_string.h"
#define ZIP_SIGNATURE "PK"
#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json"
#ifdef HAS_LZMA
#include "lzma/decompress.h"
#endif
#ifdef HAS_ZLIB
#include "zip/unzip.h"
#endif
#define ZIP_SIGNATURE "PK"
// Xz stream header: 0xFD + "7zXZ"
#define XZ_SIGNATURE "\xFD\x37\x7A\x58\x5A"
#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json"
#ifdef HAS_LZMA
#define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.xz"
#elif defined(HAS_ZLIB)
#define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.zip"
#else
#define ALLSETS_URL "https://mtgjson.com/json/AllSets.json"
@ -249,11 +260,14 @@ void LoadSetsPage::actLoadSetsFile()
QFileDialog dialog(this, tr("Load sets file"));
dialog.setFileMode(QFileDialog::ExistingFile);
QString extensions = "*.json";
#ifdef HAS_ZLIB
dialog.setNameFilter(tr("Sets JSON file (*.json *.zip)"));
#else
dialog.setNameFilter(tr("Sets JSON file (*.json)"));
extensions += " *.zip";
#endif
#ifdef HAS_LZMA
extensions += " *.xz";
#endif
dialog.setNameFilter(tr("Sets JSON file (%1)").arg(extensions));
if (!fileLineEdit->text().isEmpty() && QFile::exists(fileLineEdit->text())) {
dialog.selectFile(fileLineEdit->text());
@ -383,7 +397,32 @@ void LoadSetsPage::readSetsFromByteArray(QByteArray data)
progressBar->show();
// unzip the file if needed
if (data.startsWith(ZIP_SIGNATURE)) {
if (data.startsWith(XZ_SIGNATURE)) {
#ifdef HAS_LZMA
// zipped file
auto *inBuffer = new QBuffer(&data);
auto *outBuffer = new QBuffer(this);
inBuffer->open(QBuffer::ReadOnly);
outBuffer->open(QBuffer::WriteOnly);
XzDecompressor xz;
if (!xz.decompress(inBuffer, outBuffer)) {
zipDownloadFailed(tr("Xz extraction failed."));
return;
}
future = QtConcurrent::run(wizard()->importer, &OracleImporter::readSetsFromByteArray, outBuffer->data());
watcher.setFuture(future);
return;
#else
zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files."));
wizard()->enableButtons();
setEnabled(true);
progressLabel->hide();
progressBar->hide();
return;
#endif
} else if (data.startsWith(ZIP_SIGNATURE)) {
#ifdef HAS_ZLIB
// zipped file
auto *inBuffer = new QBuffer(&data);