[WIP] Basic mtgjsonv4 support (#3458)

* Basic mtgjsonv4 support

* Fix set type

* [WIP] Oracle: use zx instead of zip

* clanfigy fixes

* Fix reading last block of xz

* Added back zip support

* [WIP] adding xz on ci + fixes

* typo

* resolve conflict

* Make gcc an happy puppy

* test appveyor build

* appveyor maybe

* Appveyor: add  xz bindir

* Update ssl version (the old one is not available anymore)

* Windows is a really shitty platform to code on.

* test vcpkg

* again

* gosh

* nowarn

* warning 2

* static

* Maybe

* cmake fix

* fsck this pain

* FindWin32SslRuntime: add vcpkg path

* Appveyor: cache support, force usable of openssl from vcpkg

* updated as suggested

* ouch

* Import card uuids and expose this property as !uuid! for card image download

* Minor style fixes

* address changed URL

.appveyor.yml

@@ -24,66 +24,25 @@ clone_depth: 50    #same as travis, see https://www.appveyor.com/blog/2014/06/04
 image: Visual Studio 2017
 
 cache:
-    - c:\openssl-release
+    - c:\Tools\vcpkg\installed
-    - c:\protobuf-release
-    - c:\zlib-release
-# TODO: set dependency on ps skript file (in ./ci) / "-> appveyor.yml" maybe not ideal
-# that way when we update specific cached tools there (like protobuf or zlib), cache will be newly created automatically
-# https://www.appveyor.com/docs/build-cache/#cleaning-up-cache
-
 
 environment:
-    openssl_ver: 1.0.2p
-    protobuf_ver: 3.6.1
-    zlib_ver: 1.2.11
-
     matrix:
         - target_arch: win64
           qt_ver: 5.9\msvc2017_64
           cmake_generator: Visual Studio 15 2017 Win64
           cmake_toolset: v141,host=x64
-          vc_arch: amd64
+          vcpkg_arch: x64
 
         - target_arch: win32
           qt_ver: 5.9\msvc2015 # Qt doesn't provide a msvc2017_32
           cmake_generator: Visual Studio 15 2017
           cmake_toolset: v141
-          vc_arch: amd64_x86
+          vcpkg_arch: x86
-
 
 install:
-    - ps: |
+    - vcpkg remove --outdated --recurse
-        if (Test-Path c:\openssl-release) {
+    - vcpkg install openssl protobuf liblzma zlib --triplet %vcpkg_arch%-windows
-            echo "using openssl from cache"
-        } else {
-            if ($env:target_arch -eq "win64") {    # 64bit filename
-                # echo "downloading 64bit version of openssl"
-                Invoke-WebRequest "https://indy.fulgan.com/SSL/openssl-$env:openssl_ver-x64_86-win64.zip" -OutFile c:\openssl-$env:openssl_ver.zip
-            } else {    # 32bit filename
-                # echo "downloading 32bit version of openssl"
-                Invoke-WebRequest "https://indy.fulgan.com/SSL/openssl-$env:openssl_ver-i386-win32.zip" -OutFile c:\openssl-$env:openssl_ver.zip
-            }
-            Expand-Archive -Path c:\openssl-$env:openssl_ver.zip -DestinationPath c:\openssl-release
-            Set-Location -Path C:\openssl-release
-        }
-        if (Test-Path c:\protobuf-release) {
-            echo "using protobuf from cache"
-        } else {
-            Invoke-WebRequest "https://github.com/protocolbuffers/protobuf/releases/download/v$env:protobuf_ver/protobuf-cpp-$env:protobuf_ver.zip" -OutFile c:\protobuf-cpp-$env:protobuf_ver.zip
-            Expand-Archive -Path c:\protobuf-cpp-$env:protobuf_ver.zip -DestinationPath c:\
-            Set-Location -Path C:\protobuf-$env:protobuf_ver\cmake
-            cmake . -G "$env:cmake_generator" -T "$env:cmake_toolset" -Dprotobuf_BUILD_TESTS=0 -Dprotobuf_MSVC_STATIC_RUNTIME=0 -DCMAKE_INSTALL_PREFIX=c:/protobuf-release
-            msbuild INSTALL.vcxproj /p:Configuration=Release
-        }
-        if (Test-Path c:\zlib-release) {
-            echo "using zlib from cache"
-        } else {
-            Invoke-WebRequest "https://github.com/madler/zlib/archive/v$env:zlib_ver.zip" -OutFile c:\zlib-$env:zlib_ver.zip
-            Expand-Archive -Path c:\zlib-$env:zlib_ver.zip -DestinationPath c:\
-            Set-Location -Path C:\zlib-$env:zlib_ver
-            cmake . -G "$env:cmake_generator" -T "$env:cmake_toolset" -DCMAKE_INSTALL_PREFIX=c:/zlib-release
-            msbuild INSTALL.vcxproj /p:Configuration=Release
-        }
 
 services:
     - mysql
@@ -92,13 +51,10 @@ build_script:
     - ps: |
         New-Item -ItemType directory -Path $env:APPVEYOR_BUILD_FOLDER\build
         Set-Location -Path $env:APPVEYOR_BUILD_FOLDER\build
-        $zlibdir = "c:\zlib-release"
+        $vcpkgbindir = "C:\Tools\vcpkg\installed\$env:vcpkg_arch-windows\bin"
-        $openssldir = "C:\openssl-release"
-        $protodir = "c:\protobuf-release"
-        $protoc = "c:\protobuf-release\bin\protoc.exe"
         $mysqldll = "c:\Program Files\MySQL\MySQL Server 5.7\lib\libmysql.dll"
         cmake --version
-        cmake .. -G "$env:cmake_generator" -T "$env:cmake_toolset" "-DCMAKE_PREFIX_PATH=c:/Qt/$env:qt_ver;$protodir;$zlibdir;$openssldir" "-DWITH_SERVER=1" "-DPROTOBUF_PROTOC_EXECUTABLE=$protoc" "-DMYSQLCLIENT_LIBRARIES=$mysqldll"
+        cmake .. -G "$env:cmake_generator" -T "$env:cmake_toolset" "-DCMAKE_TOOLCHAIN_FILE=c:/tools/vcpkg/scripts/buildsystems/vcpkg.cmake" "-DCMAKE_PREFIX_PATH=c:/Qt/$env:qt_ver;$vcpkgbindir" "-DWITH_SERVER=1" "-DMYSQLCLIENT_LIBRARIES=$mysqldll"
     - msbuild PACKAGE.vcxproj /p:Configuration=Release
     - ps: |
         $exe = dir -name *.exe
@@ -110,7 +66,7 @@ build_script:
         (New-Object PSObject | Add-Member -PassThru NoteProperty bin $new_name | Add-Member -PassThru NoteProperty cmake $cmake_name | Add-Member -PassThru NoteProperty commit $env:APPVEYOR_REPO_COMMIT) | ConvertTo-JSON | Out-File -FilePath "latest-$env:target_arch" -Encoding ASCII
         Push-AppveyorArtifact "latest-$env:target_arch"
         $version = $matches['content']
-        
+
 test: off
 
 

.ci/Fedora29/Dockerfile

@@ -15,4 +15,5 @@ RUN dnf install -y \
         sqlite-devel \
         wget \
         zlib-devel \
+        xz-devel \
     && dnf clean all

.ci/UbuntuBionic/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         ccache \
         cmake \
+        liblzma-dev \
         libprotobuf-dev \
         libqt5multimedia5-plugins \
         libqt5svg5-dev \

.travis.yml

@@ -15,6 +15,7 @@ matrix:
         packages:
         - libprotobuf-dev
         - protobuf-compiler
+        - liblzma-dev
         - qt5-default
         - qttools5-dev
         - qttools5-dev-tools
@@ -87,9 +88,7 @@ matrix:
     env: HOMEBREW_NO_AUTO_UPDATE=1
     cache: ccache
     before_install:
-      - brew install ccache
+      - brew install ccache protobuf qt xz
-      - brew install protobuf
-      - brew install qt
     script: bash ./.ci/travis-compile.sh --server --install --debug
 
   - name: macOS (Release)
@@ -99,9 +98,7 @@ matrix:
     env: HOMEBREW_NO_AUTO_UPDATE=1
     cache: ccache
     before_install:
-      - brew install ccache
+      - brew install ccache protobuf qt xz
-      - brew install protobuf
-      - brew install qt
     script: bash ./.ci/travis-compile.sh --server --package "$TRAVIS_OS_NAME" --release
 
 

CMakeLists.txt

@@ -99,7 +99,8 @@ option(WARNING_AS_ERROR "Treat warnings as errors in debug builds" ON)
 IF(MSVC)
     # Visual Studio:
     # Maximum optimization
-    set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD")
+    # Disable warning C4251
+    set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD /wd4251")
     # Generate complete debugging information
     #set(CMAKE_CXX_FLAGS_DEBUG "/Zi")
 ELSEIF (CMAKE_COMPILER_IS_GNUCXX)

README.md

@@ -87,8 +87,9 @@ Dependencies: *(for minimum requirements search our [CMake file](https://github.
 - [protobuf](https://github.com/google/protobuf)
 - [CMake](https://www.cmake.org/)
 
-Oracle can optionally use zlib to load zipped files:
+Oracle can optionally use zlib and xz to load compressed files:
 - [zlib](https://www.zlib.net/)
+- [xz](https://tukaani.org/xz/)
 
 To compile:
 

clangify.sh

@@ -15,7 +15,8 @@ include=("common" \
 exclude=("cockatrice/src/qt-json" \
 "servatrice/src/smtp" \
 "common/sfmt" \
-"oracle/src/zip")
+"oracle/src/zip" \
+"oracle/src/lzma")
 exts=("cpp" "h")
 cf_cmd="clang-format"
 branch="origin/master"

cmake/FindWin32SslRuntime.cmake

@@ -2,7 +2,7 @@
 # will be needed by Qt in order to access https urls.
 
 if (WIN32)
-  # Get standard installation paths for OpenSSL under Windows 
+  # Get standard installation paths for OpenSSL under Windows
 
   # http://www.slproweb.com/products/Win32OpenSSL.html
 
@@ -15,6 +15,7 @@ if (WIN32)
       )
     file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles)
     set(_OPENSSL_ROOT_PATHS
+      "C:/Tools/vcpkg/installed/x64-windows/bin"
       "${_programfiles}/OpenSSL-Win64"
       "C:/OpenSSL-Win64/"
       )
@@ -28,6 +29,7 @@ if (WIN32)
       )
     file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles)
     set(_OPENSSL_ROOT_PATHS
+      "C:/Tools/vcpkg/installed/x86-windows/bin"
       "${_programfiles}/OpenSSL"
       "${_programfiles}/OpenSSL-Win32"
       "C:/OpenSSL/"

cockatrice/CMakeLists.txt

@@ -254,6 +254,8 @@ if(WIN32)
     set(plugin_dest_dir Plugins)
     set(qtconf_dest_dir .)
 
+    install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll")
+
     # qt5 plugins: audio, iconengines, imageformats, platforms, printsupport
     install(DIRECTORY "${QT_PLUGINS_DIR}/" DESTINATION ${plugin_dest_dir} COMPONENT Runtime
         FILES_MATCHING REGEX "(audio|iconengines|imageformats|platforms|printsupport)/.*[^d]\\.dll")

cockatrice/src/carddatabase.cpp

@@ -224,13 +224,14 @@ CardInfo::CardInfo(const QString &_name,
                    const SetList &_sets,
                    const QStringMap &_customPicURLs,
                    MuidMap _muIds,
+                   QStringMap _uuIds,
                    QStringMap _collectorNumbers,
                    QStringMap _rarities)
     : name(_name), isToken(_isToken), sets(_sets), manacost(_manacost), cmc(_cmc), cardtype(_cardtype),
       powtough(_powtough), text(_text), colors(_colors), relatedCards(_relatedCards),
       reverseRelatedCards(_reverseRelatedCards), setsNames(), upsideDownArt(_upsideDownArt), loyalty(_loyalty),
-      customPicURLs(_customPicURLs), muIds(std::move(_muIds)), collectorNumbers(std::move(_collectorNumbers)),
+      customPicURLs(_customPicURLs), muIds(std::move(_muIds)), uuIds(std::move(_uuIds)),
-      rarities(std::move(_rarities)), cipt(_cipt), tableRow(_tableRow)
+      collectorNumbers(std::move(_collectorNumbers)), rarities(std::move(_rarities)), cipt(_cipt), tableRow(_tableRow)
 {
     pixmapCacheKey = QLatin1String("card_") + name;
     simpleName = CardInfo::simplifyName(name);
@@ -260,12 +261,13 @@ CardInfoPtr CardInfo::newInstance(const QString &_name,
                                   const SetList &_sets,
                                   const QStringMap &_customPicURLs,
                                   MuidMap _muIds,
+                                  QStringMap _uuIds,
                                   QStringMap _collectorNumbers,
                                   QStringMap _rarities)
 {
     CardInfoPtr ptr(new CardInfo(_name, _isToken, _manacost, _cmc, _cardtype, _powtough, _text, _colors, _relatedCards,
                                  _reverseRelatedCards, _upsideDownArt, _loyalty, _cipt, _tableRow, _sets,
-                                 _customPicURLs, std::move(_muIds), std::move(_collectorNumbers),
+                                 _customPicURLs, std::move(_muIds), std::move(_uuIds), std::move(_collectorNumbers),
                                  std::move(_rarities)));
     ptr->setSmartPointer(ptr);
 
@@ -440,6 +442,7 @@ void CardDatabase::addCard(CardInfoPtr card)
             QString setName = set->getCorrectedShortName();
             sameCard->setSet(set);
             sameCard->setMuId(setName, card->getMuId(setName));
+            sameCard->setUuId(setName, card->getUuId(setName));
             sameCard->setRarity(setName, card->getRarity(setName));
             sameCard->setSetNumber(setName, card->getCollectorNumber(setName));
         }

cockatrice/src/carddatabase.h

@@ -149,6 +149,7 @@ private:
     QString loyalty;
     QStringMap customPicURLs;
     MuidMap muIds;
+    QStringMap uuIds;
     QStringMap collectorNumbers;
     QStringMap rarities;
     bool cipt;
@@ -172,7 +173,8 @@ public:
                       int _tableRow = 0,
                       const SetList &_sets = SetList(),
                       const QStringMap &_customPicURLs = QStringMap(),
-                      MuidMap muids = MuidMap(),
+                      MuidMap _muids = MuidMap(),
+                      QStringMap _uuIds = QStringMap(),
                       QStringMap _collectorNumbers = QStringMap(),
                       QStringMap _rarities = QStringMap());
     ~CardInfo() override;
@@ -193,7 +195,8 @@ public:
                                    int _tableRow = 0,
                                    const SetList &_sets = SetList(),
                                    const QStringMap &_customPicURLs = QStringMap(),
-                                   MuidMap muids = MuidMap(),
+                                   MuidMap _muids = MuidMap(),
+                                   QStringMap _uuIds = QStringMap(),
                                    QStringMap _collectorNumbers = QStringMap(),
                                    QStringMap _rarities = QStringMap());
 
@@ -310,6 +313,10 @@ public:
     {
         return muIds.value(set);
     }
+    QString getUuId(const QString &set) const
+    {
+        return uuIds.value(set);
+    }
     QString getCollectorNumber(const QString &set) const
     {
         return collectorNumbers.value(set);
@@ -344,6 +351,10 @@ public:
     {
         muIds.insert(_set, _muId);
     }
+    void setUuId(const QString &_set, const QString &_uuId)
+    {
+        uuIds.insert(_set, _uuId);
+    }
     void setSetNumber(const QString &_set, const QString &_setNumber)
     {
         collectorNumbers.insert(_set, _setNumber);

cockatrice/src/carddbparser/cockatricexml3.cpp

@@ -133,7 +133,7 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
             QList<CardRelation *> relatedCards, reverseRelatedCards;
             QStringMap customPicURLs;
             MuidMap muids;
-            QStringMap collectorNumbers, rarities;
+            QStringMap uuids, collectorNumbers, rarities;
             SetList sets;
             int tableRow = 0;
             bool cipt = false;
@@ -164,6 +164,10 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
                         muids[setName] = attrs.value("muId").toString().toInt();
                     }
 
+                    if (attrs.hasAttribute("muId")) {
+                        uuids[setName] = attrs.value("uuId").toString();
+                    }
+
                     if (attrs.hasAttribute("picURL")) {
                         customPicURLs[setName] = attrs.value("picURL").toString();
                     }
@@ -232,7 +236,7 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml)
 
             CardInfoPtr newCard = CardInfo::newInstance(
                 name, isToken, manacost, cmc, type, pt, text, colors, relatedCards, reverseRelatedCards, upsideDown,
-                loyalty, cipt, tableRow, sets, customPicURLs, muids, collectorNumbers, rarities);
+                loyalty, cipt, tableRow, sets, customPicURLs, muids, uuids, collectorNumbers, rarities);
             emit addCard(newCard);
         }
     }
@@ -274,6 +278,7 @@ static QXmlStreamWriter &operator<<(QXmlStreamWriter &xml, const CardInfoPtr &in
         tmpSet = sets[i]->getShortName();
         xml.writeAttribute("rarity", info->getRarity(tmpSet));
         xml.writeAttribute("muId", QString::number(info->getMuId(tmpSet)));
+        xml.writeAttribute("uuId", info->getUuId(tmpSet));
 
         tmpString = info->getCollectorNumber(tmpSet);
         if (!tmpString.isEmpty()) {

cockatrice/src/decklistmodel.cpp

@@ -324,7 +324,7 @@ QModelIndex DeckListModel::addCard(const QString &cardName, const QString &zoneN
             // and default values for all fields
             info = CardInfo::newInstance(cardName, false, nullptr, nullptr, "unknown", nullptr, nullptr, QStringList(),
                                          QList<CardRelation *>(), QList<CardRelation *>(), false, 0, false, 0,
-                                         SetList(), QStringMap(), MuidMap(), QStringMap(), QStringMap());
+                                         SetList(), QStringMap(), MuidMap(), QStringMap(), QStringMap(), QStringMap());
         } else {
             return {};
         }

cockatrice/src/pictureloader.cpp

@@ -267,6 +267,7 @@ QString PictureToLoad::transformUrl(QString urlTemplate) const
 
     if (set) {
         transformMap["!cardid!"] = QString::number(card->getMuId(set->getShortName()));
+        transformMap["!uuid!"] = card->getUuId(set->getShortName());
         transformMap["!collectornumber!"] = card->getCollectorNumber(set->getShortName());
         transformMap["!setcode!"] = set->getShortName();
         transformMap["!setcode_lower!"] = set->getShortName().toLower();
@@ -274,6 +275,7 @@ QString PictureToLoad::transformUrl(QString urlTemplate) const
         transformMap["!setname_lower!"] = set->getLongName().toLower();
     } else {
         transformMap["!cardid!"] = QString();
+        transformMap["!uuid!"] = QString();
         transformMap["!collectornumber!"] = QString();
         transformMap["!setcode!"] = QString();
         transformMap["!setcode_lower!"] = QString();

doc/cards.xsd

@@ -39,6 +39,7 @@
                                             <xs:simpleContent>
                                                 <xs:extension base="xs:string">
                                                     <xs:attribute type="xs:int" name="muId" use="optional" />
+                                                    <xs:attribute type="xs:string" name="uuId"  use="optional" />
                                                     <xs:attribute type="xs:anyURI" name="picURL" use="optional" />
                                                     <xs:attribute type="xs:string" name="num"  use="optional" />
                                                     <xs:attribute type="xs:string" name="rarity" use="optional" />

oracle/CMakeLists.txt

@@ -85,18 +85,33 @@ IF(ZLIB_FOUND)
         src/zip/unzip.cpp
         src/zip/zipglobal.cpp
     )
-
 ELSE()
     MESSAGE(STATUS "Oracle: zlib not found; ZIP support disabled")
 ENDIF()
 
+# LibLZMA is required to support xz files
+FIND_PACKAGE(LibLZMA)
+IF(LIBLZMA_FOUND)
+    INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS})
+    ADD_DEFINITIONS("-DHAS_LZMA")
+
+    set(oracle_SOURCES ${oracle_SOURCES}
+        src/lzma/decompress.cpp
+    )
+ELSE()
+    MESSAGE(STATUS "Oracle: LibLZMA not found; xz support disabled")
+ENDIF()
+
 # Build oracle binary and link it
 ADD_EXECUTABLE(oracle WIN32 MACOSX_BUNDLE ${oracle_SOURCES} ${oracle_QM} ${oracle_RESOURCES_RCC} ${oracle_MOC_SRCS})
+TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES})
 
 IF(ZLIB_FOUND)
-    TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES} ${ZLIB_LIBRARIES})
+    TARGET_LINK_LIBRARIES(oracle ${ZLIB_LIBRARIES})
-ELSE()
+ENDIF()
-    TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES})
+
+IF(LIBLZMA_FOUND)
+    TARGET_LINK_LIBRARIES(oracle ${LIBLZMA_LIBRARIES})
 ENDIF()
 
 if(UNIX)
@@ -163,14 +178,8 @@ IF(WIN32)
     set(plugin_dest_dir Plugins)
     set(qtconf_dest_dir .)
     list(APPEND libSearchDirs ${QT_LIBRARY_DIR})
-    IF(ZLIB_FOUND)
+
-        # look for dll in the bin/ directory (gnuwin32 package)
+    install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll")
-        get_filename_component(ZLIB_DLL_DIR "${ZLIB_INCLUDE_DIRS}/../bin/" REALPATH)
-        list(APPEND libSearchDirs ${ZLIB_DLL_DIR})
-        # look for dll in the lib/ directory (nuget package)
-        get_filename_component(ZLIB_DLL_DIR "${ZLIB_LIBRARY}" DIRECTORY)
-        list(APPEND libSearchDirs ${ZLIB_DLL_DIR})
-    ENDIF()
 
     # qt5 plugins: iconengines, platforms
 

oracle/src/lzma/decompress.cpp

@@ -0,0 +1,250 @@
+/*
+ * Simple routing to extract a single file from a xz archive
+ * Heavily based from doc/examples/02_decompress.c obtained from
+ * the official xz git repository: git.tukaani.org/xz.git
+ * The license from the original file header follows
+ *
+ * Author:     Lasse Collin
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+
+#include <lzma.h>
+#include <QDebug>
+
+#include "decompress.h"
+
+XzDecompressor::XzDecompressor(QObject *parent)
+    : QObject(parent)
+{
+
+}
+
+bool XzDecompressor::decompress(QBuffer *in, QBuffer *out)
+{
+	lzma_stream strm = LZMA_STREAM_INIT;
+	bool success;
+
+	if (!init_decoder(&strm)) {
+		return false;
+	}
+
+	success = internal_decompress(&strm, in, out);
+
+	// Free the memory allocated for the decoder. This only needs to be
+	// done after the last file.
+	lzma_end(&strm);
+
+	return success;
+}
+
+bool XzDecompressor::init_decoder(lzma_stream *strm)
+{
+	// Initialize a .xz decoder. The decoder supports a memory usage limit
+	// and a set of flags.
+	//
+	// The memory usage of the decompressor depends on the settings used
+	// to compress a .xz file. It can vary from less than a megabyte to
+	// a few gigabytes, but in practice (at least for now) it rarely
+	// exceeds 65 MiB because that's how much memory is required to
+	// decompress files created with "xz -9". Settings requiring more
+	// memory take extra effort to use and don't (at least for now)
+	// provide significantly better compression in most cases.
+	//
+	// Memory usage limit is useful if it is important that the
+	// decompressor won't consume gigabytes of memory. The need
+	// for limiting depends on the application. In this example,
+	// no memory usage limiting is used. This is done by setting
+	// the limit to UINT64_MAX.
+	//
+	// The .xz format allows concatenating compressed files as is:
+	//
+	//     echo foo | xz > foobar.xz
+	//     echo bar | xz >> foobar.xz
+	//
+	// When decompressing normal standalone .xz files, LZMA_CONCATENATED
+	// should always be used to support decompression of concatenated
+	// .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop
+	// after the first .xz stream. This can be useful when .xz data has
+	// been embedded inside another file format.
+	//
+	// Flags other than LZMA_CONCATENATED are supported too, and can
+	// be combined with bitwise-or. See lzma/container.h
+	// (src/liblzma/api/lzma/container.h in the source package or e.g.
+	// /usr/include/lzma/container.h depending on the install prefix)
+	// for details.
+	lzma_ret ret = lzma_stream_decoder(
+			strm, UINT64_MAX, LZMA_CONCATENATED);
+
+	// Return successfully if the initialization went fine.
+	if (ret == LZMA_OK)
+		return true;
+
+	// Something went wrong. The possible errors are documented in
+	// lzma/container.h (src/liblzma/api/lzma/container.h in the source
+	// package or e.g. /usr/include/lzma/container.h depending on the
+	// install prefix).
+	//
+	// Note that LZMA_MEMLIMIT_ERROR is never possible here. If you
+	// specify a very tiny limit, the error will be delayed until
+	// the first headers have been parsed by a call to lzma_code().
+	const char *msg;
+	switch (ret) {
+	case LZMA_MEM_ERROR:
+		msg = "Memory allocation failed";
+		break;
+
+	case LZMA_OPTIONS_ERROR:
+		msg = "Unsupported decompressor flags";
+		break;
+
+	default:
+		// This is most likely LZMA_PROG_ERROR indicating a bug in
+		// this program or in liblzma. It is inconvenient to have a
+		// separate error message for errors that should be impossible
+		// to occur, but knowing the error code is important for
+		// debugging. That's why it is good to print the error code
+		// at least when there is no good error message to show.
+		msg = "Unknown error, possibly a bug";
+		break;
+	}
+
+	qDebug() << "Error initializing the decoder:" << msg << "(error code " << ret << ")";
+	return false;
+}
+
+
+bool XzDecompressor::internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out)
+{
+	// When LZMA_CONCATENATED flag was used when initializing the decoder,
+	// we need to tell lzma_code() when there will be no more input.
+	// This is done by setting action to LZMA_FINISH instead of LZMA_RUN
+	// in the same way as it is done when encoding.
+	//
+	// When LZMA_CONCATENATED isn't used, there is no need to use
+	// LZMA_FINISH to tell when all the input has been read, but it
+	// is still OK to use it if you want. When LZMA_CONCATENATED isn't
+	// used, the decoder will stop after the first .xz stream. In that
+	// case some unused data may be left in strm->next_in.
+	lzma_action action = LZMA_RUN;
+
+	uint8_t inbuf[BUFSIZ];
+	uint8_t outbuf[BUFSIZ];
+	qint64 bytesAvailable;
+
+	strm->next_in = NULL;
+	strm->avail_in = 0;
+	strm->next_out = outbuf;
+	strm->avail_out = sizeof(outbuf);
+	while (true) {
+		if (strm->avail_in == 0) {
+			strm->next_in = inbuf;
+			bytesAvailable = in->bytesAvailable();
+			if(bytesAvailable == 0) {
+				// Once the end of the input file has been reached,
+				// we need to tell lzma_code() that no more input
+				// will be coming. As said before, this isn't required
+				// if the LZMA_CONCATENATED flag isn't used when
+				// initializing the decoder.
+				action = LZMA_FINISH;
+			} else if(bytesAvailable >= BUFSIZ) {
+				in->read((char*) inbuf, BUFSIZ);
+				strm->avail_in = BUFSIZ;
+			} else {
+				in->read((char*) inbuf, bytesAvailable);
+				strm->avail_in = bytesAvailable;
+			}
+		}
+
+		lzma_ret ret = lzma_code(strm, action);
+
+		if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
+			qint64 write_size = sizeof(outbuf) - strm->avail_out;
+
+			if (out->write((char *) outbuf, write_size) != write_size) {
+				qDebug() << "Write error";
+				return false;
+			}
+
+			strm->next_out = outbuf;
+			strm->avail_out = sizeof(outbuf);
+		}
+
+		if (ret != LZMA_OK) {
+			// Once everything has been decoded successfully, the
+			// return value of lzma_code() will be LZMA_STREAM_END.
+			//
+			// It is important to check for LZMA_STREAM_END. Do not
+			// assume that getting ret != LZMA_OK would mean that
+			// everything has gone well or that when you aren't
+			// getting more output it must have successfully
+			// decoded everything.
+			if (ret == LZMA_STREAM_END)
+				return true;
+
+			// It's not LZMA_OK nor LZMA_STREAM_END,
+			// so it must be an error code. See lzma/base.h
+			// (src/liblzma/api/lzma/base.h in the source package
+			// or e.g. /usr/include/lzma/base.h depending on the
+			// install prefix) for the list and documentation of
+			// possible values. Many values listen in lzma_ret
+			// enumeration aren't possible in this example, but
+			// can be made possible by enabling memory usage limit
+			// or adding flags to the decoder initialization.
+			const char *msg;
+			switch (ret) {
+			case LZMA_MEM_ERROR:
+				msg = "Memory allocation failed";
+				break;
+
+			case LZMA_FORMAT_ERROR:
+				// .xz magic bytes weren't found.
+				msg = "The input is not in the .xz format";
+				break;
+
+			case LZMA_OPTIONS_ERROR:
+				// For example, the headers specify a filter
+				// that isn't supported by this liblzma
+				// version (or it hasn't been enabled when
+				// building liblzma, but no-one sane does
+				// that unless building liblzma for an
+				// embedded system). Upgrading to a newer
+				// liblzma might help.
+				//
+				// Note that it is unlikely that the file has
+				// accidentally became corrupt if you get this
+				// error. The integrity of the .xz headers is
+				// always verified with a CRC32, so
+				// unintentionally corrupt files can be
+				// distinguished from unsupported files.
+				msg = "Unsupported compression options";
+				break;
+
+			case LZMA_DATA_ERROR:
+				msg = "Compressed file is corrupt";
+				break;
+
+			case LZMA_BUF_ERROR:
+				// Typically this error means that a valid
+				// file has got truncated, but it might also
+				// be a damaged part in the file that makes
+				// the decoder think the file is truncated.
+				// If you prefer, you can use the same error
+				// message for this as for LZMA_DATA_ERROR.
+				msg = "Compressed file is truncated or "
+						"otherwise corrupt";
+				break;
+
+			default:
+				// This is most likely LZMA_PROG_ERROR.
+				msg = "Unknown error, possibly a bug";
+				break;
+			}
+
+			qDebug() << "Decoder error:" << msg << "(error code " << ret << ")";
+			return false;
+		}
+	}
+}
+

oracle/src/lzma/decompress.h

@@ -0,0 +1,19 @@
+#ifndef XZ_DECOMPRESS_H
+#define XZ_DECOMPRESS_H
+
+#include <lzma.h>
+#include <QBuffer>
+
+class XzDecompressor : public QObject
+{
+    Q_OBJECT
+public:
+    XzDecompressor(QObject *parent = 0);
+    ~XzDecompressor() { };
+    bool decompress(QBuffer *in, QBuffer *out);
+private:
+    bool init_decoder(lzma_stream *strm);
+    bool internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out);
+};
+
+#endif

oracle/src/oracleimporter.cpp

@@ -19,7 +19,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
     setsMap = QtJson::Json::parse(QString(data), ok).toMap();
     if (!ok) {
         qDebug() << "error: QtJson::Json::parse()";
-        return 0;
+        return false;
     }
 
     QListIterator<QVariant> it(setsMap.values());
@@ -33,7 +33,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data)
 
     while (it.hasNext()) {
         map = it.next().toMap();
-        edition = map.value("code").toString();
+        edition = map.value("code").toString().toUpper();
         editionLong = map.value("name").toString();
         editionCards = map.value("cards");
         setType = map.value("type").toString();
@@ -57,6 +57,7 @@ CardInfoPtr OracleImporter::addCard(const QString &setName,
                                     QString cardName,
                                     bool isToken,
                                     int cardId,
+                                    QString &cardUuId,
                                     QString &setNumber,
                                     QString &cardCost,
                                     QString &cmc,
@@ -124,30 +125,13 @@ CardInfoPtr OracleImporter::addCard(const QString &setName,
         cards.insert(cardName, card);
     }
     card->setMuId(setName, cardId);
+    card->setUuId(setName, cardUuId);
     card->setSetNumber(setName, setNumber);
     card->setRarity(setName, rarity);
 
     return card;
 }
 
-void OracleImporter::extractColors(const QStringList &in, QStringList &out)
-{
-    foreach (QString c, in) {
-        if (c == "White")
-            out << "W";
-        else if (c == "Blue")
-            out << "U";
-        else if (c == "Black")
-            out << "B";
-        else if (c == "Red")
-            out << "R";
-        else if (c == "Green")
-            out << "G";
-        else
-            qDebug() << "error: unknown color:" << c;
-    }
-}
-
 int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
 {
     int cards = 0;
@@ -164,6 +148,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
     QList<CardRelation *> relatedCards;
     QList<CardRelation *> reverseRelatedCards; // dummy
     int cardId;
+    QString cardUuId;
     QString setNumber;
     QString rarity;
     QString cardLoyalty;
@@ -173,15 +158,20 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
     while (it.hasNext()) {
         map = it.next().toMap();
 
+        /* Currently used layouts are:
+         * augment, double_faced_token, flip, host, leveler, meld, normal, planar,
+         * saga, scheme, split, token, transform, vanguard
+         */
         QString layout = map.value("layout").toString();
 
         // don't import tokens from the json file
         if (layout == "token")
             continue;
 
-        if (layout == "split" || layout == "aftermath") {
+        // Aftermath card layout seems to have been integrated in "split"
+        if (layout == "split") {
             // Enqueue split card for later handling
-            cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0;
+            cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0;
             if (cardId)
                 splitCards.insertMulti(cardId, map);
             continue;
@@ -190,16 +180,18 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
         // normal cards handling
         cardName = map.contains("name") ? map.value("name").toString() : QString("");
         cardCost = map.contains("manaCost") ? map.value("manaCost").toString() : QString("");
-        cmc = map.contains("cmc") ? map.value("cmc").toString() : QString("0");
+        cmc = map.contains("convertedManaCost") ? map.value("convertedManaCost").toString() : QString("0");
         cardType = map.contains("type") ? map.value("type").toString() : QString("");
         cardPT = map.contains("power") || map.contains("toughness")
                      ? map.value("power").toString() + QString('/') + map.value("toughness").toString()
                      : QString("");
         cardText = map.contains("text") ? map.value("text").toString() : QString("");
-        cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0;
+        cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0;
+        cardUuId = map.contains("uuid") ? map.value("uuid").toString() : QString("");
         setNumber = map.contains("number") ? map.value("number").toString() : QString("");
         rarity = map.contains("rarity") ? map.value("rarity").toString() : QString("");
         cardLoyalty = map.contains("loyalty") ? map.value("loyalty").toString() : QString("");
+        colors = map.contains("colors") ? map.value("colors").toStringList() : QStringList();
         relatedCards = QList<CardRelation *>();
         if (map.contains("names"))
             foreach (const QString &name, map.value("names").toStringList()) {
@@ -214,11 +206,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
             upsideDown = false;
         }
 
-        colors.clear();
-        extractColors(map.value("colors").toStringList(), colors);
-
         CardInfoPtr card =
-            addCard(set->getShortName(), cardName, false, cardId, setNumber, cardCost, cmc, cardType, cardPT,
+            addCard(set->getShortName(), cardName, false, cardId, cardUuId, setNumber, cardCost, cmc, cardType, cardPT,
                     cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
 
         if (!set->contains(card)) {
@@ -250,6 +239,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
         cardType = "";
         cardPT = "";
         cardText = "";
+        cardUuId = "";
         setNumber = "";
         rarity = "";
         cardLoyalty = "";
@@ -269,10 +259,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
                     cardCost += prefix;
                 cardCost += map.value("manaCost").toString();
             }
-            if (map.contains("cmc")) {
+            if (map.contains("convertedManaCost")) {
                 if (!cmc.isEmpty())
                     cmc += prefix;
-                cmc += map.value("cmc").toString();
+                cmc += map.value("convertedManaCost").toString();
             }
             if (map.contains("type")) {
                 if (!cardType.isEmpty())
@@ -289,6 +279,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
                     cardText += prefix2;
                 cardText += map.value("text").toString();
             }
+            if (map.contains("uuid")) {
+                if (cardUuId.isEmpty())
+                    cardUuId = map.value("uuid").toString();
+            }
             if (map.contains("number")) {
                 if (setNumber.isEmpty())
                     setNumber = map.value("number").toString();
@@ -298,7 +292,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
                     rarity = map.value("rarity").toString();
             }
 
-            extractColors(map.value("colors").toStringList(), colors);
+            colors << map.value("colors").toStringList();
         }
 
         colors.removeDuplicates();
@@ -313,8 +307,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data)
 
         // add the card
         CardInfoPtr card =
-            addCard(set->getShortName(), cardName, false, muid, setNumber, cardCost, cmc, cardType, cardPT, cardLoyalty,
+            addCard(set->getShortName(), cardName, false, muid, cardUuId, setNumber, cardCost, cmc, cardType, cardPT,
-                    cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
+                    cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity);
 
         if (!set->contains(card)) {
             card->addToSet(set);

oracle/src/oracleimporter.h

@@ -61,6 +61,7 @@ private:
                         QString cardName,
                         bool isToken,
                         int cardId,
+                        QString &cardUuId,
                         QString &setNumber,
                         QString &cardCost,
                         QString &cmc,
@@ -93,7 +94,6 @@ public:
     }
 
 protected:
-    void extractColors(const QStringList &in, QStringList &out);
     void sortColors(QStringList &colors);
 };
 

oracle/src/oraclewizard.cpp

@@ -26,11 +26,22 @@
 #include "settingscache.h"
 #include "version_string.h"
 
-#define ZIP_SIGNATURE "PK"
+#ifdef HAS_LZMA
-#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json"
+#include "lzma/decompress.h"
+#endif
 
 #ifdef HAS_ZLIB
 #include "zip/unzip.h"
+#endif
+
+#define ZIP_SIGNATURE "PK"
+// Xz stream header: 0xFD + "7zXZ"
+#define XZ_SIGNATURE "\xFD\x37\x7A\x58\x5A"
+#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json"
+
+#ifdef HAS_LZMA
+#define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.xz"
+#elif defined(HAS_ZLIB)
 #define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.zip"
 #else
 #define ALLSETS_URL "https://mtgjson.com/json/AllSets.json"
@@ -249,11 +260,14 @@ void LoadSetsPage::actLoadSetsFile()
     QFileDialog dialog(this, tr("Load sets file"));
     dialog.setFileMode(QFileDialog::ExistingFile);
 
+    QString extensions = "*.json";
 #ifdef HAS_ZLIB
-    dialog.setNameFilter(tr("Sets JSON file (*.json *.zip)"));
+    extensions += " *.zip";
-#else
-    dialog.setNameFilter(tr("Sets JSON file (*.json)"));
 #endif
+#ifdef HAS_LZMA
+    extensions += " *.xz";
+#endif
+    dialog.setNameFilter(tr("Sets JSON file (%1)").arg(extensions));
 
     if (!fileLineEdit->text().isEmpty() && QFile::exists(fileLineEdit->text())) {
         dialog.selectFile(fileLineEdit->text());
@@ -383,7 +397,32 @@ void LoadSetsPage::readSetsFromByteArray(QByteArray data)
     progressBar->show();
 
     // unzip the file if needed
-    if (data.startsWith(ZIP_SIGNATURE)) {
+    if (data.startsWith(XZ_SIGNATURE)) {
+#ifdef HAS_LZMA
+        // zipped file
+        auto *inBuffer = new QBuffer(&data);
+        auto *outBuffer = new QBuffer(this);
+        inBuffer->open(QBuffer::ReadOnly);
+        outBuffer->open(QBuffer::WriteOnly);
+        XzDecompressor xz;
+        if (!xz.decompress(inBuffer, outBuffer)) {
+            zipDownloadFailed(tr("Xz extraction failed."));
+            return;
+        }
+
+        future = QtConcurrent::run(wizard()->importer, &OracleImporter::readSetsFromByteArray, outBuffer->data());
+        watcher.setFuture(future);
+        return;
+#else
+        zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files."));
+
+        wizard()->enableButtons();
+        setEnabled(true);
+        progressLabel->hide();
+        progressBar->hide();
+        return;
+#endif
+    } else if (data.startsWith(ZIP_SIGNATURE)) {
 #ifdef HAS_ZLIB
         // zipped file
         auto *inBuffer = new QBuffer(&data);

servatrice/CMakeLists.txt

@@ -164,6 +164,8 @@ if(WIN32)
     set(plugin_dest_dir Plugins)
     set(qtconf_dest_dir .)
 
+    install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll")
+
     # qt5 plugins: platforms, sqldrivers/mysql
     install(DIRECTORY "${QT_PLUGINS_DIR}/" DESTINATION ${plugin_dest_dir} COMPONENT Runtime
         FILES_MATCHING REGEX "(platforms/.*|sqldrivers/qsqlmysql)\\.dll"