mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-06-25 08:03:54 -07:00
Try to parallelize stuff.
This commit is contained in:
parent
aa24502129
commit
cc7deef83e
4 changed files with 310 additions and 288 deletions
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
#include "../../../game/cards/card_database_manager.h"
|
#include "../../../game/cards/card_database_manager.h"
|
||||||
#include "../../../settings/cache_settings.h"
|
#include "../../../settings/cache_settings.h"
|
||||||
|
#include "picture_loader_worker_work.h"
|
||||||
|
|
||||||
#include <QBuffer>
|
#include <QBuffer>
|
||||||
#include <QDirIterator>
|
#include <QDirIterator>
|
||||||
|
|
@ -9,6 +10,7 @@
|
||||||
#include <QNetworkDiskCache>
|
#include <QNetworkDiskCache>
|
||||||
#include <QNetworkReply>
|
#include <QNetworkReply>
|
||||||
#include <QThread>
|
#include <QThread>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
// Card back returned by gatherer when card is not found
|
// Card back returned by gatherer when card is not found
|
||||||
QStringList PictureLoaderWorker::md5Blacklist = QStringList() << "db0c48db407a907c16ade38de048a441";
|
QStringList PictureLoaderWorker::md5Blacklist = QStringList() << "db0c48db407a907c16ade38de048a441";
|
||||||
|
|
@ -59,178 +61,35 @@ PictureLoaderWorker::~PictureLoaderWorker()
|
||||||
pictureLoaderThread->deleteLater();
|
pictureLoaderThread->deleteLater();
|
||||||
}
|
}
|
||||||
|
|
||||||
void PictureLoaderWorker::processLoadQueue()
|
QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWorkerWork *worker)
|
||||||
{
|
{
|
||||||
if (loadQueueRunning) {
|
// Check if the request leads to a cached redirect
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
loadQueueRunning = true;
|
|
||||||
while (true) {
|
|
||||||
mutex.lock();
|
|
||||||
if (loadQueue.isEmpty()) {
|
|
||||||
mutex.unlock();
|
|
||||||
loadQueueRunning = false;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
cardBeingLoaded = loadQueue.takeFirst();
|
|
||||||
mutex.unlock();
|
|
||||||
|
|
||||||
QString setName = cardBeingLoaded.getSetName();
|
|
||||||
QString cardName = cardBeingLoaded.getCard()->getName();
|
|
||||||
QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName();
|
|
||||||
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardName << " set: " << setName << "]: Trying to load picture";
|
|
||||||
|
|
||||||
if (CardDatabaseManager::getInstance()->isProviderIdForPreferredPrinting(
|
|
||||||
cardName, cardBeingLoaded.getCard()->getPixmapCacheKey())) {
|
|
||||||
if (cardImageExistsOnDisk(setName, correctedCardName)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardName << " set: " << setName << "]: No custom picture, trying to download";
|
|
||||||
cardsToDownload.append(cardBeingLoaded);
|
|
||||||
cardBeingLoaded.clear();
|
|
||||||
if (!downloadRunning) {
|
|
||||||
startNextPicDownload();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool PictureLoaderWorker::cardImageExistsOnDisk(QString &setName, QString &correctedCardname)
|
|
||||||
{
|
|
||||||
QImage image;
|
|
||||||
QImageReader imgReader;
|
|
||||||
imgReader.setDecideFormatFromContent(true);
|
|
||||||
QList<QString> picsPaths = QList<QString>();
|
|
||||||
QDirIterator it(customPicsPath, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
|
|
||||||
|
|
||||||
// Recursively check all subdirectories of the CUSTOM folder
|
|
||||||
while (it.hasNext()) {
|
|
||||||
QString thisPath(it.next());
|
|
||||||
QFileInfo thisFileInfo(thisPath);
|
|
||||||
|
|
||||||
if (thisFileInfo.isFile() &&
|
|
||||||
(thisFileInfo.fileName() == correctedCardname || thisFileInfo.completeBaseName() == correctedCardname ||
|
|
||||||
thisFileInfo.baseName() == correctedCardname)) {
|
|
||||||
picsPaths << thisPath; // Card found in the CUSTOM directory, somewhere
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!setName.isEmpty()) {
|
|
||||||
picsPaths << picsPath + "/" + setName + "/" + correctedCardname
|
|
||||||
// We no longer store downloaded images there, but don't just ignore
|
|
||||||
// stuff that old versions have put there.
|
|
||||||
<< picsPath + "/downloadedPics/" + setName + "/" + correctedCardname;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Iterates through the list of paths, searching for images with the desired
|
|
||||||
// name with any QImageReader-supported
|
|
||||||
// extension
|
|
||||||
for (const auto &_picsPath : picsPaths) {
|
|
||||||
imgReader.setFileName(_picsPath);
|
|
||||||
if (imgReader.read(&image)) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << correctedCardname << " set: " << setName << "]: Picture found on disk.";
|
|
||||||
imageLoaded(cardBeingLoaded.getCard(), image);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
imgReader.setFileName(_picsPath + ".full");
|
|
||||||
if (imgReader.read(&image)) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << correctedCardname << " set: " << setName << "]: Picture.full found on disk.";
|
|
||||||
imageLoaded(cardBeingLoaded.getCard(), image);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
imgReader.setFileName(_picsPath + ".xlhq");
|
|
||||||
if (imgReader.read(&image)) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << correctedCardname << " set: " << setName << "]: Picture.xlhq found on disk.";
|
|
||||||
imageLoaded(cardBeingLoaded.getCard(), image);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void PictureLoaderWorker::startNextPicDownload()
|
|
||||||
{
|
|
||||||
if (cardsToDownload.isEmpty()) {
|
|
||||||
cardBeingDownloaded.clear();
|
|
||||||
downloadRunning = false;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
downloadRunning = true;
|
|
||||||
|
|
||||||
cardBeingDownloaded = cardsToDownload.takeFirst();
|
|
||||||
|
|
||||||
QString picUrl = cardBeingDownloaded.getCurrentUrl();
|
|
||||||
|
|
||||||
if (picUrl.isEmpty()) {
|
|
||||||
downloadRunning = false;
|
|
||||||
picDownloadFailed();
|
|
||||||
} else {
|
|
||||||
QUrl url(picUrl);
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace() << "[card: " << cardBeingDownloaded.getCard()->getCorrectedName()
|
|
||||||
<< " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: Trying to fetch picture from url " << url.toDisplayString();
|
|
||||||
makeRequest(url);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void PictureLoaderWorker::picDownloadFailed()
|
|
||||||
{
|
|
||||||
/* Take advantage of short circuiting here to call the nextUrl until one
|
|
||||||
is not available. Only once nextUrl evaluates to false will this move
|
|
||||||
on to nextSet. If the Urls for a particular card are empty, this will
|
|
||||||
effectively go through the sets for that card. */
|
|
||||||
if (cardBeingDownloaded.nextUrl() || cardBeingDownloaded.nextSet()) {
|
|
||||||
mutex.lock();
|
|
||||||
loadQueue.prepend(cardBeingDownloaded);
|
|
||||||
mutex.unlock();
|
|
||||||
} else {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getCorrectedName()
|
|
||||||
<< " set: " << cardBeingDownloaded.getSetName() << "]: Picture NOT found, "
|
|
||||||
<< (picDownload ? "download failed" : "downloads disabled")
|
|
||||||
<< ", no more url combinations to try: BAILING OUT";
|
|
||||||
imageLoaded(cardBeingDownloaded.getCard(), QImage());
|
|
||||||
cardBeingDownloaded.clear();
|
|
||||||
}
|
|
||||||
emit startLoadQueue();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool PictureLoaderWorker::imageIsBlackListed(const QByteArray &picData)
|
|
||||||
{
|
|
||||||
QString md5sum = QCryptographicHash::hash(picData, QCryptographicHash::Md5).toHex();
|
|
||||||
return md5Blacklist.contains(md5sum);
|
|
||||||
}
|
|
||||||
|
|
||||||
QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url)
|
|
||||||
{
|
|
||||||
// Check if the redirect is cached
|
|
||||||
QUrl cachedRedirect = getCachedRedirect(url);
|
QUrl cachedRedirect = getCachedRedirect(url);
|
||||||
if (!cachedRedirect.isEmpty()) {
|
if (!cachedRedirect.isEmpty()) {
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
qCDebug(PictureLoaderWorkerLog).nospace()
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getCorrectedName()
|
<< "[card: " << cardBeingDownloaded.getCard()->getCorrectedName()
|
||||||
<< " set: " << cardBeingDownloaded.getSetName() << "]: Using cached redirect for " << url.toDisplayString()
|
<< " set: " << cardBeingDownloaded.getSetName() << "]: Using cached redirect for " << url.toDisplayString()
|
||||||
<< " to " << cachedRedirect.toDisplayString();
|
<< " to " << cachedRedirect.toDisplayString();
|
||||||
return makeRequest(cachedRedirect); // Use the cached redirect
|
return makeRequest(cachedRedirect, worker); // Use the cached redirect
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// It's not cached so we actually have to make the request
|
||||||
|
|
||||||
QNetworkRequest req(url);
|
QNetworkRequest req(url);
|
||||||
|
|
||||||
|
// If the user requests not to "download card images on the fly", we have to cache this network request
|
||||||
|
|
||||||
if (!picDownload) {
|
if (!picDownload) {
|
||||||
req.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysCache);
|
req.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysCache);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Make the actual network request
|
||||||
|
|
||||||
QNetworkReply *reply = networkManager->get(req);
|
QNetworkReply *reply = networkManager->get(req);
|
||||||
|
|
||||||
|
// Connect an additional check to the finished signal to check if the request was redirected and if so, save it to
|
||||||
|
// the redirection cache
|
||||||
|
|
||||||
connect(reply, &QNetworkReply::finished, this, [this, reply, url]() {
|
connect(reply, &QNetworkReply::finished, this, [this, reply, url]() {
|
||||||
QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
|
QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
|
||||||
|
|
||||||
|
|
@ -250,9 +109,17 @@ QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url)
|
||||||
reply->deleteLater();
|
reply->deleteLater();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
connect(reply, &QNetworkReply::finished, worker, [reply, worker]() { worker->picDownloadFinished(reply); });
|
||||||
|
|
||||||
return reply;
|
return reply;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void PictureLoaderWorker::enqueueImageLoad(const CardInfoPtr &card)
|
||||||
|
{
|
||||||
|
auto worker = new PictureLoaderWorkerWork(this, card);
|
||||||
|
Q_UNUSED(worker);
|
||||||
|
}
|
||||||
|
|
||||||
void PictureLoaderWorker::cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl)
|
void PictureLoaderWorker::cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl)
|
||||||
{
|
{
|
||||||
redirectCache[originalUrl] = qMakePair(redirectUrl, QDateTime::currentDateTimeUtc());
|
redirectCache[originalUrl] = qMakePair(redirectUrl, QDateTime::currentDateTimeUtc());
|
||||||
|
|
@ -315,129 +182,6 @@ void PictureLoaderWorker::cleanStaleEntries()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
|
|
||||||
{
|
|
||||||
bool isFromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
|
|
||||||
|
|
||||||
if (reply->error()) {
|
|
||||||
if (isFromCache) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: Removing corrupted cache file for url " << reply->url().toDisplayString() << " and retrying ("
|
|
||||||
<< reply->errorString() << ")";
|
|
||||||
|
|
||||||
networkManager->cache()->remove(reply->url());
|
|
||||||
|
|
||||||
makeRequest(reply->url());
|
|
||||||
} else {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: " << (picDownload ? "Download" : "Cache search") << " failed for url "
|
|
||||||
<< reply->url().toDisplayString() << " (" << reply->errorString() << ")";
|
|
||||||
|
|
||||||
picDownloadFailed();
|
|
||||||
startNextPicDownload();
|
|
||||||
}
|
|
||||||
|
|
||||||
reply->deleteLater();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// List of status codes from https://doc.qt.io/qt-6/qnetworkreply.html#redirected
|
|
||||||
int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
|
|
||||||
if (statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 305 || statusCode == 307 ||
|
|
||||||
statusCode == 308) {
|
|
||||||
QUrl redirectUrl = reply->header(QNetworkRequest::LocationHeader).toUrl();
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: following " << (isFromCache ? "cached redirect" : "redirect") << " to "
|
|
||||||
<< redirectUrl.toDisplayString();
|
|
||||||
makeRequest(redirectUrl);
|
|
||||||
reply->deleteLater();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// peek is used to keep the data in the buffer for use by QImageReader
|
|
||||||
const QByteArray &picData = reply->peek(reply->size());
|
|
||||||
|
|
||||||
if (imageIsBlackListed(picData)) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: Picture found, but blacklisted, will consider it as not found";
|
|
||||||
|
|
||||||
picDownloadFailed();
|
|
||||||
reply->deleteLater();
|
|
||||||
startNextPicDownload();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
QImage testImage;
|
|
||||||
|
|
||||||
QImageReader imgReader;
|
|
||||||
imgReader.setDecideFormatFromContent(true);
|
|
||||||
imgReader.setDevice(reply);
|
|
||||||
|
|
||||||
bool logSuccessMessage = false;
|
|
||||||
|
|
||||||
static const int riffHeaderSize = 12; // RIFF_HEADER_SIZE from webp/format_constants.h
|
|
||||||
auto replyHeader = reply->peek(riffHeaderSize);
|
|
||||||
|
|
||||||
if (replyHeader.startsWith("RIFF") && replyHeader.endsWith("WEBP")) {
|
|
||||||
auto imgBuf = QBuffer(this);
|
|
||||||
imgBuf.setData(reply->readAll());
|
|
||||||
|
|
||||||
auto movie = QMovie(&imgBuf);
|
|
||||||
movie.start();
|
|
||||||
movie.stop();
|
|
||||||
|
|
||||||
imageLoaded(cardBeingDownloaded.getCard(), movie.currentImage());
|
|
||||||
logSuccessMessage = true;
|
|
||||||
} else if (imgReader.read(&testImage)) {
|
|
||||||
imageLoaded(cardBeingDownloaded.getCard(), testImage);
|
|
||||||
logSuccessMessage = true;
|
|
||||||
} else {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: Possible " << (isFromCache ? "cached" : "downloaded") << " picture at "
|
|
||||||
<< reply->url().toDisplayString() << " could not be loaded: " << reply->errorString();
|
|
||||||
|
|
||||||
picDownloadFailed();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (logSuccessMessage) {
|
|
||||||
qCDebug(PictureLoaderWorkerLog).nospace()
|
|
||||||
<< "[card: " << cardBeingDownloaded.getCard()->getName() << " set: " << cardBeingDownloaded.getSetName()
|
|
||||||
<< "]: Image successfully " << (isFromCache ? "loaded from cached" : "downloaded from") << " url "
|
|
||||||
<< reply->url().toDisplayString();
|
|
||||||
}
|
|
||||||
|
|
||||||
reply->deleteLater();
|
|
||||||
startNextPicDownload();
|
|
||||||
}
|
|
||||||
|
|
||||||
void PictureLoaderWorker::enqueueImageLoad(CardInfoPtr card)
|
|
||||||
{
|
|
||||||
QMutexLocker locker(&mutex);
|
|
||||||
|
|
||||||
// avoid queueing the same card more than once
|
|
||||||
if (!card || card == cardBeingLoaded.getCard() || card == cardBeingDownloaded.getCard()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const PictureToLoad &pic : loadQueue) {
|
|
||||||
if (pic.getCard() == card)
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const PictureToLoad &pic : cardsToDownload) {
|
|
||||||
if (pic.getCard() == card)
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
loadQueue.append(PictureToLoad(card));
|
|
||||||
emit startLoadQueue();
|
|
||||||
}
|
|
||||||
|
|
||||||
void PictureLoaderWorker::picDownloadChanged()
|
void PictureLoaderWorker::picDownloadChanged()
|
||||||
{
|
{
|
||||||
QMutexLocker locker(&mutex);
|
QMutexLocker locker(&mutex);
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
#define PICTURE_LOADER_WORKER_H
|
#define PICTURE_LOADER_WORKER_H
|
||||||
|
|
||||||
#include "../../../game/cards/card_database.h"
|
#include "../../../game/cards/card_database.h"
|
||||||
|
#include "picture_loader_worker_work.h"
|
||||||
#include "picture_to_load.h"
|
#include "picture_to_load.h"
|
||||||
|
|
||||||
#include <QLoggingCategory>
|
#include <QLoggingCategory>
|
||||||
|
|
@ -24,9 +25,12 @@ public:
|
||||||
explicit PictureLoaderWorker();
|
explicit PictureLoaderWorker();
|
||||||
~PictureLoaderWorker() override;
|
~PictureLoaderWorker() override;
|
||||||
|
|
||||||
void enqueueImageLoad(CardInfoPtr card);
|
void enqueueImageLoad(const CardInfoPtr &card);
|
||||||
void clearNetworkCache();
|
void clearNetworkCache();
|
||||||
|
|
||||||
|
public slots:
|
||||||
|
QNetworkReply *makeRequest(const QUrl &url, PictureLoaderWorkerWork *workThread);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
static QStringList md5Blacklist;
|
static QStringList md5Blacklist;
|
||||||
|
|
||||||
|
|
@ -42,10 +46,7 @@ private:
|
||||||
PictureToLoad cardBeingLoaded;
|
PictureToLoad cardBeingLoaded;
|
||||||
PictureToLoad cardBeingDownloaded;
|
PictureToLoad cardBeingDownloaded;
|
||||||
bool picDownload, downloadRunning, loadQueueRunning;
|
bool picDownload, downloadRunning, loadQueueRunning;
|
||||||
void startNextPicDownload();
|
|
||||||
bool cardImageExistsOnDisk(QString &setName, QString &correctedCardName);
|
|
||||||
bool imageIsBlackListed(const QByteArray &);
|
|
||||||
QNetworkReply *makeRequest(const QUrl &url);
|
|
||||||
void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl);
|
void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl);
|
||||||
QUrl getCachedRedirect(const QUrl &originalUrl) const;
|
QUrl getCachedRedirect(const QUrl &originalUrl) const;
|
||||||
void loadRedirectCache();
|
void loadRedirectCache();
|
||||||
|
|
@ -53,13 +54,8 @@ private:
|
||||||
void cleanStaleEntries();
|
void cleanStaleEntries();
|
||||||
|
|
||||||
private slots:
|
private slots:
|
||||||
void picDownloadFinished(QNetworkReply *reply);
|
|
||||||
void picDownloadFailed();
|
|
||||||
|
|
||||||
void picDownloadChanged();
|
void picDownloadChanged();
|
||||||
void picsPathChanged();
|
void picsPathChanged();
|
||||||
public slots:
|
|
||||||
void processLoadQueue();
|
|
||||||
|
|
||||||
signals:
|
signals:
|
||||||
void startLoadQueue();
|
void startLoadQueue();
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,235 @@
|
||||||
|
#include "picture_loader_worker_work.h"
|
||||||
|
|
||||||
|
#include "../../../game/cards/card_database_manager.h"
|
||||||
|
#include "../../../settings/cache_settings.h"
|
||||||
|
|
||||||
|
#include <QBuffer>
|
||||||
|
#include <QDirIterator>
|
||||||
|
#include <QLoggingCategory>
|
||||||
|
#include <QMovie>
|
||||||
|
#include <QNetworkDiskCache>
|
||||||
|
#include <QNetworkReply>
|
||||||
|
#include <QThread>
|
||||||
|
|
||||||
|
Q_LOGGING_CATEGORY(PictureLoaderWorkerWorkLog, "picture_loader.worker");
|
||||||
|
|
||||||
|
// Card back returned by gatherer when card is not found
|
||||||
|
QStringList PictureLoaderWorkerWork::md5Blacklist = QStringList() << "db0c48db407a907c16ade38de048a441";
|
||||||
|
|
||||||
|
PictureLoaderWorkerWork::PictureLoaderWorkerWork(PictureLoaderWorker *_worker, CardInfoPtr toLoad)
|
||||||
|
: QThread(nullptr), worker(_worker), cardToDownload(toLoad)
|
||||||
|
{
|
||||||
|
connect(this, SIGNAL(startLoadQueue()), this, SLOT(processLoadQueue()), Qt::QueuedConnection);
|
||||||
|
|
||||||
|
connect(networkManager, SIGNAL(finished(QNetworkReply *)), this, SLOT(picDownloadFinished(QNetworkReply *)));
|
||||||
|
|
||||||
|
pictureLoaderThread = new QThread;
|
||||||
|
pictureLoaderThread->start(QThread::LowPriority);
|
||||||
|
moveToThread(pictureLoaderThread);
|
||||||
|
startNextPicDownload();
|
||||||
|
}
|
||||||
|
|
||||||
|
PictureLoaderWorkerWork::~PictureLoaderWorkerWork()
|
||||||
|
{
|
||||||
|
pictureLoaderThread->deleteLater();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool PictureLoaderWorkerWork::cardImageExistsOnDisk(QString &setName, QString &correctedCardname)
|
||||||
|
{
|
||||||
|
QImage image;
|
||||||
|
QImageReader imgReader;
|
||||||
|
imgReader.setDecideFormatFromContent(true);
|
||||||
|
QList<QString> picsPaths = QList<QString>();
|
||||||
|
QDirIterator it(SettingsCache::instance().getCustomPicsPath(), QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
|
||||||
|
|
||||||
|
// Recursively check all subdirectories of the CUSTOM folder
|
||||||
|
while (it.hasNext()) {
|
||||||
|
QString thisPath(it.next());
|
||||||
|
QFileInfo thisFileInfo(thisPath);
|
||||||
|
|
||||||
|
if (thisFileInfo.isFile() &&
|
||||||
|
(thisFileInfo.fileName() == correctedCardname || thisFileInfo.completeBaseName() == correctedCardname ||
|
||||||
|
thisFileInfo.baseName() == correctedCardname)) {
|
||||||
|
picsPaths << thisPath; // Card found in the CUSTOM directory, somewhere
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!setName.isEmpty()) {
|
||||||
|
picsPaths << SettingsCache::instance().getPicsPath() + "/" + setName + "/" + correctedCardname
|
||||||
|
// We no longer store downloaded images there, but don't just ignore
|
||||||
|
// stuff that old versions have put there.
|
||||||
|
<< SettingsCache::instance().getPicsPath() + "/downloadedPics/" + setName + "/" + correctedCardname;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iterates through the list of paths, searching for images with the desired
|
||||||
|
// name with any QImageReader-supported
|
||||||
|
// extension
|
||||||
|
for (const auto &_picsPath : picsPaths) {
|
||||||
|
imgReader.setFileName(_picsPath);
|
||||||
|
if (imgReader.read(&image)) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << correctedCardname << " set: " << setName << "]: Picture found on disk.";
|
||||||
|
imageLoaded(cardToDownload.getCard(), image);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
imgReader.setFileName(_picsPath + ".full");
|
||||||
|
if (imgReader.read(&image)) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace() << "PictureLoader: [card: " << correctedCardname << " set: " << setName
|
||||||
|
<< "]: Picture.full found on disk.";
|
||||||
|
imageLoaded(cardToDownload.getCard(), image);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
imgReader.setFileName(_picsPath + ".xlhq");
|
||||||
|
if (imgReader.read(&image)) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace() << "PictureLoader: [card: " << correctedCardname << " set: " << setName
|
||||||
|
<< "]: Picture.xlhq found on disk.";
|
||||||
|
imageLoaded(cardToDownload.getCard(), image);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void PictureLoaderWorkerWork::startNextPicDownload()
|
||||||
|
{
|
||||||
|
QString picUrl = cardToDownload.getCurrentUrl();
|
||||||
|
|
||||||
|
if (picUrl.isEmpty()) {
|
||||||
|
downloadRunning = false;
|
||||||
|
picDownloadFailed();
|
||||||
|
} else {
|
||||||
|
QUrl url(picUrl);
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getCorrectedName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: Trying to fetch picture from url "
|
||||||
|
<< url.toDisplayString();
|
||||||
|
emit requestImageDownload(url, this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void PictureLoaderWorkerWork::picDownloadFailed()
|
||||||
|
{
|
||||||
|
/* Take advantage of short-circuiting here to call the nextUrl until one
|
||||||
|
is not available. Only once nextUrl evaluates to false will this move
|
||||||
|
on to nextSet. If the Urls for a particular card are empty, this will
|
||||||
|
effectively go through the sets for that card. */
|
||||||
|
if (cardToDownload.nextUrl() || cardToDownload.nextSet()) {
|
||||||
|
startNextPicDownload();
|
||||||
|
} else {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getCorrectedName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: Picture NOT found, "
|
||||||
|
<< (picDownload ? "download failed" : "downloads disabled")
|
||||||
|
<< ", no more url combinations to try: BAILING OUT";
|
||||||
|
imageLoaded(cardToDownload.getCard(), QImage());
|
||||||
|
cardToDownload.clear();
|
||||||
|
}
|
||||||
|
emit startLoadQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
|
||||||
|
{
|
||||||
|
bool isFromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
|
||||||
|
|
||||||
|
if (reply->error()) {
|
||||||
|
if (isFromCache) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: Removing corrupted cache file for url "
|
||||||
|
<< reply->url().toDisplayString() << " and retrying (" << reply->errorString() << ")";
|
||||||
|
|
||||||
|
networkManager->cache()->remove(reply->url());
|
||||||
|
|
||||||
|
requestImageDownload(reply->url(), this);
|
||||||
|
} else {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: " << (picDownload ? "Download" : "Cache search")
|
||||||
|
<< " failed for url " << reply->url().toDisplayString() << " (" << reply->errorString() << ")";
|
||||||
|
|
||||||
|
picDownloadFailed();
|
||||||
|
startNextPicDownload();
|
||||||
|
}
|
||||||
|
|
||||||
|
reply->deleteLater();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// List of status codes from https://doc.qt.io/qt-6/qnetworkreply.html#redirected
|
||||||
|
int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
|
||||||
|
if (statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 305 || statusCode == 307 ||
|
||||||
|
statusCode == 308) {
|
||||||
|
QUrl redirectUrl = reply->header(QNetworkRequest::LocationHeader).toUrl();
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: following "
|
||||||
|
<< (isFromCache ? "cached redirect" : "redirect") << " to " << redirectUrl.toDisplayString();
|
||||||
|
requestImageDownload(redirectUrl, this);
|
||||||
|
reply->deleteLater();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// peek is used to keep the data in the buffer for use by QImageReader
|
||||||
|
const QByteArray &picData = reply->peek(reply->size());
|
||||||
|
|
||||||
|
if (imageIsBlackListed(picData)) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace() << "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName()
|
||||||
|
<< "]: Picture found, but blacklisted, will consider it as not found";
|
||||||
|
|
||||||
|
picDownloadFailed();
|
||||||
|
reply->deleteLater();
|
||||||
|
startNextPicDownload();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
QImage testImage;
|
||||||
|
|
||||||
|
QImageReader imgReader;
|
||||||
|
imgReader.setDecideFormatFromContent(true);
|
||||||
|
imgReader.setDevice(reply);
|
||||||
|
|
||||||
|
bool logSuccessMessage = false;
|
||||||
|
|
||||||
|
static const int riffHeaderSize = 12; // RIFF_HEADER_SIZE from webp/format_constants.h
|
||||||
|
auto replyHeader = reply->peek(riffHeaderSize);
|
||||||
|
|
||||||
|
if (replyHeader.startsWith("RIFF") && replyHeader.endsWith("WEBP")) {
|
||||||
|
auto imgBuf = QBuffer(this);
|
||||||
|
imgBuf.setData(reply->readAll());
|
||||||
|
|
||||||
|
auto movie = QMovie(&imgBuf);
|
||||||
|
movie.start();
|
||||||
|
movie.stop();
|
||||||
|
|
||||||
|
imageLoaded(cardToDownload.getCard(), movie.currentImage());
|
||||||
|
logSuccessMessage = true;
|
||||||
|
} else if (imgReader.read(&testImage)) {
|
||||||
|
imageLoaded(cardToDownload.getCard(), testImage);
|
||||||
|
logSuccessMessage = true;
|
||||||
|
} else {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: Possible " << (isFromCache ? "cached" : "downloaded")
|
||||||
|
<< " picture at " << reply->url().toDisplayString() << " could not be loaded: " << reply->errorString();
|
||||||
|
|
||||||
|
picDownloadFailed();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (logSuccessMessage) {
|
||||||
|
qCDebug(PictureLoaderWorkerWorkLog).nospace()
|
||||||
|
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName()
|
||||||
|
<< " set: " << cardToDownload.getSetName() << "]: Image successfully "
|
||||||
|
<< (isFromCache ? "loaded from cached" : "downloaded from") << " url " << reply->url().toDisplayString();
|
||||||
|
}
|
||||||
|
|
||||||
|
reply->deleteLater();
|
||||||
|
startNextPicDownload();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool PictureLoaderWorkerWork::imageIsBlackListed(const QByteArray &picData)
|
||||||
|
{
|
||||||
|
QString md5sum = QCryptographicHash::hash(picData, QCryptographicHash::Md5).toHex();
|
||||||
|
return md5Blacklist.contains(md5sum);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
#ifndef PICTURE_LOADER_WORKER_WORK_H
|
||||||
|
#define PICTURE_LOADER_WORKER_WORK_H
|
||||||
|
|
||||||
|
#include "../../../game/cards/card_database.h"
|
||||||
|
#include "picture_loader_worker.h"
|
||||||
|
#include "picture_to_load.h"
|
||||||
|
|
||||||
|
#include <QMutex>
|
||||||
|
#include <QNetworkAccessManager>
|
||||||
|
#include <QObject>
|
||||||
|
#include <QThread>
|
||||||
|
|
||||||
|
#define REDIRECT_HEADER_NAME "redirects"
|
||||||
|
#define REDIRECT_ORIGINAL_URL "original"
|
||||||
|
#define REDIRECT_URL "redirect"
|
||||||
|
#define REDIRECT_TIMESTAMP "timestamp"
|
||||||
|
#define REDIRECT_CACHE_FILENAME "cache.ini"
|
||||||
|
|
||||||
|
class PictureLoaderWorker;
|
||||||
|
class PictureLoaderWorkerWork : public QThread
|
||||||
|
{
|
||||||
|
Q_OBJECT
|
||||||
|
public:
|
||||||
|
explicit PictureLoaderWorkerWork(PictureLoaderWorker *worker, CardInfoPtr toLoad);
|
||||||
|
~PictureLoaderWorkerWork() override;
|
||||||
|
public slots:
|
||||||
|
void picDownloadFinished(QNetworkReply *reply);
|
||||||
|
void picDownloadFailed();
|
||||||
|
|
||||||
|
private:
|
||||||
|
static QStringList md5Blacklist;
|
||||||
|
PictureLoaderWorker *worker;
|
||||||
|
QThread *pictureLoaderThread;
|
||||||
|
QNetworkAccessManager *networkManager;
|
||||||
|
PictureToLoad cardToDownload;
|
||||||
|
bool picDownload, downloadRunning, loadQueueRunning;
|
||||||
|
void startNextPicDownload();
|
||||||
|
bool cardImageExistsOnDisk(QString &setName, QString &correctedCardName);
|
||||||
|
bool imageIsBlackListed(const QByteArray &);
|
||||||
|
|
||||||
|
signals:
|
||||||
|
void startLoadQueue();
|
||||||
|
void imageLoaded(CardInfoPtr card, const QImage &image);
|
||||||
|
void requestImageDownload(const QUrl &url, PictureLoaderWorkerWork *instance);
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // PICTURE_LOADER_WORKER_WORK_H
|
||||||
Loading…
Add table
Add a link
Reference in a new issue