[PictureLoader] Properly run reply processing on Work's thread

This commit is contained in:
RickyRister 2025-06-25 12:23:50 -07:00
parent ed82106359
commit c103580f8d
4 changed files with 77 additions and 49 deletions

View file

@ -92,34 +92,7 @@ QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWo
QNetworkReply *reply = networkManager->get(req); QNetworkReply *reply = networkManager->get(req);
// Connect reply handling // Connect reply handling
connect(reply, &QNetworkReply::finished, this, [this, reply, url, worker]() { connect(reply, &QNetworkReply::finished, worker, [reply, worker] { worker->handleNetworkReply(reply); });
QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectTarget.isValid()) {
QUrl redirectUrl = redirectTarget.toUrl();
if (redirectUrl.isRelative()) {
redirectUrl = url.resolved(redirectUrl);
}
cacheRedirect(url, redirectUrl);
}
if (reply->error() == QNetworkReply::NoError) {
worker->picDownloadFinished(reply);
emit imageLoadSuccessful(url, worker);
// If we hit a cached image, we get to make another request for free.
if (reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool()) {
if (!requestLoadQueue.isEmpty()) {
auto request = requestLoadQueue.takeFirst();
makeRequest(request.first, request.second);
}
}
} else if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 429) {
qInfo() << "Too many requests.";
} else {
worker->picDownloadFinished(reply);
}
reply->deleteLater();
});
return reply; return reply;
} }
@ -127,12 +100,20 @@ QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWo
void PictureLoaderWorker::processQueuedRequests() void PictureLoaderWorker::processQueuedRequests()
{ {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
processSingleRequest();
}
}
/**
* Immediately processes a single queued request
*/
void PictureLoaderWorker::processSingleRequest()
{
if (!requestLoadQueue.isEmpty()) { if (!requestLoadQueue.isEmpty()) {
auto request = requestLoadQueue.takeFirst(); auto request = requestLoadQueue.takeFirst();
makeRequest(request.first, request.second); makeRequest(request.first, request.second);
} }
} }
}
void PictureLoaderWorker::enqueueImageLoad(const CardInfoPtr &card) void PictureLoaderWorker::enqueueImageLoad(const CardInfoPtr &card)
{ {
@ -176,6 +157,11 @@ void PictureLoaderWorker::cacheRedirect(const QUrl &originalUrl, const QUrl &red
// saveRedirectCache(); // saveRedirectCache();
} }
void PictureLoaderWorker::removedCachedUrl(const QUrl &url)
{
networkManager->cache()->remove(url);
}
QUrl PictureLoaderWorker::getCachedRedirect(const QUrl &originalUrl) const QUrl PictureLoaderWorker::getCachedRedirect(const QUrl &originalUrl) const
{ {
if (redirectCache.contains(originalUrl)) { if (redirectCache.contains(originalUrl)) {

View file

@ -38,7 +38,10 @@ public:
public slots: public slots:
QNetworkReply *makeRequest(const QUrl &url, PictureLoaderWorkerWork *workThread); QNetworkReply *makeRequest(const QUrl &url, PictureLoaderWorkerWork *workThread);
void processQueuedRequests(); void processQueuedRequests();
void processSingleRequest();
void imageLoadedSuccessfully(const CardInfoPtr &card, const QImage &image); void imageLoadedSuccessfully(const CardInfoPtr &card, const QImage &image);
void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl);
void removedCachedUrl(const QUrl &url);
private: private:
static QStringList md5Blacklist; static QStringList md5Blacklist;
@ -56,7 +59,6 @@ private:
PictureLoaderLocal *localLoader; PictureLoaderLocal *localLoader;
QSet<CardInfoPtr> currentlyLoading; // for deduplication purposes QSet<CardInfoPtr> currentlyLoading; // for deduplication purposes
void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl);
QUrl getCachedRedirect(const QUrl &originalUrl) const; QUrl getCachedRedirect(const QUrl &originalUrl) const;
void loadRedirectCache(); void loadRedirectCache();
void saveRedirectCache() const; void saveRedirectCache() const;

View file

@ -19,10 +19,11 @@ PictureLoaderWorkerWork::PictureLoaderWorkerWork(const PictureLoaderWorker *work
: QObject(nullptr), cardToDownload(toLoad), picDownload(SettingsCache::instance().getPicDownload()) : QObject(nullptr), cardToDownload(toLoad), picDownload(SettingsCache::instance().getPicDownload())
{ {
// Hook up signals to the orchestrator // Hook up signals to the orchestrator
connect(this, &PictureLoaderWorkerWork::requestImageDownload, worker, &PictureLoaderWorker::queueRequest, connect(this, &PictureLoaderWorkerWork::requestImageDownload, worker, &PictureLoaderWorker::queueRequest);
Qt::QueuedConnection); connect(this, &PictureLoaderWorkerWork::cachedImageHit, worker, &PictureLoaderWorker::processSingleRequest);
connect(this, &PictureLoaderWorkerWork::imageLoaded, worker, &PictureLoaderWorker::imageLoadedSuccessfully, connect(this, &PictureLoaderWorkerWork::urlRedirected, worker, &PictureLoaderWorker::cacheRedirect);
Qt::QueuedConnection); connect(this, &PictureLoaderWorkerWork::cachedUrlInvalidated, worker, &PictureLoaderWorker::removedCachedUrl);
connect(this, &PictureLoaderWorkerWork::imageLoaded, worker, &PictureLoaderWorker::imageLoadedSuccessfully);
// Hook up signals to settings // Hook up signals to settings
connect(&SettingsCache::instance(), SIGNAL(picDownloadChanged()), this, SLOT(picDownloadChanged())); connect(&SettingsCache::instance(), SIGNAL(picDownloadChanged()), this, SLOT(picDownloadChanged()));
@ -78,24 +79,56 @@ void PictureLoaderWorkerWork::picDownloadFailed()
} }
} }
/**
*
* @param reply The reply. Takes ownership of the object
*/
void PictureLoaderWorkerWork::handleNetworkReply(QNetworkReply *reply)
{
QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectTarget.isValid()) {
QUrl url = reply->request().url();
QUrl redirectUrl = redirectTarget.toUrl();
if (redirectUrl.isRelative()) {
redirectUrl = url.resolved(redirectUrl);
}
emit urlRedirected(url, redirectUrl);
}
if (reply->error()) {
handleFailedReply(reply);
} else {
handleSuccessfulReply(reply);
// If we hit a cached image, we get to make another request for free.
if (reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool()) {
emit cachedImageHit();
}
}
reply->deleteLater();
}
static bool imageIsBlackListed(const QByteArray &picData) static bool imageIsBlackListed(const QByteArray &picData)
{ {
QString md5sum = QCryptographicHash::hash(picData, QCryptographicHash::Md5).toHex(); QString md5sum = QCryptographicHash::hash(picData, QCryptographicHash::Md5).toHex();
return MD5_BLACKLIST.contains(md5sum); return MD5_BLACKLIST.contains(md5sum);
} }
void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply) void PictureLoaderWorkerWork::handleFailedReply(const QNetworkReply *reply)
{ {
if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 429) {
qCWarning(PictureLoaderWorkerWorkLog) << "Too many requests.";
} else {
bool isFromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool(); bool isFromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
if (reply->error()) {
if (isFromCache) { if (isFromCache) {
qCDebug(PictureLoaderWorkerWorkLog).nospace() qCDebug(PictureLoaderWorkerWorkLog).nospace()
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName() << "PictureLoader: [card: " << cardToDownload.getCard()->getName()
<< " set: " << cardToDownload.getSetName() << "]: Removing corrupted cache file for url " << " set: " << cardToDownload.getSetName() << "]: Removing corrupted cache file for url "
<< reply->url().toDisplayString() << " and retrying (" << reply->errorString() << ")"; << reply->url().toDisplayString() << " and retrying (" << reply->errorString() << ")";
networkManager->cache()->remove(reply->url()); emit cachedUrlInvalidated(reply->url());
emit requestImageDownload(reply->url(), this); emit requestImageDownload(reply->url(), this);
} else { } else {
@ -106,10 +139,12 @@ void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
picDownloadFailed(); picDownloadFailed();
} }
reply->deleteLater();
return;
} }
}
void PictureLoaderWorkerWork::handleSuccessfulReply(QNetworkReply *reply)
{
bool isFromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
// List of status codes from https://doc.qt.io/qt-6/qnetworkreply.html#redirected // List of status codes from https://doc.qt.io/qt-6/qnetworkreply.html#redirected
int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
@ -120,7 +155,6 @@ void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
<< "PictureLoader: [card: " << cardToDownload.getCard()->getName() << "PictureLoader: [card: " << cardToDownload.getCard()->getName()
<< " set: " << cardToDownload.getSetName() << "]: following " << " set: " << cardToDownload.getSetName() << "]: following "
<< (isFromCache ? "cached redirect" : "redirect") << " to " << redirectUrl.toDisplayString(); << (isFromCache ? "cached redirect" : "redirect") << " to " << redirectUrl.toDisplayString();
reply->deleteLater();
emit requestImageDownload(redirectUrl, this); emit requestImageDownload(redirectUrl, this);
return; return;
} }
@ -134,7 +168,6 @@ void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
<< " set: " << cardToDownload.getSetName() << " set: " << cardToDownload.getSetName()
<< "]: Picture found, but blacklisted, will consider it as not found"; << "]: Picture found, but blacklisted, will consider it as not found";
reply->deleteLater();
picDownloadFailed(); picDownloadFailed();
return; return;
} }
@ -147,7 +180,6 @@ void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
<< " set: " << cardToDownload.getSetName() << "]: Possible " << (isFromCache ? "cached" : "downloaded") << " set: " << cardToDownload.getSetName() << "]: Possible " << (isFromCache ? "cached" : "downloaded")
<< " picture at " << reply->url().toDisplayString() << " could not be loaded: " << reply->errorString(); << " picture at " << reply->url().toDisplayString() << " could not be loaded: " << reply->errorString();
reply->deleteLater();
picDownloadFailed(); picDownloadFailed();
} else { } else {
qCDebug(PictureLoaderWorkerWorkLog).nospace() qCDebug(PictureLoaderWorkerWorkLog).nospace()
@ -155,7 +187,6 @@ void PictureLoaderWorkerWork::picDownloadFinished(QNetworkReply *reply)
<< " set: " << cardToDownload.getSetName() << "]: Image successfully " << " set: " << cardToDownload.getSetName() << "]: Image successfully "
<< (isFromCache ? "loaded from cached" : "downloaded from") << " url " << reply->url().toDisplayString(); << (isFromCache ? "loaded from cached" : "downloaded from") << " url " << reply->url().toDisplayString();
reply->deleteLater();
concludeImageLoad(image); concludeImageLoad(image);
} }
} }

View file

@ -30,7 +30,7 @@ public:
PictureToLoad cardToDownload; PictureToLoad cardToDownload;
public slots: public slots:
void picDownloadFinished(QNetworkReply *reply); void handleNetworkReply(QNetworkReply *reply);
private: private:
QThread *pictureLoaderThread; QThread *pictureLoaderThread;
@ -39,6 +39,8 @@ private:
void startNextPicDownload(); void startNextPicDownload();
void picDownloadFailed(); void picDownloadFailed();
void handleFailedReply(const QNetworkReply *reply);
void handleSuccessfulReply(QNetworkReply *reply);
QImage tryLoadImageFromReply(QNetworkReply *reply); QImage tryLoadImageFromReply(QNetworkReply *reply);
void concludeImageLoad(const QImage &image); void concludeImageLoad(const QImage &image);
@ -53,6 +55,13 @@ signals:
*/ */
void imageLoaded(CardInfoPtr card, const QImage &image); void imageLoaded(CardInfoPtr card, const QImage &image);
void requestImageDownload(const QUrl &url, PictureLoaderWorkerWork *instance); void requestImageDownload(const QUrl &url, PictureLoaderWorkerWork *instance);
/**
* We hit a cached image from the network cache; we get to make another request for free.
*/
void cachedImageHit();
void urlRedirected(const QUrl &originalUrl, const QUrl &redirectUrl);
void cachedUrlInvalidated(const QUrl &url);
}; };
#endif // PICTURE_LOADER_WORKER_WORK_H #endif // PICTURE_LOADER_WORKER_WORK_H