Rate limit requests beforehand.

This commit is contained in:
Lukas Brübach 2025-01-24 13:09:13 +01:00
parent efa76939ac
commit 1d4bdc202d
2 changed files with 44 additions and 4 deletions

View file

@ -52,6 +52,9 @@ PictureLoaderWorker::PictureLoaderWorker()
pictureLoaderThread = new QThread; pictureLoaderThread = new QThread;
pictureLoaderThread->start(QThread::LowPriority); pictureLoaderThread->start(QThread::LowPriority);
moveToThread(pictureLoaderThread); moveToThread(pictureLoaderThread);
connect(&requestTimer, &QTimer::timeout, this, &PictureLoaderWorker::processQueuedRequests);
requestTimer.setSingleShot(true);
} }
PictureLoaderWorker::~PictureLoaderWorker() PictureLoaderWorker::~PictureLoaderWorker()
@ -61,12 +64,32 @@ PictureLoaderWorker::~PictureLoaderWorker()
QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWorkerWork *worker) QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWorkerWork *worker)
{ {
constexpr int urlRequestDelayMs = 100; // Minimum delay between requests to the same URL (in milliseconds)
// Check if we're rate-limited globally
if (rateLimited) { if (rateLimited) {
// Queue the request if currently rate-limited
requestQueue.append(qMakePair(url, worker)); requestQueue.append(qMakePair(url, worker));
return nullptr; // No immediate request return nullptr;
} }
// Check if a request was recently made to this URL
QDateTime now = QDateTime::currentDateTimeUtc();
if (lastRequestTime.contains(url)) {
int elapsedMs = lastRequestTime[url].msecsTo(now);
if (elapsedMs < urlRequestDelayMs) {
// Queue the request if too soon to reissue
requestQueue.append(qMakePair(url, worker));
if (!requestTimer.isActive()) {
requestTimer.start(urlRequestDelayMs);
}
return nullptr;
}
}
// Update the last request time for this URL
lastRequestTime[url] = now;
// Check for cached redirects
QUrl cachedRedirect = getCachedRedirect(url); QUrl cachedRedirect = getCachedRedirect(url);
if (!cachedRedirect.isEmpty()) { if (!cachedRedirect.isEmpty()) {
return makeRequest(cachedRedirect, worker); return makeRequest(cachedRedirect, worker);
@ -79,6 +102,7 @@ QNetworkReply *PictureLoaderWorker::makeRequest(const QUrl &url, PictureLoaderWo
QNetworkReply *reply = networkManager->get(req); QNetworkReply *reply = networkManager->get(req);
// Connect reply handling
connect(reply, &QNetworkReply::finished, this, [this, reply, url, worker]() { connect(reply, &QNetworkReply::finished, this, [this, reply, url, worker]() {
QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute); QVariant redirectTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectTarget.isValid()) { if (redirectTarget.isValid()) {
@ -135,12 +159,26 @@ void PictureLoaderWorker::processQueuedRequests()
qWarning() << "Resuming queued requests"; qWarning() << "Resuming queued requests";
rateLimited = false; rateLimited = false;
while (!requestQueue.isEmpty()) { if (!requestQueue.isEmpty()) {
QPair<QUrl, PictureLoaderWorkerWork *> request = requestQueue.takeFirst(); // Process requests one by one, respecting the URL delay
auto request = requestQueue.takeFirst();
QDateTime now = QDateTime::currentDateTimeUtc();
if (lastRequestTime.contains(request.first)) {
int elapsedMs = lastRequestTime[request.first].msecsTo(now);
if (elapsedMs < 100) {
// Delay request if needed
requestQueue.prepend(request); // Put back in the queue
requestTimer.start(100 - elapsedMs); // Schedule for the remaining time
return;
}
}
makeRequest(request.first, request.second); makeRequest(request.first, request.second);
} }
} }
void PictureLoaderWorker::enqueueImageLoad(const CardInfoPtr &card) void PictureLoaderWorker::enqueueImageLoad(const CardInfoPtr &card)
{ {
auto worker = new PictureLoaderWorkerWork(this, card); auto worker = new PictureLoaderWorkerWork(this, card);

View file

@ -54,6 +54,8 @@ private:
bool rateLimited = false; bool rateLimited = false;
QTimer rateLimitTimer; QTimer rateLimitTimer;
QList<QPair<QUrl, PictureLoaderWorkerWork *>> requestQueue; QList<QPair<QUrl, PictureLoaderWorkerWork *>> requestQueue;
QHash<QUrl, QDateTime> lastRequestTime; // Tracks the last request time for each URL
QTimer requestTimer; // Timer for processing delayed requests
void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl); void cacheRedirect(const QUrl &originalUrl, const QUrl &redirectUrl);
QUrl getCachedRedirect(const QUrl &originalUrl) const; QUrl getCachedRedirect(const QUrl &originalUrl) const;