[PictureLoader] Schedule exponential backoff on 429 - Too many request handler and call failed on fail (#7053)

Took 16 minutes

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>

libcockatrice_utility/CMakeLists.txt

@@ -6,7 +6,7 @@ set(CMAKE_AUTOUIC ON)
 set(CMAKE_AUTORCC ON)
 
 set(UTILITY_SOURCES libcockatrice/utility/expression.cpp libcockatrice/utility/levenshtein.cpp
-                    libcockatrice/utility/passwordhasher.cpp
+                    libcockatrice/utility/passwordhasher.cpp libcockatrice/utility/server_rate_limiter.cpp
 )
 
 set(UTILITY_HEADERS
@@ -21,6 +21,7 @@ set(UTILITY_HEADERS
     libcockatrice/utility/clamped_arithmetic.h
     libcockatrice/utility/zone_names.h
     libcockatrice/utility/days_years_between.h
+    libcockatrice/utility/server_rate_limiter.h
 )
 
 add_library(libcockatrice_utility STATIC ${UTILITY_SOURCES} ${UTILITY_HEADERS})

libcockatrice_utility/libcockatrice/utility/server_rate_limiter.cpp

@@ -0,0 +1,85 @@
+#include "server_rate_limiter.h"
+
+bool ServerRateLimiter::isRateLimited(const QString &host, const QDateTime &now) const
+{
+    auto it = backoffUntil.constFind(host);
+    return it != backoffUntil.constEnd() && now < it.value();
+}
+
+int ServerRateLimiter::rounds(const QString &host) const
+{
+    return retryRounds.value(host, 0);
+}
+
+QDateTime ServerRateLimiter::deadline(const QString &host) const
+{
+    return backoffUntil.value(host);
+}
+
+QDateTime ServerRateLimiter::earliestDeadline(const QDateTime &now) const
+{
+    QDateTime earliest;
+    for (auto it = backoffUntil.cbegin(); it != backoffUntil.cend(); ++it) {
+        if (now < it.value() && (!earliest.isValid() || it.value() < earliest)) {
+            earliest = it.value();
+        }
+    }
+    return earliest;
+}
+
+QDateTime ServerRateLimiter::on429(const QString &host, const QDateTime &now, qint64 retryAfterMs)
+{
+    QDateTime existing = backoffUntil.value(host);
+    int round = retryRounds.value(host, 0);
+
+    if (!existing.isValid() || now >= existing) {
+        if (last429.value(host).isValid() && now >= last429.value(host).addMSecs(RESET_GRACE_MS)) {
+            round = 0;
+        }
+        round = qMin(round + 1, MAX_RETRIES);
+        retryRounds.insert(host, round);
+    }
+    last429.insert(host, now);
+
+    qint64 delay;
+    if (round >= MAX_RETRIES) {
+        delay = EXHAUSTED_BACKOFF_MS;
+    } else {
+        delay = qMax<qint64>(MIN_429_BACKOFF_MS, retryAfterMs);
+        delay = qMin<qint64>(delay, MAX_BACKOFF_MS);
+    }
+
+    QDateTime deadline = now.addMSecs(delay);
+    if (existing.isValid() && existing > deadline) {
+        deadline = existing; // never shorten an active backoff
+    }
+    backoffUntil.insert(host, deadline);
+
+    return deadline;
+}
+
+void ServerRateLimiter::onSuccess(const QString &host)
+{
+    backoffUntil.remove(host);
+    retryRounds.remove(host);
+    last429.remove(host);
+}
+
+void ServerRateLimiter::clearExpired(const QDateTime &now)
+{
+    auto it = backoffUntil.begin();
+    while (it != backoffUntil.end()) {
+        if (now < it.value()) {
+            ++it;
+            continue;
+        }
+
+        QString host = it.key();
+        bool budgetStillFresh = last429.value(host).isValid() && now < last429.value(host).addMSecs(RESET_GRACE_MS);
+        it = backoffUntil.erase(it);
+        if (!budgetStillFresh) {
+            retryRounds.remove(host);
+            last429.remove(host);
+        }
+    }
+}

libcockatrice_utility/libcockatrice/utility/server_rate_limiter.h

@@ -0,0 +1,73 @@
+#ifndef SERVER_RATE_LIMITER_H
+#define SERVER_RATE_LIMITER_H
+
+#include <QDateTime>
+#include <QMap>
+#include <QString>
+
+/**
+ * @class ServerRateLimiter
+ * @ingroup Utility
+ * @brief Tracks per-server backoff state triggered by HTTP 429 responses.
+ *
+ * Keeps a monotonic backoff deadline per host and a retry-round budget that
+ * escalates once per backoff window (a burst of concurrent 429s counts as a
+ * single round). The budget is refreshed after a long period without 429s so a
+ * server is not permanently blacklisted after a past overload.
+ */
+class ServerRateLimiter
+{
+public:
+    static constexpr int MAX_RETRIES = 5;              ///< Max 429 rounds per host before the budget is exhausted
+    static constexpr int MIN_429_BACKOFF_MS = 30000;   ///< Minimum wait after a 429 (scryfall documented cool-down)
+    static constexpr int MAX_BACKOFF_MS = 60000;       ///< Cap for a single backoff period
+    static constexpr int EXHAUSTED_BACKOFF_MS = 60000; ///< Cool-down once the retry budget is exhausted
+    static constexpr qint64 RESET_GRACE_MS = 300000;   ///< Idle time after which the retry budget refreshes
+
+    /**
+     * @brief Checks whether a host is currently in backoff.
+     * @param host The host to check
+     * @param now The current time
+     * @return True if requests to the host should be paused
+     */
+    [[nodiscard]] bool isRateLimited(const QString &host, const QDateTime &now) const;
+
+    /** @return The number of consecutive 429 rounds recorded for the host. */
+    [[nodiscard]] int rounds(const QString &host) const;
+
+    /** @return The current backoff deadline for the host, or an invalid QDateTime. */
+    [[nodiscard]] QDateTime deadline(const QString &host) const;
+
+    /** @return The earliest active backoff deadline across all hosts, or an invalid QDateTime. */
+    [[nodiscard]] QDateTime earliestDeadline(const QDateTime &now) const;
+
+    /**
+     * @brief Registers a 429 response for the given host.
+     * @param host The host that returned 429
+     * @param now The time the response was received
+     * @param retryAfterMs A Retry-After hint in milliseconds, or 0 if absent
+     * @return The new backoff deadline for the host
+     *
+     * The round counter only escalates once the previous backoff window has
+     * fully passed, so concurrent 429s from a burst do not consume the whole
+     * budget. The deadline is extended monotonically and never shortened.
+     */
+    QDateTime on429(const QString &host, const QDateTime &now, qint64 retryAfterMs = 0);
+
+    /** @brief Clears all penalty state for a host after a successful request. */
+    void onSuccess(const QString &host);
+
+    /**
+     * @brief Removes expired backoffs, refreshing the round budget of hosts
+     *        that have been idle past the reset grace period.
+     * @param now The current time
+     */
+    void clearExpired(const QDateTime &now);
+
+private:
+    QMap<QString, QDateTime> backoffUntil; ///< When each host may be contacted again
+    QMap<QString, int> retryRounds;        ///< Consecutive 429 rounds per host
+    QMap<QString, QDateTime> last429;      ///< When each host was last rate limited
+};
+
+#endif // SERVER_RATE_LIMITER_H