[PictureLoader] Schedule exponential backoff on 429 - Too many request handler and call failed on fail (#7053)

Took 16 minutes

Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>

tests/CMakeLists.txt

@@ -9,6 +9,7 @@ add_test(NAME test_age_formatting COMMAND test_age_formatting)
 add_test(NAME password_hash_test COMMAND password_hash_test)
 add_test(NAME server_card_counter_test COMMAND server_card_counter_test)
 add_test(NAME server_counter_test COMMAND server_counter_test)
+add_test(NAME server_rate_limiter_test COMMAND server_rate_limiter_test)
 
 add_test(NAME deck_hash_performance_test COMMAND deck_hash_performance_test)
 set_tests_properties(deck_hash_performance_test PROPERTIES TIMEOUT 5)
@@ -23,6 +24,7 @@ add_executable(password_hash_test password_hash_test.cpp)
 add_executable(deck_hash_performance_test deck_hash_performance_test.cpp)
 add_executable(server_card_counter_test server_card_counter_test.cpp)
 add_executable(server_counter_test server_counter_test.cpp)
+add_executable(server_rate_limiter_test server_rate_limiter_test.cpp)
 
 find_package(GTest)
 
@@ -57,6 +59,7 @@ if(NOT GTEST_FOUND)
   add_dependencies(deck_hash_performance_test gtest)
   add_dependencies(server_card_counter_test gtest)
   add_dependencies(server_counter_test gtest)
+  add_dependencies(server_rate_limiter_test gtest)
 endif()
 
 include_directories(${GTEST_INCLUDE_DIRS})
@@ -81,6 +84,9 @@ target_link_libraries(
 target_link_libraries(
   server_counter_test libcockatrice_network Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}
 )
+target_link_libraries(
+  server_rate_limiter_test libcockatrice_utility Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}
+)
 
 add_subdirectory(card_zone_algorithms)
 add_subdirectory(carddatabase)

tests/server_rate_limiter_test.cpp

@@ -0,0 +1,151 @@
+#include "gtest/gtest.h"
+#include <QDateTime>
+#include <QTimeZone>
+#include <libcockatrice/utility/server_rate_limiter.h>
+
+namespace
+{
+
+const QString HOST = "api.scryfall.com";
+
+QDateTime timeAt(qint64 secsFromEpoch)
+{
+    return QDateTime::fromMSecsSinceEpoch(secsFromEpoch * 1000, QTimeZone::UTC);
+}
+
+TEST(ServerRateLimiterTest, First429SetsThirtySecondBackoff)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    QDateTime deadline = limiter.on429(HOST, now);
+
+    EXPECT_TRUE(limiter.isRateLimited(HOST, now));
+    EXPECT_EQ(now.addSecs(30), deadline);
+    EXPECT_EQ(1, limiter.rounds(HOST));
+    EXPECT_FALSE(limiter.isRateLimited(HOST, deadline));
+}
+
+TEST(ServerRateLimiterTest, RetryAfterHeaderIsHonoredWhenLarger)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    QDateTime deadline = limiter.on429(HOST, now, 45 * 1000);
+
+    EXPECT_EQ(now.addSecs(45), deadline);
+}
+
+TEST(ServerRateLimiterTest, RetryAfterSmallerThanFloorIsIgnored)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    QDateTime deadline = limiter.on429(HOST, now, 5 * 1000);
+
+    EXPECT_EQ(now.addSecs(30), deadline);
+}
+
+TEST(ServerRateLimiterTest, Concurrent429sDoNotEscalateOrShorten)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    QDateTime first = limiter.on429(HOST, now);
+    limiter.on429(HOST, now.addMSecs(100));
+    limiter.on429(HOST, now.addMSecs(200));
+
+    // The burst counts as a single round, but each 429 slides the deadline forward.
+    EXPECT_EQ(1, limiter.rounds(HOST));
+    EXPECT_EQ(now.addMSecs(30200), limiter.deadline(HOST));
+    EXPECT_TRUE(limiter.deadline(HOST) >= first);
+}
+
+TEST(ServerRateLimiterTest, EscalatesAfterWindowPasses)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    limiter.on429(HOST, now);
+    limiter.on429(HOST, now.addSecs(31));
+
+    EXPECT_EQ(2, limiter.rounds(HOST));
+}
+
+TEST(ServerRateLimiterTest, BudgetExhaustionAppliesCooldown)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    QDateTime deadline;
+    for (int i = 0; i < ServerRateLimiter::MAX_RETRIES; ++i) {
+        deadline = limiter.on429(HOST, now.addSecs(31 * i));
+    }
+
+    EXPECT_EQ(ServerRateLimiter::MAX_RETRIES, limiter.rounds(HOST));
+    EXPECT_EQ(now.addSecs(31 * (ServerRateLimiter::MAX_RETRIES - 1) + 60), deadline);
+
+    // A 429 inside the exhausted cooldown does not escalate, but keeps sliding the cooldown forward.
+    limiter.on429(HOST, now.addSecs(31 * (ServerRateLimiter::MAX_RETRIES - 1) + 30));
+    EXPECT_EQ(ServerRateLimiter::MAX_RETRIES, limiter.rounds(HOST));
+    EXPECT_EQ(now.addSecs(31 * (ServerRateLimiter::MAX_RETRIES - 1) + 30 + 60), limiter.deadline(HOST));
+}
+
+TEST(ServerRateLimiterTest, SuccessClearsPenalty)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    limiter.on429(HOST, now);
+    limiter.onSuccess(HOST);
+
+    EXPECT_EQ(0, limiter.rounds(HOST));
+    EXPECT_FALSE(limiter.isRateLimited(HOST, now));
+}
+
+TEST(ServerRateLimiterTest, ClearExpiredRefreshesStaleBudget)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    limiter.on429(HOST, now);
+    limiter.clearExpired(now.addSecs(400));
+
+    EXPECT_EQ(0, limiter.rounds(HOST));
+    EXPECT_FALSE(limiter.isRateLimited(HOST, now.addSecs(400)));
+}
+
+TEST(ServerRateLimiterTest, ClearExpiredKeepsRecentBudget)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    for (int i = 0; i < ServerRateLimiter::MAX_RETRIES; ++i) {
+        limiter.on429(HOST, now.addSecs(31 * i));
+    }
+    limiter.clearExpired(now.addSecs(200));
+
+    EXPECT_EQ(ServerRateLimiter::MAX_RETRIES, limiter.rounds(HOST));
+    EXPECT_FALSE(limiter.isRateLimited(HOST, now.addSecs(200)));
+}
+
+TEST(ServerRateLimiterTest, EarliestDeadlineAcrossHosts)
+{
+    ServerRateLimiter limiter;
+    QDateTime now = timeAt(1000);
+
+    limiter.on429("api.scryfall.com", now, 40 * 1000);
+    limiter.on429("gatherer.wizards.com", now.addSecs(5));
+
+    EXPECT_EQ(now.addSecs(35), limiter.earliestDeadline(now));
+    EXPECT_EQ(now.addSecs(40), limiter.deadline("api.scryfall.com"));
+    EXPECT_EQ(now.addSecs(40), limiter.earliestDeadline(now.addSecs(36)));
+}
+
+} // namespace
+
+int main(int argc, char **argv)
+{
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}