mirror of
https://github.com/Cockatrice/Cockatrice.git
synced 2026-09-22 09:35:08 -07:00
[PictureLoader] Add user-configurable per-host request caps (#7287)
* [PictureLoader] Add user-configurable per-host request caps Picture downloads were throttled to a uniform 10 requests/second per host with no way to tune a specific server. A rate-limited API host (Scryfall caps at 10 req/s) can trip 429s during bursts, and CDN hosts with no rate limit were throttled needlessly. Introduce developer-owned per-host caps that users can only ever lower, never raise, exposed in the download settings page: - DownloadSettings::DEVELOPER_HOST_CAPS sets the ceiling per host (api.scryfall.com 9, cards.scryfall.io unlimited, others 10). - A new hostRequestLimits setting stores user overrides in downloads.ini; clampHostRequestLimit() bounds them to [1, devCap] so a user can reduce api.scryfall.com to 5 but never raise it above 9. - The picture worker seeds, halves on 429, and recovers its sustained per-host allowance against the effective ceiling instead of the global maximum, and skips per-host accounting entirely for unlocked hosts (cards.scryfall.io) while global pacing and 429 backoff still apply. - The deck editor settings page gains one spinbox per known host, each clamped to its developer cap. * [PictureLoader] Let unlocked hosts skip dispatch pacing; adjust limits per URL Two refinements to the per-host request caps: - Unlocked hosts (UNLIMITED_HOST_QUOTA, e.g. cards.scryfall.io) no longer wait on the 100ms dispatch pacing or consume the global per-second quota. dispatchQueuedRequest fires their queued requests back-to-back, bounded only by their 429 backoff window and Qt's per-host connection pool, so an unthrottled CDN is not artificially slowed. - The deck editor download settings page replaces the static grid of one spinbox per known host with an "Adjust Rate Limit" toolbar action on the URL list. It picks the host out of the selected URL and clamps the entry against the developer cap table (including for user-added URLs). Also fixes a review finding: resetRequestQuota could write the UNLIMITED_HOST_QUOTA sentinel (-1) into the sustained per-host quota when a host became unlocked mid-run, permanently poisoning its allowance. Stale entries for unlocked hosts are now dropped, and the per-second seed is clamped against the effective ceiling so a lowered limit applies immediately. * [PictureLoader] Cap unlocked host bursts and adapt them to 429s * [PictureLoader] Store per-host limits readably and show them per URL * [PictureLoader] Make dispatch and rate-limit bookkeeping key on the real host Addresses ZeldaZach's round-4 review nits: - Dispatch now resolves the cached-redirect chain before the in-flight gate, so a redirect learned after a URL was queued can no longer bypass the MAX_IN_FLIGHT_PER_HOST cap and drain the whole queue onto the redirect target, which may carry its own developer cap. processSingleRequest does the same so the allowance math keys on the host that is actually hit. - The per-host in-flight slot is released when the reply is destroyed (with the worker as the connection context) rather than on a 'finished' connection bound to the work object, so an aborted reply or a work object deleted while a reply is pending can never permanently shrink the fast path's concurrency. - storeSettings only prunes limits for hosts with neither a URL nor a developer cap, so throttles on redirect targets (api.scryfall.com -> cards.scryfall.io) survive URL removal. - Unlocked hosts are offered 0..UNLOCKED_HOST_LIMIT_MAX (50) in the rate limit dialog, matching clampHostRequestLimit() and the documented hand-editable range, so values written into downloads.ini are no longer silently rewritten on the next edit. --------- Co-authored-by: Lukas Brübach <Bruebach.Lukas@bdosecurity.de>
This commit is contained in:
BruebachL
GitHub
parent
3d5eb84d81
commit
14acf3bf64
7 changed files with 496 additions and 34 deletions
|
|
@ -10,7 +10,9 @@
|
|||
#include <QInputDialog>
|
||||
#include <QLineEdit>
|
||||
#include <QMessageBox>
|
||||
#include <QSet>
|
||||
#include <QToolBar>
|
||||
#include <QUrl>
|
||||
#include <libcockatrice/settings/download_settings.h>
|
||||
#include <libcockatrice/settings/paths_settings.h>
|
||||
#include <libcockatrice/settings/personal_settings.h>
|
||||
|
|
@ -51,7 +53,9 @@ DeckEditorSettingsPage::DeckEditorSettingsPage()
|
|||
urlList->setDragDropMode(QAbstractItemView::InternalMove);
|
||||
connect(urlList->model(), &QAbstractItemModel::rowsMoved, this, &DeckEditorSettingsPage::urlListChanged);
|
||||
|
||||
urlList->addItems(SettingsCache::instance().downloads().getAllURLs());
|
||||
for (const QString &url : SettingsCache::instance().downloads().getAllURLs()) {
|
||||
addUrlItem(url);
|
||||
}
|
||||
|
||||
aAdd = new QAction(this);
|
||||
aAdd->setIcon(themePixmap(QStringLiteral("icons/increment")));
|
||||
|
|
@ -65,11 +69,16 @@ DeckEditorSettingsPage::DeckEditorSettingsPage()
|
|||
aRemove->setIcon(themePixmap(QStringLiteral("icons/decrement")));
|
||||
connect(aRemove, &QAction::triggered, this, &DeckEditorSettingsPage::actRemoveURL);
|
||||
|
||||
aRateLimit = new QAction(this);
|
||||
aRateLimit->setIcon(themePixmap(QStringLiteral("icons/cogwheel")));
|
||||
connect(aRateLimit, &QAction::triggered, this, &DeckEditorSettingsPage::actAdjustRateLimit);
|
||||
|
||||
auto *urlToolBar = new QToolBar;
|
||||
urlToolBar->setOrientation(Qt::Vertical);
|
||||
urlToolBar->addAction(aAdd);
|
||||
urlToolBar->addAction(aRemove);
|
||||
urlToolBar->addAction(aEdit);
|
||||
urlToolBar->addAction(aRateLimit);
|
||||
urlToolBar->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
|
||||
|
||||
auto *urlListLayout = new QHBoxLayout;
|
||||
|
|
@ -117,7 +126,9 @@ void DeckEditorSettingsPage::resetDownloadedURLsButtonClicked()
|
|||
{
|
||||
SettingsCache::instance().downloads().resetToDefaultURLs();
|
||||
urlList->clear();
|
||||
urlList->addItems(SettingsCache::instance().downloads().getAllURLs());
|
||||
for (const QString &url : SettingsCache::instance().downloads().getAllURLs()) {
|
||||
addUrlItem(url);
|
||||
}
|
||||
QMessageBox::information(this, tr("Success"), tr("Download URLs have been reset."));
|
||||
}
|
||||
|
||||
|
|
@ -126,7 +137,7 @@ void DeckEditorSettingsPage::actAddURL()
|
|||
bool ok;
|
||||
QString msg = QInputDialog::getText(this, tr("Add URL"), tr("URL:"), QLineEdit::Normal, QString(), &ok);
|
||||
if (ok) {
|
||||
urlList->addItem(msg);
|
||||
addUrlItem(msg);
|
||||
storeSettings();
|
||||
}
|
||||
}
|
||||
|
|
@ -141,12 +152,14 @@ void DeckEditorSettingsPage::actRemoveURL()
|
|||
|
||||
void DeckEditorSettingsPage::actEditURL()
|
||||
{
|
||||
if (urlList->currentItem()) {
|
||||
QString oldText = urlList->currentItem()->text();
|
||||
QListWidgetItem *item = urlList->currentItem();
|
||||
if (item) {
|
||||
const QString oldText = urlForItem(item);
|
||||
bool ok;
|
||||
QString msg = QInputDialog::getText(this, tr("Edit URL"), tr("URL:"), QLineEdit::Normal, oldText, &ok);
|
||||
if (ok) {
|
||||
urlList->currentItem()->setText(msg);
|
||||
item->setData(Qt::UserRole, msg);
|
||||
item->setText(urlLabel(msg));
|
||||
storeSettings();
|
||||
}
|
||||
}
|
||||
|
|
@ -158,10 +171,133 @@ void DeckEditorSettingsPage::storeSettings()
|
|||
|
||||
QStringList downloadUrls;
|
||||
for (int i = 0; i < urlList->count(); i++) {
|
||||
qInfo() << "Priority" << i << ":" << urlList->item(i)->text();
|
||||
downloadUrls << urlList->item(i)->text();
|
||||
const QString url = urlForItem(urlList->item(i));
|
||||
qInfo() << "Priority" << i << ":" << url;
|
||||
downloadUrls << url;
|
||||
}
|
||||
SettingsCache::instance().downloads().setDownloadUrls(downloadUrls);
|
||||
|
||||
// Drop per-host limits whose host is no longer referenced by any configured URL, so removing
|
||||
// a URL doesn't leave a stale throttle behind that reactivates if the host is re-added.
|
||||
QSet<QString> usedHosts;
|
||||
for (const QString &url : downloadUrls) {
|
||||
const QString host = QUrl(url).host();
|
||||
if (!host.isEmpty()) {
|
||||
usedHosts.insert(host);
|
||||
}
|
||||
}
|
||||
QHash<QString, int> limits = SettingsCache::instance().downloads().getHostRequestLimits();
|
||||
bool limitsChanged = false;
|
||||
for (auto it = limits.begin(); it != limits.end();) {
|
||||
// Prune only limits for hosts that are neither referenced by a configured URL nor carry a
|
||||
// developer cap. Capped hosts are often redirect targets (e.g. api.scryfall.com redirects
|
||||
// to cards.scryfall.io) that never appear in the URL list, yet they are exactly the hosts
|
||||
// the throttle applies to, so dropping them when a URL is removed would silently re-enable
|
||||
// free-running traffic to a rate-sensitive server.
|
||||
if (!usedHosts.contains(it.key()) && !DownloadSettings::getDeveloperHostCaps().contains(it.key())) {
|
||||
it = limits.erase(it);
|
||||
limitsChanged = true;
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
if (limitsChanged) {
|
||||
SettingsCache::instance().downloads().setHostRequestLimits(limits);
|
||||
}
|
||||
|
||||
refreshUrlItems();
|
||||
}
|
||||
|
||||
QListWidgetItem *DeckEditorSettingsPage::addUrlItem(const QString &url)
|
||||
{
|
||||
auto *item = new QListWidgetItem(urlLabel(url));
|
||||
item->setData(Qt::UserRole, url);
|
||||
urlList->addItem(item);
|
||||
return item;
|
||||
}
|
||||
|
||||
QString DeckEditorSettingsPage::urlForItem(const QListWidgetItem *item) const
|
||||
{
|
||||
return item->data(Qt::UserRole).toString();
|
||||
}
|
||||
|
||||
QString DeckEditorSettingsPage::urlLabel(const QString &url) const
|
||||
{
|
||||
const QString host = QUrl(url).host();
|
||||
if (host.isEmpty()) {
|
||||
return url;
|
||||
}
|
||||
|
||||
const QHash<QString, int> limits = SettingsCache::instance().downloads().getHostRequestLimits();
|
||||
const int devCap =
|
||||
DownloadSettings::getDeveloperHostCaps().value(host, DownloadSettings::DEFAULT_HOST_REQUEST_LIMIT);
|
||||
if (devCap == DownloadSettings::UNLIMITED_HOST_QUOTA && !limits.contains(host)) {
|
||||
return tr("%1 (unlimited)").arg(url);
|
||||
}
|
||||
|
||||
const int requested = limits.value(
|
||||
host, devCap == DownloadSettings::UNLIMITED_HOST_QUOTA ? DownloadSettings::DEFAULT_HOST_REQUEST_LIMIT : devCap);
|
||||
const int effective = SettingsCache::instance().downloads().clampHostRequestLimit(host, requested);
|
||||
return tr("%1 (%2/s)").arg(url).arg(effective);
|
||||
}
|
||||
|
||||
void DeckEditorSettingsPage::refreshUrlItems()
|
||||
{
|
||||
for (int i = 0; i < urlList->count(); ++i) {
|
||||
QListWidgetItem *item = urlList->item(i);
|
||||
item->setText(urlLabel(urlForItem(item)));
|
||||
}
|
||||
}
|
||||
|
||||
void DeckEditorSettingsPage::actAdjustRateLimit()
|
||||
{
|
||||
if (urlList->currentItem() == nullptr) {
|
||||
QMessageBox::information(this, tr("Adjust Rate Limit"), tr("Select a URL in the list first."));
|
||||
return;
|
||||
}
|
||||
|
||||
const QString host = QUrl(urlForItem(urlList->currentItem())).host();
|
||||
if (host.isEmpty()) {
|
||||
QMessageBox::information(this, tr("Adjust Rate Limit"), tr("The selected URL does not have a valid host."));
|
||||
return;
|
||||
}
|
||||
|
||||
const QHash<QString, int> &devCaps = DownloadSettings::getDeveloperHostCaps();
|
||||
const QHash<QString, int> currentLimits = SettingsCache::instance().downloads().getHostRequestLimits();
|
||||
const int devCap = devCaps.value(host, DownloadSettings::DEFAULT_HOST_REQUEST_LIMIT);
|
||||
const bool unlocked = devCap == DownloadSettings::UNLIMITED_HOST_QUOTA;
|
||||
|
||||
bool ok = false;
|
||||
int minimum;
|
||||
int maximum;
|
||||
int defaultValue;
|
||||
QString prompt;
|
||||
if (unlocked) {
|
||||
minimum = 0; // 0 means "unlimited"
|
||||
maximum = DownloadSettings::UNLOCKED_HOST_LIMIT_MAX;
|
||||
defaultValue = currentLimits.value(host, 0);
|
||||
prompt = tr("Requests per second (0 = unlimited, fastest; up to %1):").arg(maximum);
|
||||
} else {
|
||||
minimum = DownloadSettings::MIN_HOST_REQUEST_LIMIT;
|
||||
maximum = devCap;
|
||||
defaultValue = currentLimits.value(host, devCap);
|
||||
prompt = tr("Requests per second (developer maximum is %1):").arg(maximum);
|
||||
}
|
||||
|
||||
const int value = QInputDialog::getInt(this, tr("Adjust Rate Limit for %1").arg(host), prompt, defaultValue,
|
||||
minimum, maximum, 1, &ok);
|
||||
if (!ok) {
|
||||
return;
|
||||
}
|
||||
|
||||
QHash<QString, int> limits = currentLimits;
|
||||
if (unlocked ? value == 0 : value == devCap) {
|
||||
limits.remove(host);
|
||||
} else {
|
||||
limits.insert(host, value);
|
||||
}
|
||||
SettingsCache::instance().downloads().setHostRequestLimits(limits);
|
||||
refreshUrlItems();
|
||||
}
|
||||
|
||||
void DeckEditorSettingsPage::urlListChanged(const QModelIndex &, int, int, const QModelIndex &, int)
|
||||
|
|
@ -244,4 +380,8 @@ void DeckEditorSettingsPage::retranslateUi()
|
|||
aAdd->setText(tr("Add New URL"));
|
||||
aEdit->setText(tr("Edit URL"));
|
||||
aRemove->setText(tr("Remove URL"));
|
||||
}
|
||||
aRateLimit->setText(tr("Adjust Rate Limit"));
|
||||
|
||||
// The per-URL rate limit suffixes are translated, so refresh them when the language changes.
|
||||
refreshUrlItems();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ private slots:
|
|||
void actAddURL();
|
||||
void actRemoveURL();
|
||||
void actEditURL();
|
||||
void actAdjustRateLimit();
|
||||
void resetDownloadedURLsButtonClicked();
|
||||
|
||||
private:
|
||||
|
|
@ -34,7 +35,7 @@ private:
|
|||
QLabel urlLinkLabel;
|
||||
QCheckBox picDownloadCheckBox;
|
||||
QListWidget *urlList;
|
||||
QAction *aAdd, *aEdit, *aRemove;
|
||||
QAction *aAdd, *aEdit, *aRemove, *aRateLimit;
|
||||
QCheckBox mcDownloadSpoilersCheckBox;
|
||||
QLabel msDownloadSpoilersLabel;
|
||||
QGroupBox *mpGeneralGroupBox;
|
||||
|
|
@ -46,6 +47,18 @@ private:
|
|||
QLabel infoOnSpoilersLabel;
|
||||
QPushButton *mpSpoilerPathButton;
|
||||
QPushButton *updateNowButton;
|
||||
|
||||
/** @brief Adds a list item for the given URL, storing the raw URL alongside its displayed label. */
|
||||
QListWidgetItem *addUrlItem(const QString &url);
|
||||
|
||||
/** @brief Returns the raw URL stored on a list item. */
|
||||
[[nodiscard]] QString urlForItem(const QListWidgetItem *item) const;
|
||||
|
||||
/** @brief Returns the display label for a URL, including its current effective rate limit. */
|
||||
[[nodiscard]] QString urlLabel(const QString &url) const;
|
||||
|
||||
/** @brief Refreshes the displayed label of every URL item after limits or settings change. */
|
||||
void refreshUrlItems();
|
||||
};
|
||||
|
||||
#endif // COCKATRICE_DECK_EDITOR_SETTINGS_PAGE_H
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue