[Client/Server/Protocol] Surface live metrics in the Developer tab

Extend Response_GetServerStats with live counters from the in-process
MetricsRegistry: cards in games, event loop stall totals/worst,
total commands processed, average command time, active command types,
and game-start count/duration. Add a repeated CommandStats message
carrying per-command breakdowns (kind, extension number, resolved
protobuf name, count, total ms) for every type that has seen at
least one sample.

Server-side cmdGetServerStats() populates all new fields after the
existing DB uptime snapshot query, resolving protobuf extension names
via the descriptor pool for human-readable labels like
session/Command_Ping.

Expand TabDeveloper with two tables: an overview section (existing
DB stats plus the new live metrics) and a per-command breakdown table
(Command / Count / Total ms / Avg ms) sorted by total_ms descending
so the hottest commands surface first.

Took 55 minutes

Took 47 seconds

cockatrice/CMakeLists.txt

@@ -380,6 +380,7 @@ set(cockatrice_SOURCES
     src/interface/widgets/tabs/tab_card_art_rules.cpp
     src/interface/widgets/tabs/tab_deck_editor.cpp
     src/interface/widgets/tabs/tab_deck_storage.cpp
+    src/interface/widgets/tabs/tab_developer.cpp
     src/interface/widgets/tabs/tab_game.cpp
     src/interface/widgets/tabs/tab_home.cpp
     src/interface/widgets/tabs/tab_logs.cpp

cockatrice/src/interface/pixel_map_generator.cpp

@@ -14,6 +14,7 @@
 #define DEFAULT_COLOR_MODERATOR_LEFT "#ffffff";
 #define DEFAULT_COLOR_MODERATOR_RIGHT "#000000";
 #define DEFAULT_COLOR_ADMIN "#ff2701";
+#define DEFAULT_COLOR_DEVELOPER "#800020"
 
 /**
  * Clamps an svg render size so that rendering does not exceed a multiple of the requested size.
@@ -359,6 +360,8 @@ QIcon UserLevelPixmapGenerator::generateIconDefault(int height,
 
     if (userLevel.testFlag(ServerInfo_User::IsAdmin)) {
         colorLeft = DEFAULT_COLOR_ADMIN;
+    } else if (userLevel.testFlag(ServerInfo_User::IsDeveloper)) {
+        colorLeft = DEFAULT_COLOR_DEVELOPER;
     } else if (userLevel.testFlag(ServerInfo_User::IsModerator)) {
         colorLeft = DEFAULT_COLOR_MODERATOR_LEFT;
         colorRight = DEFAULT_COLOR_MODERATOR_RIGHT;

cockatrice/src/interface/widgets/server/user/user_context_menu.cpp

@@ -51,6 +51,8 @@ UserContextMenu::UserContextMenu(TabSupervisor *_tabSupervisor, QWidget *parent,
     aDemoteFromMod = new QAction(QString(), this);
     aPromoteToJudge = new QAction(QString(), this);
     aDemoteFromJudge = new QAction(QString(), this);
+    aPromoteToDeveloper = new QAction(QString(), this);
+    aDemoteFromDeveloper = new QAction(QString(), this);
     aGetAdminNotes = new QAction(QString(), this);
     aInvestigateUser = new QAction(QString(), this);
 
@@ -76,6 +78,8 @@ void UserContextMenu::retranslateUi()
     aDemoteFromMod->setText(tr("Dem&ote user from moderator"));
     aPromoteToJudge->setText(tr("Promote user to &judge"));
     aDemoteFromJudge->setText(tr("Demote user from judge"));
+    aPromoteToDeveloper->setText(tr("Promote user to &developer"));
+    aDemoteFromDeveloper->setText(tr("Demote user from de&veloper"));
     aGetAdminNotes->setText(tr("View admin notes"));
     aInvestigateUser->setText(tr("Investigate user"));
 }
@@ -268,7 +272,7 @@ void UserContextMenu::adjustMod_processUserResponse(const Response &resp, const
     const Command_AdjustMod &cmd = commandContainer.admin_command(0).GetExtension(Command_AdjustMod::ext);
 
     if (resp.response_code() == Response::RespOk) {
-        if (cmd.should_be_mod() || cmd.should_be_judge()) {
+        if (cmd.should_be_mod() || cmd.should_be_judge() || cmd.should_be_developer()) {
             QMessageBox::information(static_cast<QWidget *>(parent()), tr("Success"),
                                      tr("Successfully promoted user."));
         } else {
@@ -276,7 +280,7 @@ void UserContextMenu::adjustMod_processUserResponse(const Response &resp, const
         }
 
     } else {
-        if (cmd.should_be_mod() || cmd.should_be_judge()) {
+        if (cmd.should_be_mod() || cmd.should_be_judge() || cmd.should_be_developer()) {
             QMessageBox::information(static_cast<QWidget *>(parent()), tr("Failed"), tr("Failed to promote user."));
         } else {
             QMessageBox::information(static_cast<QWidget *>(parent()), tr("Failed"), tr("Failed to demote user."));
@@ -437,6 +441,15 @@ void UserContextMenu::showContextMenu(const QPoint &pos,
                    (tabSupervisor->getUserInfo()->user_level() & ServerInfo_User::IsAdmin)) {
             menu->addAction(aPromoteToJudge);
         }
+
+        if (userLevel.testFlag(ServerInfo_User::IsDeveloper) &&
+            (tabSupervisor->getUserInfo()->user_level() & ServerInfo_User::IsAdmin)) {
+            menu->addAction(aDemoteFromDeveloper);
+
+        } else if (userLevel.testFlag(ServerInfo_User::IsRegistered) &&
+                   (tabSupervisor->getUserInfo()->user_level() & ServerInfo_User::IsAdmin)) {
+            menu->addAction(aPromoteToDeveloper);
+        }
     }
     aDetails->setEnabled(true);
     aChat->setEnabled(anotherUser && online);
@@ -455,6 +468,10 @@ void UserContextMenu::showContextMenu(const QPoint &pos,
     aInvestigateUser->setEnabled(anotherUser);
     aPromoteToMod->setEnabled(anotherUser);
     aDemoteFromMod->setEnabled(anotherUser);
+    aPromoteToJudge->setEnabled(anotherUser);
+    aDemoteFromJudge->setEnabled(anotherUser);
+    aPromoteToDeveloper->setEnabled(anotherUser);
+    aDemoteFromDeveloper->setEnabled(anotherUser);
 
     QAction *actionClicked = menu->exec(pos);
     if (actionClicked == nullptr) {
@@ -489,6 +506,8 @@ void UserContextMenu::showContextMenu(const QPoint &pos,
         execAdjustMod(userName, actionClicked == aPromoteToMod);
     } else if (actionClicked == aPromoteToJudge || actionClicked == aDemoteFromJudge) {
         execAdjustJudge(userName, actionClicked == aPromoteToJudge);
+    } else if (actionClicked == aPromoteToDeveloper || actionClicked == aDemoteFromDeveloper) {
+        execAdjustDeveloper(userName, actionClicked == aPromoteToDeveloper);
     } else if (actionClicked == aBanHistory) {
         execBanHistory(userName);
     } else if (actionClicked == aWarnUser) {
@@ -698,4 +717,14 @@ void UserContextMenu::execAdjustJudge(const QString &userName, bool shouldBeJudg
     PendingCommand *pend = client->prepareAdminCommand(cmd);
     connect(pend, &PendingCommand::finished, this, &UserContextMenu::adjustMod_processUserResponse);
     client->sendCommand(pend);
+}
+
+void UserContextMenu::execAdjustDeveloper(const QString &userName, bool shouldBeDeveloper)
+{
+    Command_AdjustMod cmd;
+    cmd.set_user_name(userName.toStdString());
+    cmd.set_should_be_developer(shouldBeDeveloper);
+    PendingCommand *pend = client->prepareAdminCommand(cmd);
+    connect(pend, &PendingCommand::finished, this, &UserContextMenu::adjustMod_processUserResponse);
+    client->sendCommand(pend);
 }

cockatrice/src/interface/widgets/server/user/user_context_menu.h

@@ -45,6 +45,7 @@ private:
     QAction *aBan, *aBanHistory;
     QAction *aPromoteToMod, *aDemoteFromMod;
     QAction *aPromoteToJudge, *aDemoteFromJudge;
+    QAction *aPromoteToDeveloper, *aDemoteFromDeveloper;
     QAction *aWarnUser, *aWarnHistory;
     QAction *aGetAdminNotes;
     std::function<QList<GameInviteOption>()> gameInviteLinkProvider;
@@ -123,6 +124,7 @@ public:
     void execInvestigateUser(const QString &userName);
     void execAdjustMod(const QString &userName, bool shouldBeMod);
     void execAdjustJudge(const QString &userName, bool shouldBeJudge);
+    void execAdjustDeveloper(const QString &userName, bool shouldBeDeveloper);
 
 private:
     void execInvite(const QString &userName, const GameInviteOption &option);

cockatrice/src/interface/widgets/server/user/user_info_box.cpp

@@ -122,6 +122,8 @@ void UserInfoBox::updateInfo(const ServerInfo_User &user)
     QString userLevelText;
     if (userLevel.testFlag(ServerInfo_User::IsAdmin)) {
         userLevelText = tr("Administrator");
+    } else if (userLevel.testFlag(ServerInfo_User::IsDeveloper)) {
+        userLevelText = tr("Developer");
     } else if (userLevel.testFlag(ServerInfo_User::IsModerator)) {
         userLevelText = tr("Moderator");
     } else if (userLevel.testFlag(ServerInfo_User::IsRegistered)) {

cockatrice/src/interface/widgets/server/user/user_info_popup.cpp

@@ -245,6 +245,9 @@ void UserInfoHeaderWidget::paintEvent(QPaintEvent *)
         if (level.testFlag(ServerInfo_User::IsAdmin)) {
             return QColor(245, 158, 11);
         }
+        if (level.testFlag(ServerInfo_User::IsDeveloper)) {
+            return QColor(185, 28, 28);
+        }
         if (level.testFlag(ServerInfo_User::IsModerator)) {
             return QColor(59, 130, 246);
         }
@@ -300,6 +303,8 @@ void UserInfoHeaderWidget::paintEvent(QPaintEvent *)
     } badge;
     if (level.testFlag(ServerInfo_User::IsAdmin)) {
         badge = {"ADMIN", QColor(245, 158, 11)};
+    } else if (level.testFlag(ServerInfo_User::IsDeveloper)) {
+        badge = {"DEV", QColor(185, 28, 28)};
     } else if (level.testFlag(ServerInfo_User::IsModerator)) {
         badge = {"MOD", QColor(59, 130, 246)};
     } else if (level.testFlag(ServerInfo_User::IsJudge)) {

cockatrice/src/interface/widgets/server/user/user_list_painter.cpp

@@ -49,6 +49,8 @@ QColor UserListPainter::getAccentColor(const UserLevelFlags &userLevel, bool onl
 
     if (userLevel.testFlag(ServerInfo_User::IsAdmin)) {
         accentColor = QColor(245, 158, 11);
+    } else if (userLevel.testFlag(ServerInfo_User::IsDeveloper)) {
+        accentColor = QColor(185, 28, 28);
     } else if (userLevel.testFlag(ServerInfo_User::IsModerator)) {
         accentColor = QColor(59, 130, 246);
     } else if (userLevel.testFlag(ServerInfo_User::IsJudge)) {
@@ -299,6 +301,8 @@ QList<UserListPainter::Badge> UserListPainter::buildBadges(const UserLevelFlags
 
     if (userLevel.testFlag(ServerInfo_User::IsAdmin)) {
         badges << Badge{"ADMIN", QColor(245, 158, 11)};
+    } else if (userLevel.testFlag(ServerInfo_User::IsDeveloper)) {
+        badges << Badge{"DEV", QColor(185, 28, 28)};
     } else if (userLevel.testFlag(ServerInfo_User::IsModerator)) {
         badges << Badge{"MOD", QColor(59, 130, 246)};
     } else if (userLevel.testFlag(ServerInfo_User::IsJudge)) {
@@ -385,9 +389,9 @@ void UserListPainter::paint(QPainter *painter,
     const QString userName = QString::fromStdString(userInfo.name());
     const QString privLevel = QString::fromStdString(userInfo.privlevel());
     const QColor accentColor = getAccentColor(userLevel, online);
-    const bool hasRole = userLevel.testFlag(ServerInfo_User::IsAdmin) ||
-                         userLevel.testFlag(ServerInfo_User::IsModerator) ||
-                         userLevel.testFlag(ServerInfo_User::IsJudge);
+    const bool hasRole =
+        userLevel.testFlag(ServerInfo_User::IsAdmin) || userLevel.testFlag(ServerInfo_User::IsDeveloper) ||
+        userLevel.testFlag(ServerInfo_User::IsModerator) || userLevel.testFlag(ServerInfo_User::IsJudge);
     const QRectF cardRect = QRectF(rect).adjusted(3, 2, -3, -2);
     const int cardRight = getCardRight(option, rect);
 

cockatrice/src/interface/widgets/server/user/user_list_widget.cpp

@@ -234,9 +234,11 @@ bool UserListTWI::operator<(const QTreeWidgetItem &other) const
     const auto &lhsUserLevelFlags = UserLevelFlags(data(0, Qt::UserRole).toInt());
     const auto &rhsUserLevelFlags = UserLevelFlags(other.data(0, Qt::UserRole).toInt());
 
-    // Admins & Mods need no additional comparison checks, just to see if they're an admin or a moderator
+    // Admins, Developers & Mods need no additional comparison checks, just to see if they're an admin, a developer
+    // or a moderator
     static const QList<ServerInfo_User_UserLevelFlag> userLevelWithNoOtherPrefOrder = {
-        ServerInfo_User_UserLevelFlag_IsAdmin, ServerInfo_User_UserLevelFlag_IsModerator};
+        ServerInfo_User_UserLevelFlag_IsAdmin, ServerInfo_User_UserLevelFlag_IsDeveloper,
+        ServerInfo_User_UserLevelFlag_IsModerator};
     for (const auto &userLevelEntry : userLevelWithNoOtherPrefOrder) {
         if (lhsUserLevelFlags.testFlag(userLevelEntry) &&
             lhsUserLevelFlags.testFlag(userLevelEntry) == rhsUserLevelFlags.testFlag(userLevelEntry)) {

cockatrice/src/interface/widgets/tabs/tab_developer.cpp

@@ -0,0 +1,213 @@
+/**
+ * @file tab_developer.cpp
+ * @ingroup ServerTabs
+ */
+//! \todo Document this file.
+
+#include "tab_developer.h"
+
+#include <QDateTime>
+#include <QHBoxLayout>
+#include <QHeaderView>
+#include <QLabel>
+#include <QPushButton>
+#include <QTableWidget>
+#include <QVBoxLayout>
+#include <algorithm>
+#include <libcockatrice/network/client/abstract/abstract_client.h>
+#include <libcockatrice/protocol/pb/command_get_server_stats.pb.h>
+#include <libcockatrice/protocol/pb/response_get_server_stats.pb.h>
+#include <libcockatrice/protocol/pending_command.h>
+
+TabDeveloper::TabDeveloper(TabSupervisor *_tabSupervisor, AbstractClient *_client)
+    : Tab(_tabSupervisor), client(_client)
+{
+    statsTable = new QTableWidget(0, 2);
+    statsTable->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);
+    statsTable->setEditTriggers(QAbstractItemView::NoEditTriggers);
+    statsTable->setSelectionBehavior(QAbstractItemView::SelectRows);
+    statsTable->setSelectionMode(QAbstractItemView::SingleSelection);
+    statsTable->verticalHeader()->setVisible(false);
+    statsTable->horizontalHeader()->setSectionResizeMode(0, QHeaderView::Interactive);
+    statsTable->horizontalHeader()->setSectionResizeMode(1, QHeaderView::Interactive);
+    statsTable->horizontalHeader()->setStretchLastSection(true);
+
+    commandTable = new QTableWidget(0, 4);
+    commandTable->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding);
+    commandTable->setEditTriggers(QAbstractItemView::NoEditTriggers);
+    commandTable->setSelectionBehavior(QAbstractItemView::SelectRows);
+    commandTable->setSelectionMode(QAbstractItemView::SingleSelection);
+    commandTable->verticalHeader()->setVisible(false);
+    commandTable->horizontalHeader()->setSectionResizeMode(0, QHeaderView::Interactive);
+    commandTable->horizontalHeader()->setSectionResizeMode(1, QHeaderView::Interactive);
+    commandTable->horizontalHeader()->setSectionResizeMode(2, QHeaderView::Interactive);
+    commandTable->horizontalHeader()->setSectionResizeMode(3, QHeaderView::Interactive);
+
+    statusLabel = new QLabel;
+
+    refreshButton = new QPushButton;
+    refreshButton->setAutoDefault(true);
+    connect(refreshButton, &QPushButton::clicked, this, &TabDeveloper::refreshClicked);
+
+    auto *buttonLayout = new QHBoxLayout;
+    buttonLayout->addWidget(statusLabel, 1, Qt::AlignLeft);
+    buttonLayout->addWidget(refreshButton, 0, Qt::AlignRight);
+
+    auto *tableLayout = new QHBoxLayout;
+    tableLayout->addWidget(statsTable, 1);
+    tableLayout->addWidget(commandTable, 2);
+
+    auto *mainLayout = new QVBoxLayout;
+    mainLayout->addLayout(tableLayout, 1);
+    mainLayout->addLayout(buttonLayout);
+
+    auto *central = new QWidget;
+    central->setLayout(mainLayout);
+    setCentralWidget(central);
+
+    retranslateUi();
+}
+
+void TabDeveloper::retranslateUi()
+{
+    refreshButton->setText(tr("Refresh server stats"));
+    statsTable->setHorizontalHeaderLabels(QString(tr("Statistic;Value")).split(";"));
+    commandTable->setHorizontalHeaderLabels(QString(tr("Command;Count;Total ms;Avg ms")).split(";"));
+    if (statsTable->rowCount() == 0) {
+        statusLabel->clear();
+    }
+}
+
+QString TabDeveloper::formatBytes(quint64 bytes)
+{
+    const quint64 kib = 1024;
+    const quint64 mib = 1024 * kib;
+    const quint64 gib = 1024 * mib;
+    if (bytes >= gib) {
+        return tr("%1 GiB").arg(QString::number(bytes / static_cast<double>(gib), 'f', 2));
+    }
+    if (bytes >= mib) {
+        return tr("%1 MiB").arg(QString::number(bytes / static_cast<double>(mib), 'f', 2));
+    }
+    if (bytes >= kib) {
+        return tr("%1 KiB").arg(QString::number(bytes / static_cast<double>(kib), 'f', 2));
+    }
+    return tr("%1 bytes").arg(bytes);
+}
+
+QString TabDeveloper::formatDurationMs(qint64 ms)
+{
+    if (ms >= 1000) {
+        return tr("%1 s").arg(QString::number(ms / 1000.0, 'f', 2));
+    }
+    return tr("%1 ms").arg(ms);
+}
+
+void TabDeveloper::appendStatRow(const QString &name, const QString &value)
+{
+    const int row = statsTable->rowCount();
+    statsTable->insertRow(row);
+    statsTable->setItem(row, 0, new QTableWidgetItem(name));
+    statsTable->setItem(row, 1, new QTableWidgetItem(value));
+}
+
+void TabDeveloper::appendSeparatorRow(const QString &sectionTitle)
+{
+    const int row = statsTable->rowCount();
+    statsTable->insertRow(row);
+    auto *labelItem = new QTableWidgetItem(sectionTitle);
+    auto font = labelItem->font();
+    font.setBold(true);
+    labelItem->setFont(font);
+    labelItem->setFlags(labelItem->flags() & ~Qt::ItemIsSelectable);
+    statsTable->setItem(row, 0, labelItem);
+    statsTable->setItem(row, 1, new QTableWidgetItem(QString()));
+}
+
+void TabDeveloper::refreshClicked()
+{
+    Command_GetServerStats cmd;
+    PendingCommand *pend = client->prepareDeveloperCommand(cmd);
+    connect(pend, &PendingCommand::finished, this, &TabDeveloper::serverStatsResponse);
+    client->sendCommand(pend);
+}
+
+void TabDeveloper::serverStatsResponse(const Response &resp)
+{
+    if (resp.response_code() != Response::RespOk) {
+        statusLabel->setText(tr("Failed to collect server statistics."));
+        return;
+    }
+
+    const Response_GetServerStats &response = resp.GetExtension(Response_GetServerStats::ext);
+
+    statsTable->setRowCount(0);
+
+    // Overview section
+    appendStatRow(tr("Registered users online"), QString::number(response.users_count()));
+    appendStatRow(tr("Moderators online"), QString::number(response.mods_count()));
+    appendStatRow(tr("Games running"), QString::number(response.games_count()));
+    appendStatRow(tr("Traffic sent (last tick)"), formatBytes(response.tx_bytes()));
+    appendStatRow(tr("Traffic received (last tick)"), formatBytes(response.rx_bytes()));
+
+    const qint64 uptime = static_cast<qint64>(response.uptime_secs());
+    const int days = static_cast<int>(uptime / 86400);
+    const int hours = static_cast<int>((uptime % 86400) / 3600);
+    const int minutes = static_cast<int>((uptime % 3600) / 60);
+    appendStatRow(tr("Server uptime"), days > 0 ? tr("%1d %2h %3m").arg(days).arg(hours).arg(minutes)
+                                                : tr("%1h %2m").arg(hours).arg(minutes));
+
+    const QDateTime snapshotTime = QDateTime::fromSecsSinceEpoch(static_cast<qint64>(response.timest()));
+    appendStatRow(tr("Snapshot taken"), snapshotTime.toLocalTime().toString("yyyy-MM-dd HH:mm"));
+
+    // Live metrics section
+    appendSeparatorRow(tr("Live Metrics"));
+    appendStatRow(tr("Cards in live games"), QString::number(response.cards_in_games()));
+    appendStatRow(tr("Total commands processed"), QString::number(response.total_commands()));
+
+    if (response.total_commands() > 0) {
+        const double avgMs = static_cast<double>(response.total_command_time_ms()) / response.total_commands();
+        appendStatRow(tr("Avg command time"), QString::number(avgMs, 'f', 2) + " ms");
+    }
+    appendStatRow(tr("Active command types"), QString::number(response.active_command_types()));
+
+    appendStatRow(tr("Event loop stalls"), QString::number(response.eventloop_stalls_total()));
+    appendStatRow(tr("Last stall overshoot"), formatDurationMs(response.eventloop_last_stall_ms()));
+    appendStatRow(tr("Worst stall overshoot"), formatDurationMs(response.eventloop_max_stall_ms()));
+
+    if (response.game_start_count() > 0) {
+        appendStatRow(tr("Game starts"), QString::number(response.game_start_count()));
+        const double avgStartMs = static_cast<double>(response.game_start_total_ms()) / response.game_start_count();
+        appendStatRow(tr("Avg game start time"), QString::number(avgStartMs, 'f', 1) + " ms");
+    }
+
+    // Per-command breakdown table
+    QList<CommandStats> sortedStats(response.command_stats().begin(), response.command_stats().end());
+    std::sort(sortedStats.begin(), sortedStats.end(),
+              [](const auto &a, const auto &b) { return a.total_ms() > b.total_ms(); });
+
+    commandTable->setRowCount(0);
+    for (const auto &cs : sortedStats) {
+        const int row = commandTable->rowCount();
+        commandTable->insertRow(row);
+        commandTable->setItem(row, 0, new QTableWidgetItem(QString::fromStdString(cs.command_name())));
+
+        auto *countItem = new QTableWidgetItem(QString::number(cs.count()));
+        countItem->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter);
+        commandTable->setItem(row, 1, countItem);
+
+        auto *totalItem = new QTableWidgetItem(QString::number(cs.total_ms()));
+        totalItem->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter);
+        commandTable->setItem(row, 2, totalItem);
+
+        const double avg = cs.count() > 0 ? static_cast<double>(cs.total_ms()) / cs.count() : 0.0;
+        auto *avgItem = new QTableWidgetItem(QString::number(avg, 'f', 2));
+        avgItem->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter);
+        commandTable->setItem(row, 3, avgItem);
+    }
+    commandTable->resizeColumnsToContents();
+    statsTable->resizeColumnsToContents();
+    commandTable->resizeColumnsToContents();
+
+    statusLabel->setText(tr("Updated %1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd HH:mm")));
+}

cockatrice/src/interface/widgets/tabs/tab_developer.h

@@ -0,0 +1,46 @@
+/**
+ * @file tab_developer.h
+ * @ingroup ServerTabs
+ */
+//! \todo Document this file.
+
+#ifndef TAB_DEVELOPER_H
+#define TAB_DEVELOPER_H
+
+#include "tab.h"
+
+class AbstractClient;
+class QLabel;
+class QPushButton;
+class QTableWidget;
+class Response;
+
+class TabDeveloper : public Tab
+{
+    Q_OBJECT
+private:
+    AbstractClient *client;
+    QTableWidget *statsTable;
+    QTableWidget *commandTable;
+    QPushButton *refreshButton;
+    QLabel *statusLabel;
+
+    void appendStatRow(const QString &name, const QString &value);
+    void appendSeparatorRow(const QString &sectionTitle);
+    static QString formatBytes(quint64 bytes);
+    static QString formatDurationMs(qint64 ms);
+
+private slots:
+    void refreshClicked();
+    void serverStatsResponse(const Response &resp);
+
+public:
+    explicit TabDeveloper(TabSupervisor *_tabSupervisor, AbstractClient *_client);
+    void retranslateUi() override;
+    [[nodiscard]] QString getTabText() const override
+    {
+        return tr("Developer");
+    }
+};
+
+#endif

cockatrice/src/interface/widgets/tabs/tab_logs.cpp

@@ -14,12 +14,14 @@
 #include <QTabWidget>
 #include <QTableWidget>
 #include <libcockatrice/network/client/abstract/abstract_client.h>
+#include <libcockatrice/protocol/pb/command_get_log_history.pb.h>
 #include <libcockatrice/protocol/pb/moderator_commands.pb.h>
 #include <libcockatrice/protocol/pb/response_viewlog_history.pb.h>
 #include <libcockatrice/protocol/pending_command.h>
 #include <libcockatrice/utility/string_limits.h>
 
-TabLog::TabLog(TabSupervisor *_tabSupervisor, AbstractClient *_client) : Tab(_tabSupervisor), client(_client)
+TabLog::TabLog(TabSupervisor *_tabSupervisor, AbstractClient *_client, bool _canUseDeveloperCommands)
+    : Tab(_tabSupervisor), client(_client), canUseDeveloperCommands(_canUseDeveloperCommands)
 {
     roomTable = new QTableWidget();
     roomTable->setColumnCount(6);
@@ -117,7 +119,26 @@ void TabLog::getClicked()
     };
     cmd.set_date_range(dateRange);
     cmd.set_maximum_results(maximumResults->value());
-    PendingCommand *pend = client->prepareModeratorCommand(cmd);
+
+    PendingCommand *pend;
+    if (canUseDeveloperCommands) {
+        // Developers query logs through the developer command family.
+        Command_GetLogHistory devCmd;
+        devCmd.set_user_name(cmd.user_name());
+        devCmd.set_ip_address(cmd.ip_address());
+        devCmd.set_game_name(cmd.game_name());
+        devCmd.set_game_id(cmd.game_id());
+        devCmd.set_message(cmd.message());
+        for (int i = 0; i < cmd.log_location_size(); ++i) {
+            devCmd.add_log_location(cmd.log_location(i));
+        }
+        devCmd.set_date_range(cmd.date_range());
+        devCmd.set_maximum_results(cmd.maximum_results());
+        pend = client->prepareDeveloperCommand(devCmd);
+    } else {
+        pend = client->prepareModeratorCommand(cmd);
+    }
+
     connect(pend, &PendingCommand::finished, this, &TabLog::viewLogHistory_processResponse);
     client->sendCommand(pend);
 }

cockatrice/src/interface/widgets/tabs/tab_logs.h

@@ -33,6 +33,7 @@ class TabLog : public Tab
     Q_OBJECT
 private:
     AbstractClient *client;
+    bool canUseDeveloperCommands;
     QLabel *labelFindUserName, *labelFindIPAddress, *labelFindGameName, *labelFindGameID, *labelMessage, *labelMaximum,
         *labelDescription;
     LineEditUnfocusable *findUsername, *findIPAddress, *findGameName, *findGameID, *findMessage;
@@ -58,7 +59,7 @@ private slots:
     void restartLayout();
 
 public:
-    TabLog(TabSupervisor *_tabSupervisor, AbstractClient *_client);
+    TabLog(TabSupervisor *_tabSupervisor, AbstractClient *_client, bool _canUseDeveloperCommands = false);
     ~TabLog() override;
     void retranslateUi() override;
     [[nodiscard]] QString getTabText() const override

cockatrice/src/interface/widgets/tabs/tab_moderation.cpp

@@ -381,6 +381,9 @@ void TabModeration::moderatorLoginsResponse(const Response &response)
         if (login.user_level() & ServerInfo_User::IsAdmin) {
             levels << tr("Admin");
         }
+        if (login.user_level() & ServerInfo_User::IsDeveloper) {
+            levels << tr("Developer");
+        }
         if (login.user_level() & ServerInfo_User::IsModerator) {
             levels << tr("Moderator");
         }

cockatrice/src/interface/widgets/tabs/tab_supervisor.cpp

@@ -15,6 +15,7 @@
 #include "tab_card_art_rules.h"
 #include "tab_deck_editor.h"
 #include "tab_deck_storage.h"
+#include "tab_developer.h"
 #include "tab_game.h"
 #include "tab_home.h"
 #include "tab_logs.h"
@@ -118,7 +119,7 @@ void CloseButton::paintEvent(QPaintEvent * /*event*/)
 TabSupervisor::TabSupervisor(AbstractClient *_client, QMenu *tabsMenu, QWidget *parent)
     : QTabWidget(parent), userInfo(nullptr), client(_client), tabsMenu(tabsMenu), tabVisualDeckStorage(nullptr),
       tabServer(nullptr), tabAccount(nullptr), tabDeckStorage(nullptr), tabReplays(nullptr), tabAdmin(nullptr),
-      tabLog(nullptr), tabReport(nullptr), tabModeration(nullptr), isLocalGame(false)
+      tabLog(nullptr), tabReport(nullptr), tabModeration(nullptr), tabDeveloper(nullptr), isLocalGame(false)
 {
     setElideMode(Qt::ElideRight);
     setMovable(true);
@@ -204,6 +205,10 @@ TabSupervisor::TabSupervisor(AbstractClient *_client, QMenu *tabsMenu, QWidget *
     aTabModeration->setCheckable(true);
     connect(aTabModeration, &QAction::triggered, this, &TabSupervisor::actTabModeration);
 
+    aTabDeveloper = new QAction(this);
+    aTabDeveloper->setCheckable(true);
+    connect(aTabDeveloper, &QAction::triggered, this, &TabSupervisor::actTabDeveloper);
+
     connect(&SettingsCache::instance().shortcuts(), &ShortcutsSettings::shortCutChanged, this,
             &TabSupervisor::refreshShortcuts);
     refreshShortcuts();
@@ -245,6 +250,7 @@ void TabSupervisor::retranslateUi()
     aTabLog->setText(tr("Logs"));
     aTabReport->setText(tr("Report Queue"));
     aTabModeration->setText(tr("Moderation"));
+    aTabDeveloper->setText(tr("Developer"));
 
     // tabs
     QList<Tab *> tabs;
@@ -256,6 +262,7 @@ void TabSupervisor::retranslateUi()
     tabs.append(tabLog);
     tabs.append(tabReport);
     tabs.append(tabModeration);
+    tabs.append(tabDeveloper);
     QMapIterator<int, TabRoom *> roomIterator(roomTabs);
     while (roomIterator.hasNext()) {
         tabs.append(roomIterator.next().value());
@@ -523,6 +530,19 @@ void TabSupervisor::start(const ServerInfo_User &_userInfo)
         openTabCardArtRules();
     }
 
+    if (userInfo->user_level() & ServerInfo_User::IsDeveloper) {
+        tabsMenu->addSeparator();
+        tabsMenu->addAction(aTabDeveloper);
+        // Developers without moderation rights get log access through their
+        // own role. Moderators already have the Logs entry from above.
+        if (!(userInfo->user_level() & ServerInfo_User::IsModerator)) {
+            tabsMenu->addAction(aTabLog);
+            if (SettingsCache::instance().tabs().getTabLogOpen()) {
+                openTabLog();
+            }
+        }
+    }
+
     retranslateUi();
 }
 
@@ -535,6 +555,7 @@ void TabSupervisor::startLocal(const QList<AbstractClient *> &_clients)
     tabLog = nullptr;
     tabReport = nullptr;
     tabModeration = nullptr;
+    tabDeveloper = nullptr;
     isLocalGame = true;
     userInfo = new ServerInfo_User;
     localClients = _clients;
@@ -582,6 +603,9 @@ void TabSupervisor::stop()
         if (tabModeration) {
             tabModeration->close();
         }
+        if (tabDeveloper) {
+            tabDeveloper->close();
+        }
     }
 
     QList<Tab *> tabsToDelete;
@@ -810,7 +834,10 @@ void TabSupervisor::actTabLog(bool checked)
 
 void TabSupervisor::openTabLog()
 {
-    tabLog = new TabLog(this, client);
+    // Developers without moderation rights query logs through the developer
+    // command family, so tell the tab which family to use.
+    const bool isDeveloper = (userInfo->user_level() & ServerInfo_User::IsDeveloper) != 0;
+    tabLog = new TabLog(this, client, isDeveloper);
     myAddTab(tabLog, aTabLog);
     connect(tabLog, &QObject::destroyed, this, [this] {
         tabLog = nullptr;
@@ -872,6 +899,27 @@ void TabSupervisor::openTabModeration(const QString &userName)
     aTabModeration->setChecked(true);
 }
 
+void TabSupervisor::actTabDeveloper(bool checked)
+{
+    if (checked && !tabDeveloper) {
+        openTabDeveloper();
+        setCurrentWidget(tabDeveloper);
+    } else if (!checked && tabDeveloper) {
+        tabDeveloper->closeRequest();
+    }
+}
+
+void TabSupervisor::openTabDeveloper()
+{
+    tabDeveloper = new TabDeveloper(this, client);
+    myAddTab(tabDeveloper, aTabDeveloper);
+    connect(tabDeveloper, &QObject::destroyed, this, [this] {
+        tabDeveloper = nullptr;
+        aTabDeveloper->setChecked(false);
+    });
+    aTabDeveloper->setChecked(true);
+}
+
 void TabSupervisor::updatePingTime(int value, int max)
 {
     if (!tabServer) {

cockatrice/src/interface/widgets/tabs/tab_supervisor.h

@@ -45,6 +45,7 @@ class TabReport;
 class TabModeration;
 class TabAccount;
 class TabDeckEditor;
+class TabDeveloper;
 class TabLog;
 class RoomEvent;
 class GameEventContainer;
@@ -108,6 +109,7 @@ private:
     TabLog *tabLog;
     TabReport *tabReport;
     TabModeration *tabModeration;
+    TabDeveloper *tabDeveloper;
     QMap<int, TabRoom *> roomTabs;
     QMap<int, TabGame *> gameTabs;
     QList<TabGame *> replayTabs;
@@ -117,7 +119,7 @@ private:
 
     QAction *aTabHome, *aTabDeckEditor, *aTabVisualDeckEditor, *aTabEdhRec, *aTabArchidekt, *aTabVisualDeckStorage,
         *aTabVisualDatabaseDisplay, *aTabServer, *aTabAccount, *aTabDeckStorage, *aTabReplays, *aTabAdmin,
-        *aTabCardArtRules, *aTabLog, *aTabReport, *aTabModeration;
+        *aTabCardArtRules, *aTabLog, *aTabReport, *aTabModeration, *aTabDeveloper;
 
     int myAddTab(Tab *tab, QAction *manager = nullptr);
     void addCloseButtonToTab(Tab *tab, int tabIndex, QAction *manager);
@@ -207,6 +209,7 @@ private slots:
     void actTabLog(bool checked);
     void actTabReport(bool checked);
     void actTabModeration(bool checked);
+    void actTabDeveloper(bool checked);
 
     void openTabVisualDeckStorage();
     void openTabHome();
@@ -218,6 +221,7 @@ private slots:
     void openTabCardArtRules();
     void openTabLog();
     void openTabReport();
+    void openTabDeveloper();
 
     void updateCurrent(int index);
     void updatePingTime(int value, int max);

libcockatrice_network/libcockatrice/network/client/abstract/abstract_client.cpp

@@ -253,3 +253,11 @@ PendingCommand *AbstractClient::prepareAdminCommand(const ::google::protobuf::Me
     c->GetReflection()->MutableMessage(c, cmd.GetDescriptor()->FindExtensionByName("ext"))->CopyFrom(cmd);
     return new PendingCommand(cont);
 }
+
+PendingCommand *AbstractClient::prepareDeveloperCommand(const ::google::protobuf::Message &cmd)
+{
+    CommandContainer cont;
+    DeveloperCommand *c = cont.add_developer_command();
+    c->GetReflection()->MutableMessage(c, cmd.GetDescriptor()->FindExtensionByName("ext"))->CopyFrom(cmd);
+    return new PendingCommand(cont);
+}

libcockatrice_network/libcockatrice/network/client/abstract/abstract_client.h

@@ -173,6 +173,7 @@ public:
     static PendingCommand *prepareRoomCommand(const ::google::protobuf::Message &cmd, int roomId);
     static PendingCommand *prepareModeratorCommand(const ::google::protobuf::Message &cmd);
     static PendingCommand *prepareAdminCommand(const ::google::protobuf::Message &cmd);
+    static PendingCommand *prepareDeveloperCommand(const ::google::protobuf::Message &cmd);
 
     QMap<QString, bool> clientFeatures;
 };

libcockatrice_network/libcockatrice/network/server/remote/game/server_abstract_player.cpp

@@ -66,6 +66,15 @@ Server_AbstractPlayer::Server_AbstractPlayer(Server_Game *_game,
 
 Server_AbstractPlayer::~Server_AbstractPlayer() = default;
 
+int Server_AbstractPlayer::getCardCount() const
+{
+    int result = 0;
+    for (auto *zone : zones) {
+        result += zone->getCards().size();
+    }
+    return result;
+}
+
 void Server_AbstractPlayer::prepareDestroy()
 {
     delete deck;

libcockatrice_network/libcockatrice/network/server/remote/game/server_abstract_player.h

@@ -43,6 +43,8 @@ public:
                           Server_AbstractUserInterface *_handler);
     ~Server_AbstractPlayer() override;
     void prepareDestroy() override;
+    /// Total cards across all of this player's zones. The caller must hold the game's mutex.
+    int getCardCount() const;
     const DeckList *getDeckList() const
     {
         return deck;

libcockatrice_network/libcockatrice/network/server/remote/game/server_game.cpp

@@ -32,6 +32,7 @@
 #include "server_spectator.h"
 
 #include <QDebug>
+#include <QElapsedTimer>
 #include <QRegularExpression>
 #include <QTimer>
 #include <google/protobuf/descriptor.h>
@@ -238,6 +239,17 @@ int Server_Game::getPlayerCount() const
     return participants.size() - getSpectatorCount();
 }
 
+int Server_Game::getCardsInGame() const
+{
+    QMutexLocker locker(&gameMutex);
+
+    int result = 0;
+    for (auto *player : getPlayers()) {
+        result += player->getCardCount();
+    }
+    return result;
+}
+
 int Server_Game::getSpectatorCount() const
 {
     QMutexLocker locker(&gameMutex);
@@ -330,6 +342,9 @@ void Server_Game::doStartGameIfReady(bool forceStartGame)
         }
     }
 
+    // Only actual starts are timed. The early returns above are no-ops.
+    QElapsedTimer startupTimer;
+    startupTimer.start();
     players = getPlayers(); // players could have been kicked, get new list of players
     if (lifecycleStrategy->onGameStarting(this) == Server_GameLifecycleStrategy::StartAction::Handled) {
         locker.unlock();
@@ -373,6 +388,7 @@ void Server_Game::doStartGameIfReady(bool forceStartGame)
 
     activePlayer = -1;
     nextTurn();
+    room->getServer()->observeGameStartDurationMs(startupTimer.nsecsElapsed() / 1000000);
 
     locker.unlock();
 

libcockatrice_network/libcockatrice/network/server/remote/game/server_game.h

@@ -123,6 +123,8 @@ public:
         return gameStarted;
     }
     int getPlayerCount() const;
+    /// Total cards across all players' zones. Takes gameMutex itself.
+    int getCardsInGame() const;
     int getSpectatorCount() const;
     QMap<int, Server_AbstractPlayer *> getPlayers() const;
     Server_AbstractPlayer *getPlayer(int id) const;

libcockatrice_network/libcockatrice/network/server/remote/server.h

@@ -180,6 +180,11 @@ public:
     {
         return false;
     }
+    /// Called once per actual game start with how long bringing every player's
+    /// zones online took, so servers can spot deck sizes that wedge threads.
+    virtual void observeGameStartDurationMs(qint64 /* elapsedMs */)
+    {
+    }
 
     Server_DatabaseInterface *getDatabaseInterface() const;
     int getNextLocalGameId()

libcockatrice_network/libcockatrice/network/server/remote/server_protocolhandler.cpp

@@ -388,6 +388,33 @@ Response::ResponseCode Server_ProtocolHandler::processAdminCommandContainer(cons
     return finalResponseCode;
 }
 
+Response::ResponseCode Server_ProtocolHandler::processDeveloperCommandContainer(const CommandContainer &cont,
+                                                                                ResponseContainer &rc)
+{
+    if (!userInfo) {
+        return Response::RespLoginNeeded;
+    }
+    if (!(userInfo->user_level() & ServerInfo_User::IsDeveloper)) {
+        return Response::RespLoginNeeded;
+    }
+
+    resetIdleTimer();
+
+    Response::ResponseCode finalResponseCode = Response::RespOk;
+    for (int i = cont.developer_command_size() - 1; i >= 0; --i) {
+        Response::ResponseCode resp = Response::RespInvalidCommand;
+        const DeveloperCommand &sc = cont.developer_command(i);
+        const int num = getPbExtension(sc);
+        logDebugMessage(getSafeDebugString(sc));
+
+        resp = processExtendedDeveloperCommand(num, sc, rc);
+        if (resp != Response::RespOk) {
+            finalResponseCode = resp;
+        }
+    }
+    return finalResponseCode;
+}
+
 void Server_ProtocolHandler::processCommandContainer(const CommandContainer &cont)
 {
     // Command processing must be disabled after prepareDestroy() has been called.
@@ -410,6 +437,8 @@ void Server_ProtocolHandler::processCommandContainer(const CommandContainer &con
         finalResponseCode = processModeratorCommandContainer(cont, responseContainer);
     } else if (cont.admin_command_size()) {
         finalResponseCode = processAdminCommandContainer(cont, responseContainer);
+    } else if (cont.developer_command_size()) {
+        finalResponseCode = processDeveloperCommandContainer(cont, responseContainer);
     } else {
         finalResponseCode = Response::RespInvalidCommand;
     }
@@ -454,11 +483,12 @@ void Server_ProtocolHandler::pingClockTimeout()
         prepareDestroy();
     }
 
-    // PrivLevel users, Moderators, and Admins are not subject to the server idle timeout policy
+    // PrivLevel users, Moderators, Admins, and Developers are not subject to the server idle timeout policy
     const bool hasPrivLevel = userInfo && QString::fromStdString(userInfo->privlevel()).toLower() != "none";
-    const bool isModOrAdmin =
-        userInfo && (userInfo->user_level() & (ServerInfo_User::IsModerator | ServerInfo_User::IsAdmin));
-    if (!hasPrivLevel && !isModOrAdmin) {
+    const bool isStaff =
+        userInfo && (userInfo->user_level() &
+                     (ServerInfo_User::IsModerator | ServerInfo_User::IsAdmin | ServerInfo_User::IsDeveloper));
+    if (!hasPrivLevel && !isStaff) {
         if ((server->getIdleClientTimeout() > 0) && (idleClientWarningSent)) {
             if (timeRunning - lastActionReceived > server->getIdleClientTimeout()) {
                 prepareDestroy();

libcockatrice_network/libcockatrice/network/server/remote/server_protocolhandler.h

@@ -27,6 +27,7 @@ class CommandContainer;
 class SessionCommand;
 class ModeratorCommand;
 class AdminCommand;
+class DeveloperCommand;
 
 class Command_Ping;
 class Command_Login;
@@ -98,6 +99,12 @@ private:
     {
         return Response::RespFunctionNotAllowed;
     }
+    Response::ResponseCode processDeveloperCommandContainer(const CommandContainer &cont, ResponseContainer &rc);
+    virtual Response::ResponseCode
+    processExtendedDeveloperCommand(int /* cmdType */, const DeveloperCommand & /* cmd */, ResponseContainer & /* rc */)
+    {
+        return Response::RespFunctionNotAllowed;
+    }
 
     void resetIdleTimer();
 private slots:
@@ -129,7 +136,7 @@ public:
         return timeRunning - lastDataReceived;
     }
     bool addSaidMessageSize(int size);
-    void processCommandContainer(const CommandContainer &cont);
+    virtual void processCommandContainer(const CommandContainer &cont);
 
     void sendProtocolItem(const Response &item);
     void sendProtocolItem(const SessionEvent &item);

libcockatrice_protocol/libcockatrice/protocol/pb/CMakeLists.txt

@@ -22,6 +22,8 @@ set(PROTO_FILES
     command_del_counter.proto
     command_delete_arrow.proto
     command_draw_cards.proto
+    command_get_log_history.proto
+    command_get_server_stats.proto
     command_dump_zone.proto
     command_flip_card.proto
     command_game_say.proto
@@ -71,6 +73,7 @@ set(PROTO_FILES
     context_ready_start.proto
     context_set_sideboard_lock.proto
     context_undo_draw.proto
+    developer_commands.proto
     event_add_to_list.proto
     event_attach_card.proto
     event_change_zone_properties.proto
@@ -141,6 +144,7 @@ set(PROTO_FILES
     response_forgotpasswordrequest.proto
     response_get_admin_notes.proto
     response_get_games_of_user.proto
+    response_get_server_stats.proto
     response_get_user_info.proto
     response_join_room.proto
     response_list_users.proto

libcockatrice_protocol/libcockatrice/protocol/pb/admin_commands.proto

@@ -37,6 +37,7 @@ message Command_AdjustMod {
     required string user_name = 1;
     optional bool should_be_mod = 2;
     optional bool should_be_judge = 3;
+    optional bool should_be_developer = 4;
 }
 
 message Command_ResetUserPassword {

libcockatrice_protocol/libcockatrice/protocol/pb/command_get_log_history.proto

@@ -0,0 +1,19 @@
+syntax = "proto2";
+import "developer_commands.proto";
+
+// Developer counterpart of Command_ViewLogHistory: identical query fields, but
+// routed through the developer command family so developers never need the
+// moderator command container.
+message Command_GetLogHistory {
+    extend DeveloperCommand {
+        optional Command_GetLogHistory ext = 1001;
+    }
+    optional string user_name = 1;       // user that created message
+    optional string ip_address = 2;      // ip address of user that created message
+    optional string game_name = 3;       // client id of user that created the message
+    optional string game_id = 4;         // game number the message was sent to
+    optional string message = 5;         // raw message that was sent
+    repeated string log_location = 6;    // destination of message (ex: main room, game room, private chat)
+    required uint32 date_range = 7;      // the length of time (in minutes) to look back for
+    optional uint32 maximum_results = 8; // the maximum number of query results
+}

libcockatrice_protocol/libcockatrice/protocol/pb/command_get_server_stats.proto

@@ -0,0 +1,8 @@
+syntax = "proto2";
+import "developer_commands.proto";
+
+message Command_GetServerStats {
+    extend DeveloperCommand {
+        optional Command_GetServerStats ext = 1000;
+    }
+}

libcockatrice_protocol/libcockatrice/protocol/pb/commands.proto

@@ -4,6 +4,7 @@ import "game_commands.proto";
 import "room_commands.proto";
 import "moderator_commands.proto";
 import "admin_commands.proto";
+import "developer_commands.proto";
 
 message CommandContainer {
     optional uint64 cmd_id = 1;
@@ -16,4 +17,5 @@ message CommandContainer {
     repeated RoomCommand room_command = 102;
     repeated ModeratorCommand moderator_command = 103;
     repeated AdminCommand admin_command = 104;
+    repeated DeveloperCommand developer_command = 105;
 }

libcockatrice_protocol/libcockatrice/protocol/pb/developer_commands.proto

@@ -0,0 +1,8 @@
+syntax = "proto2";
+message DeveloperCommand {
+    enum DeveloperCommandType {
+        GET_SERVER_STATS = 1000;
+        VIEWLOG_HISTORY = 1001;
+    }
+    extensions 100 to max;
+}

libcockatrice_protocol/libcockatrice/protocol/pb/response.proto

@@ -77,6 +77,7 @@ message Response {
         FORGOT_PASSWORD_REQUEST = 1016; // Response to password reset request
         PASSWORD_SALT = 1017;           // Response containing password salt
         GET_ADMIN_NOTES = 1018;         // Response with admin notes
+        GET_SERVER_STATS = 1019;        // Response with server status statistics
         REPLAY_LIST = 1100;             // Response listing replays
         REPLAY_DOWNLOAD = 1101;         // Response for replay download
         REPLAY_GET_CODE = 1102;         // Response containing replay code

libcockatrice_protocol/libcockatrice/protocol/pb/response_get_server_stats.proto

@@ -0,0 +1,41 @@
+syntax = "proto2";
+import "response.proto";
+
+message CommandStats {
+    optional uint32 kind_index = 1;       // 0=session, 1=room, 2=game, 3=moderator, 4=admin
+    optional uint32 extension_number = 2; // protobuf extension number within the kind
+    optional string command_name = 3;     // e.g. "session/Command_Ping"
+    optional uint64 count = 4;            // number of times observed
+    optional uint64 total_ms = 5;         // cumulative processing milliseconds
+}
+
+message Response_GetServerStats {
+    extend Response {
+        optional Response_GetServerStats ext = 1220;
+    }
+
+    optional uint64 users_count = 1;
+    optional uint64 mods_count = 2;
+    optional uint64 games_count = 3;
+
+    // Traffic recorded during the last status update tick
+    optional uint64 tx_bytes = 4;
+    optional uint64 rx_bytes = 5;
+
+    optional uint64 uptime_secs = 6;
+    optional uint64 timest = 7; // unix timestamp of the snapshot
+
+    // Live metrics from MetricsRegistry (reset on server restart)
+    optional uint64 cards_in_games = 8;
+    optional uint64 eventloop_stalls_total = 9;
+    optional uint64 eventloop_last_stall_ms = 10;
+    optional uint64 eventloop_max_stall_ms = 11;
+    optional uint64 total_commands = 12;
+    optional uint64 total_command_time_ms = 13;
+    optional int32 active_command_types = 14;
+    optional uint64 game_start_count = 15;
+    optional uint64 game_start_total_ms = 16;
+
+    // Per-command breakdown (only types with count > 0)
+    repeated CommandStats command_stats = 20;
+}

libcockatrice_protocol/libcockatrice/protocol/pb/serverinfo_user.proto

@@ -8,6 +8,7 @@ message ServerInfo_User {
         IsModerator = 4;
         IsAdmin = 8;
         IsJudge = 16;
+        IsDeveloper = 32;
     };
     message PawnColorsOverride {
         optional string left_side = 1;

servatrice/CMakeLists.txt

@@ -6,7 +6,9 @@ project(Servatrice VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${
 
 set(servatrice_SOURCES
     src/email_parser.cpp
+    src/event_loop_watchdog.cpp
     src/main.cpp
+    src/metrics_registry.cpp
     src/servatrice.cpp
     src/servatrice_connection_pool.cpp
     src/servatrice_database_interface.cpp

servatrice/servatrice.ini.example

@@ -382,6 +382,17 @@ max_reports_per_day=10
 ; Maximum number of report comments a single user can post per hour; default is 30; set to 0 to disable the limit
 max_comments_per_hour=30
 
+[metrics]
+; Command containers that take longer than this many milliseconds are logged
+; as slow commands. Set to 0 to disable the log line.
+slow_command_ms=500
+
+; Each socket pool thread runs a watchdog heartbeat. If a heartbeat arrives
+; this many milliseconds late, the stall is logged and exposed as
+; servatrice_eventloop_* metrics in the Developer tab. Set to 0 to disable
+; the watchdogs.
+stall_warn_ms=2000
+
 [logging]
 ; Admin/Moderators can query the stored logs for information when looking up reports by various players. This
 ; option can allow or disallow them from doing so.

servatrice/src/event_loop_watchdog.cpp

@@ -0,0 +1,34 @@
+/**
+ * @file event_loop_watchdog.cpp
+ * @ingroup Servatrice
+ */
+
+#include "event_loop_watchdog.h"
+
+#include "servatrice.h"
+
+#include <QTimer>
+
+EventLoopWatchdog::EventLoopWatchdog(Servatrice *_servatrice, QString _threadName)
+    : QObject(nullptr), servatrice(_servatrice), threadName(std::move(_threadName))
+{
+}
+
+void EventLoopWatchdog::start()
+{
+    heartbeatTimer = new QTimer(this);
+    sinceLastTick.start();
+    connect(heartbeatTimer, &QTimer::timeout, this, &EventLoopWatchdog::checkHeartbeat);
+    heartbeatTimer->start(HeartbeatIntervalMs);
+}
+
+void EventLoopWatchdog::checkHeartbeat()
+{
+    const qint64 elapsedMs = sinceLastTick.restart();
+    const qint64 overshootMs = qMax<qint64>(0, elapsedMs - HeartbeatIntervalMs);
+    if (overshootMs < servatrice->getMetricsStallWarnMs()) {
+        return;
+    }
+
+    servatrice->observeEventLoopStall(threadName, overshootMs);
+}

servatrice/src/event_loop_watchdog.h

@@ -0,0 +1,50 @@
+/**
+ * @file event_loop_watchdog.h
+ * @ingroup Servatrice
+ */
+
+#ifndef EVENT_LOOP_WATCHDOG_H
+#define EVENT_LOOP_WATCHDOG_H
+
+#include <QElapsedTimer>
+#include <QObject>
+#include <QString>
+
+class Servatrice;
+class QTimer;
+
+/**
+ * @brief Detects blocked or overloaded worker event loops.
+ *
+ * One instance lives in each socket pool thread. A heartbeat timer tick that
+ * arrives late means the loop spent that time elsewhere: busy work, a queued
+ * slot, or a hard wedge. Overshoots past the configured threshold bump
+ * lock-free counters on the metrics registry and log one warning per stall,
+ * so a stuck pool thread becomes visible instead of silent lag.
+ */
+class EventLoopWatchdog : public QObject
+{
+    Q_OBJECT
+public:
+    /// How often the heartbeat expects to fire. Small enough to catch short stalls.
+    static constexpr int HeartbeatIntervalMs = 500;
+
+    EventLoopWatchdog(Servatrice *_servatrice, QString _threadName);
+
+    /**
+     * Starts the heartbeat timer. Must be invoked queued after the instance
+     * was moved to its target thread so the timer lives there too.
+     */
+    void start();
+
+private slots:
+    void checkHeartbeat();
+
+private:
+    Servatrice *servatrice;
+    QString threadName;
+    QElapsedTimer sinceLastTick;
+    QTimer *heartbeatTimer = nullptr;
+};
+
+#endif

servatrice/src/metrics_registry.cpp

@@ -0,0 +1,142 @@
+#include "metrics_registry.h"
+
+#include <QList>
+
+int MetricsRegistry::bucketIndexFor(qint64 elapsedMs)
+{
+    const int lastFiniteBucket = static_cast<int>(BucketBounds.size()) - 1;
+    int bucket = 0;
+    while (bucket < lastFiniteBucket && elapsedMs > BucketBounds[static_cast<size_t>(bucket)]) {
+        ++bucket;
+    }
+    return bucket;
+}
+
+void MetricsRegistry::appendCumulativeBuckets(QString &out,
+                                              const QString &bucketLine,
+                                              const std::array<std::atomic<qint64>, BucketCount> &buckets)
+{
+    // Cumulative buckets are required by the Prometheus histogram convention.
+    qint64 cumulative = 0;
+    for (int bucket = 0; bucket < static_cast<int>(BucketBounds.size()); ++bucket) {
+        cumulative += buckets[static_cast<size_t>(bucket)].load(std::memory_order_relaxed);
+        out += QStringLiteral("%1,le=\"%2\"} %3\n")
+                   .arg(bucketLine)
+                   .arg(BucketBounds[static_cast<size_t>(bucket)])
+                   .arg(cumulative);
+    }
+    cumulative += buckets[BucketCount - 1].load(std::memory_order_relaxed);
+    out += QStringLiteral("%1,le=\"+Inf\"} %2\n").arg(bucketLine).arg(cumulative);
+}
+
+void MetricsRegistry::observeCommand(int typeId, qint64 elapsedMs)
+{
+    if (typeId < 0 || typeId >= MaxTypes) {
+        typeId = MaxTypes - 1; // overflow slot keeps misrouted ids visible
+    }
+    if (elapsedMs < 0) {
+        elapsedMs = 0;
+    }
+
+    TypeStats &stats = slotFor(typeId);
+    stats.count.fetch_add(1, std::memory_order_relaxed);
+    stats.totalMs.fetch_add(elapsedMs, std::memory_order_relaxed);
+    totalCommandsCounter.fetch_add(1, std::memory_order_relaxed);
+    totalTimeCounter.fetch_add(elapsedMs, std::memory_order_relaxed);
+    stats.buckets[static_cast<size_t>(bucketIndexFor(elapsedMs))].fetch_add(1, std::memory_order_relaxed);
+}
+
+void MetricsRegistry::observeGameStartDurationMs(qint64 elapsedMs)
+{
+    if (elapsedMs < 0) {
+        elapsedMs = 0;
+    }
+
+    gameStartStats.count.fetch_add(1, std::memory_order_relaxed);
+    gameStartStats.totalMs.fetch_add(elapsedMs, std::memory_order_relaxed);
+    gameStartStats.buckets[static_cast<size_t>(bucketIndexFor(elapsedMs))].fetch_add(1, std::memory_order_relaxed);
+}
+
+MetricsRegistry::TypeStats &MetricsRegistry::slotFor(int typeId)
+{
+    return typeSlots[static_cast<size_t>(typeId)];
+}
+
+const MetricsRegistry::TypeStats &MetricsRegistry::slotFor(int typeId) const
+{
+    return typeSlots[static_cast<size_t>(typeId)];
+}
+
+int MetricsRegistry::activeTypeCount() const
+{
+    int active = 0;
+    for (int type = 0; type < MaxTypes; ++type) {
+        if (slotFor(type).count.load(std::memory_order_relaxed) > 0) {
+            ++active;
+        }
+    }
+    return active;
+}
+
+QString MetricsRegistry::toPrometheusText(const std::function<QString(int)> &nameForType,
+                                          const QHash<QString, qint64> &gauges) const
+{
+    QString out;
+    out.reserve(4096);
+
+    for (auto it = gauges.constBegin(); it != gauges.constEnd(); ++it) {
+        out += QStringLiteral("# TYPE %1 gauge\n").arg(it.key());
+        out += QStringLiteral("%1 %2\n").arg(it.key()).arg(it.value());
+    }
+
+    // Cumulative buckets are required by the Prometheus histogram convention.
+    out += QLatin1String("# TYPE servatrice_commands_duration_ms histogram\n");
+
+    for (int type = 0; type < MaxTypes; ++type) {
+        const TypeStats &stats = slotFor(type);
+        const qint64 count = stats.count.load(std::memory_order_relaxed);
+        if (count == 0) {
+            continue;
+        }
+
+        const QString label = nameForType ? nameForType(type) : QString::number(type);
+        appendCumulativeBuckets(out, QStringLiteral("servatrice_commands_duration_ms_bucket{command=\"%1\"").arg(label),
+                                stats.buckets);
+
+        out += QStringLiteral("servatrice_commands_duration_ms_sum{command=\"%1\"} %2\n")
+                   .arg(label)
+                   .arg(stats.totalMs.load(std::memory_order_relaxed));
+        out += QStringLiteral("servatrice_commands_duration_ms_count{command=\"%1\"} %2\n").arg(label).arg(count);
+    }
+
+    const qint64 startCount = gameStartStats.count.load(std::memory_order_relaxed);
+    if (startCount > 0) {
+        out += QLatin1String("# TYPE servatrice_game_start_duration_ms histogram\n");
+        appendCumulativeBuckets(out, QLatin1String("servatrice_game_start_duration_ms_bucket"), gameStartStats.buckets);
+        out += QStringLiteral("servatrice_game_start_duration_ms_sum %1\n")
+                   .arg(gameStartStats.totalMs.load(std::memory_order_relaxed));
+        out += QStringLiteral("servatrice_game_start_duration_ms_count %1\n").arg(startCount);
+    }
+
+    return out;
+}
+
+QList<MetricsRegistry::ActiveTypeStats> MetricsRegistry::collectActiveStats() const
+{
+    QList<ActiveTypeStats> result;
+    for (int type = 0; type < MaxTypes; ++type) {
+        const TypeStats &stats = slotFor(type);
+        const qint64 count = stats.count.load(std::memory_order_relaxed);
+        if (count == 0) {
+            continue;
+        }
+        result.append({type, count, stats.totalMs.load(std::memory_order_relaxed)});
+    }
+    return result;
+}
+
+MetricsRegistry::GameStartSnapshot MetricsRegistry::getGameStartSnapshot() const
+{
+    return {gameStartStats.count.load(std::memory_order_relaxed),
+            gameStartStats.totalMs.load(std::memory_order_relaxed)};
+}

servatrice/src/metrics_registry.h

@@ -0,0 +1,135 @@
+/**
+ * @file metrics_registry.h
+ * @ingroup Servatrice
+ */
+
+#ifndef METRICS_REGISTRY_H
+#define METRICS_REGISTRY_H
+
+#include <QHash>
+#include <QString>
+#include <array>
+#include <atomic>
+#include <functional>
+
+/**
+ * @brief Lock-free accumulation of command processing statistics.
+ *
+ * observeCommand() is called once per processed command from whichever socket
+ * thread handled it. It uses relaxed atomic adds on preallocated storage only,
+ * so it introduces no locks, allocations, or shared cache-line ping-pong
+ * beyond the unavoidable counter updates.
+ *
+ * Reading happens rarely (metrics scraping), accepts momentary tears between
+ * related counters, and therefore also needs no synchronization.
+ */
+class MetricsRegistry
+{
+public:
+    /**
+     * Extension numbers are only unique per command kind, so recorded ids
+     * combine the kind index with the protobuf extension number.
+     */
+    static constexpr int KindStride = 2048;
+
+    static constexpr int NumKinds = 5;
+
+    static constexpr const char *KindNames[NumKinds] = {"session", "room", "game", "moderator", "admin"};
+
+    /// Upper bound on distinct command type ids (see typeIdFor).
+    static constexpr int MaxTypes = NumKinds * KindStride;
+
+    /// Histogram bucket upper bounds in milliseconds. Anything above the last
+    /// bound lands in the trailing +Inf bucket. Constexpr so the recording
+    /// hot path never allocates.
+    static constexpr std::array<qint64, 11> BucketBounds{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000};
+
+    static int typeIdFor(int kindIndex, int extensionNumber)
+    {
+        return kindIndex * KindStride + extensionNumber;
+    }
+
+    void observeCommand(int typeId, qint64 elapsedMs);
+
+    /**
+     * Records how long one game start took to bring every player's zones
+     * online. Kept separate from command timings because it is triggered by
+     * the server itself and can dwarf any single command when decks are huge.
+     */
+    void observeGameStartDurationMs(qint64 elapsedMs);
+
+    /// Total number of observed commands across all types.
+    qint64 totalCommands() const
+    {
+        return totalCommandsCounter.load(std::memory_order_relaxed);
+    }
+
+    /// Cumulative processing milliseconds across all types.
+    qint64 totalTimeMs() const
+    {
+        return totalTimeCounter.load(std::memory_order_relaxed);
+    }
+
+    /// Number of distinct type slots that have seen at least one sample.
+    int activeTypeCount() const;
+
+    struct ActiveTypeStats
+    {
+        int typeId;
+        qint64 count;
+        qint64 totalMs;
+    };
+
+    /**
+     * Returns stats for every type slot that has seen at least one sample.
+     * The caller-provided @p labelForType maps a numeric type id to a stable
+     * human-readable label. Pass nullptr to skip label resolution.
+     */
+    QList<ActiveTypeStats> collectActiveStats() const;
+
+    struct GameStartSnapshot
+    {
+        qint64 count;
+        qint64 totalMs;
+    };
+
+    GameStartSnapshot getGameStartSnapshot() const;
+
+    /**
+     * @brief Renders all recorded data in Prometheus text exposition format.
+     *
+     * @param nameForType maps a numeric command type id to a stable label
+     * value. Ids without a mapping are rendered as their number.
+     * @param gauges simple name/value pairs emitted as gauge samples.
+     */
+    QString toPrometheusText(const std::function<QString(int)> &nameForType,
+                             const QHash<QString, qint64> &gauges) const;
+
+private:
+    static constexpr int BucketCount = static_cast<int>(BucketBounds.size()) + 1; ///< bounds + the +Inf bucket
+
+    struct TypeStats
+    {
+        std::atomic<qint64> count{0};
+        std::atomic<qint64> totalMs{0};
+        std::array<std::atomic<qint64>, BucketCount> buckets{};
+    };
+
+    TypeStats &slotFor(int typeId);
+    const TypeStats &slotFor(int typeId) const;
+
+    /// Index of the histogram bucket the sample falls into. The last index is +Inf.
+    static int bucketIndexFor(qint64 elapsedMs);
+
+    /// Appends one series of cumulative +Inf-terminated buckets to @p out.
+    static void appendCumulativeBuckets(QString &out,
+                                        const QString &bucketLine,
+                                        const std::array<std::atomic<qint64>, BucketCount> &buckets);
+
+    std::array<TypeStats, MaxTypes> typeSlots{};
+    TypeStats gameStartStats{};
+    std::atomic<qint64> totalCommandsCounter{0};
+    std::atomic<qint64> totalTimeCounter{0};
+};
+
+#endif

servatrice/src/servatrice.cpp

@@ -20,6 +20,7 @@
 #include "servatrice.h"
 
 #include "email_parser.h"
+#include "event_loop_watchdog.h"
 #include "isl_interface.h"
 #include "main.h"
 #include "servatrice_connection_pool.h"
@@ -38,6 +39,7 @@
 #include <QStringList>
 #include <QTimer>
 #include <QUrl>
+#include <game/server_game.h>
 #include <iostream>
 #include <libcockatrice/deck_list/deck_list.h>
 #include <libcockatrice/protocol/featureset.h>
@@ -63,6 +65,7 @@ Servatrice_GameServer::Servatrice_GameServer(Servatrice *_server,
         server->addDatabaseInterface(newThread, newDatabaseInterface);
 
         newThread->start();
+        server->watchWorkerThread(newThread);
         QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
                                   Q_ARG(QSqlDatabase, _sqlDatabase));
 
@@ -131,6 +134,7 @@ Servatrice_WebsocketGameServer::Servatrice_WebsocketGameServer(Servatrice *_serv
         server->addDatabaseInterface(newThread, newDatabaseInterface);
 
         newThread->start();
+        server->watchWorkerThread(newThread);
         QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
                                   Q_ARG(QSqlDatabase, _sqlDatabase));
 
@@ -470,9 +474,60 @@ bool Servatrice::initServer()
     }
 
     setRequiredFeatures(getRequiredFeatures());
+
+    // METRICS (always active. Slow-command logging and stall watchdogs are
+    // controlled by their respective thresholds below)
+    metricsSlowCommandMs = settingsCache->value("metrics/slow_command_ms", 500).toInt();
+    metricsStallWarnMs = qMax(0, settingsCache->value("metrics/stall_warn_ms", 2000).toInt());
+
     return true;
 }
 
+void Servatrice::observeGameStartDurationMs(qint64 elapsedMs)
+{
+    metricsRegistry.observeGameStartDurationMs(elapsedMs);
+}
+
+void Servatrice::observeEventLoopStall(const QString &threadName, qint64 overshootMs)
+{
+    eventLoopStallsTotal.fetch_add(1, std::memory_order_relaxed);
+    eventLoopLastStallMs.store(overshootMs, std::memory_order_relaxed);
+    qint64 prevMax = eventLoopMaxStallMs.load(std::memory_order_relaxed);
+    while (overshootMs > prevMax &&
+           !eventLoopMaxStallMs.compare_exchange_weak(prevMax, overshootMs, std::memory_order_relaxed)) {
+        // retry until the max is at least as high as the new sample
+    }
+
+    qWarning() << "Event loop stall in" << threadName << "- heartbeat overshot by" << overshootMs << "ms";
+}
+
+void Servatrice::watchWorkerThread(QThread *thread)
+{
+    if (metricsStallWarnMs <= 0) {
+        return; // watchdogs disabled via metrics/stall_warn_ms = 0
+    }
+
+    auto *watchdog = new EventLoopWatchdog(this, thread->objectName());
+    connect(thread, &QThread::finished, watchdog, &QObject::deleteLater);
+    watchdog->moveToThread(thread);
+    QMetaObject::invokeMethod(watchdog, &EventLoopWatchdog::start, Qt::QueuedConnection);
+}
+
+qint64 Servatrice::getCardsInGamesTotal() const
+{
+    qint64 total = 0;
+    QReadLocker roomsLocker(&roomsLock); // locking order: roomsLock before gamesLock/gameMutex
+    QMapIterator<int, Server_Room *> roomIterator(rooms);
+    while (roomIterator.hasNext()) {
+        Server_Room *room = roomIterator.next().value();
+        QReadLocker gamesLocker(&room->gamesLock);
+        for (auto *game : room->getGames()) {
+            total += game->getCardsInGame();
+        }
+    }
+    return total;
+}
+
 void Servatrice::addDatabaseInterface(QThread *thread, Servatrice_DatabaseInterface *databaseInterface)
 {
     databaseInterfaces.insert(thread, databaseInterface);
@@ -728,6 +783,7 @@ void Servatrice::incTxBytes(quint64 num)
     txBytesMutex.lock();
     txBytes += num;
     txBytesMutex.unlock();
+    txBytesTotal.fetch_add(num, std::memory_order_relaxed);
 }
 
 void Servatrice::incRxBytes(quint64 num)
@@ -735,6 +791,7 @@ void Servatrice::incRxBytes(quint64 num)
     rxBytesMutex.lock();
     rxBytes += num;
     rxBytesMutex.unlock();
+    rxBytesTotal.fetch_add(num, std::memory_order_relaxed);
 }
 
 void Servatrice::shutdownTimeout()

servatrice/src/servatrice.h

@@ -20,6 +20,8 @@
 #ifndef SERVATRICE_H
 #define SERVATRICE_H
 
+#include "metrics_registry.h"
+
 #include <QDateTime>
 #include <QHostAddress>
 #include <QMetaType>
@@ -30,6 +32,7 @@
 #include <QSslKey>
 #include <QTcpServer>
 #include <QWebSocketServer>
+#include <atomic>
 #include <libcockatrice/protocol/pb/response_report_stats.pb.h>
 #include <memory>
 #include <server.h>
@@ -170,6 +173,14 @@ private:
     int uptime;
     QMutex txBytesMutex, rxBytesMutex;
     quint64 txBytes, rxBytes;
+    std::atomic<quint64> txBytesTotal{0}; ///< cumulative bytes sent since process start
+    std::atomic<quint64> rxBytesTotal{0}; ///< cumulative bytes received since process start
+    MetricsRegistry metricsRegistry;
+    int metricsSlowCommandMs = 500;
+    int metricsStallWarnMs = 2000;
+    std::atomic<qint64> eventLoopStallsTotal{0}; ///< heartbeat overshoots past the warn threshold
+    std::atomic<qint64> eventLoopLastStallMs{0}; ///< overshoot of the most recent stall
+    std::atomic<qint64> eventLoopMaxStallMs{0};  ///< worst overshoot seen since process start
 
     QString shutdownReason;
     int shutdownMinutes;
@@ -286,6 +297,58 @@ public:
     void incRxBytes(quint64 num);
     void addDatabaseInterface(QThread *thread, Servatrice_DatabaseInterface *databaseInterface);
 
+    // Metrics (see [metrics] section in servatrice.ini.example)
+    MetricsRegistry &getMetricsRegistry()
+    {
+        return metricsRegistry;
+    }
+    quint64 getTxBytesTotal() const
+    {
+        return txBytesTotal.load(std::memory_order_relaxed);
+    }
+    quint64 getRxBytesTotal() const
+    {
+        return rxBytesTotal.load(std::memory_order_relaxed);
+    }
+    int getUptimeSeconds() const
+    {
+        return uptime;
+    }
+    /**
+     * Sums cards across all zones of all running games. Locks rooms and games
+     * briefly per level, so scrape-time cost grows with live game count only.
+     */
+    qint64 getCardsInGamesTotal() const;
+    int getMetricsSlowCommandMs() const
+    {
+        return metricsSlowCommandMs;
+    }
+    /// Heartbeat overshoot that counts as a stall. A value of 0 disables the watchdogs.
+    int getMetricsStallWarnMs() const
+    {
+        return metricsStallWarnMs;
+    }
+    void observeGameStartDurationMs(qint64 elapsedMs) override;
+    qint64 getEventLoopStallsTotal() const
+    {
+        return eventLoopStallsTotal.load(std::memory_order_relaxed);
+    }
+    qint64 getEventLoopLastStallMs() const
+    {
+        return eventLoopLastStallMs.load(std::memory_order_relaxed);
+    }
+    qint64 getEventLoopMaxStallMs() const
+    {
+        return eventLoopMaxStallMs.load(std::memory_order_relaxed);
+    }
+    /// Records one heartbeat overshoot and logs a single warning for it.
+    void observeEventLoopStall(const QString &threadName, qint64 overshootMs);
+    /**
+     * Installs an EventLoopWatchdog in @p thread. Called once per socket pool
+     * thread right after it starts.
+     */
+    void watchWorkerThread(QThread *thread);
+
     bool islConnectionExists(int _serverId) const;
     void addIslInterface(int _serverId, IslInterface *interface);
     void removeIslInterface(int _serverId);

servatrice/src/servatrice_database_interface.cpp

@@ -641,6 +641,10 @@ ServerInfo_User Servatrice_DatabaseInterface::evalUserQueryResult(const QSqlQuer
         userLevel |= ServerInfo_User::IsJudge;
     }
 
+    if (is_admin & 8) {
+        userLevel |= ServerInfo_User::IsDeveloper;
+    }
+
     result.set_user_level(userLevel);
 
     const QString country = query->value(3).toString();
@@ -1448,7 +1452,7 @@ QList<ServerInfo_ModeratorLogin> Servatrice_DatabaseInterface::getModeratorLastL
     QSqlQuery *query = prepareQuery("SELECT u.name, u.admin, UNIX_TIMESTAMP(a.last_login) "
                                     "FROM {prefix}_users u "
                                     "LEFT JOIN {prefix}_user_analytics a ON a.id = u.id "
-                                    "WHERE (u.admin & 7) <> 0 ORDER BY u.name");
+                                    "WHERE (u.admin & 15) <> 0 ORDER BY u.name");
 
     if (!execSqlQuery(query)) {
         qCWarning(DatabaseInterfaceLog) << "Failed to collect moderator login information: SQL Error";
@@ -1469,6 +1473,9 @@ QList<ServerInfo_ModeratorLogin> Servatrice_DatabaseInterface::getModeratorLastL
         if (isAdmin & 4) {
             userLevel |= ServerInfo_User::IsJudge;
         }
+        if (isAdmin & 8) {
+            userLevel |= ServerInfo_User::IsDeveloper;
+        }
         loginDetails.set_user_level(userLevel);
 
         if (!query->value(2).isNull()) {

servatrice/src/serversocketinterface.cpp

@@ -30,6 +30,7 @@
 
 #include <QDateTime>
 #include <QDebug>
+#include <QElapsedTimer>
 #include <QHostAddress>
 #include <QJsonDocument>
 #include <QJsonObject>
@@ -39,14 +40,18 @@
 #include <QSqlQuery>
 #include <QString>
 #include <game/server_player.h>
+#include <google/protobuf/descriptor.h>
 #include <iostream>
 #include <libcockatrice/deck_list/deck_list.h>
+#include <libcockatrice/protocol/get_pb_extension.h>
 #include <libcockatrice/protocol/pb/command_deck_del.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_del_dir.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_download.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_list.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_new_dir.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_upload.pb.h>
+#include <libcockatrice/protocol/pb/command_get_log_history.pb.h>
+#include <libcockatrice/protocol/pb/command_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_delete_match.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download_by_game_id.pb.h>
@@ -80,6 +85,7 @@
 #include <libcockatrice/protocol/pb/response_deck_upload.pb.h>
 #include <libcockatrice/protocol/pb/response_forgotpasswordrequest.pb.h>
 #include <libcockatrice/protocol/pb/response_get_admin_notes.pb.h>
+#include <libcockatrice/protocol/pb/response_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/response_moderator_last_logins.pb.h>
 #include <libcockatrice/protocol/pb/response_password_salt.pb.h>
 #include <libcockatrice/protocol/pb/response_register.pb.h>
@@ -190,6 +196,41 @@ void AbstractServerSocketInterface::logDebugMessage(const QString &message)
     logger->logMessage(message, this);
 }
 
+void AbstractServerSocketInterface::processCommandContainer(const CommandContainer &cont)
+{
+    QElapsedTimer timer;
+    timer.start();
+    Server_ProtocolHandler::processCommandContainer(cont);
+    const qint64 elapsedMs = timer.nsecsElapsed() / 1000000;
+
+    // A container usually holds a single command. When several are batched,
+    // each is attributed the container's total processing time.
+    for (const auto &cmd : cont.session_command()) {
+        servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(0, getPbExtension(cmd)), elapsedMs);
+    }
+    for (const auto &cmd : cont.room_command()) {
+        servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(1, getPbExtension(cmd)), elapsedMs);
+    }
+    for (const auto &cmd : cont.game_command()) {
+        servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(2, getPbExtension(cmd)), elapsedMs);
+    }
+    for (const auto &cmd : cont.moderator_command()) {
+        servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(3, getPbExtension(cmd)), elapsedMs);
+    }
+    for (const auto &cmd : cont.admin_command()) {
+        servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(4, getPbExtension(cmd)), elapsedMs);
+    }
+
+    const int slowCommandMs = servatrice->getMetricsSlowCommandMs();
+    if (slowCommandMs > 0 && elapsedMs >= slowCommandMs) {
+        const ServerInfo_User *info = getUserInfo();
+        const QString user = authState == PasswordRight && info ? QString::fromStdString(info->name())
+                                                                : QStringLiteral("unauthenticated");
+        qCWarning(AbstractServerSocketInterfaceLog) << "slow command container from" << user << "processed in"
+                                                    << elapsedMs << "ms (" << cont.ByteSizeLong() << "bytes)";
+    }
+}
+
 Response::ResponseCode AbstractServerSocketInterface::processExtendedSessionCommand(int cmdType,
                                                                                     const SessionCommand &cmd,
                                                                                     ResponseContainer &rc)
@@ -337,6 +378,38 @@ AbstractServerSocketInterface::processExtendedAdminCommand(int cmdType, const Ad
     }
 }
 
+// DEVELOPER FUNCTIONS.
+// Permission is checked by processDeveloperCommandContainer. Only stats-style
+// queries live here, never community moderation or server administration.
+Response::ResponseCode AbstractServerSocketInterface::processExtendedDeveloperCommand(int cmdType,
+                                                                                      const DeveloperCommand &cmd,
+                                                                                      ResponseContainer &rc)
+{
+    switch ((DeveloperCommand::DeveloperCommandType)cmdType) {
+        case DeveloperCommand::GET_SERVER_STATS:
+            return cmdGetServerStats(cmd.GetExtension(Command_GetServerStats::ext), rc);
+        case DeveloperCommand::VIEWLOG_HISTORY: {
+            // Same query as the moderator log view, just carried by the
+            // developer command family.
+            const Command_GetLogHistory &devCmd = cmd.GetExtension(Command_GetLogHistory::ext);
+            Command_ViewLogHistory modCmd;
+            modCmd.set_user_name(devCmd.user_name());
+            modCmd.set_ip_address(devCmd.ip_address());
+            modCmd.set_game_name(devCmd.game_name());
+            modCmd.set_game_id(devCmd.game_id());
+            modCmd.set_message(devCmd.message());
+            for (int i = 0; i < devCmd.log_location_size(); ++i) {
+                modCmd.add_log_location(devCmd.log_location(i));
+            }
+            modCmd.set_date_range(devCmd.date_range());
+            modCmd.set_maximum_results(devCmd.maximum_results());
+            return cmdGetLogHistory(modCmd, rc);
+        }
+        default:
+            return Response::RespFunctionNotAllowed;
+    }
+}
+
 Response::ResponseCode AbstractServerSocketInterface::cmdAddToList(const Command_AddToList &cmd, ResponseContainer &rc)
 {
     if (authState != PasswordRight) {
@@ -1643,6 +1716,83 @@ Response::ResponseCode AbstractServerSocketInterface::cmdReportUserInfo(const Co
     return Response::RespOk;
 }
 
+Response::ResponseCode AbstractServerSocketInterface::cmdGetServerStats(const Command_GetServerStats & /*cmd */,
+                                                                        ResponseContainer &rc)
+{
+    if (!sqlInterface->checkSql()) {
+        return Response::RespInternalError;
+    }
+
+    // Servatrice::statusUpdate() periodically snapshots server health into the
+    // uptime table. Serve the freshest snapshot for this server.
+    QSqlQuery *query = sqlInterface->prepareQuery(
+        "SELECT users_count, mods_count, games_count, tx_bytes, rx_bytes, uptime, UNIX_TIMESTAMP(timest) "
+        "FROM {prefix}_uptime WHERE id_server = :id_server ORDER BY timest DESC LIMIT 1");
+    query->bindValue(":id_server", servatrice->getServerID());
+    if (!sqlInterface->execSqlQuery(query)) {
+        return Response::RespInternalError;
+    }
+
+    auto *re = new Response_GetServerStats;
+    if (query->next()) {
+        re->set_users_count(query->value(0).toUInt());
+        re->set_mods_count(query->value(1).toUInt());
+        re->set_games_count(query->value(2).toUInt());
+        re->set_tx_bytes(query->value(3).toUInt());
+        re->set_rx_bytes(query->value(4).toUInt());
+        re->set_uptime_secs(query->value(5).toUInt());
+        re->set_timest(query->value(6).toUInt());
+    }
+
+    // Live metrics from the in-process MetricsRegistry (resets on server restart)
+    re->set_cards_in_games(static_cast<google::protobuf::uint64>(servatrice->getCardsInGamesTotal()));
+    re->set_eventloop_stalls_total(static_cast<google::protobuf::uint64>(servatrice->getEventLoopStallsTotal()));
+    re->set_eventloop_last_stall_ms(static_cast<google::protobuf::uint64>(servatrice->getEventLoopLastStallMs()));
+    re->set_eventloop_max_stall_ms(static_cast<google::protobuf::uint64>(servatrice->getEventLoopMaxStallMs()));
+    re->set_total_commands(static_cast<google::protobuf::uint64>(servatrice->getMetricsRegistry().totalCommands()));
+    re->set_total_command_time_ms(
+        static_cast<google::protobuf::uint64>(servatrice->getMetricsRegistry().totalTimeMs()));
+    re->set_active_command_types(servatrice->getMetricsRegistry().activeTypeCount());
+
+    const auto gameStart = servatrice->getMetricsRegistry().getGameStartSnapshot();
+    re->set_game_start_count(static_cast<google::protobuf::uint64>(gameStart.count));
+    re->set_game_start_total_ms(static_cast<google::protobuf::uint64>(gameStart.totalMs));
+
+    // Per-command breakdown: resolve protobuf extension names via the descriptor pool
+    static const char *messageNames[] = {"SessionCommand", "RoomCommand", "GameCommand", "ModeratorCommand",
+                                         "AdminCommand"};
+    const auto activeStats = servatrice->getMetricsRegistry().collectActiveStats();
+    for (const auto &stat : activeStats) {
+        const int kind = stat.typeId / MetricsRegistry::KindStride;
+        const int number = stat.typeId % MetricsRegistry::KindStride;
+
+        QString label;
+        if (kind >= 0 && kind < MetricsRegistry::NumKinds) {
+            const google::protobuf::DescriptorPool *pool = google::protobuf::DescriptorPool::generated_pool();
+            const google::protobuf::Descriptor *message = pool->FindMessageTypeByName(messageNames[kind]);
+            const google::protobuf::FieldDescriptor *extension =
+                message ? pool->FindExtensionByNumber(message, number) : nullptr;
+            if (extension) {
+                label = QString::fromLatin1(MetricsRegistry::KindNames[kind]) + QStringLiteral("/") +
+                        QString::fromStdString(std::string(extension->message_type()->name()));
+            }
+        }
+        if (label.isEmpty()) {
+            label = QString::number(stat.typeId);
+        }
+
+        CommandStats *cs = re->add_command_stats();
+        cs->set_kind_index(static_cast<google::protobuf::uint32>(kind));
+        cs->set_extension_number(static_cast<google::protobuf::uint32>(number));
+        cs->set_command_name(label.toStdString());
+        cs->set_count(static_cast<google::protobuf::uint64>(stat.count));
+        cs->set_total_ms(static_cast<google::protobuf::uint64>(stat.totalMs));
+    }
+
+    rc.setResponseExtension(re);
+    return Response::RespOk;
+}
+
 Response::ResponseCode AbstractServerSocketInterface::cmdReportStats(const Command_ReportStats & /*cmd */,
                                                                      ResponseContainer &rc)
 {
@@ -3215,7 +3365,7 @@ bool AbstractServerSocketInterface::removeAdminFlagFromUser(const QString &userN
     if (user) {
         Event_ConnectionClosed event;
         event.set_reason(Event_ConnectionClosed::DEMOTED);
-        event.set_reason_str("Your moderator and/or judge status has been revoked.");
+        event.set_reason_str("Your moderator, judge, and/or developer status has been revoked.");
         event.set_end_time(QDateTime::currentDateTime().toSecsSinceEpoch());
 
         SessionEvent *se = user->prepareSessionEvent(event);
@@ -3257,6 +3407,18 @@ Response::ResponseCode AbstractServerSocketInterface::cmdAdjustMod(const Command
         }
     }
 
+    if (cmd.has_should_be_developer()) {
+        if (cmd.should_be_developer()) {
+            if (!addAdminFlagToUser(userName, 8)) {
+                return Response::RespInternalError;
+            }
+        } else {
+            if (!removeAdminFlagFromUser(userName, 8)) {
+                return Response::RespInternalError;
+            }
+        }
+    }
+
     return Response::RespOk;
 }
 

servatrice/src/serversocketinterface.h

@@ -24,6 +24,8 @@
 #include <QMutex>
 #include <QTcpSocket>
 #include <QWebSocket>
+#include <libcockatrice/protocol/pb/command_get_log_history.pb.h>
+#include <libcockatrice/protocol/pb/command_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download_by_game_id.pb.h>
 #include <libcockatrice/protocol/pb/command_report.pb.h>
 #include <libcockatrice/protocol/pb/command_report_add_comment.pb.h>
@@ -80,6 +82,7 @@ signals:
 protected:
     void logDebugMessage(const QString &message) override;
     bool tooManyRegistrationAttempts(const QString &ipAddress);
+    void processCommandContainer(const CommandContainer &cont) override;
 
     virtual void writeToSocket(QByteArray &data) = 0;
     virtual void flushSocket() = 0;
@@ -151,6 +154,8 @@ private:
     processExtendedModeratorCommand(int cmdType, const ModeratorCommand &cmd, ResponseContainer &rc) override;
     Response::ResponseCode
     processExtendedAdminCommand(int cmdType, const AdminCommand &cmd, ResponseContainer &rc) override;
+    Response::ResponseCode
+    processExtendedDeveloperCommand(int cmdType, const DeveloperCommand &cmd, ResponseContainer &rc) override;
 
     Response::ResponseCode cmdAccountEdit(const Command_AccountEdit &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdAccountImage(const Command_AccountImage &cmd, ResponseContainer &rc);
@@ -172,6 +177,8 @@ private:
     Response::ResponseCode cmdResetUserPassword(const Command_ResetUserPassword &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdRemoveUserAvatar(const Command_RemoveUserAvatar &cmd, ResponseContainer &rc);
 
+    Response::ResponseCode cmdGetServerStats(const Command_GetServerStats &cmd, ResponseContainer &rc);
+
     bool addAdminFlagToUser(const QString &user, int flag);
     bool removeAdminFlagFromUser(const QString &user, int flag);
 

tests/CMakeLists.txt

@@ -14,6 +14,7 @@ add_test(NAME server_rate_limiter_test COMMAND server_rate_limiter_test)
 add_test(NAME warning_categories_test COMMAND warning_categories_test)
 add_test(NAME lag_monitor_test COMMAND lag_monitor_test)
 add_test(NAME latency_tracker_test COMMAND latency_tracker_test)
+add_test(NAME metrics_registry_test COMMAND metrics_registry_test)
 
 add_test(NAME deck_hash_performance_test COMMAND deck_hash_performance_test)
 set_tests_properties(deck_hash_performance_test PROPERTIES TIMEOUT 15)
@@ -34,6 +35,7 @@ add_executable(warning_categories_test warning_categories_test.cpp)
 add_executable(lag_monitor_test ${CMAKE_SOURCE_DIR}/cockatrice/src/client/lag_monitor.cpp lag_monitor_test.cpp)
 target_include_directories(lag_monitor_test PRIVATE ${CMAKE_SOURCE_DIR}/cockatrice/src)
 add_executable(latency_tracker_test latency_tracker_test.cpp)
+add_executable(metrics_registry_test ../servatrice/src/metrics_registry.cpp metrics_registry_test.cpp)
 
 find_package(GTest)
 
@@ -73,6 +75,7 @@ if(NOT GTEST_FOUND)
   add_dependencies(warning_categories_test gtest)
   add_dependencies(lag_monitor_test gtest)
   add_dependencies(latency_tracker_test gtest)
+  add_dependencies(metrics_registry_test gtest)
 endif()
 
 include_directories(${GTEST_INCLUDE_DIRS})
@@ -111,6 +114,8 @@ target_link_libraries(lag_monitor_test Threads::Threads ${GTEST_BOTH_LIBRARIES}
 target_link_libraries(
   latency_tracker_test libcockatrice_network Threads::Threads ${GTEST_BOTH_LIBRARIES} ${TEST_QT_MODULES}
 )
+target_include_directories(metrics_registry_test PRIVATE ${CMAKE_SOURCE_DIR}/servatrice/src)
+target_link_libraries(metrics_registry_test ${TEST_QT_MODULES} Threads::Threads ${GTEST_BOTH_LIBRARIES})
 
 add_subdirectory(card_zone_algorithms)
 add_subdirectory(carddatabase)

tests/metrics_registry_test.cpp

@@ -0,0 +1,131 @@
+#include <QRegularExpression>
+#include <gtest/gtest.h>
+#include <metrics_registry.h>
+
+TEST(MetricsRegistryTest, EmptyRegistryProducesNoHistogramLines)
+{
+    MetricsRegistry registry;
+
+    EXPECT_EQ(0, registry.totalCommands());
+    EXPECT_EQ(0, registry.totalTimeMs());
+    EXPECT_EQ(0, registry.activeTypeCount());
+
+    const QString text = registry.toPrometheusText([](int) { return QString("x"); }, {});
+    // The family TYPE declaration may stand alone. What must not exist is a
+    // histogram sample without data behind it.
+    EXPECT_FALSE(text.contains(QRegularExpression("servatrice_commands_duration_ms_(bucket|sum|count)")));
+}
+
+TEST(MetricsRegistryTest, SingleSampleIsRecordedInTotalsAndBuckets)
+{
+    MetricsRegistry registry;
+    registry.observeCommand(MetricsRegistry::typeIdFor(0, 1000), 7);
+
+    EXPECT_EQ(1, registry.totalCommands());
+    EXPECT_EQ(7, registry.totalTimeMs());
+    EXPECT_EQ(1, registry.activeTypeCount());
+
+    const QString text = registry.toPrometheusText(
+        [](int typeId) {
+            return QString("%1/%2").arg(typeId / MetricsRegistry::KindStride).arg(typeId % MetricsRegistry::KindStride);
+        },
+        {});
+    // 7ms falls into the le="10" bucket. Smaller buckets stay empty
+    EXPECT_TRUE(text.contains("# TYPE servatrice_commands_duration_ms histogram\n"));
+    EXPECT_TRUE(text.contains(",le=\"10\"} 1"));
+    EXPECT_TRUE(text.contains(",le=\"5\"} 0"));
+    EXPECT_TRUE(text.contains("_sum{command=\"0/1000\"} 7"));
+    EXPECT_TRUE(text.contains("_count{command=\"0/1000\"} 1"));
+}
+
+TEST(MetricsRegistryTest, BucketsAreCumulative)
+{
+    MetricsRegistry registry;
+    registry.observeCommand(MetricsRegistry::typeIdFor(0, 1000), 2);
+    registry.observeCommand(MetricsRegistry::typeIdFor(0, 1000), 30);
+
+    const QString text = registry.toPrometheusText([](int) { return QString("cmd"); }, {});
+
+    // cumulative counts: <=25 -> 1 sample, <=50 -> 2 samples
+    EXPECT_TRUE(text.contains(",le=\"25\"} 1\n"));
+    EXPECT_TRUE(text.contains(",le=\"50\"} 2\n"));
+    EXPECT_TRUE(text.contains(",le=\"+Inf\"} 2\n"));
+}
+
+TEST(MetricsRegistryTest, KindEncodingSeparatesSameExtensionNumber)
+{
+    MetricsRegistry registry;
+    const int sessionPing = MetricsRegistry::typeIdFor(0, 1000);
+    const int roomLeaveRoom = MetricsRegistry::typeIdFor(1, 1000);
+    ASSERT_NE(sessionPing, roomLeaveRoom);
+
+    registry.observeCommand(sessionPing, 1);
+    registry.observeCommand(roomLeaveRoom, 5000);
+
+    EXPECT_EQ(2, registry.activeTypeCount());
+}
+
+TEST(MetricsRegistryTest, OutOfRangeIdsLandInOverflowSlot)
+{
+    MetricsRegistry registry;
+    registry.observeCommand(-1, 4);
+    registry.observeCommand(MetricsRegistry::MaxTypes + 12345, 4);
+
+    EXPECT_EQ(2, registry.totalCommands());
+    EXPECT_EQ(1, registry.activeTypeCount()); // both collapsed into one slot
+    EXPECT_EQ(8, registry.totalTimeMs());
+}
+
+TEST(MetricsRegistryTest, NegativeDurationsAreClamped)
+{
+    MetricsRegistry registry;
+    registry.observeCommand(MetricsRegistry::typeIdFor(0, 1000), -50);
+
+    EXPECT_EQ(0, registry.totalTimeMs());
+}
+
+TEST(MetricsRegistryTest, GaugesAndLabelEscapingAreRendered)
+{
+    MetricsRegistry registry;
+
+    QHash<QString, qint64> gauges;
+    gauges.insert("servatrice_users_current", 42);
+
+    const QString text = registry.toPrometheusText(nullptr, gauges);
+    EXPECT_TRUE(text.contains("# TYPE servatrice_users_current gauge\n"));
+    EXPECT_TRUE(text.contains("servatrice_users_current 42\n"));
+}
+
+TEST(MetricsRegistryTest, UnnamedTypesFallBackToNumericLabel)
+{
+    MetricsRegistry registry;
+    registry.observeCommand(MetricsRegistry::typeIdFor(2, 1042), 9);
+
+    const QString text = registry.toPrometheusText(nullptr, {});
+    EXPECT_TRUE(text.contains("{command=\"" + QString::number(MetricsRegistry::typeIdFor(2, 1042)) + "\"}"));
+}
+
+TEST(MetricsRegistryTest, GameStartHistogramOnlyAppearsAfterSamples)
+{
+    MetricsRegistry registry;
+    EXPECT_FALSE(registry.toPrometheusText(nullptr, {}).contains("servatrice_game_start_duration_ms"));
+
+    registry.observeGameStartDurationMs(120);
+    const QString text = registry.toPrometheusText(nullptr, {});
+    // 120ms falls into the le="250" bucket
+    EXPECT_TRUE(text.contains("# TYPE servatrice_game_start_duration_ms histogram\n"));
+    EXPECT_TRUE(text.contains(",le=\"100\"} 0\n"));
+    EXPECT_TRUE(text.contains(",le=\"250\"} 1\n"));
+    EXPECT_TRUE(text.contains("servatrice_game_start_duration_ms_sum 120\n"));
+    EXPECT_TRUE(text.contains("servatrice_game_start_duration_ms_count 1\n"));
+}
+
+TEST(MetricsRegistryTest, GameStartHistogramIsSeparateFromCommandTotals)
+{
+    MetricsRegistry registry;
+    registry.observeGameStartDurationMs(10);
+
+    EXPECT_EQ(0, registry.totalCommands());
+    EXPECT_EQ(0, registry.totalTimeMs());
+    EXPECT_EQ(0, registry.activeTypeCount());
+}