Merge branch 'master' into tooomm-patch-33

servatrice/CMakeLists.txt

@@ -6,7 +6,9 @@ project(Servatrice VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${
 
 set(servatrice_SOURCES
     src/email_parser.cpp
+    src/event_loop_watchdog.cpp
     src/main.cpp
+    src/metrics_registry.cpp
     src/servatrice.cpp
     src/servatrice_connection_pool.cpp
     src/servatrice_database_interface.cpp
@@ -95,6 +97,8 @@ set(DESKTOPDIR
 # Build servatrice binary and link it
 add_executable(servatrice MACOSX_BUNDLE ${servatrice_MOC_SRCS} ${servatrice_RESOURCES_RCC} ${servatrice_SOURCES})
 
+target_precompile_headers(servatrice PRIVATE "${CMAKE_SOURCE_DIR}/cmake/pch/qtcore_pch.h")
+
 if(CMAKE_HOST_SYSTEM MATCHES "FreeBSD")
   target_link_libraries(
     servatrice libcockatrice_deck_list libcockatrice_network_server_remote Threads::Threads ${SERVATRICE_QT_MODULES}

servatrice/servatrice.ini.example

@@ -382,6 +382,19 @@ max_reports_per_day=10
 ; Maximum number of report comments a single user can post per hour; default is 30; set to 0 to disable the limit
 max_comments_per_hour=30
 
+[metrics]
+; Command containers that take longer than this many milliseconds are logged
+; as slow commands. Set to 0 to disable the log line. A latency spike produces
+; one warning per slow container with no rate limiting of its own -- a bad
+; patch can briefly flood the log, which is how you notice it.
+slow_command_ms=500
+
+; Each socket pool thread runs a watchdog heartbeat. If a heartbeat arrives
+; this many milliseconds late, the stall is logged and exposed as
+; servatrice_eventloop_* metrics in the Developer tab. Set to 0 to disable
+; the watchdogs.
+stall_warn_ms=2000
+
 [logging]
 ; Admin/Moderators can query the stored logs for information when looking up reports by various players. This
 ; option can allow or disallow them from doing so.

servatrice/servatrice.sql

@@ -25,6 +25,9 @@ INSERT INTO cockatrice_schema_version VALUES(36);
 -- users and user data tables
 CREATE TABLE IF NOT EXISTS `cockatrice_users` (
   `id` int(7) unsigned zerofill NOT NULL auto_increment,
+  -- Bitfield of staff levels: 1 = admin (implies moderator), 2 = moderator,
+  -- 4 = judge, 8 = developer. Operators set these by hand with
+  -- "UPDATE cockatrice_users SET admin = ...".
   `admin` tinyint(1) NOT NULL,
   `name` varchar(35) NOT NULL,
   `realname` varchar(255) NOT NULL,

servatrice/src/event_loop_watchdog.cpp

@@ -0,0 +1,34 @@
+/**
+ * @file event_loop_watchdog.cpp
+ * @ingroup Servatrice
+ */
+
+#include "event_loop_watchdog.h"
+
+#include "servatrice.h"
+
+#include <QTimer>
+
+EventLoopWatchdog::EventLoopWatchdog(Servatrice *_servatrice, QString _threadName)
+    : QObject(nullptr), servatrice(_servatrice), threadName(std::move(_threadName))
+{
+}
+
+void EventLoopWatchdog::start()
+{
+    heartbeatTimer = new QTimer(this);
+    sinceLastTick.start();
+    connect(heartbeatTimer, &QTimer::timeout, this, &EventLoopWatchdog::checkHeartbeat);
+    heartbeatTimer->start(HeartbeatIntervalMs);
+}
+
+void EventLoopWatchdog::checkHeartbeat()
+{
+    const qint64 elapsedMs = sinceLastTick.restart();
+    const qint64 overshootMs = qMax<qint64>(0, elapsedMs - HeartbeatIntervalMs);
+    if (overshootMs < servatrice->getMetricsStallWarnMs()) {
+        return;
+    }
+
+    servatrice->observeEventLoopStall(threadName, overshootMs);
+}

servatrice/src/event_loop_watchdog.h

@@ -0,0 +1,50 @@
+/**
+ * @file event_loop_watchdog.h
+ * @ingroup Servatrice
+ */
+
+#ifndef EVENT_LOOP_WATCHDOG_H
+#define EVENT_LOOP_WATCHDOG_H
+
+#include <QElapsedTimer>
+#include <QObject>
+#include <QString>
+
+class Servatrice;
+class QTimer;
+
+/**
+ * @brief Detects blocked or overloaded worker event loops.
+ *
+ * One instance lives in each socket pool thread. A heartbeat timer tick that
+ * arrives late means the loop spent that time elsewhere: busy work, a queued
+ * slot, or a hard wedge. Overshoots past the configured threshold bump
+ * lock-free counters on the metrics registry and log one warning per stall,
+ * so a stuck pool thread becomes visible instead of silent lag.
+ */
+class EventLoopWatchdog : public QObject
+{
+    Q_OBJECT
+public:
+    /// How often the heartbeat expects to fire. Small enough to catch short stalls.
+    static constexpr int HeartbeatIntervalMs = 500;
+
+    EventLoopWatchdog(Servatrice *_servatrice, QString _threadName);
+
+    /**
+     * Starts the heartbeat timer. Must be invoked queued after the instance
+     * was moved to its target thread so the timer lives there too.
+     */
+    void start();
+
+private slots:
+    void checkHeartbeat();
+
+private:
+    Servatrice *servatrice;
+    QString threadName;
+    QElapsedTimer sinceLastTick;
+    QTimer *heartbeatTimer = nullptr;
+};
+
+#endif

servatrice/src/main.cpp

@@ -33,6 +33,7 @@
 #include <QtGlobal>
 #include <iostream>
 #include <libcockatrice/rng/rng_sfmt.h>
+#include <libcockatrice/utility/cryptoutil.h>
 #include <libcockatrice/utility/passwordhasher.h>
 
 RNG_Abstract *rng;
@@ -169,7 +170,7 @@ int main(int argc, char *argv[])
 
     signalhandler = new SignalHandler();
 
-    rng = new RNG_SFMT;
+    rng = new RNG_SFMT(CryptoUtil::randomUInt64());
 
     std::cerr << "Servatrice " << VERSION_STRING << " starting." << std::endl;
     std::cerr << "-------------------------" << std::endl;

servatrice/src/metrics_registry.cpp

@@ -0,0 +1,70 @@
+#include "metrics_registry.h"
+
+#include <QList>
+
+void MetricsRegistry::observeCommand(int typeId, qint64 elapsedMs)
+{
+    if (typeId < 0 || typeId >= MaxTypes) {
+        typeId = MaxTypes - 1; // overflow slot keeps misrouted ids visible
+    }
+    if (elapsedMs < 0) {
+        elapsedMs = 0;
+    }
+
+    TypeStats &stats = slotFor(typeId);
+    stats.count.fetch_add(1, std::memory_order_relaxed);
+    stats.totalMs.fetch_add(elapsedMs, std::memory_order_relaxed);
+    totalCommandsCounter.fetch_add(1, std::memory_order_relaxed);
+    totalTimeCounter.fetch_add(elapsedMs, std::memory_order_relaxed);
+}
+
+void MetricsRegistry::observeGameStartDurationMs(qint64 elapsedMs)
+{
+    if (elapsedMs < 0) {
+        elapsedMs = 0;
+    }
+
+    gameStartStats.count.fetch_add(1, std::memory_order_relaxed);
+    gameStartStats.totalMs.fetch_add(elapsedMs, std::memory_order_relaxed);
+}
+
+MetricsRegistry::TypeStats &MetricsRegistry::slotFor(int typeId)
+{
+    return typeSlots[static_cast<size_t>(typeId)];
+}
+
+const MetricsRegistry::TypeStats &MetricsRegistry::slotFor(int typeId) const
+{
+    return typeSlots[static_cast<size_t>(typeId)];
+}
+
+int MetricsRegistry::activeTypeCount() const
+{
+    int active = 0;
+    for (int type = 0; type < MaxTypes; ++type) {
+        if (slotFor(type).count.load(std::memory_order_relaxed) > 0) {
+            ++active;
+        }
+    }
+    return active;
+}
+
+QList<MetricsRegistry::ActiveTypeStats> MetricsRegistry::collectActiveStats() const
+{
+    QList<ActiveTypeStats> result;
+    for (int type = 0; type < MaxTypes; ++type) {
+        const TypeStats &stats = slotFor(type);
+        const qint64 count = stats.count.load(std::memory_order_relaxed);
+        if (count == 0) {
+            continue;
+        }
+        result.append({type, count, stats.totalMs.load(std::memory_order_relaxed)});
+    }
+    return result;
+}
+
+MetricsRegistry::GameStartSnapshot MetricsRegistry::getGameStartSnapshot() const
+{
+    return {gameStartStats.count.load(std::memory_order_relaxed),
+            gameStartStats.totalMs.load(std::memory_order_relaxed)};
+}

servatrice/src/metrics_registry.h

@@ -0,0 +1,120 @@
+/**
+ * @file metrics_registry.h
+ * @ingroup Servatrice
+ */
+
+#ifndef METRICS_REGISTRY_H
+#define METRICS_REGISTRY_H
+
+#include <QList>
+#include <QString>
+#include <array>
+#include <atomic>
+
+/**
+ * @brief Lock-free accumulation of command processing statistics.
+ *
+ * observeCommand() is called once per processed command from whichever socket
+ * thread handled it. It uses relaxed atomic adds on preallocated storage only,
+ * so it introduces no locks, allocations, or shared cache-line ping-pong
+ * beyond the unavoidable counter updates.
+ *
+ * Reading happens rarely (metrics scraping), accepts momentary tears between
+ * related counters, and therefore also needs no synchronization.
+ *
+ * Only counts and totals are retained. An earlier Prometheus-style cumulative
+ * histogram (per-type, time-bucketed) was cut because nothing in the server
+ * ever wrote it out; it belongs to the future /metrics exporter that needs it.
+ */
+class MetricsRegistry
+{
+public:
+    /**
+     * Extension numbers are only unique per command kind, so recorded ids
+     * combine the kind index with the protobuf extension number.
+     *
+     * The stride is only as wide as it needs to be: 1280 is the first round
+     * number above the largest extension actually in use (ModeratorCommand =
+     * 1206) and keeps the preallocated TypeStats array small. Bump it if a new
+     * command exceeds it.
+     */
+    static constexpr int KindStride = 1280;
+
+    static constexpr int NumKinds = 6;
+
+    static constexpr const char *KindNames[NumKinds] = {"session", "room", "game", "moderator", "admin", "developer"};
+
+    /// Upper bound on distinct command type ids (see typeIdFor).
+    static constexpr int MaxTypes = NumKinds * KindStride;
+
+    /// Guard against typeIdFor() overflowing into the neighbouring kind's slots.
+    static_assert(KindStride > 1206, "KindStride must exceed the highest command extension number in use");
+
+    static int typeIdFor(int kindIndex, int extensionNumber)
+    {
+        return kindIndex * KindStride + extensionNumber;
+    }
+
+    void observeCommand(int typeId, qint64 elapsedMs);
+
+    /**
+     * Records how long one game start took to bring every player's zones
+     * online. Kept separate from command timings because it is triggered by
+     * the server itself and can dwarf any single command when decks are huge.
+     */
+    void observeGameStartDurationMs(qint64 elapsedMs);
+
+    /// Total number of observed commands across all types.
+    qint64 totalCommands() const
+    {
+        return totalCommandsCounter.load(std::memory_order_relaxed);
+    }
+
+    /// Cumulative processing milliseconds across all types.
+    qint64 totalTimeMs() const
+    {
+        return totalTimeCounter.load(std::memory_order_relaxed);
+    }
+
+    /// Number of distinct type slots that have seen at least one sample.
+    int activeTypeCount() const;
+
+    struct ActiveTypeStats
+    {
+        int typeId;
+        qint64 count;
+        qint64 totalMs;
+    };
+
+    /**
+     * Returns stats for every type slot that has seen at least one sample.
+     * Callers resolve the numeric type id to a human-readable label via
+     * typeIdFor()/KindNames as needed.
+     */
+    QList<ActiveTypeStats> collectActiveStats() const;
+
+    struct GameStartSnapshot
+    {
+        qint64 count;
+        qint64 totalMs;
+    };
+
+    GameStartSnapshot getGameStartSnapshot() const;
+
+private:
+    struct TypeStats
+    {
+        std::atomic<qint64> count{0};
+        std::atomic<qint64> totalMs{0};
+    };
+
+    TypeStats &slotFor(int typeId);
+    const TypeStats &slotFor(int typeId) const;
+
+    std::array<TypeStats, MaxTypes> typeSlots{};
+    TypeStats gameStartStats{};
+    std::atomic<qint64> totalCommandsCounter{0};
+    std::atomic<qint64> totalTimeCounter{0};
+};
+
+#endif

servatrice/src/servatrice.cpp

@@ -20,6 +20,7 @@
 #include "servatrice.h"
 
 #include "email_parser.h"
+#include "event_loop_watchdog.h"
 #include "isl_interface.h"
 #include "main.h"
 #include "servatrice_connection_pool.h"
@@ -38,6 +39,7 @@
 #include <QStringList>
 #include <QTimer>
 #include <QUrl>
+#include <game/server_game.h>
 #include <iostream>
 #include <libcockatrice/deck_list/deck_list.h>
 #include <libcockatrice/protocol/featureset.h>
@@ -63,6 +65,7 @@ Servatrice_GameServer::Servatrice_GameServer(Servatrice *_server,
         server->addDatabaseInterface(newThread, newDatabaseInterface);
 
         newThread->start();
+        server->watchWorkerThread(newThread);
         QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
                                   Q_ARG(QSqlDatabase, _sqlDatabase));
 
@@ -86,7 +89,6 @@ void Servatrice_GameServer::incomingConnection(qintptr socketDescriptor)
     Servatrice_ConnectionPool *pool = findLeastUsedConnectionPool();
 
     auto ssi = new TcpServerSocketInterface(server, pool->getDatabaseInterface());
-    connect(ssi, SIGNAL(incTxBytes(qint64)), this, SLOT(incTxBytes(qint64)));
     ssi->moveToThread(pool->thread());
     pool->addClient();
     connect(ssi, SIGNAL(destroyed()), pool, SLOT(removeClient()));
@@ -131,6 +133,7 @@ Servatrice_WebsocketGameServer::Servatrice_WebsocketGameServer(Servatrice *_serv
         server->addDatabaseInterface(newThread, newDatabaseInterface);
 
         newThread->start();
+        server->watchWorkerThread(newThread);
         QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
                                   Q_ARG(QSqlDatabase, _sqlDatabase));
 
@@ -156,7 +159,6 @@ void Servatrice_WebsocketGameServer::onNewConnection()
     Servatrice_ConnectionPool *pool = findLeastUsedConnectionPool();
 
     auto ssi = new WebsocketServerSocketInterface(server, pool->getDatabaseInterface());
-    connect(ssi, SIGNAL(incTxBytes(quint64)), this, SLOT(incTxBytes(quint64)));
     /*
      * Due to a Qt limitation, websockets can't be moved to another thread.
      * This will hopefully change in Qt6 if QtWebSocket will be integrated in QtNetwork
@@ -226,6 +228,13 @@ bool Servatrice::initServer()
 {
 
     serverId = getServerID();
+
+    // METRICS (always active. Slow-command logging and stall watchdogs are
+    // controlled by their respective thresholds below). Read up front so the
+    // values are available before any pool thread is started and watchdogged.
+    metricsSlowCommandMs = settingsCache->value("metrics/slow_command_ms", 500).toInt();
+    metricsStallWarnMs = qMax(0, settingsCache->value("metrics/stall_warn_ms", 2000).toInt());
+
     if (getAuthenticationMethodString() == "sql") {
         qDebug() << "Authenticating method: sql";
         authenticationMethod = AuthenticationSql;
@@ -470,9 +479,55 @@ bool Servatrice::initServer()
     }
 
     setRequiredFeatures(getRequiredFeatures());
+
     return true;
 }
 
+void Servatrice::observeGameStartDurationMs(qint64 elapsedMs)
+{
+    metricsRegistry.observeGameStartDurationMs(elapsedMs);
+}
+
+void Servatrice::observeEventLoopStall(const QString &threadName, qint64 overshootMs)
+{
+    eventLoopStallsTotal.fetch_add(1, std::memory_order_relaxed);
+    eventLoopLastStallMs.store(overshootMs, std::memory_order_relaxed);
+    qint64 prevMax = eventLoopMaxStallMs.load(std::memory_order_relaxed);
+    while (overshootMs > prevMax &&
+           !eventLoopMaxStallMs.compare_exchange_weak(prevMax, overshootMs, std::memory_order_relaxed)) {
+        // retry until the max is at least as high as the new sample
+    }
+
+    qWarning() << "Event loop stall in" << threadName << "- heartbeat overshot by" << overshootMs << "ms";
+}
+
+void Servatrice::watchWorkerThread(QThread *thread)
+{
+    if (metricsStallWarnMs <= 0) {
+        return; // watchdogs disabled via metrics/stall_warn_ms = 0
+    }
+
+    auto *watchdog = new EventLoopWatchdog(this, thread->objectName());
+    connect(thread, &QThread::finished, watchdog, &QObject::deleteLater);
+    watchdog->moveToThread(thread);
+    QMetaObject::invokeMethod(watchdog, &EventLoopWatchdog::start, Qt::QueuedConnection);
+}
+
+qint64 Servatrice::getCardsInGamesTotal() const
+{
+    qint64 total = 0;
+    QReadLocker roomsLocker(&roomsLock); // locking order: roomsLock before gamesLock/gameMutex
+    QMapIterator<int, Server_Room *> roomIterator(rooms);
+    while (roomIterator.hasNext()) {
+        Server_Room *room = roomIterator.next().value();
+        QReadLocker gamesLocker(&room->gamesLock);
+        for (auto *game : room->getGames()) {
+            total += game->getCardsInGame();
+        }
+    }
+    return total;
+}
+
 void Servatrice::addDatabaseInterface(QThread *thread, Servatrice_DatabaseInterface *databaseInterface)
 {
     databaseInterfaces.insert(thread, databaseInterface);

servatrice/src/servatrice.h

@@ -20,6 +20,8 @@
 #ifndef SERVATRICE_H
 #define SERVATRICE_H
 
+#include "metrics_registry.h"
+
 #include <QDateTime>
 #include <QHostAddress>
 #include <QMetaType>
@@ -30,6 +32,7 @@
 #include <QSslKey>
 #include <QTcpServer>
 #include <QWebSocketServer>
+#include <atomic>
 #include <libcockatrice/protocol/pb/response_report_stats.pb.h>
 #include <memory>
 #include <server.h>
@@ -170,6 +173,12 @@ private:
     int uptime;
     QMutex txBytesMutex, rxBytesMutex;
     quint64 txBytes, rxBytes;
+    MetricsRegistry metricsRegistry;
+    int metricsSlowCommandMs = 500;
+    int metricsStallWarnMs = 2000;
+    std::atomic<qint64> eventLoopStallsTotal{0}; ///< heartbeat overshoots past the warn threshold
+    std::atomic<qint64> eventLoopLastStallMs{0}; ///< overshoot of the most recent stall
+    std::atomic<qint64> eventLoopMaxStallMs{0};  ///< worst overshoot seen since process start
 
     QString shutdownReason;
     int shutdownMinutes;
@@ -286,6 +295,49 @@ public:
     void incRxBytes(quint64 num);
     void addDatabaseInterface(QThread *thread, Servatrice_DatabaseInterface *databaseInterface);
 
+    // Metrics (see [metrics] section in servatrice.ini.example)
+    MetricsRegistry &getMetricsRegistry()
+    {
+        return metricsRegistry;
+    }
+    /**
+     * Sums cards across all zones of all running games. Each game takes its
+     * own gameMutex -- the hot per-game lock every game action contends on --
+     * and then iterates every player's zones, so the scrape cost is really
+     * O(total cards in play) plus one mutex acquisition per live game. Keep
+     * scrapes infrequent in big multiplayer rooms.
+     */
+    qint64 getCardsInGamesTotal() const;
+    int getMetricsSlowCommandMs() const
+    {
+        return metricsSlowCommandMs;
+    }
+    /// Heartbeat overshoot that counts as a stall. A value of 0 disables the watchdogs.
+    int getMetricsStallWarnMs() const
+    {
+        return metricsStallWarnMs;
+    }
+    void observeGameStartDurationMs(qint64 elapsedMs) override;
+    qint64 getEventLoopStallsTotal() const
+    {
+        return eventLoopStallsTotal.load(std::memory_order_relaxed);
+    }
+    qint64 getEventLoopLastStallMs() const
+    {
+        return eventLoopLastStallMs.load(std::memory_order_relaxed);
+    }
+    qint64 getEventLoopMaxStallMs() const
+    {
+        return eventLoopMaxStallMs.load(std::memory_order_relaxed);
+    }
+    /// Records one heartbeat overshoot and logs a single warning for it.
+    void observeEventLoopStall(const QString &threadName, qint64 overshootMs);
+    /**
+     * Installs an EventLoopWatchdog in @p thread. Called once per socket pool
+     * thread right after it starts.
+     */
+    void watchWorkerThread(QThread *thread);
+
     bool islConnectionExists(int _serverId) const;
     void addIslInterface(int _serverId, IslInterface *interface);
     void removeIslInterface(int _serverId);

servatrice/src/servatrice_database_interface.cpp

@@ -641,6 +641,10 @@ ServerInfo_User Servatrice_DatabaseInterface::evalUserQueryResult(const QSqlQuer
         userLevel |= ServerInfo_User::IsJudge;
     }
 
+    if (is_admin & 8) {
+        userLevel |= ServerInfo_User::IsDeveloper;
+    }
+
     result.set_user_level(userLevel);
 
     const QString country = query->value(3).toString();
@@ -1448,7 +1452,7 @@ QList<ServerInfo_ModeratorLogin> Servatrice_DatabaseInterface::getModeratorLastL
     QSqlQuery *query = prepareQuery("SELECT u.name, u.admin, UNIX_TIMESTAMP(a.last_login) "
                                     "FROM {prefix}_users u "
                                     "LEFT JOIN {prefix}_user_analytics a ON a.id = u.id "
-                                    "WHERE (u.admin & 7) <> 0 ORDER BY u.name");
+                                    "WHERE (u.admin & 15) <> 0 ORDER BY u.name");
 
     if (!execSqlQuery(query)) {
         qCWarning(DatabaseInterfaceLog) << "Failed to collect moderator login information: SQL Error";
@@ -1469,6 +1473,9 @@ QList<ServerInfo_ModeratorLogin> Servatrice_DatabaseInterface::getModeratorLastL
         if (isAdmin & 4) {
             userLevel |= ServerInfo_User::IsJudge;
         }
+        if (isAdmin & 8) {
+            userLevel |= ServerInfo_User::IsDeveloper;
+        }
         loginDetails.set_user_level(userLevel);
 
         if (!query->value(2).isNull()) {
@@ -1480,6 +1487,38 @@ QList<ServerInfo_ModeratorLogin> Servatrice_DatabaseInterface::getModeratorLastL
     return results;
 }
 
+Servatrice_DatabaseInterface::UptimeSnapshot Servatrice_DatabaseInterface::getLatestUptimeSnapshot(int serverId)
+{
+    UptimeSnapshot snapshot;
+
+    if (!checkSql()) {
+        return snapshot;
+    }
+
+    QSqlQuery *query = prepareQuery("SELECT users_count, mods_count, games_count, tx_bytes, rx_bytes, uptime, "
+                                    "UNIX_TIMESTAMP(timest) FROM {prefix}_uptime "
+                                    "WHERE id_server = :id_server ORDER BY timest DESC LIMIT 1");
+    query->bindValue(":id_server", serverId);
+
+    if (!execSqlQuery(query)) {
+        qCWarning(DatabaseInterfaceLog) << "Failed to collect server stats snapshot: SQL Error";
+        return snapshot;
+    }
+
+    if (query->next()) {
+        snapshot.valid = true;
+        snapshot.usersCount = query->value(0).toULongLong();
+        snapshot.modsCount = query->value(1).toULongLong();
+        snapshot.gamesCount = query->value(2).toULongLong();
+        snapshot.txBytes = query->value(3).toULongLong();
+        snapshot.rxBytes = query->value(4).toULongLong();
+        snapshot.uptimeSecs = query->value(5).toULongLong();
+        snapshot.timest = query->value(6).toULongLong();
+    }
+
+    return snapshot;
+}
+
 bool Servatrice_DatabaseInterface::removeUserAvatar(const QString &userName)
 {
     if (!checkSql()) {

servatrice/src/servatrice_database_interface.h

@@ -140,6 +140,21 @@ public:
     QList<ServerInfo_UserSession> getUserSessions(const QString &userName, int limit);
     QList<ServerInfo_UserAlt> getUserAlts(const QString &userName);
     QList<ServerInfo_ModeratorLogin> getModeratorLastLogins();
+
+    // Uptime snapshot as recorded by Servatrice::statusUpdate() into the
+    // {prefix}_uptime table. valid is false when no snapshot exists yet.
+    struct UptimeSnapshot
+    {
+        bool valid = false;
+        quint64 usersCount = 0;
+        quint64 modsCount = 0;
+        quint64 gamesCount = 0;
+        quint64 txBytes = 0;
+        quint64 rxBytes = 0;
+        quint64 uptimeSecs = 0;
+        quint64 timest = 0;
+    };
+    UptimeSnapshot getLatestUptimeSnapshot(int serverId);
     bool removeUserAvatar(const QString &userName);
     bool addForgotPassword(const QString &user);
     bool removeForgotPassword(const QString &user) override;

servatrice/src/serversocketinterface.cpp

@@ -30,6 +30,7 @@
 
 #include <QDateTime>
 #include <QDebug>
+#include <QElapsedTimer>
 #include <QHostAddress>
 #include <QJsonDocument>
 #include <QJsonObject>
@@ -39,14 +40,17 @@
 #include <QSqlQuery>
 #include <QString>
 #include <game/server_player.h>
+#include <google/protobuf/descriptor.h>
 #include <iostream>
 #include <libcockatrice/deck_list/deck_list.h>
+#include <libcockatrice/protocol/get_pb_extension.h>
 #include <libcockatrice/protocol/pb/command_deck_del.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_del_dir.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_download.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_list.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_new_dir.pb.h>
 #include <libcockatrice/protocol/pb/command_deck_upload.pb.h>
+#include <libcockatrice/protocol/pb/command_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_delete_match.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download_by_game_id.pb.h>
@@ -80,6 +84,7 @@
 #include <libcockatrice/protocol/pb/response_deck_upload.pb.h>
 #include <libcockatrice/protocol/pb/response_forgotpasswordrequest.pb.h>
 #include <libcockatrice/protocol/pb/response_get_admin_notes.pb.h>
+#include <libcockatrice/protocol/pb/response_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/response_moderator_last_logins.pb.h>
 #include <libcockatrice/protocol/pb/response_password_salt.pb.h>
 #include <libcockatrice/protocol/pb/response_register.pb.h>
@@ -190,6 +195,77 @@ void AbstractServerSocketInterface::logDebugMessage(const QString &message)
     logger->logMessage(message, this);
 }
 
+void AbstractServerSocketInterface::processCommandContainer(const CommandContainer &cont)
+{
+    QElapsedTimer timer;
+    timer.start();
+    Server_ProtocolHandler::processCommandContainer(cont);
+    const qint64 elapsedMs = timer.nsecsElapsed() / 1000000;
+
+    // The base dispatch is an if/else-if chain — at most one family is
+    // actually processed.  Recording every family in the container would
+    // let an unauthenticated client stampforge developer/moderator/admin
+    // samples by batching them alongside a session command the server
+    // actually runs.  Mirror the base's selection and skip entirely when
+    // deleted or when no family matched.
+    if (deleted) {
+        return;
+    }
+
+    // When getPbExtension returns -1 (no extension set) and the kind is
+    // non-zero, typeIdFor wraps into the previous kind's range instead of
+    // hitting the typeId < 0 guard in observeCommand.  Skip such entries.
+    int kind = -1;
+    if (cont.game_command_size()) {
+        kind = 2;
+    } else if (cont.room_command_size()) {
+        kind = 1;
+    } else if (cont.session_command_size()) {
+        kind = 0;
+    } else if (cont.moderator_command_size()) {
+        kind = 3;
+    } else if (cont.admin_command_size()) {
+        kind = 4;
+    } else if (cont.developer_command_size()) {
+        kind = 5;
+    }
+
+    if (kind >= 0) {
+        auto recordDispatched = [&](int familyKind, const auto &cmds) {
+            for (const auto &cmd : cmds) {
+                const int ext = getPbExtension(cmd);
+                if (ext >= 0) {
+                    servatrice->getMetricsRegistry().observeCommand(MetricsRegistry::typeIdFor(familyKind, ext),
+                                                                    elapsedMs);
+                }
+            }
+        };
+
+        if (kind == 0) {
+            recordDispatched(kind, cont.session_command());
+        } else if (kind == 1) {
+            recordDispatched(kind, cont.room_command());
+        } else if (kind == 2) {
+            recordDispatched(kind, cont.game_command());
+        } else if (kind == 3) {
+            recordDispatched(kind, cont.moderator_command());
+        } else if (kind == 4) {
+            recordDispatched(kind, cont.admin_command());
+        } else {
+            recordDispatched(kind, cont.developer_command());
+        }
+    }
+
+    const int slowCommandMs = servatrice->getMetricsSlowCommandMs();
+    if (slowCommandMs > 0 && elapsedMs >= slowCommandMs) {
+        const ServerInfo_User *info = getUserInfo();
+        const QString user = authState == PasswordRight && info ? QString::fromStdString(info->name())
+                                                                : QStringLiteral("unauthenticated");
+        qCWarning(AbstractServerSocketInterfaceLog) << "slow command container from" << user << "processed in"
+                                                    << elapsedMs << "ms (" << cont.ByteSizeLong() << "bytes)";
+    }
+}
+
 Response::ResponseCode AbstractServerSocketInterface::processExtendedSessionCommand(int cmdType,
                                                                                     const SessionCommand &cmd,
                                                                                     ResponseContainer &rc)
@@ -284,7 +360,7 @@ Response::ResponseCode AbstractServerSocketInterface::processExtendedModeratorCo
         case ModeratorCommand::REPORT_RESOLVE:
             return cmdReportResolve(cmd.GetExtension(Command_ReportResolve::ext), rc);
         case ModeratorCommand::VIEWLOG_HISTORY:
-            return cmdGetLogHistory(cmd.GetExtension(Command_ViewLogHistory::ext), rc);
+            return cmdGetLogHistory(cmd.GetExtension(Command_ViewLogHistory::ext), rc, true);
         case ModeratorCommand::GRANT_REPLAY_ACCESS:
             return cmdGrantReplayAccess(cmd.GetExtension(Command_GrantReplayAccess::ext), rc);
         case ModeratorCommand::REPLAY_DOWNLOAD_BY_GAME_ID:
@@ -337,6 +413,26 @@ AbstractServerSocketInterface::processExtendedAdminCommand(int cmdType, const Ad
     }
 }
 
+// DEVELOPER FUNCTIONS.
+// Permission is checked by processDeveloperCommandContainer. Only stats-style
+// queries live here, never community moderation or server administration.
+Response::ResponseCode AbstractServerSocketInterface::processExtendedDeveloperCommand(int cmdType,
+                                                                                      const DeveloperCommand &cmd,
+                                                                                      ResponseContainer &rc)
+{
+    switch ((DeveloperCommand::DeveloperCommandType)cmdType) {
+        case DeveloperCommand::GET_SERVER_STATS:
+            return cmdGetServerStats(cmd.GetExtension(Command_GetServerStats::ext), rc);
+        case DeveloperCommand::VIEWLOG_HISTORY: {
+            // Same query as the moderator log view, carried by the developer
+            // command family, but narrows out private chats and sender IPs.
+            return cmdGetLogHistory(cmd.GetExtension(Command_ViewLogHistory::dev_ext), rc, false);
+        }
+        default:
+            return Response::RespFunctionNotAllowed;
+    }
+}
+
 Response::ResponseCode AbstractServerSocketInterface::cmdAddToList(const Command_AddToList &cmd, ResponseContainer &rc)
 {
     if (authState != PasswordRight) {
@@ -1020,12 +1116,13 @@ Response::ResponseCode AbstractServerSocketInterface::cmdReplaySubmitCode(const
 // MODERATOR FUNCTIONS.
 // May be called by admins and moderators. Permission is checked by the calling function.
 Response::ResponseCode AbstractServerSocketInterface::cmdGetLogHistory(const Command_ViewLogHistory &cmd,
-                                                                       ResponseContainer &rc)
+                                                                       ResponseContainer &rc,
+                                                                       bool allowPrivateChat)
 {
 
     QList<ServerInfo_ChatMessage> messageList;
     QString userName = nameFromStdString(cmd.user_name());
-    QString ipAddress = nameFromStdString(cmd.ip_address());
+    QString ipAddress = allowPrivateChat ? nameFromStdString(cmd.ip_address()) : QString();
     QString gameName = nameFromStdString(cmd.game_name());
     QString gameID = nameFromStdString(cmd.game_id());
     QString message = textFromStdString(cmd.message());
@@ -1040,11 +1137,21 @@ Response::ResponseCode AbstractServerSocketInterface::cmdGetLogHistory(const Com
         if (nameFromStdString(cmd.log_location(i)).simplified() == "game") {
             gameType = true;
         }
-        if (nameFromStdString(cmd.log_location(i)).simplified() == "chat") {
+        if (nameFromStdString(cmd.log_location(i)).simplified() == "chat" && allowPrivateChat) {
             chatType = true;
         }
     }
 
+    // For callers that must not see private conversations, never leave the
+    // target-type filter empty: if the request only asked for "chat" (or for
+    // nothing at all) the query below would carry no target_type restriction
+    // and would return every row, private messages included. Fall back to the
+    // game/room diagnostics the caller is allowed to see.
+    if (!allowPrivateChat && !gameType && !roomType) {
+        gameType = true;
+        roomType = true;
+    }
+
     int dateRange = cmd.date_range();
     int maximumResults = cmd.maximum_results();
 
@@ -1054,7 +1161,11 @@ Response::ResponseCode AbstractServerSocketInterface::cmdGetLogHistory(const Com
         QListIterator<ServerInfo_ChatMessage> messageIterator(sqlInterface->getMessageLogHistory(
             userName, ipAddress, gameName, gameID, message, chatType, gameType, roomType, dateRange, maximumResults));
         while (messageIterator.hasNext()) {
-            re->add_log_message()->CopyFrom(messageIterator.next());
+            ServerInfo_ChatMessage chatMessage = messageIterator.next();
+            if (!allowPrivateChat) {
+                chatMessage.clear_sender_ip();
+            }
+            re->add_log_message()->CopyFrom(chatMessage);
         }
     } else {
         ServerInfo_ChatMessage chatMessage;
@@ -1643,6 +1754,79 @@ Response::ResponseCode AbstractServerSocketInterface::cmdReportUserInfo(const Co
     return Response::RespOk;
 }
 
+Response::ResponseCode AbstractServerSocketInterface::cmdGetServerStats(const Command_GetServerStats & /*cmd */,
+                                                                        ResponseContainer &rc)
+{
+    if (!sqlInterface->checkSql()) {
+        return Response::RespInternalError;
+    }
+
+    // Servatrice::statusUpdate() periodically snapshots server health into the
+    // uptime table. Serve the freshest snapshot for this server.
+    const auto snapshot = sqlInterface->getLatestUptimeSnapshot(servatrice->getServerID());
+    if (!snapshot.valid) {
+        // No snapshot yet (fresh server, or statusUpdate() has not ticked).
+        return Response::RespInternalError;
+    }
+
+    auto *re = new Response_GetServerStats;
+    re->set_users_count(snapshot.usersCount);
+    re->set_mods_count(snapshot.modsCount);
+    re->set_games_count(snapshot.gamesCount);
+    re->set_tx_bytes(snapshot.txBytes);
+    re->set_rx_bytes(snapshot.rxBytes);
+    re->set_uptime_secs(snapshot.uptimeSecs);
+    re->set_timest(snapshot.timest);
+
+    // Live metrics from the in-process MetricsRegistry (resets on server restart)
+    re->set_cards_in_games(static_cast<google::protobuf::uint64>(servatrice->getCardsInGamesTotal()));
+    re->set_eventloop_stalls_total(static_cast<google::protobuf::uint64>(servatrice->getEventLoopStallsTotal()));
+    re->set_eventloop_last_stall_ms(static_cast<google::protobuf::uint64>(servatrice->getEventLoopLastStallMs()));
+    re->set_eventloop_max_stall_ms(static_cast<google::protobuf::uint64>(servatrice->getEventLoopMaxStallMs()));
+    re->set_total_commands(static_cast<google::protobuf::uint64>(servatrice->getMetricsRegistry().totalCommands()));
+    re->set_total_command_time_ms(
+        static_cast<google::protobuf::uint64>(servatrice->getMetricsRegistry().totalTimeMs()));
+    re->set_active_command_types(servatrice->getMetricsRegistry().activeTypeCount());
+
+    const auto gameStart = servatrice->getMetricsRegistry().getGameStartSnapshot();
+    re->set_game_start_count(static_cast<google::protobuf::uint64>(gameStart.count));
+    re->set_game_start_total_ms(static_cast<google::protobuf::uint64>(gameStart.totalMs));
+
+    // Per-command breakdown: resolve protobuf extension names via the descriptor pool
+    static const char *messageNames[] = {"SessionCommand",   "RoomCommand",  "GameCommand",
+                                         "ModeratorCommand", "AdminCommand", "DeveloperCommand"};
+    const auto activeStats = servatrice->getMetricsRegistry().collectActiveStats();
+    for (const auto &stat : activeStats) {
+        const int kind = stat.typeId / MetricsRegistry::KindStride;
+        const int number = stat.typeId % MetricsRegistry::KindStride;
+
+        QString label;
+        if (kind >= 0 && kind < MetricsRegistry::NumKinds) {
+            const google::protobuf::DescriptorPool *pool = google::protobuf::DescriptorPool::generated_pool();
+            const google::protobuf::Descriptor *message = pool->FindMessageTypeByName(messageNames[kind]);
+            const google::protobuf::FieldDescriptor *extension =
+                message ? pool->FindExtensionByNumber(message, number) : nullptr;
+            if (extension) {
+                label = QString::fromLatin1(MetricsRegistry::KindNames[kind]) + QStringLiteral("/") +
+                        QString::fromStdString(std::string(extension->message_type()->name()));
+            }
+        }
+        if (label.isEmpty()) {
+            label = QString::number(stat.typeId);
+        }
+
+        CommandStats *cs = re->add_command_stats();
+        cs->set_kind_index(static_cast<google::protobuf::uint32>(kind));
+        cs->set_extension_number(static_cast<google::protobuf::uint32>(number));
+        cs->set_command_name(label.toStdString());
+        cs->set_count(static_cast<google::protobuf::uint64>(stat.count));
+        cs->set_total_ms(static_cast<google::protobuf::uint64>(stat.totalMs));
+    }
+
+    rc.setResponseExtension(re);
+    return Response::RespOk;
+}
+
 Response::ResponseCode AbstractServerSocketInterface::cmdReportStats(const Command_ReportStats & /*cmd */,
                                                                      ResponseContainer &rc)
 {
@@ -3215,7 +3399,7 @@ bool AbstractServerSocketInterface::removeAdminFlagFromUser(const QString &userN
     if (user) {
         Event_ConnectionClosed event;
         event.set_reason(Event_ConnectionClosed::DEMOTED);
-        event.set_reason_str("Your moderator and/or judge status has been revoked.");
+        event.set_reason_str("Your moderator, judge, and/or developer status has been revoked.");
         event.set_end_time(QDateTime::currentDateTime().toSecsSinceEpoch());
 
         SessionEvent *se = user->prepareSessionEvent(event);
@@ -3257,6 +3441,18 @@ Response::ResponseCode AbstractServerSocketInterface::cmdAdjustMod(const Command
         }
     }
 
+    if (cmd.has_should_be_developer()) {
+        if (cmd.should_be_developer()) {
+            if (!addAdminFlagToUser(userName, 8)) {
+                return Response::RespInternalError;
+            }
+        } else {
+            if (!removeAdminFlagFromUser(userName, 8)) {
+                return Response::RespInternalError;
+            }
+        }
+    }
+
     return Response::RespOk;
 }
 

servatrice/src/serversocketinterface.h

@@ -24,6 +24,7 @@
 #include <QMutex>
 #include <QTcpSocket>
 #include <QWebSocket>
+#include <libcockatrice/protocol/pb/command_get_server_stats.pb.h>
 #include <libcockatrice/protocol/pb/command_replay_download_by_game_id.pb.h>
 #include <libcockatrice/protocol/pb/command_report.pb.h>
 #include <libcockatrice/protocol/pb/command_report_add_comment.pb.h>
@@ -80,6 +81,7 @@ signals:
 protected:
     void logDebugMessage(const QString &message) override;
     bool tooManyRegistrationAttempts(const QString &ipAddress);
+    void processCommandContainer(const CommandContainer &cont) override;
 
     virtual void writeToSocket(QByteArray &data) = 0;
     virtual void flushSocket() = 0;
@@ -115,7 +117,8 @@ private:
     Response::ResponseCode cmdBanFromServer(const Command_BanFromServer &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdReportList(const Command_ReportList &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdWarnUser(const Command_WarnUser &cmd, ResponseContainer &rc);
-    Response::ResponseCode cmdGetLogHistory(const Command_ViewLogHistory &cmd, ResponseContainer &rc);
+    Response::ResponseCode
+    cmdGetLogHistory(const Command_ViewLogHistory &cmd, ResponseContainer &rc, bool allowPrivateChat);
     Response::ResponseCode cmdGetBanHistory(const Command_GetBanHistory &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdGetWarnList(const Command_GetWarnList &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdGetWarnHistory(const Command_GetWarnHistory &cmd, ResponseContainer &rc);
@@ -151,6 +154,8 @@ private:
     processExtendedModeratorCommand(int cmdType, const ModeratorCommand &cmd, ResponseContainer &rc) override;
     Response::ResponseCode
     processExtendedAdminCommand(int cmdType, const AdminCommand &cmd, ResponseContainer &rc) override;
+    Response::ResponseCode
+    processExtendedDeveloperCommand(int cmdType, const DeveloperCommand &cmd, ResponseContainer &rc) override;
 
     Response::ResponseCode cmdAccountEdit(const Command_AccountEdit &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdAccountImage(const Command_AccountImage &cmd, ResponseContainer &rc);
@@ -172,6 +177,8 @@ private:
     Response::ResponseCode cmdResetUserPassword(const Command_ResetUserPassword &cmd, ResponseContainer &rc);
     Response::ResponseCode cmdRemoveUserAvatar(const Command_RemoveUserAvatar &cmd, ResponseContainer &rc);
 
+    Response::ResponseCode cmdGetServerStats(const Command_GetServerStats &cmd, ResponseContainer &rc);
+
     bool addAdminFlagToUser(const QString &user, int flag);
     bool removeAdminFlagFromUser(const QString &user, int flag);