jeirmeister/Cockatrice
1
0
Fork 0
mirror of https://github.com/Cockatrice/Cockatrice.git synced 2026-09-22 01:25:10 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

[Server] Instrument command processing, game starts, and event loops

Browse source 
Add a lock-free MetricsRegistry that accumulates per-command processing
times in preallocated histogram slots (one per protobuf command type,
bucketed at 1/5/10/25/50/100/250/500/1000/2500/5000 ms +Inf). The
hot-path observeCommand() uses only relaxed atomic adds — no locks,
no allocations, no cache-line ping-pong beyond the unavoidable counter
updates.

Wire the registry into AbstractServerSocketInterface::processCommandContainer()
so every processed command is attributed with its container's wall-clock
time. When a container exceeds metrics/slow_command_ms (default 500),
a warning is logged including the connected username.

Add an EventLoopWatchdog heartbeat that runs on every socket pool thread.
If a heartbeat overshoots metrics/stall_warn_ms (default 2000 ms), the
overshoot is recorded in atomic counters and a warning is logged. Both
thresholds are configurable in servatrice.ini; setting stall_warn_ms to 0
disables the watchdogs entirely.

Track game-start durations via a separate histogram in MetricsRegistry.
Server_Game::startGameNow() measures the time from zone creation through
player materialization and reports it via Server::observeGameStartDurationMs().

Add a live card-count gauge: Server_Game exposes getCardsInGame() and
Servatrice::getCardsInGamesTotal() sums across all running games under
the appropriate read locks.

Include a standalone metrics_registry_test (Google Test) that validates
empty registries, single/multi-sample histograms, kind encoding,
overflow-slot collapse, negative-duration clamping, gauge rendering,
and the game-start histogram separation.

Took 10 minutes
This commit is contained in:
Lukas Brübach 2026-08-25 20:42:10 +02:00
parent 2b33cbfb66
commit a542831aad
18 changed files with 749 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

57
servatrice/src/servatrice.cpp
View file

@ -20,6 +20,7 @@
#include "servatrice.h"
#include "email_parser.h"
#include "event_loop_watchdog.h"
#include "isl_interface.h"
#include "main.h"
#include "servatrice_connection_pool.h"
@ -38,6 +39,7 @@
#include <QStringList>
#include <QTimer>
#include <QUrl>
#include <game/server_game.h>
#include <iostream>
#include <libcockatrice/deck_list/deck_list.h>
#include <libcockatrice/protocol/featureset.h>
@ -63,6 +65,7 @@ Servatrice_GameServer::Servatrice_GameServer(Servatrice *_server,
server->addDatabaseInterface(newThread, newDatabaseInterface);
newThread->start();
server->watchWorkerThread(newThread);
QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
Q_ARG(QSqlDatabase, _sqlDatabase));
@ -131,6 +134,7 @@ Servatrice_WebsocketGameServer::Servatrice_WebsocketGameServer(Servatrice *_serv
server->addDatabaseInterface(newThread, newDatabaseInterface);
newThread->start();
server->watchWorkerThread(newThread);
QMetaObject::invokeMethod(newDatabaseInterface, "initDatabase", Qt::BlockingQueuedConnection,
Q_ARG(QSqlDatabase, _sqlDatabase));
@ -470,9 +474,60 @@ bool Servatrice::initServer()
}
setRequiredFeatures(getRequiredFeatures());
// METRICS (always active. Slow-command logging and stall watchdogs are
// controlled by their respective thresholds below)
metricsSlowCommandMs = settingsCache->value("metrics/slow_command_ms", 500).toInt();
metricsStallWarnMs = qMax(0, settingsCache->value("metrics/stall_warn_ms", 2000).toInt());
return true;
}
void Servatrice::observeGameStartDurationMs(qint64 elapsedMs)
{
metricsRegistry.observeGameStartDurationMs(elapsedMs);
}
void Servatrice::observeEventLoopStall(const QString &threadName, qint64 overshootMs)
{
eventLoopStallsTotal.fetch_add(1, std::memory_order_relaxed);
eventLoopLastStallMs.store(overshootMs, std::memory_order_relaxed);
qint64 prevMax = eventLoopMaxStallMs.load(std::memory_order_relaxed);
while (overshootMs > prevMax &&
!eventLoopMaxStallMs.compare_exchange_weak(prevMax, overshootMs, std::memory_order_relaxed)) {
// retry until the max is at least as high as the new sample
}
qWarning() << "Event loop stall in" << threadName << "- heartbeat overshot by" << overshootMs << "ms";
}
void Servatrice::watchWorkerThread(QThread *thread)
{
if (metricsStallWarnMs <= 0) {
return; // watchdogs disabled via metrics/stall_warn_ms = 0
}
auto *watchdog = new EventLoopWatchdog(this, thread->objectName());
connect(thread, &QThread::finished, watchdog, &QObject::deleteLater);
watchdog->moveToThread(thread);
QMetaObject::invokeMethod(watchdog, &EventLoopWatchdog::start, Qt::QueuedConnection);
}
qint64 Servatrice::getCardsInGamesTotal() const
{
qint64 total = 0;
QReadLocker roomsLocker(&roomsLock); // locking order: roomsLock before gamesLock/gameMutex
QMapIterator<int, Server_Room *> roomIterator(rooms);
while (roomIterator.hasNext()) {
Server_Room *room = roomIterator.next().value();
QReadLocker gamesLocker(&room->gamesLock);
for (auto *game : room->getGames()) {
total += game->getCardsInGame();
}
}
return total;
}
void Servatrice::addDatabaseInterface(QThread *thread, Servatrice_DatabaseInterface *databaseInterface)
{
databaseInterfaces.insert(thread, databaseInterface);
@ -728,6 +783,7 @@ void Servatrice::incTxBytes(quint64 num)
txBytesMutex.lock();
txBytes += num;
txBytesMutex.unlock();
txBytesTotal.fetch_add(num, std::memory_order_relaxed);
}
void Servatrice::incRxBytes(quint64 num)
@ -735,6 +791,7 @@ void Servatrice::incRxBytes(quint64 num)
rxBytesMutex.lock();
rxBytes += num;
rxBytesMutex.unlock();
rxBytesTotal.fetch_add(num, std::memory_order_relaxed);
}
void Servatrice::shutdownTimeout()