Prometheus monitoring support

nix/tests/test-bundle-starts.nix

@@ -73,6 +73,8 @@ in
     };
   };
 
+  # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
+  # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest
   testScript = ''
     machine.wait_for_unit("spacebar-api")
     machine.wait_for_unit("spacebar-cdn")
@@ -82,7 +84,13 @@ in
     machine.wait_for_open_port(3001)
     machine.wait_for_open_port(3002)
     machine.wait_for_open_port(3003)
-    # If well known works, its probably fine(tm)?
+
+    # this should be working
     machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client")
+
+    # check if metrics endpoint works on all services
+    machine.succeed("curl -f http://api.sb.localhost/metrics")
+    machine.succeed("curl -f http://gateway.sb.localhost/metrics")
+    machine.succeed("curl -f http://cdn.sb.localhost/metrics")
   '';
 }

package.json

@@ -115,6 +115,7 @@
         "pg-query-stream": "^4.15.0",
         "picocolors": "^1.1.1",
         "probe-image-size": "^7.3.0",
+        "prom-client": "^15.1.3",
         "reflect-metadata": "^0.2.2",
         "tslib": "^2.8.1",
         "typeorm": "^0.3.30",

src/api/Server.ts

@@ -16,15 +16,16 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util";
-import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares";
+import path from "node:path";
 import { Request, Response, Router } from "express";
-import { Server, ServerOptions } from "lambert-server";
 import morgan from "morgan";
-import path from "node:path";
+import { Server, ServerOptions } from "lambert-server";
 import { red } from "picocolors";
+import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util";
+import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares";
 import { initInstance } from "./util/handlers/Instance";
 import { route } from "./util";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 const ASSETS_FOLDER = path.join(__dirname, "..", "..", "assets");
 const PUBLIC_ASSETS_FOLDER = path.join(ASSETS_FOLDER, "public");
@@ -50,6 +51,8 @@ export class SpacebarServer extends Server {
     }
 
     async start() {
+        await Monitoring.init();
+        Monitoring.attach(this.app);
         await initDatabase();
         await Config.init();
         await initEvent();

src/bundle/Server.ts

@@ -16,21 +16,18 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import morgan from "morgan";
-
-process.on("unhandledRejection", console.error);
-process.on("uncaughtException", console.error);
-
 import http from "node:http";
+import fs from "node:fs";
+import cluster from "node:cluster";
+import morgan from "morgan";
+import express from "express";
+import { green, bold } from "picocolors";
 import * as Api from "@spacebar/api";
 import * as Gateway from "@spacebar/gateway";
 import * as Webrtc from "@spacebar/webrtc";
 import { CDNServer } from "@spacebar/cdn";
-import express from "express";
-import { green, bold } from "picocolors";
 import { Config, initDatabase } from "@spacebar/util";
-import fs from "node:fs";
-import cluster from "node:cluster";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 const app = express();
 const server = http.createServer();
@@ -58,6 +55,7 @@ process.on("SIGTERM", async () => {
 });
 
 async function main() {
+    await Monitoring.init();
     await initDatabase();
     await Config.init();
 

src/cdn/Server.ts

@@ -16,13 +16,14 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
+import path from "node:path";
+import morgan from "morgan";
 import { Server, ServerOptions } from "lambert-server";
 import { Attachment, Config, initDatabase, registerRoutes } from "@spacebar/util";
 import { CORS, BodyParser } from "@spacebar/api";
-import path from "node:path";
 import guildProfilesRoute from "./routes/guild-profiles";
-import morgan from "morgan";
 import { storage } from "./util";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 export type CDNServerOptions = ServerOptions;
 
@@ -34,6 +35,8 @@ export class CDNServer extends Server {
     }
 
     async start() {
+        await Monitoring.init();
+        Monitoring.attach(this.app);
         await initDatabase();
         await Config.init();
 

src/gateway/Server.ts

@@ -16,15 +16,14 @@
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import dotenv from "dotenv";
-dotenv.config({ quiet: true });
-import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util";
+import http from "node:http";
+import { setInterval } from "node:timers";
 import ws from "ws";
+import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util";
+import { randomString } from "@spacebar/api"; // TODO: move to util
 import { Connection, openConnections } from "./events/Connection";
-import http from "node:http";
 import { cleanupOnStartup } from "./util";
-import { randomString } from "@spacebar/api";
-import { setInterval } from "node:timers";
+import { Monitoring } from "../util/monitoring/Monitoring";
 
 export class Server {
     public ws: ws.Server;
@@ -67,7 +66,9 @@ export class Server {
                     res.setHeader("Set-Cookie", `__sb_sessid=${randomString(32)}; Secure; HttpOnly; SameSite=None; Path=/`);
                 }
                 const requestUrl = new URL(`http://${req.headers.host}${req.url}`);
-                if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") {
+                if (requestUrl.pathname === "/metrics") {
+                    return await Monitoring.handleRawRequest(req, res);
+                } else if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") {
                     if (!req.headers.authorization) {
                         return res.writeHead(401).end("Unauthorized");
                     } else {
@@ -167,6 +168,7 @@ export class Server {
     }
 
     async start(): Promise<void> {
+        await Monitoring.init();
         await initDatabase();
         await Config.init();
         await initEvent();

src/gateway/events/Connection.ts

@@ -29,18 +29,30 @@ import { Deflate, Inflate } from "fast-zlib";
 import { URL } from "node:url";
 import { Config } from "@spacebar/util";
 import { Decoder, Encoder } from "@toondepauw/node-zstd";
+import { Monitoring } from "../../util/monitoring/Monitoring";
+import { Gauge } from "prom-client";
 
 // TODO: check rate limit
 // TODO: specify rate limit in config
 // TODO: check msg max size
 
 export const openConnections: WebSocket[] = [];
 
+const openConnectionCount = Monitoring.attachMetric(
+    "spacebar_gateway_open_connection_count",
+    new Gauge({
+        name: "spacebar_gateway_open_connection_count",
+        help: "The total number of HTTP requests received",
+    }),
+);
+
 export async function Connection(this: WS.Server, socket: WebSocket, request: IncomingMessage) {
     openConnections.push(socket);
+    openConnectionCount.set(openConnections.length);
     socket.on("close", () => {
         const index = openConnections.indexOf(socket);
         if (index !== -1) openConnections.splice(index, 1);
+        openConnectionCount.set(openConnections.length);
     });
 
     const forwardedFor = Config.get().security.forwardedFor;

src/util/monitoring/Monitoring.ts

@@ -0,0 +1,85 @@
+/*
+	Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
+	Copyright (C) 2026 Spacebar and Spacebar Contributors
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published
+	by the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { IncomingMessage, ServerResponse } from "node:http";
+import * as client from "prom-client";
+import { Application, Router } from "express";
+import { Metric } from "prom-client";
+
+export class Monitoring {
+    static isInitialised = false;
+    public static async init() {
+        if (Monitoring.isInitialised) return;
+        console.log("[Monitoring] Initialising prometheus metrics");
+        client.collectDefaultMetrics({ prefix: "spacebar_" });
+        Monitoring.isInitialised = true;
+    }
+
+    public static attachMetric<T extends Metric>(name: string, metric: T): T {
+        const existingMetric = client.register.getSingleMetric(name);
+        // TODO: is there any way to *ensure* the metric is T? We're assuming that there's no conflicting definitions across the app...
+        if (existingMetric) return existingMetric as T;
+        client.register.registerMetric(metric);
+        return metric;
+    }
+
+    public static attach(app: Application) {
+        const http_request_total = new client.Counter({
+            name: "spacebar_http_request_total",
+            help: "The total number of HTTP requests received",
+            labelNames: ["path", "method", "status_code"],
+        });
+        client.register.registerMetric(http_request_total);
+
+        const http_response_rate_histogram = new client.Histogram({
+            name: "spacebar_http_duration",
+            labelNames: ["path", "method", "status_code"],
+            help: "The duration of HTTP requests in seconds",
+            buckets: [0.0, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 10],
+        });
+        client.register.registerMetric(http_response_rate_histogram);
+
+        app.use((req, res, next) => {
+            const endTimer = http_response_rate_histogram.startTimer();
+            res.on("finish", () => {
+                const path = (res.locals.lambertRouteBase ?? req.baseUrl ?? "") + req.route?.path;
+                if (!req.route?.path && req.method !== "OPTIONS") {
+                    console.log("[Monitoring] Request route path was undefined? Request path:", req.path, "Request route:", req.route);
+                }
+                endTimer({ method: req.method, path, status_code: res.statusCode });
+
+                // OPTIONS requests don't set path due to not being routed... discard unhandled ones
+                if (!path && req.method === "OPTIONS") return;
+
+                http_request_total.inc({ method: req.method, path, status_code: res.statusCode });
+            });
+            next();
+        });
+
+        app.get("/metrics", async (req, res) => {
+            res.setHeader("Content-Type", client.register.contentType);
+            const metrics = await client.register.metrics();
+            res.send(metrics);
+        });
+    }
+
+    static async handleRawRequest(req: IncomingMessage, res: ServerResponse) {
+        const metrics = await client.register.metrics();
+        res.setHeader("Content-Type", client.register.contentType).writeHead(200).end(metrics);
+    }
+}

src/util/util/lambert-server/Server.ts

@@ -54,7 +54,15 @@ export class Server {
             if (router.default) router = router.default;
             if (!router || router?.prototype?.constructor?.name !== "router") throw `File doesn't export any default router`;
 
-            this.app.use(path, <Router>router);
+            this.app.use(
+                path,
+                // TODO: I wish this middleware wasn't nessecary to preserve base path param names for monitoring...
+                (_, res, next) => {
+                    res.locals.lambertRouteBase = path;
+                    next();
+                },
+                <Router>router,
+            );
 
             if (this.options.serverInitLogging && process.env.LOG_ROUTES !== "false") console.log(`[Server] Route ${path} registered`);