diff --git a/nix/tests/test-bundle-starts.nix b/nix/tests/test-bundle-starts.nix index 2e088f6de9..852b17180f 100644 --- a/nix/tests/test-bundle-starts.nix +++ b/nix/tests/test-bundle-starts.nix @@ -73,6 +73,8 @@ in }; }; + # https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests + # https://nixos.org/manual/nixpkgs/unstable/#tester-runNixOSTest testScript = '' machine.wait_for_unit("spacebar-api") machine.wait_for_unit("spacebar-cdn") @@ -82,7 +84,13 @@ in machine.wait_for_open_port(3001) machine.wait_for_open_port(3002) machine.wait_for_open_port(3003) - # If well known works, its probably fine(tm)? + + # this should be working machine.succeed("curl -f http://api.sb.localhost/.well-known/spacebar/client") + + # check if metrics endpoint works on all services + machine.succeed("curl -f http://api.sb.localhost/metrics") + machine.succeed("curl -f http://gateway.sb.localhost/metrics") + machine.succeed("curl -f http://cdn.sb.localhost/metrics") ''; } diff --git a/package-lock.json b/package-lock.json index 0cd867fdbe..4bf4c2fdd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "pg-query-stream": "^4.15.0", "picocolors": "^1.1.1", "probe-image-size": "^7.3.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", @@ -953,6 +954,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@peculiar/asn1-schema": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.7.0.tgz", @@ -2324,6 +2334,12 @@ "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", "license": "CC0-1.0" }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/bmp-ts": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/bmp-ts/-/bmp-ts-1.0.9.tgz", @@ -5875,6 +5891,19 @@ "stream-parser": "~0.3.1" } }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -6550,6 +6579,15 @@ "node": ">=8" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/thirty-two": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz", diff --git a/package.json b/package.json index 35108b7160..de215ed2fb 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,7 @@ "pg-query-stream": "^4.15.0", "picocolors": "^1.1.1", "probe-image-size": "^7.3.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tslib": "^2.8.1", "typeorm": "^0.3.30", diff --git a/src/api/Server.ts b/src/api/Server.ts index 1d9fe2ab22..4a8c2d6bfb 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -16,15 +16,16 @@ along with this program. If not, see . */ -import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util"; -import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares"; +import path from "node:path"; import { Request, Response, Router } from "express"; -import { Server, ServerOptions } from "lambert-server"; import morgan from "morgan"; -import path from "node:path"; +import { Server, ServerOptions } from "lambert-server"; import { red } from "picocolors"; +import { Config, ConnectionConfig, ConnectionLoader, Email, JSONReplacer, WebAuthn, initDatabase, initEvent, registerRoutes, getDatabase, getRevInfoOrFail } from "@spacebar/util"; +import { Authentication, CORS, ImageProxy, BodyParser, ErrorHandler, initRateLimits, initTranslation } from "./middlewares"; import { initInstance } from "./util/handlers/Instance"; import { route } from "./util"; +import { Monitoring } from "../util/monitoring/Monitoring"; const ASSETS_FOLDER = path.join(__dirname, "..", "..", "assets"); const PUBLIC_ASSETS_FOLDER = path.join(ASSETS_FOLDER, "public"); @@ -50,6 +51,8 @@ export class SpacebarServer extends Server { } async start() { + await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); await initEvent(); diff --git a/src/bundle/Server.ts b/src/bundle/Server.ts index 91c3dcf423..129d4b6351 100644 --- a/src/bundle/Server.ts +++ b/src/bundle/Server.ts @@ -16,21 +16,18 @@ along with this program. If not, see . */ -import morgan from "morgan"; - -process.on("unhandledRejection", console.error); -process.on("uncaughtException", console.error); - import http from "node:http"; +import fs from "node:fs"; +import cluster from "node:cluster"; +import morgan from "morgan"; +import express from "express"; +import { green, bold } from "picocolors"; import * as Api from "@spacebar/api"; import * as Gateway from "@spacebar/gateway"; import * as Webrtc from "@spacebar/webrtc"; import { CDNServer } from "@spacebar/cdn"; -import express from "express"; -import { green, bold } from "picocolors"; import { Config, initDatabase } from "@spacebar/util"; -import fs from "node:fs"; -import cluster from "node:cluster"; +import { Monitoring } from "../util/monitoring/Monitoring"; const app = express(); const server = http.createServer(); @@ -58,6 +55,7 @@ process.on("SIGTERM", async () => { }); async function main() { + await Monitoring.init(); await initDatabase(); await Config.init(); diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 8260407ec3..f74368fab5 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -16,13 +16,14 @@ along with this program. If not, see . */ +import path from "node:path"; +import morgan from "morgan"; import { Server, ServerOptions } from "lambert-server"; import { Attachment, Config, initDatabase, registerRoutes } from "@spacebar/util"; import { CORS, BodyParser } from "@spacebar/api"; -import path from "node:path"; import guildProfilesRoute from "./routes/guild-profiles"; -import morgan from "morgan"; import { storage } from "./util"; +import { Monitoring } from "../util/monitoring/Monitoring"; export type CDNServerOptions = ServerOptions; @@ -34,6 +35,8 @@ export class CDNServer extends Server { } async start() { + await Monitoring.init(); + Monitoring.attach(this.app); await initDatabase(); await Config.init(); diff --git a/src/gateway/Server.ts b/src/gateway/Server.ts index 632fbc5295..9986b427b4 100644 --- a/src/gateway/Server.ts +++ b/src/gateway/Server.ts @@ -16,15 +16,14 @@ along with this program. If not, see . */ -import dotenv from "dotenv"; -dotenv.config({ quiet: true }); -import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util"; +import http from "node:http"; +import { setInterval } from "node:timers"; import ws from "ws"; +import { checkToken, closeDatabase, Config, initDatabase, initEvent, Rights } from "@spacebar/util"; +import { randomString } from "@spacebar/api"; // TODO: move to util import { Connection, openConnections } from "./events/Connection"; -import http from "node:http"; import { cleanupOnStartup } from "./util"; -import { randomString } from "@spacebar/api"; -import { setInterval } from "node:timers"; +import { Monitoring } from "../util/monitoring/Monitoring"; export class Server { public ws: ws.Server; @@ -67,7 +66,9 @@ export class Server { res.setHeader("Set-Cookie", `__sb_sessid=${randomString(32)}; Secure; HttpOnly; SameSite=None; Path=/`); } const requestUrl = new URL(`http://${req.headers.host}${req.url}`); - if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") { + if (requestUrl.pathname === "/metrics") { + return await Monitoring.handleRawRequest(req, res); + } else if (requestUrl.pathname === "/_spacebar/gateway/admin/introspect") { if (!req.headers.authorization) { return res.writeHead(401).end("Unauthorized"); } else { @@ -167,6 +168,7 @@ export class Server { } async start(): Promise { + await Monitoring.init(); await initDatabase(); await Config.init(); await initEvent(); diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts index 5876c3e15e..acef2eb326 100644 --- a/src/gateway/events/Connection.ts +++ b/src/gateway/events/Connection.ts @@ -29,6 +29,8 @@ import { Deflate, Inflate } from "fast-zlib"; import { URL } from "node:url"; import { Config } from "@spacebar/util"; import { Decoder, Encoder } from "@toondepauw/node-zstd"; +import { Monitoring } from "../../util/monitoring/Monitoring"; +import { Gauge } from "prom-client"; // TODO: check rate limit // TODO: specify rate limit in config @@ -36,11 +38,21 @@ import { Decoder, Encoder } from "@toondepauw/node-zstd"; export const openConnections: WebSocket[] = []; +const openConnectionCount = Monitoring.attachMetric( + "spacebar_gateway_open_connection_count", + new Gauge({ + name: "spacebar_gateway_open_connection_count", + help: "The total number of HTTP requests received", + }), +); + export async function Connection(this: WS.Server, socket: WebSocket, request: IncomingMessage) { openConnections.push(socket); + openConnectionCount.set(openConnections.length); socket.on("close", () => { const index = openConnections.indexOf(socket); if (index !== -1) openConnections.splice(index, 1); + openConnectionCount.set(openConnections.length); }); const forwardedFor = Config.get().security.forwardedFor; diff --git a/src/util/monitoring/Monitoring.ts b/src/util/monitoring/Monitoring.ts new file mode 100644 index 0000000000..2479267428 --- /dev/null +++ b/src/util/monitoring/Monitoring.ts @@ -0,0 +1,85 @@ +/* + Spacebar: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2026 Spacebar and Spacebar Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +import { IncomingMessage, ServerResponse } from "node:http"; +import * as client from "prom-client"; +import { Application, Router } from "express"; +import { Metric } from "prom-client"; + +export class Monitoring { + static isInitialised = false; + public static async init() { + if (Monitoring.isInitialised) return; + console.log("[Monitoring] Initialising prometheus metrics"); + client.collectDefaultMetrics({ prefix: "spacebar_" }); + Monitoring.isInitialised = true; + } + + public static attachMetric(name: string, metric: T): T { + const existingMetric = client.register.getSingleMetric(name); + // TODO: is there any way to *ensure* the metric is T? We're assuming that there's no conflicting definitions across the app... + if (existingMetric) return existingMetric as T; + client.register.registerMetric(metric); + return metric; + } + + public static attach(app: Application) { + const http_request_total = new client.Counter({ + name: "spacebar_http_request_total", + help: "The total number of HTTP requests received", + labelNames: ["path", "method", "status_code"], + }); + client.register.registerMetric(http_request_total); + + const http_response_rate_histogram = new client.Histogram({ + name: "spacebar_http_duration", + labelNames: ["path", "method", "status_code"], + help: "The duration of HTTP requests in seconds", + buckets: [0.0, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 10], + }); + client.register.registerMetric(http_response_rate_histogram); + + app.use((req, res, next) => { + const endTimer = http_response_rate_histogram.startTimer(); + res.on("finish", () => { + const path = (res.locals.lambertRouteBase ?? req.baseUrl ?? "") + req.route?.path; + if (!req.route?.path && req.method !== "OPTIONS") { + console.log("[Monitoring] Request route path was undefined? Request path:", req.path, "Request route:", req.route); + } + endTimer({ method: req.method, path, status_code: res.statusCode }); + + // OPTIONS requests don't set path due to not being routed... discard unhandled ones + if (!path && req.method === "OPTIONS") return; + + http_request_total.inc({ method: req.method, path, status_code: res.statusCode }); + }); + next(); + }); + + app.get("/metrics", async (req, res) => { + res.setHeader("Content-Type", client.register.contentType); + const metrics = await client.register.metrics(); + res.send(metrics); + }); + } + + static async handleRawRequest(req: IncomingMessage, res: ServerResponse) { + const metrics = await client.register.metrics(); + res.setHeader("Content-Type", client.register.contentType).writeHead(200).end(metrics); + } +} diff --git a/src/util/util/lambert-server/Server.ts b/src/util/util/lambert-server/Server.ts index 617b279439..4a73f7c32e 100644 --- a/src/util/util/lambert-server/Server.ts +++ b/src/util/util/lambert-server/Server.ts @@ -54,7 +54,15 @@ export class Server { if (router.default) router = router.default; if (!router || router?.prototype?.constructor?.name !== "router") throw `File doesn't export any default router`; - this.app.use(path, router); + this.app.use( + path, + // TODO: I wish this middleware wasn't nessecary to preserve base path param names for monitoring... + (_, res, next) => { + res.locals.lambertRouteBase = path; + next(); + }, + router, + ); if (this.options.serverInitLogging && process.env.LOG_ROUTES !== "false") console.log(`[Server] Route ${path} registered`); diff --git a/src/webrtc/Server.ts b/src/webrtc/Server.ts index 0f8ead2ce6..f80c22b4f5 100644 --- a/src/webrtc/Server.ts +++ b/src/webrtc/Server.ts @@ -15,14 +15,14 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -import dotenv from "dotenv"; -dotenv.config({ quiet: true }); -import { closeDatabase, Config, initDatabase, initEvent, Session, TimeSpan } from "@spacebar/util"; + import http from "node:http"; import ws from "ws"; +import { green, yellow } from "picocolors"; +import { closeDatabase, Config, initDatabase, initEvent } from "@spacebar/util"; import { Connection } from "./events/Connection"; import { loadWebRtcLibrary, mediaServer, WRTC_PORT_MAX, WRTC_PORT_MIN, WRTC_PUBLIC_IP } from "./util"; -import { green, yellow } from "picocolors"; +import { Monitoring } from "../util/monitoring/Monitoring"; export class Server { public ws: ws.Server; @@ -36,8 +36,11 @@ export class Server { if (server) this.server = server; else { - this.server = http.createServer(function (req, res) { - res.writeHead(200).end("Online"); + this.server = http.createServer(async (req, res) => { + const requestUrl = new URL(`http://${req.headers.host}${req.url}`); + if (requestUrl.pathname === "/metrics") { + return await Monitoring.handleRawRequest(req, res); + } else res.writeHead(200).end("Online"); }); } @@ -59,6 +62,7 @@ export class Server { } async start(): Promise { + await Monitoring.init(); await initDatabase(); await Config.init(); await initEvent();