diff --git a/.gitmodules b/.gitmodules index 5e3e9437a..43b90d09d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/Statsify/public-assets [submodule "assets/private"] path = assets/private - url = https://github.com/Statsify/assets \ No newline at end of file + url = https://github.com/Statsify/assets diff --git a/apps/api/package.json b/apps/api/package.json index 81d7ec0c9..a4ab97355 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -21,6 +21,7 @@ "@nestjs/platform-fastify": "^11.1.6", "@nestjs/swagger": "^11.2.0", "@sentry/node": "^7.118.0", + "@sentry/profiling-node": "^7.120.4", "@statsify/api-client": "workspace:^", "@statsify/assets": "workspace:^", "@statsify/logger": "workspace:^", diff --git a/apps/api/src/hypixel/hypixel.service.ts b/apps/api/src/hypixel/hypixel.service.ts index 426ad3e22..dbc8bcec0 100644 --- a/apps/api/src/hypixel/hypixel.service.ts +++ b/apps/api/src/hypixel/hypixel.service.ts @@ -18,8 +18,8 @@ import { } from "@statsify/schemas"; import { HttpService } from "@nestjs/axios"; import { Injectable } from "@nestjs/common"; -import { Logger } from "@statsify/logger"; -import { Observable, catchError, lastValueFrom, map, of, tap, throwError } from "rxjs"; +import { Logger, startSentrySpan } from "@statsify/logger"; +import { Observable, catchError, finalize, lastValueFrom, map, of, tap, throwError } from "rxjs"; import type { APIData } from "@statsify/util"; @Injectable() @@ -142,27 +142,31 @@ export class HypixelService { } private request(url: string, params?: Record): Observable { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const child = transaction?.startChild({ - op: "http.client", + const span = startSentrySpan({ + op: "hypixel.fetch", description: `GET ${this.httpService.axiosRef.getUri({ url })}`, + data: { + "hypixel.endpoint": url, + "http.method": "GET", + "http.route": url, + }, }); return this.httpService.get(url, { params }).pipe( tap((res) => { - child?.setHttpStatus(res.status); - child?.finish(); + if (span) Sentry.setHttpStatus(span, res.status); + span?.setAttribute("hypixel.ratelimit.limit", res.headers["ratelimit-limit"]); + span?.setAttribute("hypixel.ratelimit.remaining", res.headers["ratelimit-remaining"]); + span?.setAttribute("hypixel.ratelimit.reset", res.headers["ratelimit-reset"]); }), map((res) => res.data), - catchError((err) => - throwError( - () => - new Error(`Fetching ${url} failed with reason: ${err.message}`, { - cause: err, - }) - ) - ) + catchError((err) => throwError( + () => + new Error(`Fetching ${url} failed with reason: ${err.message}`, { + cause: err, + }) + )), + finalize(() => span?.end()) ); } } diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 648d33c1a..acbbe9ad9 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -14,12 +14,13 @@ import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger"; import { FastifyAdapter, NestFastifyApplication } from "@nestjs/platform-fastify"; import { Logger } from "@statsify/logger"; import { NestFactory } from "@nestjs/core"; -import { SentryInterceptor } from "./sentry/index.js"; +import { SentryInterceptor, instrumentMongooseQueries } from "./sentry/index.js"; import { Severity, setGlobalOptions } from "@typegoose/typegoose"; import { ValidationPipe } from "@nestjs/common"; import { config } from "@statsify/util"; import { join } from "node:path"; import { mkdir } from "node:fs/promises"; +import { nodeProfilingIntegration } from "@sentry/profiling-node"; const directory = import.meta.dirname; @@ -30,20 +31,29 @@ process.on("uncaughtException", handleError); process.on("unhandledRejection", handleError); const sentryDsn = await config("sentry.apiDsn", { required: false }); +const sentryTracesSampleRate = + await config("sentry.tracesSampleRate", { required: false }) ?? 0; +const sentryProfilesSampleRate = + await config("sentry.profilesSampleRate", { required: false }) ?? + sentryTracesSampleRate; if (sentryDsn) { Sentry.init({ dsn: sentryDsn, integrations: [ - new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true }), - new Sentry.Integrations.Mongo({ useMongoose: true }), + Sentry.httpIntegration({ spans: false, breadcrumbs: true }), + Sentry.mongoIntegration(), + nodeProfilingIntegration(), ], normalizeDepth: 3, - tracesSampleRate: await config("sentry.tracesSampleRate"), + tracesSampleRate: sentryTracesSampleRate, + profilesSampleRate: sentryProfilesSampleRate, environment: await config("environment"), }); } +instrumentMongooseQueries(); + const mediaRoot = await config("api.mediaRoot"); await mkdir(join(mediaRoot, "badges"), { recursive: true }); diff --git a/apps/api/src/leaderboards/leaderboard.service.ts b/apps/api/src/leaderboards/leaderboard.service.ts index 58a15af98..9616df573 100644 --- a/apps/api/src/leaderboards/leaderboard.service.ts +++ b/apps/api/src/leaderboards/leaderboard.service.ts @@ -6,7 +6,6 @@ * https://github.com/Statsify/statsify/blob/main/LICENSE */ -import * as Sentry from "@sentry/node"; import { Constructor, Flatten } from "@statsify/util"; import { DateTime } from "luxon"; import { InjectRedis } from "#redis"; @@ -14,6 +13,7 @@ import { Injectable, InternalServerErrorException } from "@nestjs/common"; import { type LeaderboardEnabledMetadata, getLeaderboardField, getLeaderboardFields } from "@statsify/schemas"; import { LeaderboardQuery } from "@statsify/api-client"; import { Redis } from "ioredis"; +import { withSentrySpan } from "@statsify/logger"; const DAYS_IN_WEEK = { monday: 0, @@ -38,13 +38,6 @@ export abstract class LeaderboardService { remove = false ) { const fields = getLeaderboardFields(constructor); - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const child = transaction?.startChild({ - op: "redis", - description: `add ${constructor.name} leaderboards`, - }); - const pipeline = this.redis.pipeline(); const name = constructor.name.toLowerCase(); @@ -67,9 +60,13 @@ export abstract class LeaderboardService { } } - await pipeline.exec(); - - child?.finish(); + await withSentrySpan({ + op: "redis.write", + description: `add ${constructor.name} leaderboards`, + data: { + "leaderboard.type": name, + }, + }, () => pipeline.exec()); } public async getLeaderboard( @@ -124,7 +121,8 @@ export abstract class LeaderboardService { field, top, bottom - 1, - sort + sort, + type ); const additionalFieldMetadata = additionalFields.map((k) => @@ -194,13 +192,6 @@ export abstract class LeaderboardService { fields: string[], id: string ) { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const child = transaction?.startChild({ - op: "redis", - description: `get ${constructor.name} rankings`, - }); - const pipeline = this.redis.pipeline(); const constructorName = constructor.name.toLowerCase(); @@ -221,9 +212,14 @@ export abstract class LeaderboardService { } } - const responses = await pipeline.exec(); - - child?.finish(); + const responses = await withSentrySpan({ + op: "redis.get", + description: `get ${constructor.name} rankings`, + data: { + "leaderboard.fields": fields, + "leaderboard.type": constructorName, + }, + }, () => pipeline.exec()); if (!responses) throw new InternalServerErrorException(); @@ -272,23 +268,27 @@ export abstract class LeaderboardService { field: string, top: number, bottom: number, - sort = "DESC" + sort = "DESC", + queryType?: LeaderboardQuery ) { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const child = transaction?.startChild({ - op: "redis", - description: `get ${constructor.name} leaderboards`, - }); - const name = constructor.name.toLowerCase(); - field = `${name}.${field}`; - - const scores = await (sort === "ASC" ? - this.redis.zrange(field, top, bottom, "WITHSCORES") : - this.redis.zrevrange(field, top, bottom, "WITHSCORES")); + const leaderboardField = field; + field = `${name}.${leaderboardField}`; - child?.finish(); + const scores = await withSentrySpan({ + op: "redis.get", + description: `get ${constructor.name} leaderboards`, + data: { + "leaderboard.field": leaderboardField, + "leaderboard.query_type": queryType, + "leaderboard.sort": sort, + "leaderboard.type": name, + }, + }, () => + sort === "ASC" ? + this.redis.zrange(field, top, bottom, "WITHSCORES") : + this.redis.zrevrange(field, top, bottom, "WITHSCORES") + ); const response: { id: string; score: number; index: number }[] = []; diff --git a/apps/api/src/sentry/index.ts b/apps/api/src/sentry/index.ts index 0bb0a391a..5343cd72e 100644 --- a/apps/api/src/sentry/index.ts +++ b/apps/api/src/sentry/index.ts @@ -7,3 +7,4 @@ */ export * from "./sentry.interceptor.js"; +export * from "./mongoose.js"; diff --git a/apps/api/src/sentry/mongoose.ts b/apps/api/src/sentry/mongoose.ts new file mode 100644 index 000000000..3c6a7baf3 --- /dev/null +++ b/apps/api/src/sentry/mongoose.ts @@ -0,0 +1,94 @@ +/** + * Copyright (c) Statsify + * + * This source code is licensed under the GNU GPL v3 license found in the + * LICENSE file in the root directory of this source tree. + * https://github.com/Statsify/statsify/blob/main/LICENSE + */ + +import { Aggregate, Query } from "mongoose"; +import { startSentrySpan } from "@statsify/logger"; + +let mongooseInstrumented = false; + +type InstrumentedQuery = Query & { + mongooseCollection?: { name?: string }; + op?: string; +}; + +type InstrumentedAggregate = Aggregate & { + _model?: { + collection?: { name?: string }; + modelName?: string; + }; +}; + +export function instrumentMongooseQueries() { + if (mongooseInstrumented) return; + mongooseInstrumented = true; + + instrumentQueryExec(); + instrumentAggregateExec(); +} + +function instrumentQueryExec() { + const exec = Query.prototype.exec; + + Query.prototype.exec = function instrumentedExec( + this: InstrumentedQuery, + ...args: Parameters + ): ReturnType { + const collection = this.mongooseCollection?.name ?? this.model.collection.name; + const operation = this.op ?? "query"; + const span = startSentrySpan({ + op: "mongo.query", + description: `${collection}.${operation}`, + data: { + "db.collection": collection, + "db.operation": operation, + "db.system": "mongodb", + "mongoose.model": this.model.modelName, + }, + }); + + try { + return exec.apply(this, args).finally(() => span?.end()) as ReturnType< + typeof exec + >; + } catch (error) { + span?.end(); + throw error; + } + }; +} + +function instrumentAggregateExec() { + const exec = Aggregate.prototype.exec; + + Aggregate.prototype.exec = function instrumentedExec( + this: InstrumentedAggregate, + ...args: Parameters + ): ReturnType { + const model = this["_model"]; + const collection = model?.collection?.name ?? "unknown"; + const span = startSentrySpan({ + op: "mongo.query", + description: `${collection}.aggregate`, + data: { + "db.collection": collection, + "db.operation": "aggregate", + "db.system": "mongodb", + "mongoose.model": model?.modelName ?? "unknown", + }, + }); + + try { + return exec.apply(this, args).finally(() => span?.end()) as ReturnType< + typeof exec + >; + } catch (error) { + span?.end(); + throw error; + } + }; +} diff --git a/apps/api/src/sentry/sentry.interceptor.ts b/apps/api/src/sentry/sentry.interceptor.ts index 76c868696..7de5e03c1 100644 --- a/apps/api/src/sentry/sentry.interceptor.ts +++ b/apps/api/src/sentry/sentry.interceptor.ts @@ -36,34 +36,39 @@ export class SentryInterceptor implements NestInterceptor { headers: req.headers, }); - let transaction: ReturnType | undefined; + let transaction: Sentry.Span | undefined; if (!url.pathname.includes("/skin")) { - transaction = Sentry.startTransaction({ + transaction = Sentry.startInactiveSpan({ op: "request", name: `${req.method} ${url.pathname}`, + forceTransaction: true, }); - - Sentry.configureScope((scope) => scope.setSpan(transaction)); } - return next.handle().pipe( + const response$ = next.handle().pipe( catchError((err) => { const isHttpException = err instanceof HttpException; const isInternalError = err instanceof InternalServerErrorException; if (isHttpException && !isInternalError) { - transaction?.finish(); + transaction?.end(); throw err; } Sentry.captureException(err); - transaction?.setHttpStatus(500); - transaction?.finish(); + if (transaction) Sentry.setHttpStatus(transaction, 500); + transaction?.end(); throw err; }), - tap(() => transaction?.finish()) + tap(() => transaction?.end()) ); + + return transaction ? + new Observable((subscriber) => + Sentry.withActiveSpan(transaction, () => response$.subscribe(subscriber)) + ) : + response$; } } diff --git a/apps/discord-bot/package.json b/apps/discord-bot/package.json index 7fb74f2e5..55a856b5d 100644 --- a/apps/discord-bot/package.json +++ b/apps/discord-bot/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@sentry/node": "^7.118.0", + "@sentry/profiling-node": "^7.120.4", "@statsify/api-client": "workspace:^", "@statsify/assets": "workspace:^", "@statsify/discord": "workspace:^", diff --git a/apps/discord-bot/src/commands/base.hypixel-command.ts b/apps/discord-bot/src/commands/base.hypixel-command.ts index 3d32a4f2e..6b3dd96c9 100644 --- a/apps/discord-bot/src/commands/base.hypixel-command.ts +++ b/apps/discord-bot/src/commands/base.hypixel-command.ts @@ -59,6 +59,7 @@ export interface BaseHypixelCommand { description: "", args: [PlayerArgument], cooldown: 10, + group: "hypixel", }) export abstract class BaseHypixelCommand { protected readonly apiService: ApiService; diff --git a/apps/discord-bot/src/commands/ratios/ratios.command.tsx b/apps/discord-bot/src/commands/ratios/ratios.command.tsx index 854c04db9..ce5a2c5e4 100644 --- a/apps/discord-bot/src/commands/ratios/ratios.command.tsx +++ b/apps/discord-bot/src/commands/ratios/ratios.command.tsx @@ -57,7 +57,7 @@ import { render } from "@statsify/rendering"; const args = [PlayerArgument]; -@Command({ description: (t) => t("commands.ratios") }) +@Command({ description: (t) => t("commands.ratios"), group: "hypixel" }) export class RatiosCommand { public constructor(private readonly apiService: ApiService) {} diff --git a/apps/discord-bot/src/index.ts b/apps/discord-bot/src/index.ts index 7bfbafbb6..2500730be 100644 --- a/apps/discord-bot/src/index.ts +++ b/apps/discord-bot/src/index.ts @@ -15,7 +15,8 @@ import { InteractionServer, RestClient, WebsocketShard } from "tiny-discord"; import { Logger } from "@statsify/logger"; import { VerifyCommand } from "#commands/verify.command"; import { config } from "@statsify/util"; -import { join } from "node:path"; +import { join } from "node:path"; +import { nodeProfilingIntegration } from "@sentry/profiling-node"; const directory = import.meta.dirname; @@ -26,13 +27,22 @@ process.on("uncaughtException", handleError); process.on("unhandledRejection", handleError); const sentryDsn = await config("sentry.discordBotDsn", { required: false }); +const sentryTracesSampleRate = + await config("sentry.tracesSampleRate", { required: false }) ?? 0; +const sentryProfilesSampleRate = + await config("sentry.profilesSampleRate", { required: false }) ?? + sentryTracesSampleRate; if (sentryDsn) { Sentry.init({ - dsn: sentryDsn, - integrations: [new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true })], + dsn: sentryDsn, + integrations: [ + Sentry.httpIntegration({ spans: false, breadcrumbs: true }), + nodeProfilingIntegration(), + ], normalizeDepth: 3, - tracesSampleRate: await config("sentry.tracesSampleRate"), + tracesSampleRate: sentryTracesSampleRate, + profilesSampleRate: sentryProfilesSampleRate, environment: await config("environment"), }); } diff --git a/apps/discord-bot/src/lib/command.listener.ts b/apps/discord-bot/src/lib/command.listener.ts index d42bbf4f6..aeb0ac2c3 100644 --- a/apps/discord-bot/src/lib/command.listener.ts +++ b/apps/discord-bot/src/lib/command.listener.ts @@ -63,48 +63,66 @@ export class CommandListener extends AbstractCommandListener { const [command, data, commandName] = this.getCommandAndData( parentCommand, parentData - ); - - const transaction = Sentry.startTransaction({ name: commandName, op: "command" }); - - Sentry.configureScope((scope) => scope.setSpan(transaction)); - - Sentry.setContext("command", { - command: commandName, - options: data.options, - guild: interaction.getGuildId() ?? null, - }); - - const user = await this.apiService.getUser(id); - - Sentry.setUser({ - id, - username: `${username}#${discriminator}`, - locale, - uuid: user?.uuid ?? null, - tier: user?.tier ?? UserTier.NONE, - serverMember: user?.serverMember ?? false, - theme: user?.theme ?? null, - }); - - const context = new CommandContext(this, interaction, data); - context.setUser(user); - - const preconditions = [ - this.tierPrecondition.bind(this, command, user), - this.cooldownPrecondition.bind(this, parentCommand, user, id), - ]; - - this.apiService.incrementCommand(commandName); - - return this.executeCommand({ - commandName, - command, - context, - preconditions, - message: this.getTipResponse(commandName, user), - }); - } + ); + + const [name, ...subcommandParts] = commandName.split(" "); + const group = parentCommand.group ?? "unknown"; + const subcommand = subcommandParts.join(" ") || undefined; + + const transaction = Sentry.startInactiveSpan({ + name: commandName, + op: "command.total", + forceTransaction: true, + attributes: { + "command.name": name, + "command.group": group, + "command.subcommand": subcommand, + "guild.id": interaction.getGuildId(), + }, + }); + + return Sentry.withActiveSpan(transaction, async () => { + Sentry.setContext("command", { + command: commandName, + group, + name, + subcommand: subcommand ?? null, + options: data.options, + guild: interaction.getGuildId() ?? null, + }); + + const user = await this.apiService.getUser(id); + + Sentry.setUser({ + id, + username: `${username}#${discriminator}`, + locale, + uuid: user?.uuid ?? null, + tier: user?.tier ?? UserTier.NONE, + serverMember: user?.serverMember ?? false, + theme: user?.theme ?? null, + }); + + const context = new CommandContext(this, interaction, data); + context.setUser(user); + + const preconditions = [ + this.tierPrecondition.bind(this, command, user), + this.cooldownPrecondition.bind(this, parentCommand, user, id), + ]; + + this.apiService.incrementCommand(commandName); + + return this.executeCommand({ + commandName, + command, + context, + observabilityGroup: group, + preconditions, + message: this.getTipResponse(commandName, user), + }); + }); + } private cooldownPrecondition( command: CommandResolvable, diff --git a/apps/support-bot/src/index.ts b/apps/support-bot/src/index.ts index 59a2f43e9..ad1196cfb 100644 --- a/apps/support-bot/src/index.ts +++ b/apps/support-bot/src/index.ts @@ -44,7 +44,7 @@ const sentryDsn = await config("sentry.supportBotDsn", { required: false }); if (sentryDsn) { Sentry.init({ dsn: sentryDsn, - integrations: [new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true })], + integrations: [Sentry.httpIntegration({ spans: false, breadcrumbs: true })], normalizeDepth: 3, tracesSampleRate: await config("sentry.tracesSampleRate"), environment: await config("environment"), diff --git a/apps/support-bot/src/lib/command.listener.ts b/apps/support-bot/src/lib/command.listener.ts index e1445a37f..170b5632a 100644 --- a/apps/support-bot/src/lib/command.listener.ts +++ b/apps/support-bot/src/lib/command.listener.ts @@ -59,13 +59,19 @@ export class CommandListener extends AbstractCommandListener { const user = await this.apiService.getUser(id); - const context = new CommandContext(this, interaction, data); - context.setUser(user); - - const preconditions = [this.tierPrecondition.bind(this, command, user)]; - - return this.executeCommand({ commandName, command, context, preconditions }); - } + const context = new CommandContext(this, interaction, data); + context.setUser(user); + + const preconditions = [this.tierPrecondition.bind(this, command, user)]; + + return this.executeCommand({ + commandName, + command, + context, + observabilityGroup: parentCommand.group ?? "unknown", + preconditions, + }); + } public static create( client: WebsocketShard, diff --git a/apps/verify-server/src/index.ts b/apps/verify-server/src/index.ts index 0ce9286a3..c9134a808 100644 --- a/apps/verify-server/src/index.ts +++ b/apps/verify-server/src/index.ts @@ -40,7 +40,7 @@ const sentryDsn = await config("sentry.verifyServerDsn", { required: false }); if (sentryDsn) { Sentry.init({ dsn: sentryDsn, - integrations: [new Sentry.Integrations.Mongo({ useMongoose: true })], + integrations: [Sentry.mongoIntegration()], normalizeDepth: 3, tracesSampleRate: await config("sentry.tracesSampleRate"), environment: await config("environment"), diff --git a/assets/private b/assets/private index 7bd3861f9..38a97e46e 160000 --- a/assets/private +++ b/assets/private @@ -1 +1 @@ -Subproject commit 7bd3861f93050e270dff5666aac33dd4081aa22f +Subproject commit 38a97e46ee32b939b0f617bfd66a8833c7c6e619 diff --git a/config.schema.js b/config.schema.js index 4317d8ae2..a2787face 100644 --- a/config.schema.js +++ b/config.schema.js @@ -68,6 +68,7 @@ export default { verifyServerDsn: "", supportBotDsn: "", tracesSampleRate: 1, + profilesSampleRate: 1, }, environment: "dev", }; diff --git a/packages/api-client/package.json b/packages/api-client/package.json index d911fa1fb..0cd661b49 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -14,6 +14,7 @@ "@nestjs/common": "^11.1.6", "@nestjs/swagger": "^11.2.0", "@sentry/node": "^7.118.0", + "@statsify/logger": "workspace:^", "@statsify/rendering": "workspace:^", "@statsify/schemas": "workspace:^", "@statsify/util": "workspace:^", diff --git a/packages/api-client/src/api.service.ts b/packages/api-client/src/api.service.ts index f43108367..52f72bca6 100644 --- a/packages/api-client/src/api.service.ts +++ b/packages/api-client/src/api.service.ts @@ -7,7 +7,13 @@ */ import * as Sentry from "@sentry/node"; -import axios, { AxiosInstance, AxiosRequestHeaders, Method, ResponseType } from "axios"; +import axios, { + AxiosInstance, + AxiosRequestHeaders, + AxiosResponse, + Method, + ResponseType, +} from "axios"; import { CacheLevel, GuildQuery, @@ -34,6 +40,7 @@ import { import { User, UserFooter, UserTheme } from "@statsify/schemas"; import { config } from "@statsify/util"; import { loadImage } from "@statsify/rendering"; +import { withSentrySpan } from "@statsify/logger"; interface ExtraData { headers?: AxiosRequestHeaders; @@ -41,6 +48,23 @@ interface ExtraData { responseType?: ResponseType; } +function getCacheHit(data: unknown): boolean | undefined { + if (!data || typeof data !== "object") return undefined; + + if ("cached" in data && typeof data.cached === "boolean") { + return data.cached; + } + + for (const value of Object.values(data)) { + if (value && typeof value === "object" && "cached" in value) { + const cached = (value as { cached?: unknown }).cached; + if (typeof cached === "boolean") return cached; + } + } + + return undefined; +} + // TODO: Move dtos in api to @statsify/api-client interface UpdateUser { serverMember?: boolean; @@ -288,25 +312,32 @@ export class ApiService { method: Method = "GET", { body, headers, responseType }: ExtraData = {} ): Promise { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const child = transaction?.startChild({ - op: "http.client", + const res = await withSentrySpan({ + op: "statsify.fetch", description: `${method} ${url}`, + data: { + "cache.level": typeof params?.cache === "string" ? params.cache : undefined, + "http.method": method, + "http.route": url, + }, + }, async (span) => { + const response: AxiosResponse = await this.axios.request({ + url, + method, + params, + headers, + data: body, + responseType, + }); + + if (span) Sentry.setHttpStatus(span, response.status); + + const cacheHit = getCacheHit(response.data); + if (cacheHit !== undefined) span?.setAttribute("cache.hit", cacheHit); + + return response; }); - const res = await this.axios.request({ - url, - method, - params, - headers, - data: body, - responseType, - }); - - child?.setHttpStatus(res.status); - child?.finish(); - const data = res.data; if (data.success === false) throw new Error("API request was unsuccessful"); diff --git a/packages/discord/src/command/abstract-command.listener.ts b/packages/discord/src/command/abstract-command.listener.ts index 90527fdb1..6562b424d 100644 --- a/packages/discord/src/command/abstract-command.listener.ts +++ b/packages/discord/src/command/abstract-command.listener.ts @@ -6,7 +6,6 @@ * https://github.com/Statsify/statsify/blob/main/LICENSE */ -import * as Sentry from "@sentry/node"; import { type APIUser, ApplicationCommandOptionType, @@ -17,7 +16,12 @@ import { CommandContext } from "./command.context.js"; import { ErrorMessage } from "#util/error.message"; import { type IMessage, Message } from "#messages"; import { Interaction, type InteractionAttachment } from "#interaction"; -import { Logger } from "@statsify/logger"; +import { + Logger, + getSentryTransaction, + setSentryMemoryUsage, + startSentrySpan, +} from "@statsify/logger"; import { User, UserTier } from "@statsify/schemas"; import { getAssetPath, getLogoPath } from "@statsify/assets"; import { readFileSync } from "node:fs"; @@ -38,6 +42,7 @@ export interface ExecuteCommandOptions { commandName: string; command: CommandResolvable; context: CommandContext; + observabilityGroup?: string; preconditions?: CommandPrecondition[]; message?: IMessage | Message; } @@ -126,10 +131,22 @@ export abstract class AbstractCommandListener { commandName, command, context, + observabilityGroup, preconditions = [], message, }: ExecuteCommandOptions) { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); + const transaction = getSentryTransaction(); + const commandSpan = startSentrySpan({ + op: "discord.command.execute", + description: commandName, + data: { + "command.name": command.name, + "command.group": observabilityGroup ?? command.group ?? "unknown", + "command.full_name": commandName, + "guild.id": context.getInteraction().getGuildId(), + "user.tier": context.getUser()?.tier ?? UserTier.NONE, + }, + }); try { for (const precondition of preconditions) { @@ -137,23 +154,37 @@ export abstract class AbstractCommandListener { } const response = await command.execute(context); + commandSpan?.end(); - if (typeof response !== "object") return; - transaction?.finish(); + if (typeof response !== "object") { + setSentryMemoryUsage(transaction); + transaction?.end(); + return; + } - context.reply({ + await context.reply({ ...message, ...response, }); + + setSentryMemoryUsage(transaction); + transaction?.end(); } catch (err) { if (err instanceof Message) { - transaction?.finish(); - return context.reply(err); + try { + await context.reply(err); + } finally { + setSentryMemoryUsage(transaction); + transaction?.end(); + } + + return; } this.logger.error(`An error occurred when running "${commandName}"`); this.logger.error(err); - transaction?.finish(); + setSentryMemoryUsage(transaction); + transaction?.end(); } } diff --git a/packages/discord/src/command/command.interface.ts b/packages/discord/src/command/command.interface.ts index 84ff30af1..da94e8e0c 100644 --- a/packages/discord/src/command/command.interface.ts +++ b/packages/discord/src/command/command.interface.ts @@ -27,6 +27,11 @@ export interface CommandOptions { cooldown?: number; + /** + * The product area this command belongs to. Used for observability. + */ + group?: string; + /** * The minimum user tier required to use this command. */ diff --git a/packages/discord/src/command/command.resolvable.ts b/packages/discord/src/command/command.resolvable.ts index 5279bae4c..f7b0431dc 100644 --- a/packages/discord/src/command/command.resolvable.ts +++ b/packages/discord/src/command/command.resolvable.ts @@ -33,6 +33,7 @@ export class CommandResolvable { public args: AbstractArgument[]; public cooldown: number; + public group?: string; public tier: UserTier; public preview?: string; @@ -49,6 +50,7 @@ export class CommandResolvable { methodName, tier = UserTier.NONE, preview, + group, cooldown = 10, }: CommandMetadata, target: any @@ -72,6 +74,7 @@ export class CommandResolvable { this.type = ApplicationCommandType.ChatInput; this.cooldown = cooldown; + this.group = group; const argsResolved = (args ?? [])?.map((a) => a instanceof AbstractArgument ? a : new a() diff --git a/packages/discord/src/interaction/interaction.ts b/packages/discord/src/interaction/interaction.ts index 6707683f1..6f5a84dcc 100644 --- a/packages/discord/src/interaction/interaction.ts +++ b/packages/discord/src/interaction/interaction.ts @@ -14,6 +14,7 @@ import { } from "discord-api-types/v10"; import { type IMessage, Message, getLocalizeFunction } from "#messages"; import { parseDiscordResponse } from "#util/parse-discord-error"; +import { withSentrySpan } from "@statsify/logger"; import type { InteractionServer, RestClient, @@ -154,7 +155,31 @@ export class Interaction { } private async request(options: RestClient.RequestOptions) { - const response = await this.rest.request(options); - return parseDiscordResponse(response); + const route = this.getRouteName(options.path); + + return withSentrySpan({ + op: "discord.reply", + description: `${options.method.toUpperCase()} ${route}`, + data: { + "http.method": options.method.toUpperCase(), + "http.route": route, + }, + }, async () => { + const response = await this.rest.request(options); + return parseDiscordResponse(response); + }); + } + + private getRouteName(path: string) { + return path + .replace( + /^\/interactions\/[^/]+\/[^/]+\/callback$/, + "/interactions/:interactionId/:interactionToken/callback" + ) + .replace( + /^\/webhooks\/[^/]+\/[^/]+/, + "/webhooks/:applicationId/:interactionToken" + ) + .replace(/\/messages\/[^/]+$/, "/messages/:messageId"); } } diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts index d2f715e65..03f049e9d 100644 --- a/packages/logger/src/index.ts +++ b/packages/logger/src/index.ts @@ -14,6 +14,24 @@ import type { ConsoleLoggerOptions, LogLevel, LoggerService } from "@nestjs/comm const DEFAULT_LOG_LEVELS: LogLevel[] = ["log", "error", "warn", "debug", "verbose", "fatal"]; +type SentryTagValue = boolean | number | string | null | undefined; + +type SentryLogLevel = Extract; +type SentrySpanAttributeValue = + | boolean + | number + | string + | Array + | Array + | Array; + +export interface SentrySpanOptions { + op: string; + description?: string; + data?: Record; + tags?: Record; +} + export const STATUS_COLORS = { debug: 0xC700E7, warn: 0xFAB627, @@ -34,6 +52,99 @@ const ColorByLogLevel: Record = { const isProduction = await config("environment") === "prod"; +export function getSentryTransaction() { + const activeSpan = Sentry.getActiveSpan(); + return activeSpan ? Sentry.getRootSpan(activeSpan) : undefined; +} + +function getSentrySpanAttribute(value: unknown): SentrySpanAttributeValue | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value === "boolean" || typeof value === "number" || typeof value === "string") + return value; + + if (Array.isArray(value)) { + return value.map((entry) => entry === null || entry === undefined ? entry : String(entry)); + } + + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +function getSentrySpanAttributes({ data, tags }: SentrySpanOptions) { + const attributes: Record = {}; + + for (const [key, value] of Object.entries(data ?? {})) { + attributes[key] = getSentrySpanAttribute(value); + } + + for (const [key, value] of Object.entries(tags ?? {})) { + attributes[key] = getSentrySpanAttribute(value); + } + + return attributes; +} + +function getSentrySpanOptions(options: SentrySpanOptions) { + return { + name: options.description ?? options.op, + op: options.op, + attributes: getSentrySpanAttributes(options), + }; +} + +export function startSentrySpan(options: SentrySpanOptions) { + const parentSpan = getSentryTransaction(); + + if (!parentSpan) return undefined; + + return Sentry.startInactiveSpan({ + ...getSentrySpanOptions(options), + parentSpan, + }); +} + +export async function withSentrySpan( + options: SentrySpanOptions, + callback: (span?: Sentry.Span) => Promise +): Promise { + if (!getSentryTransaction()) return callback(undefined); + + return Sentry.startSpan(getSentrySpanOptions(options), callback); +} + +export function withSentrySpanSync( + options: SentrySpanOptions, + callback: (span?: Sentry.Span) => T +): T { + if (!getSentryTransaction()) return callback(undefined); + + return Sentry.startSpan(getSentrySpanOptions(options), callback); +} + +export function setSentryMemoryUsage(span = getSentryTransaction()) { + if (!span) return; + + const { rss, heapUsed } = process.memoryUsage(); + + span.setAttribute("memory.rss.bytes", rss); + span.setAttribute("memory.heap_used.bytes", heapUsed); +} + +function stringifyMessage(message: unknown) { + if (message instanceof Error) return message.message; + if (typeof message === "string") return message; + if (typeof message !== "object" || message === null) return String(message); + + try { + return JSON.stringify(message); + } catch { + return String(message); + } +} + /** * A logger implementing the NestJS LoggerService interface. However can be used anywhere. * Outputs: {icon} {context} {time} {message} @@ -81,8 +192,7 @@ export class Logger implements LoggerService { let normalizedMessage = message; if (message instanceof Error) { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - transaction?.setStatus("internal_error"); + getSentryTransaction()?.setStatus({ code: 2, message: "internal_error" }); Sentry.captureException(message); normalizedMessage = message.stack; @@ -93,6 +203,9 @@ export class Logger implements LoggerService { ...optionalParameters, ]); + const sentryLog = this.getContextAndMessages([message, ...optionalParameters]); + + Logger.captureSentryLog(sentryLog.messages, sentryLog.context, "error"); Logger.printMessage(messages, context, "error", "stderr", "📉"); } @@ -108,6 +221,7 @@ export class Logger implements LoggerService { ...optionalParameters, ]); + Logger.captureSentryLog(messages, context, "warn"); Logger.printMessage(messages, context, "warn"); } @@ -151,8 +265,7 @@ export class Logger implements LoggerService { let normalizedMessage = message; if (message instanceof Error) { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - transaction?.setStatus("internal_error"); + getSentryTransaction()?.setStatus({ code: 2, message: "internal_error" }); Sentry.captureException(message); normalizedMessage = message.stack; @@ -163,6 +276,9 @@ export class Logger implements LoggerService { ...optionalParameters, ]); + const sentryLog = this.getContextAndMessages([message, ...optionalParameters]); + + Logger.captureSentryLog(sentryLog.messages, sentryLog.context, "fatal"); Logger.printMessage(messages, context, "fatal", "stderr", "📉"); } @@ -236,6 +352,27 @@ export class Logger implements LoggerService { process[writeStreamType].write(computedMessage); } } + + private static captureSentryLog( + messages: unknown[], + context: string | undefined, + logLevel: SentryLogLevel + ) { + if (!isProduction) return; + + for (const message of messages) { + Sentry.logger[logLevel](stringifyMessage(message), { + "logger.context": context ?? "Default", + "logger.level": logLevel, + ...(message instanceof Error ? + { + "error.name": message.name, + "error.message": message.message, + } : + {}), + }); + } + } } diff --git a/packages/rendering/package.json b/packages/rendering/package.json index 1a56658c8..737958049 100644 --- a/packages/rendering/package.json +++ b/packages/rendering/package.json @@ -22,6 +22,7 @@ "dependencies": { "@sentry/node": "^7.118.0", "@statsify/assets": "workspace:^", + "@statsify/logger": "workspace:^", "@statsify/util": "workspace:^", "@swc/helpers": "^0.5.23", "axios": "1.11.0", diff --git a/packages/rendering/src/canvas.ts b/packages/rendering/src/canvas.ts index 609cc0b68..51642248b 100644 --- a/packages/rendering/src/canvas.ts +++ b/packages/rendering/src/canvas.ts @@ -7,8 +7,61 @@ */ import { Canvas } from "skia-canvas"; +import { startSentrySpan } from "@statsify/logger"; type CanvasOptions = ConstructorParameters[2] & { gpu?: boolean }; +type CanvasToBuffer = typeof Canvas.prototype.toBuffer; + +let canvasToBufferInstrumented = false; + +function instrumentCanvasToBuffer() { + if (canvasToBufferInstrumented) return; + canvasToBufferInstrumented = true; + + const toBuffer = Canvas.prototype.toBuffer; + + Canvas.prototype.toBuffer = function instrumentedToBuffer( + this: Canvas, + ...args: Parameters + ): ReturnType { + const format = args[0]; + const isPng = format === undefined || format === "png"; + + if (!isPng) return toBuffer.apply(this, args) as ReturnType; + + const span = startSentrySpan({ + op: "png.encode", + description: "Encode canvas as PNG", + data: { + "render.width": this.width, + "render.height": this.height, + }, + }); + + let result: ReturnType; + + try { + result = toBuffer.apply(this, args) as ReturnType; + } catch (error) { + span?.end(); + throw error; + } + + if (!result || typeof (result as Promise).then !== "function") { + span?.end(); + return result; + } + + return (result as Promise) + .then((buffer) => { + span?.setAttribute("png.bytes", buffer.byteLength); + return buffer; + }) + .finally(() => span?.end()) as ReturnType; + }; +} + +instrumentCanvasToBuffer(); export function createCanvas( width?: number, diff --git a/packages/rendering/src/jsx/render.ts b/packages/rendering/src/jsx/render.ts index b1e3a1e1b..990d83e35 100644 --- a/packages/rendering/src/jsx/render.ts +++ b/packages/rendering/src/jsx/render.ts @@ -6,7 +6,6 @@ * https://github.com/Statsify/statsify/blob/main/LICENSE */ -import * as Sentry from "@sentry/node"; import type { Canvas, CanvasRenderingContext2D } from "skia-canvas"; import { Container } from "typedi"; import { FontRenderer } from "#font"; @@ -15,6 +14,7 @@ import { createCanvas } from "../canvas.js"; import { createInstructions } from "./create-instructions.js"; import { getPositionalDelta, getTotalSize } from "./util.js"; import { noop } from "@statsify/util"; +import { withSentrySpanSync } from "@statsify/logger"; import type { ComputedThemeContext, ElementNode, @@ -119,48 +119,44 @@ const renderRecursive = ( }; export function render(node: ElementNode, theme?: Theme): Canvas { - const transaction = Sentry.getCurrentHub().getScope()?.getTransaction(); - - const instructionsTransaction = transaction?.startChild({ - op: "jsx.createInstructions", + const instructions = withSentrySpanSync({ + op: "render.instructions", description: "Create instructions", - }); - - const instructions = createInstructions(node); - - instructionsTransaction?.finish(); + }, () => createInstructions(node)); const width = Math.round(getTotalSize(instructions.x)); const height = Math.round(getTotalSize(instructions.y)); - const renderTransaction = transaction?.startChild({ - op: "jsx.render", - description: "Render JSX", + return withSentrySpanSync({ + op: "render.generate", + description: "Generate render canvas", + data: { + "render.width": width, + "render.height": height, + }, + }, () => { + const canvas = createCanvas(width, height); + const ctx = canvas.getContext("2d"); + ctx.imageSmoothingEnabled = false; + + const context: ComputedThemeContext = { + renderer: noop(), + ...theme?.context, + canvasWidth: width, + canvasHeight: height, + }; + + if (!context.renderer) context.renderer = Container.get(FontRenderer); + + renderRecursive( + ctx, + context, + { ...intrinsicRenders, ...theme?.elements }, + instructions, + 0, + 0 + ); + + return canvas; }); - - const canvas = createCanvas(width, height); - const ctx = canvas.getContext("2d"); - ctx.imageSmoothingEnabled = false; - - const context: ComputedThemeContext = { - renderer: noop(), - ...theme?.context, - canvasWidth: width, - canvasHeight: height, - }; - - if (!context.renderer) context.renderer = Container.get(FontRenderer); - - renderRecursive( - ctx, - context, - { ...intrinsicRenders, ...theme?.elements }, - instructions, - 0, - 0 - ); - - renderTransaction?.finish(); - - return canvas; } diff --git a/packages/util/src/config.ts b/packages/util/src/config.ts index b3c71297e..f15a58255 100644 --- a/packages/util/src/config.ts +++ b/packages/util/src/config.ts @@ -228,6 +228,11 @@ export interface Config { * The percentage of transactions to send to Sentry */ tracesSampleRate?: number; + + /** + * The percentage of sampled transactions to profile with Sentry + */ + profilesSampleRate?: number; }; /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 183491aaa..e0ee130c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,6 +89,9 @@ importers: '@sentry/node': specifier: ^7.118.0 version: 7.120.4 + '@sentry/profiling-node': + specifier: ^7.120.4 + version: 7.120.4 '@statsify/api-client': specifier: workspace:^ version: link:../../packages/api-client @@ -162,6 +165,9 @@ importers: '@sentry/node': specifier: ^7.118.0 version: 7.120.4 + '@sentry/profiling-node': + specifier: ^7.120.4 + version: 7.120.4 '@statsify/api-client': specifier: workspace:^ version: link:../../packages/api-client @@ -469,6 +475,9 @@ importers: '@sentry/node': specifier: ^7.118.0 version: 7.120.4 + '@statsify/logger': + specifier: workspace:^ + version: link:../logger '@statsify/rendering': specifier: workspace:^ version: link:../rendering @@ -597,6 +606,9 @@ importers: '@statsify/assets': specifier: workspace:^ version: link:../assets + '@statsify/logger': + specifier: workspace:^ + version: link:../logger '@statsify/util': specifier: workspace:^ version: link:../util @@ -648,6 +660,8 @@ importers: packages/skin-renderer/pkg: {} + packages/skin-renderer/pkg: {} + packages/util: dependencies: '@swc/helpers': @@ -2690,6 +2704,11 @@ packages: resolution: {integrity: sha512-qq3wZAXXj2SRWhqErnGCSJKUhPSlZ+RGnCZjhfjHpP49KNpcd9YdPTIUsFMgeyjdh6Ew6aVCv23g1hTP0CHpYw==} engines: {node: '>=8'} + '@sentry/profiling-node@7.120.4': + resolution: {integrity: sha512-2Eb/LcYk7ohUx1KNnxcrN6hiyFTbD8Q9ffAvqtx09yJh1JhasvA+XCAcY72ONI5Aia4rCVkql9eEPSyhkmhsbA==} + engines: {node: '>=8.0.0'} + hasBin: true + '@sentry/types@7.120.4': resolution: {integrity: sha512-cUq2hSSe6/qrU6oZsEP4InMI5VVdD86aypE+ENrQ6eZEVLTCYm1w6XhW1NvIu3UuWh7gZec4a9J7AFpYxki88Q==} engines: {node: '>=8'} @@ -4745,6 +4764,10 @@ packages: sass: optional: true + node-abi@3.92.0: + resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} + engines: {node: '>=10'} + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -7463,6 +7486,11 @@ snapshots: '@sentry/types': 7.120.4 '@sentry/utils': 7.120.4 + '@sentry/profiling-node@7.120.4': + dependencies: + detect-libc: 2.1.2 + node-abi: 3.92.0 + '@sentry/types@7.120.4': {} '@sentry/utils@7.120.4': @@ -7784,9 +7812,9 @@ snapshots: fast-glob: 3.3.3 is-glob: 4.0.3 minimatch: 9.0.5 - semver: 7.7.2 - ts-api-utils: 2.1.0(typescript@6.0.3) - typescript: 6.0.3 + semver: 7.7.3 + ts-api-utils: 2.1.0(typescript@5.9.2) + typescript: 5.9.2 transitivePeerDependencies: - supports-color @@ -9014,6 +9042,21 @@ snapshots: dependencies: binary-extensions: 2.3.0 + is-boolean-object@1.2.2: + dependencies: + call-bound: 1.0.4 + has-tostringtag: 1.0.2 + + is-builtin-module@5.0.0: + dependencies: + builtin-modules: 5.0.0 + + is-bun-module@2.0.0: + dependencies: + semver: 7.7.3 + + is-callable@1.2.7: {} + is-class@0.0.9: {} is-core-module@2.16.1: @@ -9114,7 +9157,14 @@ snapshots: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 7.8.1 + semver: 7.7.3 + + jsx-ast-utils@3.3.5: + dependencies: + array-includes: 3.1.9 + array.prototype.flat: 1.3.3 + object.assign: 4.1.7 + object.values: 1.2.1 jwa@1.4.2: dependencies: @@ -9473,6 +9523,10 @@ snapshots: - '@babel/core' - babel-plugin-macros + node-abi@3.92.0: + dependencies: + semver: 7.7.3 + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 @@ -10029,7 +10083,9 @@ snapshots: semver-truncate@3.0.0: dependencies: - semver: 7.8.1 + semver: 7.7.3 + + semver@6.3.1: {} semver@7.5.4: dependencies: @@ -10037,8 +10093,7 @@ snapshots: semver@7.7.2: {} - semver@7.7.3: - optional: true + semver@7.7.3: {} semver@7.8.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 795b137c5..c9d7ac012 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,7 @@ packages: onlyBuiltDependencies: - '@nestjs/core' + - '@sentry-internal/node-cpu-profiler' - '@scarf/scarf' - '@swc/core' - esbuild