diff --git a/changelog.d/754.feature b/changelog.d/754.feature new file mode 100644 index 00000000..f67b58d7 --- /dev/null +++ b/changelog.d/754.feature @@ -0,0 +1 @@ +Add support for Sentry tracing. diff --git a/config.sample.yml b/config.sample.yml index c2cab55b..281140e6 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -157,6 +157,11 @@ widgets: publicUrl: https://example.com/widgetapi/v1/static/ branding: widgetTitle: Hookshot Configuration +sentry: + # (Optional) Configure Sentry error reporting + + dsn: https://examplePublicKey@o0.ingest.sentry.io/0 + environment: production permissions: # (Optional) Permissions for using the bridge. See docs/setup.md#permissions for help diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ffd6b673..2fd7fad9 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,6 +17,7 @@ - [GitLab Project](./usage/room_configuration/gitlab_project.md) - [JIRA Project](./usage/room_configuration/jira_project.md) - [📊 Metrics](./metrics.md) +- [Sentry](./sentry.md) # 🧑‍💻 Development - [Contributing](./contributing.md) diff --git a/docs/_site/style.css b/docs/_site/style.css index fcea28b5..c63806ba 100644 --- a/docs/_site/style.css +++ b/docs/_site/style.css @@ -29,25 +29,30 @@ /* icons for headers */ .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(2) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/feeds.png') + content: ' ' url('/matrix-hookshot/latest/icons/feeds.png'); } .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(3) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/figma.png') + content: ' ' url('/matrix-hookshot/latest/icons/figma.png'); } .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(4) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/github.png') + content: ' ' url('/matrix-hookshot/latest/icons/github.png'); } .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(5) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/gitlab.png') + content: ' ' url('/matrix-hookshot/latest/icons/gitlab.png'); } .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(6) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/jira.png') + content: ' ' url('/matrix-hookshot/latest/icons/jira.png'); } .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(7) strong:after { - content: ' ' url('/matrix-hookshot/latest/icons/webhooks.png') + content: ' ' url('/matrix-hookshot/latest/icons/webhooks.png'); } + +.chapter li:nth-child(7) > a:nth-child(1) > strong:after { + content: ' ' url('/matrix-hookshot/latest/icons/sentry.png'); +} + diff --git a/docs/icons/sentry.png b/docs/icons/sentry.png new file mode 100644 index 00000000..423b60b7 Binary files /dev/null and b/docs/icons/sentry.png differ diff --git a/docs/sentry.md b/docs/sentry.md new file mode 100644 index 00000000..0dd23037 --- /dev/null +++ b/docs/sentry.md @@ -0,0 +1,14 @@ +Sentry +====== + +Hookshot supports [Sentry](https://sentry.io/welcome/) error reporting. + +You can configure Sentry by adding the following to your config: + +```yaml +sentry: + dsn: https://examplePublicKey@o0.ingest.sentry.io/0 # The DSN for your Sentry project. + environment: production # The environment sentry is being used in. Can be omitted. +``` + +Sentry will automatically include the name of your homeserver as the `serverName` reported. diff --git a/package.json b/package.json index af0de1f4..cbad3b45 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "@octokit/auth-token": "^2.4.5", "@octokit/rest": "^18.10.0", "@octokit/webhooks": "^9.1.2", + "@sentry/node": "^7.52.1", "ajv": "^8.11.0", "axios": "^0.24.0", "cors": "^2.8.5", @@ -87,10 +88,10 @@ "@types/micromatch": "^4.0.1", "@types/mime": "^2.0.3", "@types/mocha": "^9.0.0", + "@types/node": "18", "@types/node-emoji": "^1.8.1", "@types/uuid": "^8.3.3", "@types/xml2js": "^0.4.11", - "@types/node": "18", "@typescript-eslint/eslint-plugin": "^5.4.0", "@typescript-eslint/parser": "^5.4.0", "@uiw/react-codemirror": "^4.12.3", diff --git a/src/App/BridgeApp.ts b/src/App/BridgeApp.ts index d3df66e2..7c182b61 100644 --- a/src/App/BridgeApp.ts +++ b/src/App/BridgeApp.ts @@ -5,10 +5,11 @@ import { Webhooks } from "../Webhooks"; import { MatrixSender } from "../MatrixSender"; import { UserNotificationWatcher } from "../Notifications/UserNotificationWatcher"; import { ListenerService } from "../ListenerService"; -import { Logger } from "matrix-appservice-bridge"; +import { Logger, getBridgeVersion } from "matrix-appservice-bridge"; import { LogService } from "matrix-bot-sdk"; import { getAppservice } from "../appservice"; import BotUsersManager from "../Managers/BotUsersManager"; +import * as Sentry from '@sentry/node'; Logger.configure({console: "info"}); const log = new Logger("App"); @@ -37,6 +38,17 @@ async function start() { userNotificationWatcher.start(); } + if (config.sentry) { + Sentry.init({ + dsn: config.sentry.dsn, + environment: config.sentry.environment, + release: getBridgeVersion(), + serverName: config.bridge.domain, + includeLocalVariables: true, + }); + log.info("Sentry reporting enabled"); + } + const botUsersManager = new BotUsersManager(config, appservice); const bridgeApp = new Bridge(config, listener, appservice, storage, botUsersManager); diff --git a/src/Bridge.ts b/src/Bridge.ts index b7c22698..45e2e6c0 100644 --- a/src/Bridge.ts +++ b/src/Bridge.ts @@ -15,7 +15,7 @@ import { IGitLabWebhookIssueStateEvent, IGitLabWebhookMREvent, IGitLabWebhookNot import { JiraIssueEvent, JiraIssueUpdatedEvent, JiraVersionEvent } from "./jira/WebhookTypes"; import { JiraOAuthResult } from "./jira/Types"; import { MatrixEvent, MatrixMemberContent, MatrixMessageContent } from "./MatrixEvent"; -import { MessageQueue, createMessageQueue } from "./MessageQueue"; +import { MessageQueue, MessageQueueMessageOut, createMessageQueue } from "./MessageQueue"; import { MessageSenderClient } from "./MatrixSender"; import { NotifFilter, NotificationFilterStateContent } from "./NotificationFilters"; import { NotificationProcessor } from "./NotificationsProcessor"; @@ -40,6 +40,8 @@ import { GenericWebhookEvent, GenericWebhookEventResult } from "./generic/types" import { SetupWidget } from "./Widgets/SetupWidget"; import { FeedEntry, FeedError, FeedReader, FeedSuccess } from "./feeds/FeedReader"; import PQueue from "p-queue"; +import * as Sentry from '@sentry/node'; + const log = new Logger("Bridge"); export class Bridge { @@ -772,17 +774,31 @@ export class Bridge { this.ready = true; } + private handleHookshotEvent(msg: MessageQueueMessageOut, connection: ConnType, handler: (c: ConnType, data: EventType) => Promise|unknown) { + Sentry.withScope((scope) => { + scope.setTransactionName('handleHookshotEvent'); + scope.setTags({ + eventType: msg.eventName, + roomId: connection.roomId, + }); + scope.setContext("connection", { + id: connection.connectionId, + }); + new Promise(() => handler(connection, msg.data)).catch((ex) => { + Sentry.captureException(ex, scope); + Metrics.connectionsEventFailed.inc({ event: msg.eventName, connectionId: connection.connectionId }); + log.warn(`Connection ${connection.toString()} failed to handle ${msg.eventName}:`, ex); + }); + }); + } + private async bindHandlerToQueue(event: string, connectionFetcher: (data: EventType) => ConnType[], handler: (c: ConnType, data: EventType) => Promise|unknown) { + const connectionFetcherBound = connectionFetcher.bind(this); this.queue.on(event, (msg) => { - const connections = connectionFetcher.bind(this)(msg.data); + const connections = connectionFetcherBound(msg.data); log.debug(`${event} for ${connections.map(c => c.toString()).join(', ') || '[empty]'}`); - connections.forEach(async (connection) => { - try { - await handler(connection, msg.data); - } catch (ex) { - Metrics.connectionsEventFailed.inc({ event, connectionId: connection.connectionId }); - log.warn(`Connection ${connection.toString()} failed to handle ${event}:`, ex); - } + connections.forEach((connection) => { + this.handleHookshotEvent(msg, connection, handler); }) }); } @@ -894,12 +910,24 @@ export class Bridge { if (!adminRoom) { let handled = false; for (const connection of this.connectionManager.getAllConnectionsForRoom(roomId)) { + const scope = new Sentry.Scope(); + scope.setTransactionName('onRoomMessage'); + scope.setTags({ + eventId: event.event_id, + sender: event.sender, + eventType: event.type, + roomId: connection.roomId, + }); + scope.setContext("connection", { + id: connection.connectionId, + }); try { if (connection.onMessageEvent) { handled = await connection.onMessageEvent(event, checkPermission, processedReplyMetadata); } } catch (ex) { log.warn(`Connection ${connection.toString()} failed to handle message:`, ex); + Sentry.captureException(ex, scope); } if (handled) { break; @@ -1066,6 +1094,17 @@ export class Bridge { if (!this.connectionManager.verifyStateEventForConnection(connection, state, true)) { continue; } + const scope = new Sentry.Scope(); + scope.setTransactionName('onStateUpdate'); + scope.setTags({ + eventId: event.event_id, + sender: event.sender, + eventType: event.type, + roomId: connection.roomId, + }); + scope.setContext("connection", { + id: connection.connectionId, + }); try { // Empty object == redacted if (event.content.disabled === true || Object.keys(event.content).length === 0) { @@ -1116,11 +1155,24 @@ export class Bridge { } for (const connection of this.connectionManager.getAllConnectionsForRoom(roomId)) { + if (!connection.onEvent) { + continue; + } + const scope = new Sentry.Scope(); + scope.setTransactionName('onRoomEvent'); + scope.setTags({ + eventId: event.event_id, + sender: event.sender, + eventType: event.type, + roomId: connection.roomId, + }); + scope.setContext("connection", { + id: connection.connectionId, + }); try { - if (connection.onEvent) { - await connection.onEvent(event); - } + await connection.onEvent(event); } catch (ex) { + Sentry.captureException(ex, scope); log.warn(`Connection ${connection.toString()} failed to handle onEvent:`, ex); } } diff --git a/src/ListenerService.ts b/src/ListenerService.ts index 7a36529f..6733b26b 100644 --- a/src/ListenerService.ts +++ b/src/ListenerService.ts @@ -7,7 +7,7 @@ import { errorMiddleware } from "./api"; // See https://github.com/turt2live/matrix-bot-sdk/issues/191 export type ResourceName = "webhooks"|"widgets"|"metrics"|"provisioning"; export const ResourceTypeArray: ResourceName[] = ["webhooks","widgets","metrics","provisioning"]; - +import { Handlers } from "@sentry/node"; export interface BridgeConfigListener { bindAddress?: string; port: number; @@ -30,6 +30,7 @@ export class ListenerService { } for (const listenerConfig of config) { const app = expressApp(); + app.use(Handlers.requestHandler()); this.listeners.push({ config: listenerConfig, app, @@ -74,6 +75,8 @@ export class ListenerService { listener.app.get("/live", (_, res) => res.send({ok: true})); listener.app.get("/ready", (_, res) => res.status(listener.resourcesBound ? 200 : 500).send({ready: listener.resourcesBound})); + // By default, Sentry only reports 500+ errors, which is what we want. + listener.app.use(Handlers.errorHandler()); // Always include the error handler listener.app.use((err: unknown, req: Request, res: Response, next: NextFunction) => errorMiddleware(log)(err, req, res, next)); log.info(`Listening on http://${addr}:${listener.config.port} for ${listener.config.resources.join(', ')}`) diff --git a/src/config/Config.ts b/src/config/Config.ts index c2510997..bd106aff 100644 --- a/src/config/Config.ts +++ b/src/config/Config.ts @@ -438,6 +438,11 @@ export interface BridgeConfigGoNebMigrator { goNebBotPrefix?: string; } +export interface BridgeConfigSentry { + dsn: string; + environment?: string; +} + export interface BridgeConfigRoot { bot?: BridgeConfigBot; serviceBots?: BridgeConfigServiceBot[]; @@ -459,6 +464,7 @@ export interface BridgeConfigRoot { metrics?: BridgeConfigMetrics; listeners?: BridgeConfigListener[]; goNebMigrator?: BridgeConfigGoNebMigrator; + sentry?: BridgeConfigSentry; } export class BridgeConfig { @@ -514,6 +520,9 @@ export class BridgeConfig { @configKey("go-neb migrator configuration", true) public readonly goNebMigrator?: BridgeConfigGoNebMigrator; + @configKey("Configure Sentry error reporting", true) + public readonly sentry?: BridgeConfigSentry; + @hideKey() private readonly bridgePermissions: BridgePermissions; @@ -548,6 +557,7 @@ export class BridgeConfig { } this.widgets = configData.widgets && new BridgeWidgetConfig(configData.widgets); + this.sentry = configData.sentry; // To allow DEBUG as well as debug this.logging.level = this.logging.level.toLowerCase() as "debug"|"info"|"warn"|"error"|"trace"; diff --git a/src/config/Defaults.ts b/src/config/Defaults.ts index 6ad266ac..09a3d2ae 100644 --- a/src/config/Defaults.ts +++ b/src/config/Defaults.ts @@ -145,7 +145,11 @@ export const DefaultConfigRoot: BridgeConfigRoot = { bindAddress: '0.0.0.0', resources: ['widgets'], } - ] + ], + sentry: { + dsn: "https://examplePublicKey@o0.ingest.sentry.io/0", + environment: "production" + } }; export const DefaultConfig = new BridgeConfig(DefaultConfigRoot); diff --git a/yarn.lock b/yarn.lock index 57f6ab5c..4bffb679 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1183,6 +1183,52 @@ domhandler "^4.2.0" selderee "^0.6.0" +"@sentry-internal/tracing@7.52.1": + version "7.52.1" + resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.52.1.tgz#c98823afd2f9814466fa26f24a1a54fe63b27c24" + integrity sha512-6N99rE+Ek0LgbqSzI/XpsKSLUyJjQ9nychViy+MP60p1x+hllukfTsDbNtUNrPlW0Bx+vqUrWKkAqmTFad94TQ== + dependencies: + "@sentry/core" "7.52.1" + "@sentry/types" "7.52.1" + "@sentry/utils" "7.52.1" + tslib "^1.9.3" + +"@sentry/core@7.52.1": + version "7.52.1" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.52.1.tgz#4de702937ba8944802bb06eb8dfdf089c39f6bab" + integrity sha512-36clugQu5z/9jrit1gzI7KfKbAUimjRab39JeR0mJ6pMuKLTTK7PhbpUAD4AQBs9qVeXN2c7h9SVZiSA0UDvkg== + dependencies: + "@sentry/types" "7.52.1" + "@sentry/utils" "7.52.1" + tslib "^1.9.3" + +"@sentry/node@^7.52.1": + version "7.52.1" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.52.1.tgz#3939bf47485461d990f6fe192d9fdced6ed95953" + integrity sha512-n3frjYbkY/+eZ5RTQMaipv6Hh9w3ia40GDeRK6KJQit7OLKLmXisD+FsdYzm8Jc784csSvb6HGGVgqLpO1p9Og== + dependencies: + "@sentry-internal/tracing" "7.52.1" + "@sentry/core" "7.52.1" + "@sentry/types" "7.52.1" + "@sentry/utils" "7.52.1" + cookie "^0.4.1" + https-proxy-agent "^5.0.0" + lru_map "^0.3.3" + tslib "^1.9.3" + +"@sentry/types@7.52.1": + version "7.52.1" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.52.1.tgz#bcff6d0462d9b9b7b9ec31c0068fe02d44f25da2" + integrity sha512-OMbGBPrJsw0iEXwZ2bJUYxewI1IEAU2e1aQGc0O6QW5+6hhCh+8HO8Xl4EymqwejjztuwStkl6G1qhK+Q0/Row== + +"@sentry/utils@7.52.1": + version "7.52.1" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.52.1.tgz#4a3e49b918f78dba4524c924286210259020cac5" + integrity sha512-MPt1Xu/jluulknW8CmZ2naJ53jEdtdwCBSo6fXJvOTI0SDqwIPbXDVrsnqLAhVJuIN7xbkj96nuY/VBR6S5sWg== + dependencies: + "@sentry/types" "7.52.1" + tslib "^1.9.3" + "@trysound/sax@0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" @@ -2312,7 +2358,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= -cookie@0.4.2: +cookie@0.4.2, cookie@^0.4.1: version "0.4.2" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== @@ -3580,7 +3626,7 @@ http-status-codes@^2.2.0: resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.2.0.tgz#bb2efe63d941dfc2be18e15f703da525169622be" integrity sha512-feERVo9iWxvnejp3SEfm/+oNG517npqL2/PIA8ORjyOZjGC7TwCRQsZylciLS64i6pJ0wRYz3rkXLRwbtFa8Ng== -https-proxy-agent@^5.0.1: +https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== @@ -4260,6 +4306,11 @@ lru-cache@^7.10.1: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.13.1.tgz#267a81fbd0881327c46a81c5922606a2cfe336c4" integrity sha512-CHqbAq7NFlW3RSnoWXLJBxCWaZVBrfa9UEHId2M3AW8iEBurbqduNexEUCGc3SHc6iCYXNJCDi903LajSVAEPQ== +lru_map@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" + integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ== + magic-string@^0.25.7: version "0.25.9" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.9.tgz#de7f9faf91ef8a1c91d02c2e5314c8277dbcdd1c" @@ -5868,7 +5919,7 @@ ts-node@^10.9.1: v8-compile-cache-lib "^3.0.1" yn "3.1.1" -tslib@^1.8.1: +tslib@^1.8.1, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==