Add Sentry support (#754)

* Add sentry support

* Add basic listening for errors on express.

* Additional event types

* Add a default config

* changelog

* Improve wording

* Add docs

changelog.d/754.feature

@@ -0,0 +1 @@
+Add support for Sentry tracing.

config.sample.yml

@@ -157,6 +157,11 @@ widgets:
   publicUrl: https://example.com/widgetapi/v1/static/
   branding:
     widgetTitle: Hookshot Configuration
+sentry:
+  # (Optional) Configure Sentry error reporting
+
+  dsn: https://examplePublicKey@o0.ingest.sentry.io/0
+  environment: production
 permissions:
   # (Optional) Permissions for using the bridge. See docs/setup.md#permissions for help
 

docs/SUMMARY.md

@@ -17,6 +17,7 @@
     - [GitLab Project](./usage/room_configuration/gitlab_project.md)
     - [JIRA Project](./usage/room_configuration/jira_project.md)
 - [📊 Metrics](./metrics.md)
+- [Sentry](./sentry.md)
 
 # 🧑‍💻 Development
 - [Contributing](./contributing.md)

docs/_site/style.css

@@ -29,25 +29,30 @@
 /* icons for headers */
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(2) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/feeds.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/feeds.png');
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(3) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/figma.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/figma.png');
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(4) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/github.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/github.png');
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(5) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/gitlab.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/gitlab.png');
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(6) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/jira.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/jira.png');
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(7) strong:after {
-    content: ' ' url('/matrix-hookshot/latest/icons/webhooks.png')
+    content: ' ' url('/matrix-hookshot/latest/icons/webhooks.png');
 }
+
+.chapter li:nth-child(7) > a:nth-child(1) > strong:after {
+    content: ' ' url('/matrix-hookshot/latest/icons/sentry.png');
+}
+

docs/sentry.md

@@ -0,0 +1,14 @@
+Sentry
+======
+
+Hookshot supports [Sentry](https://sentry.io/welcome/) error reporting.
+
+You can configure Sentry by adding the following to your config:
+
+```yaml
+sentry:
+  dsn: https://examplePublicKey@o0.ingest.sentry.io/0 # The DSN for your Sentry project.
+  environment: production # The environment sentry is being used in. Can be omitted.
+```
+
+Sentry will automatically include the name of your homeserver as the `serverName` reported.

package.json

@@ -46,6 +46,7 @@
     "@octokit/auth-token": "^2.4.5",
     "@octokit/rest": "^18.10.0",
     "@octokit/webhooks": "^9.1.2",
+    "@sentry/node": "^7.52.1",
     "ajv": "^8.11.0",
     "axios": "^0.24.0",
     "cors": "^2.8.5",
@@ -87,10 +88,10 @@
     "@types/micromatch": "^4.0.1",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.0.0",
+    "@types/node": "18",
     "@types/node-emoji": "^1.8.1",
     "@types/uuid": "^8.3.3",
     "@types/xml2js": "^0.4.11",
-    "@types/node": "18",
     "@typescript-eslint/eslint-plugin": "^5.4.0",
     "@typescript-eslint/parser": "^5.4.0",
     "@uiw/react-codemirror": "^4.12.3",

src/App/BridgeApp.ts

@@ -5,10 +5,11 @@ import { Webhooks } from "../Webhooks";
 import { MatrixSender } from "../MatrixSender";
 import { UserNotificationWatcher } from "../Notifications/UserNotificationWatcher";
 import { ListenerService } from "../ListenerService";
-import { Logger } from "matrix-appservice-bridge";
+import { Logger, getBridgeVersion } from "matrix-appservice-bridge";
 import { LogService } from "matrix-bot-sdk";
 import { getAppservice } from "../appservice";
 import BotUsersManager from "../Managers/BotUsersManager";
+import * as Sentry from '@sentry/node';
 
 Logger.configure({console: "info"});
 const log = new Logger("App");
@@ -37,6 +38,17 @@ async function start() {
         userNotificationWatcher.start();
     }
 
+    if (config.sentry) {
+        Sentry.init({
+            dsn: config.sentry.dsn,
+            environment: config.sentry.environment,
+            release: getBridgeVersion(),
+            serverName: config.bridge.domain,
+            includeLocalVariables: true,
+        });
+        log.info("Sentry reporting enabled");
+    }
+
     const botUsersManager = new BotUsersManager(config, appservice);
 
     const bridgeApp = new Bridge(config, listener, appservice, storage, botUsersManager);

src/Bridge.ts

@@ -15,7 +15,7 @@ import { IGitLabWebhookIssueStateEvent, IGitLabWebhookMREvent, IGitLabWebhookNot
 import { JiraIssueEvent, JiraIssueUpdatedEvent, JiraVersionEvent } from "./jira/WebhookTypes";
 import { JiraOAuthResult } from "./jira/Types";
 import { MatrixEvent, MatrixMemberContent, MatrixMessageContent } from "./MatrixEvent";
-import { MessageQueue, createMessageQueue } from "./MessageQueue";
+import { MessageQueue, MessageQueueMessageOut, createMessageQueue } from "./MessageQueue";
 import { MessageSenderClient } from "./MatrixSender";
 import { NotifFilter, NotificationFilterStateContent } from "./NotificationFilters";
 import { NotificationProcessor } from "./NotificationsProcessor";
@@ -40,6 +40,8 @@ import { GenericWebhookEvent, GenericWebhookEventResult } from "./generic/types"
 import { SetupWidget } from "./Widgets/SetupWidget";
 import { FeedEntry, FeedError, FeedReader, FeedSuccess } from "./feeds/FeedReader";
 import PQueue from "p-queue";
+import * as Sentry from '@sentry/node';
+
 const log = new Logger("Bridge");
 
 export class Bridge {
@@ -784,17 +786,31 @@ export class Bridge {
         this.ready = true;
     }
 
+    private handleHookshotEvent<EventType, ConnType extends IConnection>(msg: MessageQueueMessageOut<EventType>, connection: ConnType, handler: (c: ConnType, data: EventType) => Promise<unknown>|unknown) {
+        Sentry.withScope((scope) => {
+            scope.setTransactionName('handleHookshotEvent');
+            scope.setTags({
+                eventType: msg.eventName,
+                roomId: connection.roomId,
+            });
+            scope.setContext("connection", {
+                id: connection.connectionId,
+            });
+            new Promise(() => handler(connection, msg.data)).catch((ex) => {
+                Sentry.captureException(ex, scope);
+                Metrics.connectionsEventFailed.inc({ event: msg.eventName, connectionId: connection.connectionId });
+                log.warn(`Connection ${connection.toString()} failed to handle ${msg.eventName}:`, ex);
+            });
+        });
+    }
+
     private async bindHandlerToQueue<EventType, ConnType extends IConnection>(event: string, connectionFetcher: (data: EventType) => ConnType[], handler: (c: ConnType, data: EventType) => Promise<unknown>|unknown) {
+        const connectionFetcherBound = connectionFetcher.bind(this);
         this.queue.on<EventType>(event, (msg) => {
-            const connections = connectionFetcher.bind(this)(msg.data);
+            const connections = connectionFetcherBound(msg.data);
             log.debug(`${event} for ${connections.map(c => c.toString()).join(', ') || '[empty]'}`);
-            connections.forEach(async (connection) => {
-                try {
-                    await handler(connection, msg.data);
-                } catch (ex) {
-                    Metrics.connectionsEventFailed.inc({ event, connectionId: connection.connectionId });
-                    log.warn(`Connection ${connection.toString()} failed to handle ${event}:`, ex);
-                }
+            connections.forEach((connection) => {
+                this.handleHookshotEvent(msg, connection, handler);
             })
         });
     }
@@ -906,12 +922,24 @@ export class Bridge {
         if (!adminRoom) {
             let handled = false;
             for (const connection of this.connectionManager.getAllConnectionsForRoom(roomId)) {
+                const scope = new Sentry.Scope();
+                scope.setTransactionName('onRoomMessage');
+                scope.setTags({
+                    eventId: event.event_id,
+                    sender: event.sender,
+                    eventType: event.type,
+                    roomId: connection.roomId,
+                });
+                scope.setContext("connection", {
+                    id: connection.connectionId,
+                });
                 try {
                     if (connection.onMessageEvent) {
                         handled = await connection.onMessageEvent(event, checkPermission, processedReplyMetadata);
                     }
                 } catch (ex) {
                     log.warn(`Connection ${connection.toString()} failed to handle message:`, ex);
+                    Sentry.captureException(ex, scope);
                 }
                 if (handled) {
                     break;
@@ -1078,6 +1106,17 @@ export class Bridge {
                 if (!this.connectionManager.verifyStateEventForConnection(connection, state, true)) {
                     continue;
                 }
+                const scope = new Sentry.Scope();
+                scope.setTransactionName('onStateUpdate');
+                scope.setTags({
+                    eventId: event.event_id,
+                    sender: event.sender,
+                    eventType: event.type,
+                    roomId: connection.roomId,
+                });
+                scope.setContext("connection", {
+                    id: connection.connectionId,
+                });
                 try {
                     // Empty object == redacted
                     if (event.content.disabled === true || Object.keys(event.content).length === 0) {
@@ -1128,11 +1167,24 @@ export class Bridge {
         }
 
         for (const connection of this.connectionManager.getAllConnectionsForRoom(roomId)) {
+            if (!connection.onEvent) { 
+                continue;
+            }
+            const scope = new Sentry.Scope();
+            scope.setTransactionName('onRoomEvent');
+            scope.setTags({
+                eventId: event.event_id,
+                sender: event.sender,
+                eventType: event.type,
+                roomId: connection.roomId,
+            });
+            scope.setContext("connection", {
+                id: connection.connectionId,
+            });
             try {
-                if (connection.onEvent) {
-                    await connection.onEvent(event);
-                }
+                await connection.onEvent(event);
             } catch (ex) {
+                Sentry.captureException(ex, scope);
                 log.warn(`Connection ${connection.toString()} failed to handle onEvent:`, ex);
             }
         }

src/ListenerService.ts

@@ -7,7 +7,7 @@ import { errorMiddleware } from "./api";
 // See https://github.com/turt2live/matrix-bot-sdk/issues/191
 export type ResourceName = "webhooks"|"widgets"|"metrics"|"provisioning";
 export const ResourceTypeArray: ResourceName[] = ["webhooks","widgets","metrics","provisioning"];
-
+import { Handlers } from "@sentry/node";
 export interface BridgeConfigListener {
     bindAddress?: string;
     port: number;
@@ -30,6 +30,7 @@ export class ListenerService {
         }
         for (const listenerConfig of config) {
             const app = expressApp();
+            app.use(Handlers.requestHandler());
             this.listeners.push({
                 config: listenerConfig,
                 app,
@@ -74,6 +75,8 @@ export class ListenerService {
             listener.app.get("/live", (_, res) => res.send({ok: true}));
             listener.app.get("/ready", (_, res) => res.status(listener.resourcesBound ? 200 : 500).send({ready: listener.resourcesBound}));
 
+            // By default, Sentry only reports 500+ errors, which is what we want.
+            listener.app.use(Handlers.errorHandler());
             // Always include the error handler
             listener.app.use((err: unknown, req: Request, res: Response, next: NextFunction) => errorMiddleware(log)(err, req, res, next));
             log.info(`Listening on http://${addr}:${listener.config.port} for ${listener.config.resources.join(', ')}`)

src/config/Config.ts

@@ -438,6 +438,11 @@ export interface BridgeConfigGoNebMigrator {
     goNebBotPrefix?: string;
 }
 
+export interface BridgeConfigSentry {
+    dsn: string;
+    environment?: string;
+}
+
 export interface BridgeConfigRoot {
     bot?: BridgeConfigBot;
     serviceBots?: BridgeConfigServiceBot[];
@@ -459,6 +464,7 @@ export interface BridgeConfigRoot {
     metrics?: BridgeConfigMetrics;
     listeners?: BridgeConfigListener[];
     goNebMigrator?: BridgeConfigGoNebMigrator;
+    sentry?: BridgeConfigSentry;
 }
 
 export class BridgeConfig {
@@ -514,6 +520,9 @@ export class BridgeConfig {
     @configKey("go-neb migrator configuration", true)
     public readonly goNebMigrator?: BridgeConfigGoNebMigrator;
 
+    @configKey("Configure Sentry error reporting", true)
+    public readonly sentry?: BridgeConfigSentry;
+
     @hideKey()
     private readonly bridgePermissions: BridgePermissions;
 
@@ -548,6 +557,7 @@ export class BridgeConfig {
         }
 
         this.widgets = configData.widgets && new BridgeWidgetConfig(configData.widgets);
+        this.sentry = configData.sentry;
 
         // To allow DEBUG as well as debug
         this.logging.level = this.logging.level.toLowerCase() as "debug"|"info"|"warn"|"error"|"trace";

src/config/Defaults.ts

@@ -145,7 +145,11 @@ export const DefaultConfigRoot: BridgeConfigRoot = {
             bindAddress: '0.0.0.0',
             resources: ['widgets'],
         }
-    ]
+    ],
+    sentry: {
+        dsn: "https://examplePublicKey@o0.ingest.sentry.io/0",
+        environment: "production"
+    }
 };
 
 export const DefaultConfig = new BridgeConfig(DefaultConfigRoot);

yarn.lock

@@ -1183,6 +1183,52 @@
     domhandler "^4.2.0"
     selderee "^0.6.0"
 
+"@sentry-internal/tracing@7.52.1":
+  version "7.52.1"
+  resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.52.1.tgz#c98823afd2f9814466fa26f24a1a54fe63b27c24"
+  integrity sha512-6N99rE+Ek0LgbqSzI/XpsKSLUyJjQ9nychViy+MP60p1x+hllukfTsDbNtUNrPlW0Bx+vqUrWKkAqmTFad94TQ==
+  dependencies:
+    "@sentry/core" "7.52.1"
+    "@sentry/types" "7.52.1"
+    "@sentry/utils" "7.52.1"
+    tslib "^1.9.3"
+
+"@sentry/core@7.52.1":
+  version "7.52.1"
+  resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.52.1.tgz#4de702937ba8944802bb06eb8dfdf089c39f6bab"
+  integrity sha512-36clugQu5z/9jrit1gzI7KfKbAUimjRab39JeR0mJ6pMuKLTTK7PhbpUAD4AQBs9qVeXN2c7h9SVZiSA0UDvkg==
+  dependencies:
+    "@sentry/types" "7.52.1"
+    "@sentry/utils" "7.52.1"
+    tslib "^1.9.3"
+
+"@sentry/node@^7.52.1":
+  version "7.52.1"
+  resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.52.1.tgz#3939bf47485461d990f6fe192d9fdced6ed95953"
+  integrity sha512-n3frjYbkY/+eZ5RTQMaipv6Hh9w3ia40GDeRK6KJQit7OLKLmXisD+FsdYzm8Jc784csSvb6HGGVgqLpO1p9Og==
+  dependencies:
+    "@sentry-internal/tracing" "7.52.1"
+    "@sentry/core" "7.52.1"
+    "@sentry/types" "7.52.1"
+    "@sentry/utils" "7.52.1"
+    cookie "^0.4.1"
+    https-proxy-agent "^5.0.0"
+    lru_map "^0.3.3"
+    tslib "^1.9.3"
+
+"@sentry/types@7.52.1":
+  version "7.52.1"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.52.1.tgz#bcff6d0462d9b9b7b9ec31c0068fe02d44f25da2"
+  integrity sha512-OMbGBPrJsw0iEXwZ2bJUYxewI1IEAU2e1aQGc0O6QW5+6hhCh+8HO8Xl4EymqwejjztuwStkl6G1qhK+Q0/Row==
+
+"@sentry/utils@7.52.1":
+  version "7.52.1"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.52.1.tgz#4a3e49b918f78dba4524c924286210259020cac5"
+  integrity sha512-MPt1Xu/jluulknW8CmZ2naJ53jEdtdwCBSo6fXJvOTI0SDqwIPbXDVrsnqLAhVJuIN7xbkj96nuY/VBR6S5sWg==
+  dependencies:
+    "@sentry/types" "7.52.1"
+    tslib "^1.9.3"
+
 "@trysound/sax@0.2.0":
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad"
@@ -2312,7 +2358,7 @@ cookie-signature@1.0.6:
   resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
   integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw=
 
-cookie@0.4.2:
+cookie@0.4.2, cookie@^0.4.1:
   version "0.4.2"
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432"
   integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==
@@ -3580,7 +3626,7 @@ http-status-codes@^2.2.0:
   resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.2.0.tgz#bb2efe63d941dfc2be18e15f703da525169622be"
   integrity sha512-feERVo9iWxvnejp3SEfm/+oNG517npqL2/PIA8ORjyOZjGC7TwCRQsZylciLS64i6pJ0wRYz3rkXLRwbtFa8Ng==
 
-https-proxy-agent@^5.0.1:
+https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6"
   integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==
@@ -4260,6 +4306,11 @@ lru-cache@^7.10.1:
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.13.1.tgz#267a81fbd0881327c46a81c5922606a2cfe336c4"
   integrity sha512-CHqbAq7NFlW3RSnoWXLJBxCWaZVBrfa9UEHId2M3AW8iEBurbqduNexEUCGc3SHc6iCYXNJCDi903LajSVAEPQ==
 
+lru_map@^0.3.3:
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
+  integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==
+
 magic-string@^0.25.7:
   version "0.25.9"
   resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.9.tgz#de7f9faf91ef8a1c91d02c2e5314c8277dbcdd1c"
@@ -5868,7 +5919,7 @@ ts-node@^10.9.1:
     v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
-tslib@^1.8.1:
+tslib@^1.8.1, tslib@^1.9.3:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==