Capture, report and reraise panics in Listen

internal/sentry.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"context"
 	"github.com/getsentry/sentry-go"
+	"time"
 )
 
 // GetSentryHubFromContextOrDefault is a version of sentry.GetHubFromContext which
@@ -22,3 +23,18 @@ func GetSentryHubFromContextOrDefault(ctx context.Context) *sentry.Hub {
 	}
 	return hub
 }
+
+// ReportPanicsToSentry checks for panics by calling recover, reports any panic found to
+// sentry, and then reraises the panic. To have tracebacks included in the report, make
+// sure you call panic with an error, not a string.
+//
+// Typically, you want to call this in the form `defer internal.ReportPanicsToSentry()`.
+func ReportPanicsToSentry() {
+	panicData := recover()
+	if panicData != nil {
+		sentry.CurrentHub().Recover(panicData)
+		sentry.Flush(time.Second * 5)
+	}
+	// We still want to fail loudly here.
+	panic(panicData)
+}

sync2/handler2/handler.go

@@ -74,6 +74,7 @@ func NewHandler(
 // Listen starts all consumers
 func (h *Handler) Listen() {
 	go func() {
+		defer internal.ReportPanicsToSentry()
 		err := h.v3Sub.Listen()
 		if err != nil {
 			logger.Err(err).Msg("Failed to listen for v3 messages")

sync3/handler/handler.go

@@ -41,13 +41,13 @@ var logger = zerolog.New(os.Stdout).With().Timestamp().Logger().Output(zerolog.C
 // This is a net.http Handler for sync v3. It is responsible for pairing requests to Conns and to
 // ensure that the sync v2 poller is running for this client.
 type SyncLiveHandler struct {
-	V2         sync2.Client
-	Storage    *state.Storage
-	V2Store    *sync2.Storage
+	V2           sync2.Client
+	Storage      *state.Storage
+	V2Store      *sync2.Storage
 	V2Sub        *pubsub.V2Sub
 	EnsurePoller *EnsurePoller
 	ConnMap      *sync3.ConnMap
-	Extensions *extensions.Handler
+	Extensions   *extensions.Handler
 
 	// inserts are done by v2 poll loops, selects are done by v3 request threads
 	// but the v3 requests touch non-overlapping keys, which is a good use case for sync.Map
@@ -109,6 +109,7 @@ func (h *SyncLiveHandler) Startup(storeSnapshot *state.StartupSnapshot) error {
 // Listen starts all consumers
 func (h *SyncLiveHandler) Listen() {
 	go func() {
+		defer internal.ReportPanicsToSentry()
 		err := h.V2Sub.Listen()
 		if err != nil {
 			logger.Err(err).Msg("Failed to listen for v2 messages")