feat: implement observability of notifications subsystem (#13799)

coderd/notifications/utils_test.go

@@ -3,9 +3,12 @@ package notifications_test
 import (
 	"context"
 	"database/sql"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
 	"cdr.dev/slog"
@@ -17,6 +20,9 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbgen"
 	"github.com/coder/coder/v2/coderd/database/dbmem"
 	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/notifications"
+	"github.com/coder/coder/v2/coderd/notifications/dispatch"
+	"github.com/coder/coder/v2/coderd/notifications/types"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
 )
@@ -57,13 +63,13 @@ func defaultNotificationsConfig(method database.NotificationMethod) codersdk.Not
 	return codersdk.NotificationsConfig{
 		Method:              serpent.String(method),
 		MaxSendAttempts:     5,
-		RetryInterval:       serpent.Duration(time.Minute * 5),
-		StoreSyncInterval:   serpent.Duration(time.Second * 2),
-		StoreSyncBufferSize: 50,
-		LeasePeriod:         serpent.Duration(time.Minute * 2),
+		FetchInterval:       serpent.Duration(time.Millisecond * 100),
+		StoreSyncInterval:   serpent.Duration(time.Millisecond * 200),
+		LeasePeriod:         serpent.Duration(time.Second * 10),
+		DispatchTimeout:     serpent.Duration(time.Second * 5),
+		RetryInterval:       serpent.Duration(time.Millisecond * 50),
 		LeaseCount:          10,
-		FetchInterval:       serpent.Duration(time.Second * 10),
-		DispatchTimeout:     serpent.Duration(time.Minute),
+		StoreSyncBufferSize: 50,
 		SMTP:                codersdk.NotificationsEmailConfig{},
 		Webhook:             codersdk.NotificationsWebhookConfig{},
 	}
@@ -81,3 +87,47 @@ func createSampleUser(t *testing.T, db database.Store) database.User {
 		Username: "bob",
 	})
 }
+
+func createMetrics() *notifications.Metrics {
+	return notifications.NewMetrics(prometheus.NewRegistry())
+}
+
+type dispatchInterceptor struct {
+	handler notifications.Handler
+
+	sent        atomic.Int32
+	retryable   atomic.Int32
+	unretryable atomic.Int32
+	err         atomic.Int32
+	lastErr     atomic.Value
+}
+
+func newDispatchInterceptor(h notifications.Handler) *dispatchInterceptor {
+	return &dispatchInterceptor{handler: h}
+}
+
+func (i *dispatchInterceptor) Dispatcher(payload types.MessagePayload, title, body string) (dispatch.DeliveryFunc, error) {
+	return func(ctx context.Context, msgID uuid.UUID) (retryable bool, err error) {
+		deliveryFn, err := i.handler.Dispatcher(payload, title, body)
+		if err != nil {
+			return false, err
+		}
+
+		retryable, err = deliveryFn(ctx, msgID)
+
+		if err != nil {
+			i.err.Add(1)
+			i.lastErr.Store(err)
+		}
+
+		switch {
+		case !retryable && err == nil:
+			i.sent.Add(1)
+		case retryable:
+			i.retryable.Add(1)
+		case !retryable && err != nil:
+			i.unretryable.Add(1)
+		}
+		return retryable, err
+	}, nil
+}