diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 47e45151d7..34e01dc1b2 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -579,7 +579,11 @@ func (d *Dispatcher) runAG(ag *aggrGroup) { go ag.run(func(ctx context.Context, alerts ...*alert.Alert) bool { _, _, err := d.stage.Exec(ctx, d.logger, alerts...) if err != nil { - logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "err", err) + logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "receiver", ag.opts.Receiver, "err", err) + var ie *notify.ErrorWithIntegration + if errors.As(err, &ie) { + logger = logger.With("integration", ie.Integration) + } if errors.Is(ctx.Err(), context.Canceled) { // It is expected for the context to be canceled on // configuration reload or shutdown. In this case, the diff --git a/notify/notify_test.go b/notify/notify_test.go index a62a15df73..cb6bc03c29 100644 --- a/notify/notify_test.go +++ b/notify/notify_test.go @@ -419,6 +419,8 @@ func TestRetryStageWithError(t *testing.T) { fail, retry := true, true sent := []*types.Alert{} i := Integration{ + name: "slack", + idx: 0, notifier: notifierFunc(func(ctx context.Context, alerts ...*types.Alert) (bool, error) { if fail { fail = false @@ -429,7 +431,7 @@ func TestRetryStageWithError(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "team-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*types.Alert{ { @@ -456,6 +458,12 @@ func TestRetryStageWithError(t *testing.T) { resctx, _, err = r.Exec(ctx, promslog.NewNopLogger(), alerts...) require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "team-receiver", ie.Receiver) + require.Equal(t, "slack[0]", ie.Integration) + require.Contains(t, err.Error(), "fail to deliver notification") } func TestRetryStageWithErrorCode(t *testing.T) { @@ -517,7 +525,7 @@ func TestRetryStageWithContextCanceled(t *testing.T) { }), rs: sendResolved(false), } - r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) + r := NewRetryStage(i, "canceled-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder()) alerts := []*types.Alert{ { @@ -538,6 +546,11 @@ func TestRetryStageWithContextCanceled(t *testing.T) { require.Error(t, err) require.NotNil(t, resctx) + + var ie *ErrorWithIntegration + require.ErrorAs(t, err, &ie) + require.Equal(t, "canceled-receiver", ie.Receiver) + require.Equal(t, "test[0]", ie.Integration) } func TestRetryStageNoResolved(t *testing.T) { diff --git a/notify/retry_stage.go b/notify/retry_stage.go index 1fe08cd385..c25b63c91d 100644 --- a/notify/retry_stage.go +++ b/notify/retry_stage.go @@ -140,7 +140,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A } if iErr != nil { - return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr) + return ctx, nil, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)) } return ctx, nil, nil default: @@ -157,7 +158,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A if err != nil { r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc() if !retry { - return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err) + return ctx, alerts, NewErrorWithIntegration(r.groupName, r.integration.String(), + fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err)) } if ctx.Err() == nil { if iErr == nil || err.Error() != iErr.Error() { diff --git a/notify/util.go b/notify/util.go index fe4c9ea508..4a1ae48762 100644 --- a/notify/util.go +++ b/notify/util.go @@ -281,6 +281,31 @@ func (e *ErrorWithReason) Error() string { return e.Err.Error() } +// ErrorWithIntegration annotates a notification failure with the receiver and +// integration that failed, so callers can attach them as structured log fields. +type ErrorWithIntegration struct { + Receiver string + Integration string + Err error +} + +// NewErrorWithIntegration returns an error annotated with receiver and integration. +func NewErrorWithIntegration(receiver, integration string, err error) *ErrorWithIntegration { + return &ErrorWithIntegration{ + Receiver: receiver, + Integration: integration, + Err: err, + } +} + +func (e *ErrorWithIntegration) Error() string { + return e.Err.Error() +} + +func (e *ErrorWithIntegration) Unwrap() error { + return e.Err +} + // Reason is the failure reason. type Reason int