Skip to content

notify: add receiver and integration as structured log fields on failures #5401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
locker95 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

notify: add receiver and integration as structured log fields on failures #5401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion dispatch/dispatch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,11 @@ func (d *Dispatcher) runAG(ag *aggrGroup) {
go ag.run(func(ctx context.Context, alerts ...*alert.Alert) bool {
_, _, err := d.stage.Exec(ctx, d.logger, alerts...)
if err != nil {
logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "err", err)
logger := d.logger.With("aggrGroup", ag.GroupKey(), "num_alerts", len(alerts), "receiver", ag.opts.Receiver, "err", err)
var ie *notify.ErrorWithIntegration
if errors.As(err, &ie) {
logger = logger.With("integration", ie.Integration)
}
if errors.Is(ctx.Err(), context.Canceled) {
// It is expected for the context to be canceled on
// configuration reload or shutdown. In this case, the
Expand Down
17 changes: 15 additions & 2 deletions notify/notify_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,8 @@ func TestRetryStageWithError(t *testing.T) {
fail, retry := true, true
sent := []*types.Alert{}
i := Integration{
name: "slack",
idx: 0,
notifier: notifierFunc(func(ctx context.Context, alerts ...*types.Alert) (bool, error) {
if fail {
fail = false
Expand All @@ -429,7 +431,7 @@ func TestRetryStageWithError(t *testing.T) {
}),
rs: sendResolved(false),
}
r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder())
r := NewRetryStage(i, "team-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder())

alerts := []*types.Alert{
{
Expand All @@ -456,6 +458,12 @@ func TestRetryStageWithError(t *testing.T) {
resctx, _, err = r.Exec(ctx, promslog.NewNopLogger(), alerts...)
require.Error(t, err)
require.NotNil(t, resctx)

var ie *ErrorWithIntegration
require.ErrorAs(t, err, &ie)
require.Equal(t, "team-receiver", ie.Receiver)
require.Equal(t, "slack[0]", ie.Integration)
require.Contains(t, err.Error(), "fail to deliver notification")
}

func TestRetryStageWithErrorCode(t *testing.T) {
Expand Down Expand Up @@ -517,7 +525,7 @@ func TestRetryStageWithContextCanceled(t *testing.T) {
}),
rs: sendResolved(false),
}
r := NewRetryStage(i, "", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder())
r := NewRetryStage(i, "canceled-receiver", NewMetrics(prometheus.NewRegistry(), featurecontrol.NoopFlags{}), eventrecorder.NopRecorder())

alerts := []*types.Alert{
{
Expand All @@ -538,6 +546,11 @@ func TestRetryStageWithContextCanceled(t *testing.T) {

require.Error(t, err)
require.NotNil(t, resctx)

var ie *ErrorWithIntegration
require.ErrorAs(t, err, &ie)
require.Equal(t, "canceled-receiver", ie.Receiver)
require.Equal(t, "test[0]", ie.Integration)
}

func TestRetryStageNoResolved(t *testing.T) {
Expand Down
6 changes: 4 additions & 2 deletions notify/retry_stage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A
}

if iErr != nil {
return ctx, nil, fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr)
return ctx, nil, NewErrorWithIntegration(r.groupName, r.integration.String(),
fmt.Errorf("%s/%s: notify retry canceled after %d attempts: %w", r.groupName, r.integration.String(), i, iErr))
}
return ctx, nil, nil
default:
Expand All @@ -157,7 +158,8 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*alert.A
if err != nil {
r.metrics.numNotificationRequestsFailedTotal.WithLabelValues(r.labelValues...).Inc()
if !retry {
return ctx, alerts, fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err)
return ctx, alerts, NewErrorWithIntegration(r.groupName, r.integration.String(),
fmt.Errorf("%s/%s: notify retry canceled due to unrecoverable error after %d attempts: %w", r.groupName, r.integration.String(), i, err))
}
if ctx.Err() == nil {
if iErr == nil || err.Error() != iErr.Error() {
Expand Down
25 changes: 25 additions & 0 deletions notify/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,31 @@ func (e *ErrorWithReason) Error() string {
return e.Err.Error()
}

// ErrorWithIntegration annotates a notification failure with the receiver and
// integration that failed, so callers can attach them as structured log fields.
type ErrorWithIntegration struct {
Receiver string
Integration string
Err error
}

// NewErrorWithIntegration returns an error annotated with receiver and integration.
func NewErrorWithIntegration(receiver, integration string, err error) *ErrorWithIntegration {
return &ErrorWithIntegration{
Receiver: receiver,
Integration: integration,
Err: err,
}
}

func (e *ErrorWithIntegration) Error() string {
return e.Err.Error()
}

func (e *ErrorWithIntegration) Unwrap() error {
return e.Err
}

// Reason is the failure reason.
type Reason int

Expand Down