Add timeout option for telegram notifier

docs/configuration.md

@@ -1900,6 +1900,12 @@ attributes:
 
 # The HTTP client's configuration.
 [ http_config: <http_config> | default = global.http_config ]
+
+# The maximum time to wait for a telegram request to complete, before failing the
+# request and allowing it to be retried. The default value of 0s indicates that
+# no timeout should be applied.
+# NOTE: This will have no effect if set higher than the group_interval.
+[ timeout: <duration> | default = 0s ]
 ```
 
 ### `<victorops_config>`

notify/telegram/config.go

@@ -15,6 +15,7 @@ package telegram
 
 import (
 	"errors"
+	"time"
 
 	commoncfg "github.com/prometheus/common/config"
 
@@ -45,6 +46,10 @@ type TelegramConfig struct {
 	Message              string           `yaml:"message,omitempty" json:"message,omitempty"`
 	DisableNotifications bool             `yaml:"disable_notifications,omitempty" json:"disable_notifications,omitempty"`
 	ParseMode            string           `yaml:"parse_mode,omitempty" json:"parse_mode,omitempty"`
+
+	// Timeout is the maximum time allowed to invoke the telegram. Setting this to 0
+	// does not impose a timeout.
+	Timeout time.Duration `yaml:"timeout" json:"timeout"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

notify/telegram/telegram.go

@@ -50,6 +50,10 @@ func New(conf *TelegramConfig, t *template.Template, l *slog.Logger, httpOpts ..
 		return nil, err
 	}
 
+	if conf.Timeout > 0 {
+		httpclient.Timeout = conf.Timeout
+	}
+
 	client, err := createTelegramClient(conf.APIUrl.String(), conf.ParseMode, httpclient)
 	if err != nil {
 		return nil, err
@@ -119,6 +123,9 @@ func (n *Notifier) Notify(ctx context.Context, alert ...*types.Alert) (bool, err
 		ParseMode:             n.conf.ParseMode,
 	})
 	if err != nil {
+		if n.conf.Timeout > 0 && errors.Is(err, context.DeadlineExceeded) {
+			err = fmt.Errorf("configured telegram timeout reached (%s)", n.conf.Timeout)
+		}
 		return true, wrapWithFailureReason(notify.RedactURL(err))
 	}
 	logger.Debug("Telegram message successfully published", "message_id", message.ID, "chat_id", message.Chat.ID)

notify/telegram/telegram_test.go

@@ -16,6 +16,7 @@ package telegram
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -33,9 +34,9 @@ import (
 
 	amcommoncfg "github.com/prometheus/alertmanager/config/common"
 
+	"github.com/prometheus/alertmanager/alert"
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/notify/test"
-	"github.com/prometheus/alertmanager/types"
 )
 
 func TestTelegramUnmarshal(t *testing.T) {
@@ -182,7 +183,7 @@ func TestTelegramNotify(t *testing.T) {
 			defer cancel()
 			ctx = notify.WithGroupKey(ctx, "1")
 
-			retry, err := notifier.Notify(ctx, []*types.Alert{
+			retry, err := notifier.Notify(ctx, []*alert.Alert{
 				{
 					Alert: model.Alert{
 						Labels: model.LabelSet{
@@ -261,7 +262,7 @@ func TestTelegramNotifyFailureReason(t *testing.T) {
 			defer cancel()
 			ctx = notify.WithGroupKey(ctx, "1")
 
-			retry, err := notifier.Notify(ctx, []*types.Alert{
+			retry, err := notifier.Notify(ctx, []*alert.Alert{
 				{
 					Alert: model.Alert{
 						Labels:   model.LabelSet{"lbl1": "val1"},
@@ -307,7 +308,7 @@ func TestTelegramNotifyRedactURL(t *testing.T) {
 		defer cancel()
 		ctx = notify.WithGroupKey(ctx, "1")
 
-		retry, err := notifier.Notify(ctx, &types.Alert{
+		retry, err := notifier.Notify(ctx, &alert.Alert{
 			Alert: model.Alert{Labels: model.LabelSet{"alertname": "test"}},
 		})
 		require.True(t, retry)
@@ -344,11 +345,76 @@ func TestTelegramNotifyRedactURL(t *testing.T) {
 		defer cancel()
 		ctx = notify.WithGroupKey(ctx, "1")
 
-		retry, err := notifier.Notify(ctx, &types.Alert{
+		retry, err := notifier.Notify(ctx, &alert.Alert{
 			Alert: model.Alert{Labels: model.LabelSet{"alertname": "test"}},
 		})
 		require.True(t, retry)
 		require.Error(t, err)
 		require.NotContains(t, err.Error(), token, "bot token leaked in API error")
 	})
 }
+
+func TestTelegramTimeout(t *testing.T) {
+	token := "secret"
+
+	tests := []struct {
+		name    string
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		{
+			name:    "success",
+			latency: 100 * time.Millisecond,
+			timeout: 120 * time.Millisecond,
+			wantErr: false,
+		},
+		{
+			name:    "timeout",
+			latency: 100 * time.Millisecond,
+			timeout: 80 * time.Millisecond,
+			wantErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				require.Equal(t, "/bot"+token+"/sendMessage", r.URL.Path)
+				_, err := io.ReadAll(r.Body)
+				require.NoError(t, err)
+				time.Sleep(tc.latency)
+				w.Write([]byte(`{"ok":true,"result":{"chat":{}}}`))
+			}))
+			defer srv.Close()
+			u, _ := url.Parse(srv.URL)
+
+			cfg := &TelegramConfig{
+				Message:    "test",
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				BotToken:   commoncfg.Secret(token),
+				Timeout:    tc.timeout,
+				APIUrl:     &amcommoncfg.URL{URL: u},
+			}
+
+			notifier, err := New(cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			testAlert := &alert.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			_, err = notifier.Notify(ctx, testAlert)
+			require.Equal(t, tc.wantErr, err != nil)
+			if tc.wantErr {
+				require.EqualError(t, err, fmt.Sprintf("configured telegram timeout reached (%s)", tc.timeout))
+			}
+		})
+	}
+}