Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions backend/internal/handler/admin/setting_handler_runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,66 @@ func (h *SettingHandler) UpdateRateLimit429CooldownSettings(c *gin.Context) {
})
}

// GetOpenAI403CooldownSettings 获取 OpenAI 403 临时冷却配置
// GET /api/v1/admin/settings/openai-403-cooldown
func (h *SettingHandler) GetOpenAI403CooldownSettings(c *gin.Context) {
settings, err := h.settingService.GetOpenAI403CooldownSettings(c.Request.Context())
if err != nil {
response.ErrorFrom(c, err)
return
}

response.Success(c, dto.OpenAI403CooldownSettings{
Enabled: settings.Enabled,
CooldownMinutes: settings.CooldownMinutes,
DisableThreshold: settings.DisableThreshold,
WindowMinutes: settings.WindowMinutes,
})
}

// UpdateOpenAI403CooldownSettingsRequest 更新 OpenAI 403 临时冷却配置请求
type UpdateOpenAI403CooldownSettingsRequest struct {
Enabled bool `json:"enabled"`
CooldownMinutes int `json:"cooldown_minutes"`
DisableThreshold int `json:"disable_threshold"`
WindowMinutes int `json:"window_minutes"`
}

// UpdateOpenAI403CooldownSettings 更新 OpenAI 403 临时冷却配置
// PUT /api/v1/admin/settings/openai-403-cooldown
func (h *SettingHandler) UpdateOpenAI403CooldownSettings(c *gin.Context) {
var req UpdateOpenAI403CooldownSettingsRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.BadRequest(c, "Invalid request: "+err.Error())
return
}

settings := &service.OpenAI403CooldownSettings{
Enabled: req.Enabled,
CooldownMinutes: req.CooldownMinutes,
DisableThreshold: req.DisableThreshold,
WindowMinutes: req.WindowMinutes,
}

if err := h.settingService.SetOpenAI403CooldownSettings(c.Request.Context(), settings); err != nil {
response.BadRequest(c, err.Error())
return
}

updatedSettings, err := h.settingService.GetOpenAI403CooldownSettings(c.Request.Context())
if err != nil {
response.ErrorFrom(c, err)
return
}

response.Success(c, dto.OpenAI403CooldownSettings{
Enabled: updatedSettings.Enabled,
CooldownMinutes: updatedSettings.CooldownMinutes,
DisableThreshold: updatedSettings.DisableThreshold,
WindowMinutes: updatedSettings.WindowMinutes,
})
}

// GetPanelRateLimitSettings 获取面板 API 限流配置
// GET /api/v1/admin/settings/panel-rate-limit
func (h *SettingHandler) GetPanelRateLimitSettings(c *gin.Context) {
Expand Down
8 changes: 8 additions & 0 deletions backend/internal/handler/dto/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,14 @@ type RateLimit429CooldownSettings struct {
CooldownSeconds int `json:"cooldown_seconds"`
}

// OpenAI403CooldownSettings OpenAI 403 临时冷却配置 DTO
type OpenAI403CooldownSettings struct {
Enabled bool `json:"enabled"`
CooldownMinutes int `json:"cooldown_minutes"`
DisableThreshold int `json:"disable_threshold"`
WindowMinutes int `json:"window_minutes"`
}

// PanelRateLimitSettings 面板 API 限流配置 DTO
type PanelRateLimitSettings struct {
Enabled bool `json:"enabled"`
Expand Down
2 changes: 2 additions & 0 deletions backend/internal/server/routes/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,8 @@ func registerSettingsRoutes(admin *gin.RouterGroup, h *handler.Handlers) {
// 429默认回避配置
adminSettings.GET("/rate-limit-429-cooldown", h.Admin.Setting.GetRateLimit429CooldownSettings)
adminSettings.PUT("/rate-limit-429-cooldown", h.Admin.Setting.UpdateRateLimit429CooldownSettings)
adminSettings.GET("/openai-403-cooldown", h.Admin.Setting.GetOpenAI403CooldownSettings)
adminSettings.PUT("/openai-403-cooldown", h.Admin.Setting.UpdateOpenAI403CooldownSettings)
// 面板 API 限流配置
adminSettings.GET("/panel-rate-limit", h.Admin.Setting.GetPanelRateLimitSettings)
adminSettings.PUT("/panel-rate-limit", h.Admin.Setting.UpdatePanelRateLimitSettings)
Expand Down
3 changes: 3 additions & 0 deletions backend/internal/service/domain_constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,9 @@ const (

// SettingKeyRateLimit429CooldownSettings stores JSON config for 429 fallback cooldown handling.
SettingKeyRateLimit429CooldownSettings = "rate_limit_429_cooldown_settings"

// SettingKeyOpenAI403CooldownSettings stores JSON config for OpenAI 403 temporary cooldown handling.
SettingKeyOpenAI403CooldownSettings = "openai_403_cooldown_settings"
// SettingKeyOpenAIAPIKeyHealthBreakerSettings stores the opt-in OpenAI pool API-key breaker config.
SettingKeyOpenAIAPIKeyHealthBreakerSettings = "openai_apikey_health_breaker_settings"

Expand Down
223 changes: 223 additions & 0 deletions backend/internal/service/openai_403_cooldown_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
//go:build unit

package service

import (
"context"
"encoding/json"
"testing"
"time"

"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/stretchr/testify/require"
)

type openAI403TempRecorder struct {
*rateLimitAccountRepoStub
until time.Time
}

func (r *openAI403TempRecorder) SetTempUnschedulable(ctx context.Context, id int64, until time.Time, reason string) error {
r.until = until
return r.rateLimitAccountRepoStub.SetTempUnschedulable(ctx, id, until, reason)
}

type openAI403WindowRecorder struct {
*countingOpenAI403CounterCache
windows []int
}

func (r *openAI403WindowRecorder) IncrementOpenAI403Count(ctx context.Context, accountID int64, window int) (int64, error) {
r.windows = append(r.windows, window)
return r.countingOpenAI403CounterCache.IncrementOpenAI403Count(ctx, accountID, window)
}

func setOpenAI403TestSettings(t *testing.T, h *openAI403TestHarness, settings OpenAI403CooldownSettings) {
t.Helper()
repo := newMockSettingRepo()
data, err := json.Marshal(settings)
require.NoError(t, err)
repo.data[SettingKeyOpenAI403CooldownSettings] = string(data)
h.svc.SetSettingService(NewSettingService(repo, &config.Config{}))
}

func TestGetOpenAI403CooldownSettings_DefaultsWhenNotSet(t *testing.T) {
svc := NewSettingService(newMockSettingRepo(), &config.Config{})

settings, err := svc.GetOpenAI403CooldownSettings(context.Background())
require.NoError(t, err)
// MUTATION-SANITY: changing any OpenAI 403 default constant makes these assertions fail.
require.True(t, settings.Enabled)
require.Equal(t, 10, settings.CooldownMinutes)
require.Equal(t, 3, settings.DisableThreshold)
require.Equal(t, 180, settings.WindowMinutes)
}

func TestGetOpenAI403CooldownSettings_ClampsOutOfRange(t *testing.T) {
repo := newMockSettingRepo()
repo.data[SettingKeyOpenAI403CooldownSettings] = `{"enabled":true,"cooldown_minutes":99999,"disable_threshold":0,"window_minutes":-5}`
svc := NewSettingService(repo, &config.Config{})

settings, err := svc.GetOpenAI403CooldownSettings(context.Background())
require.NoError(t, err)
// MUTATION-SANITY: removing the read-side clamps returns 99999, 0, and -5 here.
require.Equal(t, 1440, settings.CooldownMinutes)
require.Equal(t, 1, settings.DisableThreshold)
require.Equal(t, 1, settings.WindowMinutes)
}

func TestSetOpenAI403CooldownSettings_RejectsOutOfRangeWhenEnabled(t *testing.T) {
tests := []struct {
name string
settings OpenAI403CooldownSettings
field string
}{
{
name: "cooldown_minutes",
settings: OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 0, DisableThreshold: 3, WindowMinutes: 180,
},
field: "cooldown_minutes",
},
{
name: "disable_threshold",
settings: OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 101, WindowMinutes: 180,
},
field: "disable_threshold",
},
{
name: "window_minutes",
settings: OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 3, WindowMinutes: 1441,
},
field: "window_minutes",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
svc := NewSettingService(newMockSettingRepo(), &config.Config{})

err := svc.SetOpenAI403CooldownSettings(context.Background(), &test.settings)

// MUTATION-SANITY: removing the enabled-state validation makes this call succeed.
require.Error(t, err)
require.Contains(t, err.Error(), test.field)
})
}
}

func TestSetOpenAI403CooldownSettings_NormalizesWhenDisabled(t *testing.T) {
repo := newMockSettingRepo()
svc := NewSettingService(repo, &config.Config{})
settings := &OpenAI403CooldownSettings{
Enabled: false,
CooldownMinutes: 0,
DisableThreshold: 101,
WindowMinutes: -1,
}

err := svc.SetOpenAI403CooldownSettings(context.Background(), settings)
require.NoError(t, err)
stored, err := svc.GetOpenAI403CooldownSettings(context.Background())
require.NoError(t, err)
// MUTATION-SANITY: changing the disabled branch to reject invalid values fails before this normalized result.
require.False(t, stored.Enabled)
require.Equal(t, 10, stored.CooldownMinutes)
require.Equal(t, 3, stored.DisableThreshold)
require.Equal(t, 180, stored.WindowMinutes)
}

func TestHandleOpenAI403_UsesConfiguredCooldownMinutes(t *testing.T) {
h := newOpenAI403TestHarness(t, 601, 1)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 2, DisableThreshold: 3, WindowMinutes: 180,
})
recorder := &openAI403TempRecorder{rateLimitAccountRepoStub: h.repo}
h.svc.accountRepo = recorder
before := time.Now()

require.True(t, h.handle(`{"error":{"message":"temporary edge rejection"}}`))

// MUTATION-SANITY: replacing the configured cooldown with the 10-minute constant misses this window.
require.WithinDuration(t, before.Add(2*time.Minute), recorder.until, 5*time.Second)
require.Less(t, recorder.until.Sub(before), 5*time.Minute)
}

func TestHandleOpenAI403_UsesConfiguredThreshold(t *testing.T) {
t.Run("at_threshold_disables", func(t *testing.T) {
h := newOpenAI403TestHarness(t, 602, 2)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 2, WindowMinutes: 180,
})

require.True(t, h.handle(`{"error":{"message":"workspace forbidden"}}`))
// MUTATION-SANITY: comparing against the original threshold of 3 leaves this account temporary instead.
require.Equal(t, 1, h.repo.setErrorCalls)
require.Equal(t, 0, h.repo.tempCalls)
})

t.Run("below_threshold_is_temporary", func(t *testing.T) {
h := newOpenAI403TestHarness(t, 603, 1)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 2, WindowMinutes: 180,
})

require.True(t, h.handle(`{"error":{"message":"temporary edge rejection"}}`))
// MUTATION-SANITY: reversing the configured threshold comparison permanently disables this account.
require.Equal(t, 0, h.repo.setErrorCalls)
require.Equal(t, 1, h.repo.tempCalls)
})
}

func TestHandleOpenAI403_PassesConfiguredWindowToCounter(t *testing.T) {
h := newOpenAI403TestHarness(t, 604, 1)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 3, WindowMinutes: 30,
})
counter := &openAI403WindowRecorder{countingOpenAI403CounterCache: h.counter}
h.svc.SetOpenAI403CounterCache(counter)

require.True(t, h.handle(`{"error":{"message":"temporary edge rejection"}}`))

// MUTATION-SANITY: passing the original 180-minute constant records 180 instead of 30.
require.Equal(t, []int{30}, counter.windows)
}

func TestHandleOpenAI403_DisabledSkipsAccountPenalty(t *testing.T) {
h := newOpenAI403TestHarness(t, 605, 1)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: false, CooldownMinutes: 10, DisableThreshold: 3, WindowMinutes: 180,
})

// MUTATION-SANITY: deleting the disabled guard returns true and records counter and penalty calls.
require.False(t, h.handle(`{"error":{"message":"temporary edge rejection"}}`))
h.requireNoAccountPenalty(t)
}

func TestHandleOpenAI403_FallsBackToDefaultsWithoutSettingService(t *testing.T) {
h := newOpenAI403TestHarness(t, 606, 1)
recorder := &openAI403TempRecorder{rateLimitAccountRepoStub: h.repo}
h.svc.accountRepo = recorder
counter := &openAI403WindowRecorder{countingOpenAI403CounterCache: h.counter}
h.svc.SetOpenAI403CounterCache(counter)
before := time.Now()

require.True(t, h.handle(`{"error":{"message":"temporary edge rejection"}}`))

// MUTATION-SANITY: breaking the no-service fallback changes the 10-minute duration or 180-minute window.
require.WithinDuration(t, before.Add(10*time.Minute), recorder.until, 5*time.Second)
require.Equal(t, []int{180}, counter.windows)
}

func TestHandleOpenAI403_HTMLBodyStillSkipsPenaltyWhenEnabled(t *testing.T) {
h := newOpenAI403TestHarness(t, 607, 1)
setOpenAI403TestSettings(t, h, OpenAI403CooldownSettings{
Enabled: true, CooldownMinutes: 10, DisableThreshold: 3, WindowMinutes: 180,
})

// MUTATION-SANITY: moving the settings branch ahead of the HTML guard allows account penalty side effects.
require.False(t, h.handle(openAI403HTMLBody))
h.requireNoAccountPenalty(t)
}
33 changes: 27 additions & 6 deletions backend/internal/service/ratelimit_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ const (
openAI403CooldownMinutesDefault = 10
openAI403DisableThreshold = 3
openAI403CounterWindowMinutes = 180
maxOpenAI403CooldownMinutes = 1440
maxOpenAI403DisableThreshold = 100
maxOpenAI403WindowMinutes = 1440
)

// NewRateLimitService 创建RateLimitService实例
Expand Down Expand Up @@ -985,6 +988,12 @@ func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account
return false
}

settings := s.getOpenAI403CooldownSettings(ctx, account)
if !settings.Enabled {
slog.Info("openai_403_cooldown_disabled_skip_account_penalty", "account_id", account.ID)
return false
}

msg := buildForbiddenErrorMessage(
"Access forbidden (403):",
upstreamMsg,
Expand All @@ -997,21 +1006,21 @@ func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account
return true
}

count, err := s.openAI403CounterCache.IncrementOpenAI403Count(ctx, account.ID, openAI403CounterWindowMinutes)
count, err := s.openAI403CounterCache.IncrementOpenAI403Count(ctx, account.ID, settings.WindowMinutes)
if err != nil {
slog.Warn("openai_403_increment_failed", "account_id", account.ID, "error", err)
s.handleAuthError(ctx, account, msg)
return true
}

if count >= openAI403DisableThreshold {
msg = fmt.Sprintf("%s | consecutive_403=%d/%d", msg, count, openAI403DisableThreshold)
if count >= int64(settings.DisableThreshold) {
msg = fmt.Sprintf("%s | consecutive_403=%d/%d", msg, count, settings.DisableThreshold)
s.handleAuthError(ctx, account, msg)
return true
}

until := time.Now().Add(time.Duration(openAI403CooldownMinutesDefault) * time.Minute)
reason := fmt.Sprintf("OpenAI 403 temporary cooldown (%d/%d): %s", count, openAI403DisableThreshold, msg)
until := time.Now().Add(time.Duration(settings.CooldownMinutes) * time.Minute)
reason := fmt.Sprintf("OpenAI 403 temporary cooldown (%d/%d): %s", count, settings.DisableThreshold, msg)
s.notifyAccountSchedulingBlocked(account, until, "openai_403_temp")
if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, reason); err != nil {
slog.Warn("openai_403_set_temp_unschedulable_failed", "account_id", account.ID, "error", err)
Expand All @@ -1024,11 +1033,23 @@ func (s *RateLimitService) handleOpenAI403(ctx context.Context, account *Account
"account_id", account.ID,
"until", until,
"count", count,
"threshold", openAI403DisableThreshold,
"threshold", settings.DisableThreshold,
"cooldown_minutes", settings.CooldownMinutes,
)
return true
}

func (s *RateLimitService) getOpenAI403CooldownSettings(ctx context.Context, account *Account) *OpenAI403CooldownSettings {
if s.settingService != nil {
settings, err := s.settingService.GetOpenAI403CooldownSettings(ctx)
if err == nil && settings != nil {
return settings
}
slog.Warn("openai_403_cooldown_settings_read_failed", "account_id", account.ID, "error", err)
}
return DefaultOpenAI403CooldownSettings()
}

// handleAntigravity403 处理 Antigravity 平台的 403 错误
// validation(需要验证)→ 永久 SetError(需人工去 Google 验证后恢复)
// violation(违规封号)→ 永久 SetError(需人工处理)
Expand Down
Loading
Loading