Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend/internal/service/model_rate_limit.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const (
modelRateLimitsKey = "model_rate_limits"
antigravityGeminiModelRateLimitKey = "antigravity:gemini"
openAIImageGenerationRateLimitKey = "openai:image_generation"
openAICodexSparkRateLimitReason = "openai_codex_spark_rate_limit"
// anthropicFableRateLimitKey 是 Anthropic 7d_oi(Fable 专属 7d 窗口)限流的
// 家族级 scope:命中后所有 Fable 变体(含 [1m] 等后缀)都不再调度到该账号。
anthropicFableRateLimitKey = "claude-fable-5"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,10 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont
s.rateLimitService.HandleTempUnschedulable(stateCtx, account, statusCode, responseBody, canonicalModel[0]) {
return true
}
if statusCode == http.StatusTooManyRequests && s.rateLimitService != nil && len(canonicalModel) > 0 &&
s.rateLimitService.HandleOpenAICodexSparkRateLimit(stateCtx, account, canonicalModel[0], statusCode, headers, responseBody) {
return false
}
if statusCode == http.StatusTooManyRequests {
s.markOpenAIOAuth429RateLimited(stateCtx, account, headers, responseBody)
}
Expand Down
184 changes: 182 additions & 2 deletions backend/internal/service/openai_account_runtime_block_fastpath_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,11 @@ import (

type oauth429RateLimitRepo struct {
mockAccountRepoForGemini
setRateLimitedCalls int
lastRateLimitedUntil time.Time
setRateLimitedCalls int
lastRateLimitedUntil time.Time
setModelRateLimitCalls int
lastModelRateLimitKey string
lastModelRateLimitedUntil time.Time
}

func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until time.Time) error {
Expand All @@ -25,6 +28,13 @@ func (r *oauth429RateLimitRepo) SetRateLimited(_ context.Context, _ int64, until
return nil
}

func (r *oauth429RateLimitRepo) SetModelRateLimit(_ context.Context, _ int64, scope string, until time.Time, _ ...string) error {
r.setModelRateLimitCalls++
r.lastModelRateLimitKey = scope
r.lastModelRateLimitedUntil = until
return nil
}

func TestOpenAI429FastPath_KeepsOAuthAccountSchedulableDuringRetryWindow(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
Expand Down Expand Up @@ -85,6 +95,176 @@ func TestOpenAI429FastPath_BlocksOAuthImmediatelyWhenSevenDayQuotaIsExhausted(t
require.False(t, svc.ShouldRetryOpenAIOAuth429(account, headers, nil))
}

func TestOpenAI429FastPath_SparkQuotaOnlyBlocksSparkModel(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 425, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")

shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), account, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)

require.False(t, shouldDisable)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account), "Spark quota must not create an account runtime block")
require.Equal(t, 0, repo.setRateLimitedCalls, "Spark quota must not persist account-level rate limit")
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
}

func TestOpenAI429FastPath_SparkTransient429UsesShortFallback(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 428, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "37")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")

shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), account, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)

require.False(t, shouldDisable)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Less(t, time.Until(repo.lastModelRateLimitedUntil), time.Minute)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), time.Second)
}

func TestOpenAIStream429_SparkQuotaUsesQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 429, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)

status, shouldDisable := svc.handleOpenAIStreamTerminalAccountSideEffects(
nil, account, payload, "quota exhausted", headers, "gpt-5.3-codex-spark",
)

require.Equal(t, http.StatusTooManyRequests, status)
require.False(t, shouldDisable)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}

func TestOpenAIStreamFailover_Spark429KeepsModelScope(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 432, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
headers.Set("x-codex-secondary-used-percent", "20")
headers.Set("x-codex-secondary-reset-after-seconds", "3600")
headers.Set("x-codex-secondary-window-minutes", "300")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)

failoverErr := svc.newOpenAIStreamFailoverErrorWithModel(
nil, account, false, "", payload, "quota exhausted", "gpt-5.3-codex-spark", headers,
)

require.Equal(t, http.StatusTooManyRequests, failoverErr.StatusCode)
require.Equal(t, 0, repo.setRateLimitedCalls)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}

func TestOpenAIWSErrorEvent_OrdinaryModelIgnoresHandshakeQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 430, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)

svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex")

require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
require.Zero(t, repo.setRateLimitedCalls)
}

func TestOpenAIWSErrorEvent_SparkQuotaUsesHandshakeQuotaHeaders(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
account := &Account{ID: 431, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")
payload := []byte(`{"type":"error","error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`)

svc.persistOpenAIWSRateLimitSignal(context.Background(), account, headers, payload, "rate_limit_exceeded", "rate_limit_error", "quota exhausted", "gpt-5.3-codex-spark")

require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
require.Greater(t, time.Until(repo.lastModelRateLimitedUntil), 6*24*time.Hour)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
}

func TestOpenAI429FastPath_SparkShadowQuotaStaysModelScoped(t *testing.T) {
repo := &oauth429RateLimitRepo{}
rateLimits := NewRateLimitService(repo, nil, &config.Config{}, nil, nil)
svc := &OpenAIGatewayService{rateLimitService: rateLimits}
rateLimits.SetAccountRuntimeBlocker(svc)
parentID := int64(426)
shadow := &Account{ID: 427, Platform: PlatformOpenAI, Type: AccountTypeOAuth, ParentAccountID: &parentID, QuotaDimension: QuotaDimensionSpark}
headers := http.Header{}
headers.Set("x-codex-primary-used-percent", "100")
headers.Set("x-codex-primary-reset-after-seconds", "604800")
headers.Set("x-codex-primary-window-minutes", "10080")

shouldDisable := svc.handleOpenAIAccountUpstreamError(
context.Background(), shadow, http.StatusTooManyRequests, headers,
[]byte(`{"error":{"type":"rate_limit_error","code":"rate_limit_exceeded"}}`),
"gpt-5.3-codex-spark",
)

require.False(t, shouldDisable)
require.False(t, svc.isOpenAIAccountRuntimeBlocked(shadow))
require.Equal(t, 0, repo.setRateLimitedCalls)
require.Equal(t, 1, repo.setModelRateLimitCalls)
require.Equal(t, "gpt-5.3-codex-spark", repo.lastModelRateLimitKey)
}

func TestOpenAI429FastPath_RetriesOAuthWhenNoQuotaSignalExists(t *testing.T) {
svc := &OpenAIGatewayService{}
account := &Account{ID: 424, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
Expand Down
4 changes: 2 additions & 2 deletions backend/internal/service/openai_gateway_chat_completions.go
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ func (s *OpenAIGatewayService) handleChatBufferedStreamingResponse(
}
message := openAICompatFailedResponseMessage(finalResponse)
if openAIStreamFailedEventShouldFailover(payload, message) {
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
// response.failed 到达在 HTTP 200 SSE 流上,无真实 HTTP 错误码;统一走语义
Expand Down Expand Up @@ -766,7 +766,7 @@ func (s *OpenAIGatewayService) handleChatStreamingResponse(
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
}
if !clientOutputStarted && shouldFailover {
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
return true
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
Expand Down
4 changes: 2 additions & 2 deletions backend/internal/service/openai_gateway_messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -605,7 +605,7 @@ func (s *OpenAIGatewayService) handleAnthropicBufferedStreamingResponse(
}
message := openAICompatFailedResponseMessage(finalResponse)
if openAIStreamFailedEventShouldFailover(payload, message) {
return nil, s.newOpenAIStreamFailoverError(c, account, false, requestID, payload, message, resp.Header)
return nil, s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payload, message, upstreamModel, resp.Header)
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payload, message)
// 统一走语义状态推断 + body 归一化(与 /v1/responses 路径一致),
Expand Down Expand Up @@ -1038,7 +1038,7 @@ func (s *OpenAIGatewayService) handleAnthropicStreamingResponse(
shouldFailover = openAIStreamErrorEventShouldFailover(payloadBytes, message)
}
if !clientOutputStarted && shouldFailover {
streamFailoverErr = s.newOpenAIStreamFailoverError(c, account, false, requestID, payloadBytes, message, resp.Header)
streamFailoverErr = s.newOpenAIStreamFailoverErrorWithModel(c, account, false, requestID, payloadBytes, message, upstreamModel, resp.Header)
return true
}
message = s.recordOpenAIStreamUpstreamError(c, account, false, requestID, "http_error", payloadBytes, message)
Expand Down
40 changes: 29 additions & 11 deletions backend/internal/service/openai_gateway_passthrough.go
Original file line number Diff line number Diff line change
Expand Up @@ -1558,6 +1558,7 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
payload []byte,
message string,
headers http.Header,
canonicalModel ...string,
) (int, bool) {
statusCode := openAIStreamFailureStatus(payload, message)
switch statusCode {
Expand All @@ -1571,14 +1572,17 @@ func (s *OpenAIGatewayService) handleOpenAIStreamTerminalAccountSideEffects(
if c != nil && c.Request != nil {
ctx = c.Request.Context()
}
model := firstNonEmpty(canonicalModel...)
if model == "" {
model = firstNonEmpty(gjson.GetBytes(payload, "model").String(), gjson.GetBytes(payload, "response.model").String())
}
accountHeaders := headers
if statusCode == http.StatusTooManyRequests {
// The enclosing HTTP response succeeded. Its quota snapshot describes
// normal account state and must not become the reset for a semantic 429
// carried by a stream terminal event.
accountHeaders = nil
// 普通模型的流式 429 不能继承外层 HTTP 200 的全局 quota 快照;
// 只有 OAuth/SetupToken 的 Spark 配额 429 才需要保留 headers 读取明确的 5h/7d reset。
accountHeaders = openAIWSSemantic429Headers(account, model, headers)
}
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload)
return statusCode, s.handleOpenAIAccountUpstreamError(ctx, account, statusCode, accountHeaders, payload, model)
default:
return statusCode, false
}
Expand Down Expand Up @@ -1654,6 +1658,19 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
payload []byte,
message string,
responseHeaders ...http.Header,
) *UpstreamFailoverError {
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, "", responseHeaders...)
}

func (s *OpenAIGatewayService) newOpenAIStreamFailoverErrorWithModel(
c *gin.Context,
account *Account,
passthrough bool,
upstreamRequestID string,
payload []byte,
message string,
canonicalModel string,
responseHeaders ...http.Header,
) *UpstreamFailoverError {
message = sanitizeUpstreamErrorMessage(strings.TrimSpace(message))
if message == "" {
Expand All @@ -1663,7 +1680,7 @@ func (s *OpenAIGatewayService) newOpenAIStreamFailoverError(
if len(responseHeaders) > 0 && responseHeaders[0] != nil {
headers = responseHeaders[0].Clone()
}
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers)
statusCode, shouldDisable := s.handleOpenAIStreamTerminalAccountSideEffects(c, account, payload, message, headers, canonicalModel)
// 流内 failed 事件承载于 HTTP 200;使用事件的语义状态更新账号健康,
// 再由 failover 引擎按 StatusCode/RetryableOnSameAccount 决定恢复策略。
message = s.recordOpenAIStreamUpstreamError(c, account, passthrough, upstreamRequestID, "failover", payload, message)
Expand Down Expand Up @@ -1734,6 +1751,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
terminalType string,
payload []byte,
message string,
canonicalModel ...string,
) *UpstreamFailoverError {
if account == nil || IsResponseCommitted(c) {
return nil
Expand All @@ -1751,7 +1769,7 @@ func (s *OpenAIGatewayService) nonStreamingTerminalFailureFailover(
headers = resp.Header
upstreamRequestID = strings.TrimSpace(resp.Header.Get("x-request-id"))
}
return s.newOpenAIStreamFailoverError(c, account, passthrough, upstreamRequestID, payload, message, headers)
return s.newOpenAIStreamFailoverErrorWithModel(c, account, passthrough, upstreamRequestID, payload, message, firstNonEmpty(canonicalModel...), headers)
}

func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
Expand Down Expand Up @@ -1873,7 +1891,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
return
}
if bareErrorAccountSideEffectsPending {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, bareErrorPayload, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
if clientDisconnected || !writePendingLines() {
Expand Down Expand Up @@ -2005,7 +2023,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
// account health; EOF synthesis applies the pending effect.
bareErrorAccountSideEffectsPending = true
} else {
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header)
s.handleOpenAIStreamTerminalAccountSideEffects(c, account, dataBytes, failedMessage, resp.Header, mappedModel)
bareErrorAccountSideEffectsPending = false
}
}
Expand All @@ -2020,7 +2038,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough(
}
if shouldFailover {
return resultWithUsage(),
s.newOpenAIStreamFailoverError(c, account, true, upstreamRequestID, dataBytes, failedMessage, resp.Header)
s.newOpenAIStreamFailoverErrorWithModel(c, account, true, upstreamRequestID, dataBytes, failedMessage, mappedModel, resp.Header)
}
if !cyberHit && !sawBareError {
if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, failedMessage); matched {
Expand Down Expand Up @@ -2273,7 +2291,7 @@ func (s *OpenAIGatewayService) handlePassthroughSSEToJSON(resp *http.Response, c
if compactErr := newOpenAICompactFallbackSignal(c, terminalPayload, msg); compactErr != nil {
return nil, compactErr
}
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg); failoverErr != nil {
if failoverErr := s.nonStreamingTerminalFailureFailover(c, resp, account, true, terminalType, terminalPayload, msg, mappedModel); failoverErr != nil {
return nil, failoverErr
}
return nil, s.writeOpenAINonStreamingProtocolError(resp, c, msg)
Expand Down
Loading
Loading