feat(router): treat configured finish reasons as deployment failures #41825
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -708,6 +708,20 @@ def as_output_cap(value: object) -> int | None: | |
| return cap if cap >= 0 else None | ||
|
|
||
|
|
||
| ## Exception classes a treat_finish_reason_as_failure value may name: resolved from litellm at | ||
| ## use time, validated at Router construction. | ||
| _FINISH_REASON_FAILURE_EXCEPTION_NAMES: Final = frozenset( | ||
| { | ||
| "RateLimitError", | ||
| "APIError", | ||
| "BadRequestError", | ||
| "Timeout", | ||
| "ServiceUnavailableError", | ||
| "InternalServerError", | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| class Router: | ||
| model_names: set = set() | ||
| cache_responses: bool | None = False | ||
|
|
@@ -752,6 +766,7 @@ def __init__( | |
| fallbacks: list = [], | ||
| context_window_fallbacks: list = [], | ||
| content_policy_fallbacks: list = [], | ||
| treat_finish_reason_as_failure: dict[str, str] | None = None, | ||
|
Contributor
There was a problem hiding this comment.
Author
There was a problem hiding this comment. Verified against the update path:
Contributor
There was a problem hiding this comment. Thanks for verifying. Since |
||
| model_group_alias: dict[str, str | RouterModelGroupAliasItem] | None = {}, | ||
| enable_pre_call_checks: bool = False, | ||
| enable_tag_filtering: bool = False, | ||
|
|
@@ -1072,6 +1087,33 @@ def __init__( | |
| _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks | ||
| self.validate_fallbacks(fallback_param=_content_policy_fallbacks) | ||
| self.content_policy_fallbacks = _content_policy_fallbacks | ||
|
|
||
| ## treat_finish_reason_as_failure: map a terminal finish/stop reason on a 200 response to a | ||
| ## router-understood exception class, so the mapped reason engages allowed_fails/cooldowns/ | ||
| ## fallbacks like any failure. Reason strings are matched exactly. | ||
| if treat_finish_reason_as_failure is not None: | ||
| for exception_name in treat_finish_reason_as_failure.values(): | ||
| if exception_name not in _FINISH_REASON_FAILURE_EXCEPTION_NAMES: | ||
| raise ValueError( | ||
| f"treat_finish_reason_as_failure values must be one of {sorted(_FINISH_REASON_FAILURE_EXCEPTION_NAMES)}, got {exception_name}" | ||
| ) | ||
| self.treat_finish_reason_as_failure = treat_finish_reason_as_failure | ||
| if treat_finish_reason_as_failure: | ||
| verbose_router_logger.warning( | ||
| "treat_finish_reason_as_failure applies to non-streaming responses only; a streamed 200 with the mapped stop reason is delivered unchanged." | ||
| ) | ||
| healthy_terminal_keys: Final = treat_finish_reason_as_failure.keys() & { | ||
| "stop", | ||
| "length", | ||
| "tool_calls", | ||
| "function_call", | ||
| } | ||
| if healthy_terminal_keys: | ||
| verbose_router_logger.warning( | ||
| "treat_finish_reason_as_failure keys %s are healthy terminal reasons in the mapped OpenAI set; mapping them fails successful responses. Keys are matched against provider-native stop reasons.", | ||
| sorted(healthy_terminal_keys), | ||
| ) | ||
|
|
||
| self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model | ||
| self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model | ||
| self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model | ||
|
|
@@ -2549,6 +2591,14 @@ def _completion(self, model: str, messages: list[dict[str, str]], **kwargs) -> M | |
| llm_provider="", | ||
| ) | ||
|
|
||
| ## CHECK MAPPED FINISH REASON ERROR ## | ||
| if isinstance(response, ModelResponse): | ||
| _mapped_reason = self._get_mapped_finish_reason(response) | ||
| if _mapped_reason is not None: | ||
| self._handle_mapped_finish_reason_failure( | ||
| model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs | ||
| ) | ||
|
|
||
| if ( | ||
| isinstance(response, CustomStreamWrapper) | ||
| and response.completion_stream is None | ||
|
|
@@ -3673,6 +3723,14 @@ async def _acompletion( | |
| llm_provider="", | ||
| ) | ||
|
|
||
| ## CHECK MAPPED FINISH REASON ERROR ## | ||
| if isinstance(response, ModelResponse): | ||
| _mapped_reason = self._get_mapped_finish_reason(response) | ||
| if _mapped_reason is not None: | ||
| self._handle_mapped_finish_reason_failure( | ||
| model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs | ||
| ) | ||
|
|
||
| if ( | ||
| isinstance(response, CustomStreamWrapper) | ||
| and response.completion_stream is None | ||
|
|
@@ -5390,6 +5448,17 @@ async def _ageneric_api_call_with_fallbacks_helper(self, model: str, original_ge | |
| refusal_details: Final = cast(dict, response["stop_details"]) # cast-ok: gate verified the shape | ||
| raise safeguard_refusal_error(model=model, stop_details=refusal_details) | ||
|
|
||
| if ( | ||
| self.treat_finish_reason_as_failure | ||
| and getattr(original_generic_function, "__name__", "") == "anthropic_messages" | ||
| and isinstance(response, dict) | ||
| ): | ||
| stop_reason: Final = response.get("stop_reason") | ||
| if stop_reason in self.treat_finish_reason_as_failure: | ||
| self._handle_mapped_finish_reason_failure( | ||
| model=model, deployment=deployment, reason=stop_reason, kwargs=kwargs | ||
| ) | ||
|
|
||
| self.success_calls[model_name] += 1 | ||
| verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) | ||
|
|
||
|
|
@@ -8580,6 +8649,34 @@ def _refusal_fallback_available(self, model_group: str, kwargs: Mapping[str, Any | |
| content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) | ||
| if content_policy_fallbacks is not None: | ||
| return self._has_content_policy_fallback(model_group, kwargs) | ||
| return self._generic_fallback_available(model_group, kwargs) | ||
|
|
||
| def _get_mapped_finish_reason(self, response: ModelResponse) -> str | None: | ||
| """ | ||
| The finish reason configured in treat_finish_reason_as_failure that this response carries, | ||
| or None. Checks both the mapped finish_reason and the pre-mapping value stashed in | ||
| provider_specific_fields["native_finish_reason"]. Streaming detection is a follow-up | ||
| modeled on _aanthropic_messages_streaming_iterator. | ||
| """ | ||
| if not self.treat_finish_reason_as_failure: | ||
| return None | ||
| if not (response.choices and len(response.choices) > 0): | ||
| return None | ||
|
Contributor
There was a problem hiding this comment.
Knowledge Base Used: Model invocation runtime
Author
There was a problem hiding this comment. Keeping the choices[0] read: it matches the adjacent
Contributor
There was a problem hiding this comment. That’s fair. |
||
| choice: Final = response.choices[0] | ||
| if choice.finish_reason in self.treat_finish_reason_as_failure: | ||
| return choice.finish_reason | ||
| native_reason: Final = (getattr(choice, "provider_specific_fields", None) or {}).get("native_finish_reason") | ||
| if native_reason in self.treat_finish_reason_as_failure: | ||
| return native_reason | ||
| return None | ||
|
|
||
| def _generic_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: | ||
| """ | ||
| Whether a generic fallback can serve a retry: default fallbacks set, or a generic chain | ||
| resolving for this request. Shared tail of the fallback-availability gates. | ||
| """ | ||
| if fallbacks_disabled_for_request(kwargs): | ||
| return False | ||
| if self._has_default_fallbacks(): | ||
| return True | ||
| fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) | ||
|
|
@@ -8591,6 +8688,66 @@ def _refusal_fallback_available(self, model_group: str, kwargs: Mapping[str, Any | |
| ) | ||
| return resolved is not None | ||
|
|
||
| def _handle_mapped_finish_reason_failure(self, model: str, deployment: dict, reason: str, kwargs: dict) -> None: | ||
|
Contributor
There was a problem hiding this comment.
Context Used: CLAUDE.md (source) Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!
Author
There was a problem hiding this comment. The new helper signatures use
Contributor
There was a problem hiding this comment. You're right. The new helper signatures in Tip: You can customize Greptile's behavior for this repo with |
||
| """ | ||
| Account for a mapped finish-reason failure, then raise the configured exception into the | ||
| fallback chain when a generic fallback can serve. Accounting happens before the gate: | ||
| the raise lands after the 200 came back, so litellm's failure callbacks never fire for | ||
| it, and this is the only path that parks the deployment. A deployment with no model_info | ||
| id cannot be accounted or parked, but the raise still applies to it. | ||
| """ | ||
| exception: Final = self._account_mapped_finish_reason_failure( | ||
| model=model, deployment=deployment, reason=reason, kwargs=kwargs | ||
| ) or self._finish_reason_failure_error(model=model, reason=reason) | ||
| if self._generic_fallback_available(model, kwargs): | ||
| raise exception | ||
|
|
||
| def _finish_reason_failure_error(self, model: str, reason: str) -> Exception: | ||
| """Build the exception instance configured for a mapped finish reason.""" | ||
| exception_name: Final = self.treat_finish_reason_as_failure[reason] | ||
| exception_cls: Final = getattr(litellm, exception_name) | ||
| message: Final = f"Response finished with reason '{reason}' (treat_finish_reason_as_failure)." | ||
| if exception_name == "APIError": | ||
| return exception_cls(status_code=500, message=message, llm_provider="", model=model) | ||
| return exception_cls(message=message, llm_provider="", model=model) | ||
|
|
||
| def _account_mapped_finish_reason_failure( | ||
| self, model: str, deployment: dict, reason: str, kwargs: dict | ||
| ) -> Exception | None: | ||
| """ | ||
| Count and park a mapped finish-reason failure: increment the per-minute failure counter | ||
| and set the cooldown, honoring a deployment-level cooldown_time like | ||
| deployment_callback_on_failure does (the retry-after-header tier has no counterpart | ||
| here: the exception is synthesized, it carries no response headers). Returns the built | ||
| exception so the caller can raise the same instance it accounted for, or None when the | ||
| deployment has no id to account against. | ||
| """ | ||
| model_info: Final = deployment.get("model_info") or {} | ||
| deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None | ||
| if deployment_id is None: | ||
| return None | ||
| litellm_params: Final = deployment.get("litellm_params") or {} | ||
| deployment_cooldown: Final = _first_present( | ||
| model_info if isinstance(model_info, dict) else None, litellm_params, key="cooldown_time" | ||
| ) | ||
| time_to_cooldown: Final = ( | ||
| deployment_cooldown if deployment_cooldown is not None and deployment_cooldown >= 0 else self.cooldown_time | ||
| ) | ||
| exception: Final = self._finish_reason_failure_error(model=model, reason=reason) | ||
| increment_deployment_failures_for_current_minute( | ||
| litellm_router_instance=self, | ||
| deployment_id=deployment_id, | ||
| ) | ||
| _set_cooldown_deployments( | ||
|
Contributor
There was a problem hiding this comment. Low: Caller-controlled responses poison shared deployment health Finish reasons can depend on caller-controlled input, such as prompt size or output limits. When one of those reasons is configured, a client can repeatedly trigger it to increment the deployment-wide failure counter and cool down the deployment for every tenant; with
Author
There was a problem hiding this comment. Known and deliberate. The same caller-induced parking already applies to |
||
| litellm_router_instance=self, | ||
| exception_status=exception.status_code, | ||
| original_exception=exception, | ||
| deployment=deployment_id, | ||
| time_to_cooldown=time_to_cooldown, | ||
| requested_model_group=(get_litellm_metadata_from_kwargs(kwargs) or {}).get("model_group"), | ||
| ) | ||
| return exception | ||
|
|
||
| def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: | ||
| """ | ||
| Determines if a content policy error should be raised. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Context Used: CLAUDE.md (source)
Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keeping the two module-level notes: the
##label blocks match the surrounding house style, including the## CHECK CONTENT FILTER ERROR ##checkpoints this change extends. They carry the allowlist semantics a reader cannot get from the frozenset alone.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That context is fair. These are concise semantic labels rather than narrative comments: they document the allowlist’s purpose and the healthy-reason warning, and they match the surrounding
##checkpoint style. I retract the comment; no change is needed.