feat(guardrails): Add multilingual toxicity classifier (#1335)

Author: AkhileshNegi

backend/app/api/docs/guardrails_schemas.py

@@ -449,17 +449,16 @@ class ValidatorGroup(NamedTuple):
         runs_on="input and output",
         notes=(
             _NO_PARTIAL_FIX,
-            "If you want slurs redacted *and* profanity checked, use "
-            "`content_safety` instead of running both — it orders them so a "
-            "redacted slur does not then trip the profanity check.",
+            "English-only model: common Hinglish (`kaise ho`) can score as "
+            "profane. For Hindi/Hinglish users prefer `content_safety`.",
         ),
     ),
     ValidatorType(
         type="content_safety",
         summary=(
-            "Slur redaction followed by a profanity check, as one validator. "
-            "The slur pass runs first, so text it has already cleaned is not "
-            "blanked by the profanity pass."
+            "Slur redaction plus a multilingual toxicity classifier, as one "
+            "validator. Catches abuse in English, Hindi and Romanised "
+            "Hinglish, including abuse the slur list misses."
         ),
         runs_on="input and output",
         config=(
@@ -469,13 +468,45 @@ class ValidatorGroup(NamedTuple):
                 "Lowest slur severity to redact.",
             ),
             ConfigKey(
-                "profanity_threshold",
-                {"type": "number", "minimum": 0, "maximum": 1, "default": 0.9},
+                "threshold",
+                {"type": "number", "minimum": 0, "maximum": 1, "default": 0.5},
+                (
+                    "Toxicity probability at or above which the whole message "
+                    "is replaced. Lower it to be stricter."
+                ),
+            ),
+            ConfigKey(
+                "fallback_message",
+                {"type": "string", "minLength": 1, "default": None},
                 (
-                    "Profanity probability at or above which the text fails "
-                    "and is blanked. Lower it to be stricter."
+                    "Returned in place of a toxic message, with either `fix` "
+                    "or `rephrase`. Not translated."
                 ),
             ),
+            ConfigKey(
+                "model_name",
+                {
+                    "type": "string",
+                    "default": "textdetox/bert-multilingual-toxicity-classifier",
+                },
+                "Any HuggingFace text-classification model.",
+            ),
+            ConfigKey(
+                "toxic_label",
+                {"type": "string", "default": "LABEL_1"},
+                (
+                    "Label `model_name` emits for toxic text. Change it "
+                    "together with `model_name`."
+                ),
+            ),
+        ),
+        notes=(
+            "With `fix`, a message whose only problem is a listed slur keeps "
+            "its redacted form (`[REDACTED_SLUR]`); a message the classifier "
+            "scores toxic is replaced whole by `fallback_message`. With "
+            "`rephrase`, any failure returns `fallback_message`.",
+            "Without `fallback_message`, `fix` returns an empty `safe_text` "
+            "for a toxic message, and `metadata.reason` names this validator.",
         ),
     ),
     ValidatorType(

backend/app/models/llm/request.py

@@ -479,6 +479,7 @@ class Validator(SQLModel):
 class InlineValidatorTypeEnum(str, Enum):
     """Validator types productised for tenants; mirrors GET /guardrails."""
 
+    CONTENT_SAFETY = "content_safety"
     GENDER_ASSUMPTION_BIAS = "gender_assumption_bias"
     PII_REMOVER = "pii_remover"
     TOPIC_RELEVANCE_LLM = "topic_relevance_llm"

backend/app/tests/services/llm/test_guardrails.py

@@ -542,6 +542,32 @@ def test_inline_spec_posted_shape(self, mock_fetch, mock_run) -> None:
             {"type": "pii_remover", "on_fail": "exception", "stage": "input"},
         ]
 
+    def test_content_safety_inline_spec_forwards_params(
+        self, mock_fetch, mock_run
+    ) -> None:
+        mock_run.return_value = _PASS_RESPONSE
+
+        _apply(
+            [
+                InlineValidator(
+                    type="content_safety",
+                    on_fail="rephrase",
+                    threshold=0.5,
+                    fallback_message="Please rephrase without abusive language.",
+                )
+            ]
+        )
+
+        assert mock_run.call_args.args[1] == [
+            {
+                "type": "content_safety",
+                "on_fail": "rephrase",
+                "threshold": 0.5,
+                "fallback_message": "Please rephrase without abusive language.",
+                "stage": "input",
+            }
+        ]
+
     def test_output_path_sets_output_stage(self, mock_fetch, mock_run) -> None:
         ref, row = _ref()
         mock_fetch.return_value = ([], [row])

docs/wiki/modules/llm-call.md

@@ -28,7 +28,7 @@ All paths relative to `backend/app/`.
 
 ## Key pydantic/SQLModel schemas (`models/llm/request.py`)
 - `LLMCallConfig` — one-of: saved reference (`id` + `version`) XOR ad-hoc `blob` (validator-enforced)
-- `ConfigBlob` — `completion` + optional `prompt_template` (`PromptTemplate.template`, plain string; `{{input}}` interpolation is llm-chain-only) + `input_guardrails`/`output_guardrails`. Each guardrail entry is either a `Validator` reference (`{validator_config_id}`) or an `InlineValidator` spec (`{type, on_fail, ...per-type params}`, `type` limited to `InlineValidatorTypeEnum`: gender_assumption_bias, pii_remover, topic_relevance_llm, uli_slur_match — same set as `EXPOSED_VALIDATOR_TYPES`); tagged union `GuardrailEntry`. Both carry `is_enabled` (default true). Disabled entries are never fetched or sent. `services/llm/guardrails.apply_guardrails` POSTs fetched reference rows plus inline specs (a blob is all-refs or all-inline), with `is_enabled` stripped and `stage` stamped on inline specs.
+- `ConfigBlob` — `completion` + optional `prompt_template` (`PromptTemplate.template`, plain string; `{{input}}` interpolation is llm-chain-only) + `input_guardrails`/`output_guardrails`. Each guardrail entry is either a `Validator` reference (`{validator_config_id}`) or an `InlineValidator` spec (`{type, on_fail, ...per-type params}`, `type` limited to `InlineValidatorTypeEnum`: content_safety, gender_assumption_bias, pii_remover, topic_relevance_llm, uli_slur_match — same set as `EXPOSED_VALIDATOR_TYPES`); tagged union `GuardrailEntry`. Both carry `is_enabled` (default true). Disabled entries are never fetched or sent. `services/llm/guardrails.apply_guardrails` POSTs fetched reference rows plus inline specs (a blob is all-refs or all-inline), with `is_enabled` stripped and `stage` stamped on inline specs.
 - `CompletionConfig` — discriminated union on `provider`: `KaapiCompletionConfig` (standardized params: `TextLLMParams`/`STTLLMParams`/`TTSLLMParams`), `NativeCompletionConfig` (pass-through), `ProxyCompletionConfig` (client's own endpoint)
 - `TextLLMParams` reasoning knobs: `reasoning` + `effort` (OpenAI-style), `thinking` (Anthropic adaptive-thinking container, forwarded as-is) and `thinking_level` (Gemini: `minimal`/`low`/`medium`/`high`; `minimal` is flash-only). A knob must be **declared here** to survive config save — pydantic's default extra policy is ignore, so an undeclared key is dropped silently at validation with no error.
 - `QueryParams` — per-call input + `ConversationConfig`