feat(azure.ai.agents): add invocationsModeration to RAI policies

cli/azd/extensions/azure.ai.agents/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 1.0.0-beta.11 (Unreleased)
+
+### Features Added
+
+- [[#9596]](https://github.com/Azure/azure-dev/pull/9596) Add `invocationsModeration` to `rai_policy` policies so hosted agents on the `invocations` protocol can tell the content-safety proxy where the moderatable text lives in their request and response bodies. Supports buffered and server-sent-event responses via `responseMode`, `inputPaths`, `outputPaths`, and `streamSelectors`, and is validated locally before deploy. Declaring `invocationsModeration` on an agent that does not expose the `invocations` protocol is now a validation error, since the block would otherwise be silently ignored.
+
 ## 1.0.0-beta.10 (2026-08-13)
 
 ### Features Added

cli/azd/extensions/azure.ai.agents/README.md

@@ -133,6 +133,72 @@ Details:
 > the other inline agent properties such as `codeConfiguration` and
 > `environmentVariables`.
 
+### Moderating invocations-protocol traffic
+
+For agents that expose the `invocations` protocol, the RAI policy alone is not
+enough: the content-safety proxy needs to be told **where the text lives** in the
+request and response bodies. Without that it has nothing to submit to the policy,
+so no content is actually screened. Supply an `invocationsModeration` block on the
+`rai_policy` entry:
+
+```yaml
+services:
+  my-agent:
+    host: azure.ai.agent
+    project: .
+    kind: hosted
+    name: my-agent
+    protocols:
+      - protocol: invocations
+        version: "1.0.0"
+    policies:
+      - type: rai_policy
+        raiPolicyName: /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.CognitiveServices/accounts/<account-name>/raiPolicies/<policy-name>
+        invocationsModeration:
+          responseMode: both
+          inputContentType: json
+          outputContentType: json
+          inputPaths:
+            - $.input
+          outputPaths:
+            - $.output
+          streamSelectors:
+            - eventType: response.output_text.delta
+              textField: $.delta
+```
+
+Fields:
+
+| Field | Required | Description |
+| --- | --- | --- |
+| `responseMode` | yes | `non_streaming`, `streaming`, or `both`. |
+| `inputContentType` | no | `json` (default) or `text`. |
+| `outputContentType` | no | `json` (default) or `text`. |
+| `inputPaths` | when `inputContentType` is `json` or omitted (it defaults to `json`) | JSONPath expressions selecting the request text. |
+| `outputPaths` | when `responseMode` includes non-streaming and `outputContentType` is `json` | JSONPath expressions selecting the buffered response text. |
+| `streamSelectors` | when `responseMode` includes streaming and `outputContentType` is `json` | `eventType` (required) and `textField` per server-sent event frame. |
+
+`invocationsModeration` is only valid on agents whose `protocols` list includes
+`invocations`. Declaring it elsewhere — including on an `invocations_ws`-only
+agent, which does not go through the content-safety HTTP proxy — fails validation
+rather than silently deploying a policy that never runs.
+
+> **Understanding `responseMode`:** it declares which response *shapes* the
+> container can produce, **not** "input and output". Input is always moderated.
+> For the output side the proxy inspects the actual response `Content-Type` and
+> runs exactly one gate: the SSE gate for `text/event-stream`, the buffered gate
+> otherwise. Use `both` only for containers that genuinely answer both ways —
+> if a response arrives in a shape `responseMode` did not declare, the request
+> fails closed rather than skipping moderation.
+
+Set `inputContentType`/`outputContentType` to `text` when the body is plain text;
+the whole body is then moderated and no paths are needed for that direction.
+
+As with `raiPolicyName`, the deprecated on-disk `agent.yaml` shape uses snake_case
+keys throughout this block (`invocations_moderation`, `response_mode`,
+`input_paths`, `stream_selectors`, `event_type`, and so on). The **values**
+(`non_streaming`, `streaming`, `both`, `json`, `text`) are the same in both.
+
 ## Session carry-over across deploys
 
 When a hosted agent is redeployed, Foundry assigns the agent a **new version** and

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go

@@ -90,9 +90,57 @@ const (
 	AgentEventHandlerDestinationTypeEvals AgentEventHandlerDestinationType = "evals"
 )
 
+// RaiInvocationContentType identifies how the invocations request or response body is encoded,
+// which determines how the content-safety proxy extracts the text it moderates.
+type RaiInvocationContentType string
+
+const (
+	// RaiInvocationContentTypeJSON extracts text from a JSON body using JSONPath expressions.
+	RaiInvocationContentTypeJSON RaiInvocationContentType = "json"
+	// RaiInvocationContentTypeText treats the whole body as the text to moderate.
+	RaiInvocationContentTypeText RaiInvocationContentType = "text"
+)
+
+// RaiInvocationMode declares the response shapes the agent container is able to produce.
+// It is not an "input and output" switch: at runtime the proxy inspects the actual response
+// Content-Type and runs exactly one output gate.
+type RaiInvocationMode string
+
+const (
+	// RaiInvocationModeNonStreaming declares the agent only returns buffered (non-SSE) responses.
+	RaiInvocationModeNonStreaming RaiInvocationMode = "non_streaming"
+	// RaiInvocationModeStreaming declares the agent only returns server-sent event streams.
+	RaiInvocationModeStreaming RaiInvocationMode = "streaming"
+	// RaiInvocationModeBoth declares the agent may return either shape depending on the request.
+	RaiInvocationModeBoth RaiInvocationMode = "both"
+)
+
+// SseTextSelector locates the moderatable text inside a single server-sent event frame.
+type SseTextSelector struct {
+	// EventType is the SSE event name the selector applies to.
+	EventType string `json:"event_type"`
+	// TextField is the JSONPath expression, relative to the frame payload, holding the text.
+	TextField string `json:"text_field,omitempty"`
+}
+
+// InvocationsModeration configures how the content-safety proxy extracts text from
+// invocations-protocol requests and responses so it can be submitted to the RAI policy.
+// Without it a RAI policy attached to an invocations agent has nothing to moderate.
+type InvocationsModeration struct {
+	InputContentType  RaiInvocationContentType `json:"input_content_type,omitempty"`
+	OutputContentType RaiInvocationContentType `json:"output_content_type,omitempty"`
+	ResponseMode      RaiInvocationMode        `json:"response_mode"`
+	InputPaths        []string                 `json:"input_paths,omitempty"`
+	OutputPaths       []string                 `json:"output_paths,omitempty"`
+	StreamSelectors   []SseTextSelector        `json:"stream_selectors,omitempty"`
+}
+
 // RaiConfig represents configuration for Responsible AI content filtering
 type RaiConfig struct {
 	RaiPolicyName string `json:"rai_policy_name"`
+	// InvocationsModeration is optional and only meaningful for agents that expose the
+	// invocations protocol.
+	InvocationsModeration *InvocationsModeration `json:"invocations_moderation,omitempty"`
 }
 
 // AgentDefinition is the base definition for all agent types

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go

@@ -8,6 +8,7 @@ import (
 	"maps"
 	"math"
 	"regexp"
+	"slices"
 	"strings"
 
 	"azureaiagent/internal/pkg/agents/agent_api"
@@ -91,12 +92,41 @@ func constructBuildConfig(options ...AgentBuildOption) *AgentBuildConfig {
 func mapRaiConfig(policies []Policy) *agent_api.RaiConfig {
 	for _, policy := range policies {
 		if policy.Type == PolicyTypeRai && policy.RaiPolicyName != "" {
-			return &agent_api.RaiConfig{RaiPolicyName: policy.RaiPolicyName}
+			return &agent_api.RaiConfig{
+				RaiPolicyName:         policy.RaiPolicyName,
+				InvocationsModeration: mapInvocationsModeration(policy.InvocationsModeration),
+			}
 		}
 	}
 	return nil
 }
 
+// mapInvocationsModeration translates the YAML invocations-moderation block into its
+// data-plane representation. It returns nil when the block is absent so agents that do not
+// configure it serialize exactly as before.
+func mapInvocationsModeration(moderation *InvocationsModeration) *agent_api.InvocationsModeration {
+	if moderation == nil {
+		return nil
+	}
+
+	mapped := &agent_api.InvocationsModeration{
+		InputContentType:  agent_api.RaiInvocationContentType(moderation.InputContentType),
+		OutputContentType: agent_api.RaiInvocationContentType(moderation.OutputContentType),
+		ResponseMode:      agent_api.RaiInvocationMode(moderation.ResponseMode),
+		InputPaths:        slices.Clone(moderation.InputPaths),
+		OutputPaths:       slices.Clone(moderation.OutputPaths),
+	}
+
+	for _, selector := range moderation.StreamSelectors {
+		mapped.StreamSelectors = append(mapped.StreamSelectors, agent_api.SseTextSelector{
+			EventType: selector.EventType,
+			TextField: selector.TextField,
+		})
+	}
+
+	return mapped
+}
+
 // MapEndpointAndCard maps YAML-layer endpoint and card fields to API model types
 // without requiring or validating the full agent definition. This is used by the
 // endpoint update command where only endpoint/card patching is needed.

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_test.go

@@ -6,6 +6,7 @@ package agent_yaml
 import (
 	"encoding/json"
 	"math"
+	"slices"
 	"strings"
 	"testing"
 
@@ -1729,3 +1730,151 @@ func TestMapRaiConfig(t *testing.T) {
 		t.Errorf("mapRaiConfig(p1) = %+v, want RaiPolicyName=p1", got)
 	}
 }
+
+func TestMapRaiConfig_WithInvocationsModeration(t *testing.T) {
+	t.Parallel()
+
+	got := mapRaiConfig([]Policy{{
+		Type:          PolicyTypeRai,
+		RaiPolicyName: "p1",
+		InvocationsModeration: &InvocationsModeration{
+			InputContentType:  InvocationContentTypeJSON,
+			OutputContentType: InvocationContentTypeJSON,
+			ResponseMode:      InvocationResponseModeBoth,
+			InputPaths:        []string{"$.input"},
+			OutputPaths:       []string{"$.output"},
+			StreamSelectors: []SseTextSelector{
+				{EventType: "response.output_text.delta", TextField: "$.delta"},
+			},
+		},
+	}})
+
+	if got == nil {
+		t.Fatal("mapRaiConfig returned nil")
+	}
+	moderation := got.InvocationsModeration
+	if moderation == nil {
+		t.Fatal("InvocationsModeration is nil")
+	}
+	if moderation.InputContentType != agent_api.RaiInvocationContentTypeJSON {
+		t.Errorf("InputContentType = %q, want %q",
+			moderation.InputContentType, agent_api.RaiInvocationContentTypeJSON)
+	}
+	if moderation.OutputContentType != agent_api.RaiInvocationContentTypeJSON {
+		t.Errorf("OutputContentType = %q, want %q",
+			moderation.OutputContentType, agent_api.RaiInvocationContentTypeJSON)
+	}
+	if moderation.ResponseMode != agent_api.RaiInvocationModeBoth {
+		t.Errorf("ResponseMode = %q, want %q", moderation.ResponseMode, agent_api.RaiInvocationModeBoth)
+	}
+	if !slices.Equal(moderation.InputPaths, []string{"$.input"}) {
+		t.Errorf("InputPaths = %v, want [$.input]", moderation.InputPaths)
+	}
+	if !slices.Equal(moderation.OutputPaths, []string{"$.output"}) {
+		t.Errorf("OutputPaths = %v, want [$.output]", moderation.OutputPaths)
+	}
+	if len(moderation.StreamSelectors) != 1 {
+		t.Fatalf("len(StreamSelectors) = %d, want 1", len(moderation.StreamSelectors))
+	}
+	if moderation.StreamSelectors[0].EventType != "response.output_text.delta" {
+		t.Errorf("StreamSelectors[0].EventType = %q, want response.output_text.delta",
+			moderation.StreamSelectors[0].EventType)
+	}
+	if moderation.StreamSelectors[0].TextField != "$.delta" {
+		t.Errorf("StreamSelectors[0].TextField = %q, want $.delta",
+			moderation.StreamSelectors[0].TextField)
+	}
+}
+
+func TestMapRaiConfig_WithoutInvocationsModeration(t *testing.T) {
+	t.Parallel()
+
+	got := mapRaiConfig([]Policy{{Type: PolicyTypeRai, RaiPolicyName: "p1"}})
+	if got == nil {
+		t.Fatal("mapRaiConfig returned nil")
+	}
+	if got.InvocationsModeration != nil {
+		t.Errorf("InvocationsModeration = %+v, want nil", got.InvocationsModeration)
+	}
+
+	// Agents that do not configure moderation must serialize exactly as they did before the
+	// field existed, so existing deployments are unaffected.
+	encoded, err := json.Marshal(got)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if string(encoded) != `{"rai_policy_name":"p1"}` {
+		t.Errorf("serialized rai_config = %s, want {\"rai_policy_name\":\"p1\"}", encoded)
+	}
+}
+
+func TestMapRaiConfig_InvocationsModerationSlicesAreCopied(t *testing.T) {
+	t.Parallel()
+
+	inputPaths := []string{"$.input"}
+	policies := []Policy{{
+		Type:          PolicyTypeRai,
+		RaiPolicyName: "p1",
+		InvocationsModeration: &InvocationsModeration{
+			ResponseMode: InvocationResponseModeNonStreaming,
+			InputPaths:   inputPaths,
+		},
+	}}
+
+	got := mapRaiConfig(policies)
+	inputPaths[0] = "$.mutated"
+
+	if got.InvocationsModeration.InputPaths[0] != "$.input" {
+		t.Errorf("mapped InputPaths aliases the source slice: got %q",
+			got.InvocationsModeration.InputPaths[0])
+	}
+}
+
+func TestCreateHostedAgentAPIRequest_WithInvocationsModeration(t *testing.T) {
+	t.Parallel()
+
+	agent := ContainerAgent{
+		AgentDefinition: AgentDefinition{
+			Kind: AgentKindHosted,
+			Name: "rai-agent",
+		},
+		Protocols: []ProtocolVersionRecord{{Protocol: InvocationsProtocol, Version: "1.0.0"}},
+		Policies: []Policy{{
+			Type:          PolicyTypeRai,
+			RaiPolicyName: "/subscriptions/x/raiPolicies/p",
+			InvocationsModeration: &InvocationsModeration{
+				ResponseMode: InvocationResponseModeNonStreaming,
+				InputPaths:   []string{"$.input"},
+				OutputPaths:  []string{"$.output"},
+			},
+		}},
+	}
+
+	req, err := CreateHostedAgentAPIRequest(agent, &AgentBuildConfig{ImageURL: "img:latest"})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	definition, ok := req.Definition.(agent_api.HostedAgentDefinition)
+	if !ok {
+		t.Fatalf("unexpected definition type %T", req.Definition)
+	}
+	if definition.RaiConfig == nil || definition.RaiConfig.InvocationsModeration == nil {
+		t.Fatalf("expected invocations moderation on the request, got %+v", definition.RaiConfig)
+	}
+
+	encoded, err := json.Marshal(definition.RaiConfig)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	for _, want := range []string{
+		`"invocations_moderation"`,
+		`"response_mode":"non_streaming"`,
+		`"input_paths":["$.input"]`,
+		`"output_paths":["$.output"]`,
+	} {
+		if !strings.Contains(string(encoded), want) {
+			t.Errorf("serialized rai_config %s missing %s", encoded, want)
+		}
+	}
+}