Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions cli/azd/extensions/azure.ai.agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,20 @@ effect: activity-protocol agents open the Microsoft 365 Agents Playground rather
than the Agent Inspector, and `--port 8087` on its own collides with the
inspector's own default UI port.

### Local client route telemetry

When installed from the official registry, the extension reports the
`local_client.route.selected` usage event after `azd ai agent run` resolves the
service and protocol profile. Its `ext.route` attribute is exactly one of:

- `inspector` for a non-activity agent;
- `playground` for an activity-protocol agent; or
- `suppressed` when `--no-client` or the deprecated `--no-inspector` is set.

The event is emitted before checking client availability, starting the local
agent, or launching a client. It records route selection, not successful client
launch.

## Migrating Legacy Agent Configuration

New Foundry agent projects keep the agent definition directly on the
Expand Down
2 changes: 1 addition & 1 deletion cli/azd/extensions/azure.ai.agents/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.3
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0
github.com/azure/azure-dev/cli/azd v1.28.0
github.com/azure/azure-dev/cli/azd v1.31.0
github.com/braydonk/yaml v0.9.0
github.com/drone/envsubst v1.0.3 // indirect
github.com/fatih/color v1.18.0
Expand Down
4 changes: 2 additions & 2 deletions cli/azd/extensions/azure.ai.agents/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWp
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s=
github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8=
github.com/azure/azure-dev/cli/azd v1.31.0 h1:p0U4F6w2bPrdzmzavksqfJCnlXoQu9GTQogy+6KXMmM=
github.com/azure/azure-dev/cli/azd v1.31.0/go.mod h1:HFBGeWRWhNsOoYaUcyToqaowibqcbSCfkfJfnIfI4nU=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
Expand Down
31 changes: 30 additions & 1 deletion cli/azd/extensions/azure.ai.agents/internal/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ import (
const (
agentInspectorExtensionID = "azure.ai.inspector"
agentInspectorReadyPollPeriod = 250 * time.Millisecond
localClientRouteSelectedEvent = "local_client.route.selected"
localClientRouteAttribute = "route"
localClientRouteInspector = "inspector"
localClientRoutePlayground = "playground"
localClientRouteSuppressed = "suppressed"
// defaultInspectorUIPort mirrors the default UI port of the
// azure.ai.inspector extension. The inspector extension remains the source
// of truth for the actual default: when --inspector-port is unset we do not
Expand Down Expand Up @@ -152,6 +157,8 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error {
// validation can fail without starting a process, and such a failure must
// not clear a session belonging to an already-running agent.
activityProfile := resolveActivityRunProfile(runCtx.Definition)
suppressClient := flags.noInspector || flags.noClient
reportLocalClientRouteSelected(ctx, azdClient.Telemetry(), activityProfile, suppressClient)
if err := validateInspectorPortForProfile(flags, activityProfile.IsActivity); err != nil {
return err
}
Expand All @@ -171,7 +178,6 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error {
// Resolve local-client availability before the agent starts so advisory
// port warnings can account for whether an inspector will actually launch.
// Reuse the result after proc.Start rather than issuing a second RPC.
suppressClient := flags.noInspector || flags.noClient
inspectorInstalled := false
var inspectorInstallErr error
if !activityProfile.IsActivity && !suppressClient {
Expand Down Expand Up @@ -379,6 +385,29 @@ func runRun(ctx context.Context, flags *runFlags, noPrompt bool) error {
return nil
}

func reportLocalClientRouteSelected(
ctx context.Context,
telemetry azdext.TelemetryServiceClient,
activityProfile activityRunProfile,
suppressClient bool,
) {
route := localClientRouteInspector
if suppressClient {
route = localClientRouteSuppressed
} else if activityProfile.IsActivity {
route = localClientRoutePlayground
}

if _, err := telemetry.ReportUsage(ctx, &azdext.ReportUsageRequest{
EventName: localClientRouteSelectedEvent,
Attributes: map[string]string{
localClientRouteAttribute: route,
},
Comment thread
dooriya marked this conversation as resolved.
}); err != nil {
log.Printf("run: failed to report local client route selection: %v", err)
}
}

func handleInspectorAutoLaunch(
ctx context.Context,
workflow azdext.WorkflowServiceClient,
Expand Down
85 changes: 85 additions & 0 deletions cli/azd/extensions/azure.ai.agents/internal/cmd/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"io"
"maps"
"net"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -238,6 +239,73 @@ func TestWaitForLocalPort(t *testing.T) {
})
}

func TestReportLocalClientRouteSelected(t *testing.T) {
t.Parallel()

tests := []struct {
name string
activityProfile activityRunProfile
suppressClient bool
reportErr error
wantRoute string
}{
{
name: "selects Inspector for non-activity agent",
wantRoute: localClientRouteInspector,
},
{
name: "selects Playground for activity agent",
activityProfile: activityRunProfile{IsActivity: true},
wantRoute: localClientRoutePlayground,
},
{
name: "selects suppressed for non-activity agent",
suppressClient: true,
wantRoute: localClientRouteSuppressed,
},
{
name: "suppression overrides activity route",
activityProfile: activityRunProfile{IsActivity: true},
suppressClient: true,
wantRoute: localClientRouteSuppressed,
},
{
name: "reporting failure is best effort",
reportErr: errors.New("telemetry unavailable"),
wantRoute: localClientRouteInspector,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

telemetry := &recordingTelemetryClient{err: tt.reportErr}
reportLocalClientRouteSelected(
t.Context(),
telemetry,
tt.activityProfile,
tt.suppressClient,
)

if telemetry.request == nil {
t.Fatal("expected telemetry request")
}
if telemetry.request.EventName != localClientRouteSelectedEvent {
t.Fatalf(
"event name = %q, want %q",
telemetry.request.EventName,
localClientRouteSelectedEvent,
)
}
wantAttributes := map[string]string{localClientRouteAttribute: tt.wantRoute}
if !maps.Equal(telemetry.request.Attributes, wantAttributes) {
t.Fatalf("attributes = %v, want %v", telemetry.request.Attributes, wantAttributes)
}
})
}
}

func TestLaunchInspectorUsesWorkflowCommand(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -743,6 +811,11 @@ type recordingWorkflowClient struct {
called chan struct{}
}

type recordingTelemetryClient struct {
request *azdext.ReportUsageRequest
err error
}

type lockedBuffer struct {
mu sync.Mutex
bytes.Buffer
Expand Down Expand Up @@ -772,6 +845,18 @@ func (c *recordingWorkflowClient) Run(
return &azdext.EmptyResponse{}, c.err
}

func (c *recordingTelemetryClient) ReportUsage(
_ context.Context,
request *azdext.ReportUsageRequest,
_ ...grpc.CallOption,
) (*azdext.ReportUsageResponse, error) {
c.request = request
if c.err != nil {
return nil, c.err
}
return &azdext.ReportUsageResponse{Accepted: true}, nil
}

// createVenv sets up a minimal .venv directory structure for testing.
// Returns the path to the .venv directory.
func createVenv(t *testing.T, projectDir string) string {
Expand Down
15 changes: 15 additions & 0 deletions cli/azd/extensions/azure.ai.inspector/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,21 @@ azd ai inspector launch --session-id <uuid> --conversation-id <uuid>
| `--session-id` | _(SPA mints UUID)_ | Optional explicit session ID for the SPA. |
| `--conversation-id` | _(SPA mints UUID)_ | Optional explicit conversation ID for the SPA. |

## Telemetry

When the SPA sends `setViewReady` after mounting, the extension reports this
best-effort usage event through azd:

```text
extension.event = inspector.funnel.stage
ext.stage = ui_ready
ext.outcome = succeeded
```

The event means the Inspector UI loaded. It does not mean that the UI connected
to the agent or sent a request. No ports, URLs, IDs, prompts, or responses are
included.

## Local Development

### Prerequisites
Expand Down
2 changes: 1 addition & 1 deletion cli/azd/extensions/azure.ai.inspector/extension.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ description: Browser-based inspector UI for locally running Foundry agents. (Bet
usage: azd ai inspector <command> [options]
# NOTE: Make sure version.txt is in sync with this version.
version: 1.0.0-beta.3
requiredAzdVersion: ">=1.27.0"
requiredAzdVersion: ">=1.31.0"
language: go
capabilities:
- custom-commands
Expand Down
8 changes: 5 additions & 3 deletions cli/azd/extensions/azure.ai.inspector/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@ module azureaiinspector
go 1.26.4

require (
github.com/azure/azure-dev/cli/azd v1.24.3
github.com/azure/azure-dev/cli/azd v1.31.0
github.com/cli/browser v1.3.0
github.com/gorilla/websocket v1.5.3
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.11.1
)

require (
dario.cat/mergo v1.0.2 // indirect
github.com/AlecAivazis/survey/v2 v2.3.7 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
Expand Down Expand Up @@ -75,7 +77,6 @@ require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/stretchr/testify v1.11.1 // indirect
github.com/theckman/yacspin v0.13.12 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
Expand All @@ -91,7 +92,8 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/term v0.44.0 // indirect
golang.org/x/text v0.38.0 // indirect
Expand Down
12 changes: 8 additions & 4 deletions cli/azd/extensions/azure.ai.inspector/go.sum
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8=
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc=
Expand Down Expand Up @@ -45,8 +47,8 @@ github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWp
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/azure/azure-dev/cli/azd v1.24.3 h1:r2kEr2YYLu4ImKo6nR/WjhHg/1SliN1uwmAVqnM8t3o=
github.com/azure/azure-dev/cli/azd v1.24.3/go.mod h1:YANepMw36aWA8/mQyXau6JCAG84oK0ZgfvLF8rN5asU=
github.com/azure/azure-dev/cli/azd v1.31.0 h1:p0U4F6w2bPrdzmzavksqfJCnlXoQu9GTQogy+6KXMmM=
github.com/azure/azure-dev/cli/azd v1.31.0/go.mod h1:HFBGeWRWhNsOoYaUcyToqaowibqcbSCfkfJfnIfI4nU=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o=
Expand Down Expand Up @@ -259,11 +261,13 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
30 changes: 30 additions & 0 deletions cli/azd/extensions/azure.ai.inspector/internal/cmd/inspector.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"azureaiinspector/internal/inspector"

"github.com/azure/azure-dev/cli/azd/pkg/azdext"
"github.com/cli/browser"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -98,6 +99,9 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error {
}
}

reportUsage, closeTelemetry := newUsageReporter(ctx)
defer closeTelemetry()

srv := inspector.New(inspector.Config{
Port: flags.inspectorPort,
AgentPort: flags.port,
Expand All @@ -106,6 +110,7 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error {
ConversationID: flags.conversationID,
SSESink: sseSink,
Silent: flags.silent,
ReportUsage: reportUsage,
})

url := srv.URL()
Expand Down Expand Up @@ -137,6 +142,31 @@ func runInspector(ctx context.Context, flags *inspectorFlags) error {
return srv.Start(ctx, ready)
}

func newUsageReporter(ctx context.Context) (inspector.ReportUsageFunc, func()) {
azdClient, err := azdext.NewAzdClient()
if err != nil {
log.Printf("inspector: failed to create telemetry client: %v", err)
return nil, func() {}
}

return usageReporter(ctx, azdClient.Telemetry()), azdClient.Close
}

func usageReporter(ctx context.Context, telemetry azdext.TelemetryServiceClient) inspector.ReportUsageFunc {
// Capture the command context because WebSocket request contexts do not carry
// the azd access token or parent trace metadata needed by ReportUsage.
reportUsage := func(eventName string, attributes map[string]string) {
if _, err := telemetry.ReportUsage(ctx, &azdext.ReportUsageRequest{
EventName: eventName,
Attributes: attributes,
}); err != nil {
log.Printf("inspector: failed to report %s: %v", eventName, err)
}
}

return reportUsage
}

// injectSSEEvents wraps the local agentserver SSE stream so it matches the
// Foundry SSE shape that readSSEStream expects. agentserver discriminates
// chunks via a JSON `type` field on each `data:` line and omits the
Expand Down
Loading
Loading