diff --git a/evals/azure-skills/microsoft-foundry/eval.yaml b/evals/azure-skills/microsoft-foundry/eval.yaml index 3284eeb73..710334516 100644 --- a/evals/azure-skills/microsoft-foundry/eval.yaml +++ b/evals/azure-skills/microsoft-foundry/eval.yaml @@ -19,6 +19,10 @@ scoring: threshold: 0.8 stimuli: + # ═══════════════════════════════════════════════════════════════════════════ + # Microsoft Foundry Routing + # ═══════════════════════════════════════════════════════════════════════════ + # ── ai-model-deployment-prompt ── # Jest: "invokes microsoft-foundry skill for AI model deployment prompt" # Assertions: softCheckSkill + isSkillInvoked (invocation rate ≥ 80%) @@ -29,13 +33,12 @@ stimuli: tier: smoke cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -51,13 +54,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -73,13 +75,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -95,13 +96,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -117,13 +117,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -139,13 +138,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -161,13 +159,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -183,13 +180,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -205,13 +201,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -227,13 +222,12 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: @@ -249,19 +243,21 @@ stimuli: tier: full cost: llm area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # Global: has_output - - type: completed # Global: no_runtime_failure - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" - # ── agent-optimizer-prompt ── + # ═══════════════════════════════════════════════════════════════════════════ + # Foundry Agent Optimizer + # ═══════════════════════════════════════════════════════════════════════════ + - name: "Generate evals for Agent Optimizer" prompt: | I have an azd Python hosted agent that is already wired for Foundry Agent Optimizer. @@ -273,7 +269,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -290,7 +286,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -307,14 +303,17 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: required: - microsoft-foundry - # ── agent-observe-prompts ── + # ═══════════════════════════════════════════════════════════════════════════ + # Foundry Agent Observe + # ═══════════════════════════════════════════════════════════════════════════ + - name: "Evaluate deployed Foundry agent" prompt: | Evaluate my deployed Foundry agent using the evaluation suite in its @@ -325,7 +324,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -342,7 +341,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -359,12 +358,40 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Evaluate Foundry agent quality" + prompt: "Evaluate my Foundry agent and check its quality" + tags: + type: integration + tier: full + cost: llm + area: routing graders: - type: skill-invocation config: required: - microsoft-foundry + - type: completed + + - name: "Set up Foundry agent observability" + prompt: "Set up monitoring and evaluation for my Foundry agent" + tags: + type: integration + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: completed # ═══════════════════════════════════════════════════════════════════════════ # Foundry Agent Create @@ -377,7 +404,7 @@ stimuli: tier: smoke cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -394,7 +421,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -415,7 +442,7 @@ stimuli: tier: smoke cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -432,7 +459,7 @@ stimuli: tier: full cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -453,7 +480,7 @@ stimuli: tier: smoke cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -474,7 +501,7 @@ stimuli: tier: smoke cost: llm area: routing - earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"}]' + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation config: @@ -484,7 +511,595 @@ stimuli: config: pattern: "(?i)fatal error|unhandled exception|stack trace" - # ── Foundry E2E checks ── + # ═══════════════════════════════════════════════════════════════════════════ + # Foundry Agent Trace + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Analyze Foundry agent traces" + prompt: "Analyze traces for my Foundry agent in App Insights" + tags: + type: integration + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: completed + + - name: "Find failing Foundry agent traces" + prompt: "Find failing traces and errors for my Foundry agent" + tags: + type: integration + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: completed + + # ═══════════════════════════════════════════════════════════════════════════ + # Fine-tuning + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Fine-tuning guidance" + prompt: "Help me fine-tune gpt-4.1-mini on my dataset" + tags: + type: integration + tier: full + cost: llm + area: response-quality + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"[Tt]raining"}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)training" + + - name: "RFT with a Python grader" + prompt: "Submit a reinforcement fine-tuning job with a Python grader" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "SFT model distillation" + prompt: "Distill gpt-4.1-mini into nano using supervised fine-tuning" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + # ═══════════════════════════════════════════════════════════════════════════ + # Model Deployment Capacity + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Discover model capacity across regions" + prompt: "Find available capacity for gpt-4o across all Azure regions" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Compare regional TPM capacity" + prompt: "Which Azure regions have gpt-4o available with enough TPM capacity?" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + # ═══════════════════════════════════════════════════════════════════════════ + # Model Deployment Customize + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Configure a custom model deployment" + prompt: "Deploy gpt-4o with custom SKU and capacity configuration" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Deploy with provisioned throughput" + prompt: "Deploy gpt-4o with provisioned throughput PTU in my Foundry project" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + # ═══════════════════════════════════════════════════════════════════════════ + # Model Deployment Router + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Deploy a model to an Azure project" + prompt: "Deploy gpt-4o model to my Azure project" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Route model deployment to capacity discovery" + prompt: "Where can I deploy gpt-4o? Check capacity across regions" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Route model deployment to customization" + prompt: "Deploy gpt-4o with custom SKU and capacity settings" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + # ═══════════════════════════════════════════════════════════════════════════ + # Model Deployment Optimal Region + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Deploy quickly to the optimal region" + prompt: "Deploy gpt-4o quickly to the optimal region" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + - name: "Deploy to the best highly available region" + prompt: "Deploy gpt-4o to the best available region with high availability" + tags: + type: integration + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"microsoft-foundry"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + + # ═══════════════════════════════════════════════════════════════════════════ + # Quota + # ═══════════════════════════════════════════════════════════════════════════ + + - name: "Check current quota usage" + prompt: "Use the microsoft-foundry skill to show me my current quota usage for Microsoft Foundry resources" + tags: + type: integration + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: completed + + - name: "Show quota-related commands" + prompt: "How do I check my Microsoft Foundry quota limits?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)az cognitiveservices|quota" + - type: completed + + - name: "Explain quota and TPM" + prompt: "Explain quota in Microsoft Foundry" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)TPM|Tokens Per Minute" + - type: completed + + - name: "Check quota before model deployment" + prompt: "Use the microsoft-foundry skill to check if I have enough quota to deploy GPT-4o to Microsoft Foundry" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)capacity|quota" + - type: completed + + - name: "Calculate production quota requirements" + prompt: "How much quota do I need for a production Foundry deployment?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)TPM|PTU|capacity|tokens per minute" + - type: output-matches + config: + pattern: "(?i)calculate|estimate|calculation|quantify" + - type: completed + + - name: "Request a quota increase" + prompt: "Using the microsoft-foundry quota skill, how do I request a quota increase for Microsoft Foundry?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)Azure Portal|portal" + - type: completed + + - name: "Explain quota increase justification" + prompt: "Request more TPM quota for Microsoft Foundry and explain what justification is needed" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)justification|business|reason|rationale" + - type: completed + + - name: "Monitor quota across deployments" + prompt: "Use the microsoft-foundry quota skill to monitor quota usage across all my Microsoft Foundry deployments" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)deployment|usage|quota" + - type: completed + + - name: "Track quota allocation by model" + prompt: "Show me quota allocation by model in Microsoft Foundry" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)model" + - type: output-matches + config: + pattern: "(?i)capacity|quota|allocation" + - type: completed + + - name: "Troubleshoot QuotaExceeded" + prompt: "My Microsoft Foundry deployment failed with QuotaExceeded error. Help me fix it." + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)QuotaExceeded|quota" + - type: completed + + - name: "Troubleshoot InsufficientQuota" + prompt: "I'm getting an InsufficientQuota error when deploying gpt-4o to eastus in Microsoft Foundry. Use the microsoft-foundry skill to help me troubleshoot and fix this." + tags: + type: integration + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: completed + + - name: "Troubleshoot DeploymentLimitReached" + prompt: "DeploymentLimitReached error in Microsoft Foundry, what should I do?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)delete|deployment" + - type: completed + + - name: "Address 429 rate limiting" + prompt: "Getting 429 rate limit errors from my Foundry deployment" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)429|rate limit" + - type: completed + + - name: "Plan production deployment capacity" + prompt: "Help me plan capacity for production Microsoft Foundry deployment with 1M requests per day" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)TPM|PTU|capacity|tokens per minute" + - type: output-matches + config: + pattern: "(?i)calculate|estimate|calculation|quantify" + - type: completed + + - name: "Explain quota management best practices" + prompt: "What are best practices for quota management in Microsoft Foundry?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)best practice|optimize" + - type: completed + + - name: "List model deployments and capacity" + prompt: "Use the microsoft-foundry skill to list all my Microsoft Foundry model deployments and their capacity" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: prompt + config: + scoring: binary + threshold: 1 + prompt: | + Pass if the trajectory does at least one of the following: calls an Azure MCP model deployment tool, runs an Azure CLI command concerning deployments, models, capacity, or quota, or mentions an applicable `az cognitiveservices`, `az rest`, or `az ai` command. Otherwise fail. + - type: completed + + - name: "Explain regional quota distribution" + prompt: "Using the microsoft-foundry quota skill, explain how quota works across different Azure regions for Foundry" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)region" + - type: completed + + - name: "Offer another region when quota is exhausted" + prompt: "I ran out of quota in East US for Microsoft Foundry. What are my options?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)region|location" + - type: completed + + - name: "Optimize quota allocation" + prompt: "How can I optimize my Microsoft Foundry quota allocation?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)optimize|consolidate" + - type: completed + + - name: "Free quota from unused deployments" + prompt: "I need to free up quota in Microsoft Foundry" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)delete|unused" + - type: completed + + - name: "Interpret quota usage output" + prompt: "What does the quota usage output mean in Microsoft Foundry?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)currentValue|limit" + - type: completed + + - name: "Explain the TPM quota concept" + prompt: "What is TPM in the context of Microsoft Foundry quotas?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)Tokens Per Minute|TPM" + - type: completed + + - name: "Provide quota error resolution steps" + prompt: "Walk me through fixing a quota error in Microsoft Foundry deployment" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: skill-invocation + config: + required: + - microsoft-foundry + - type: output-matches + config: + pattern: "(?i)step|check" + - type: completed + + - name: "Offer multiple quota resolution options" + prompt: "What are my options when I hit quota limits in Microsoft Foundry?" + tags: + type: integration + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)option|reduce|increase" + - type: completed + + # ═══════════════════════════════════════════════════════════════════════════ + # Foundry E2E Checks + # ═══════════════════════════════════════════════════════════════════════════ + - name: "Golden Path - Create and deploy hosted agent" constraints: max_turns: 50 @@ -516,7 +1131,6 @@ stimuli: code was not generated, no new Foundry project or model deployment was created, local testing was not run, deployment did not succeed, deployment did not use direct code deploy, the deployed agent was not actually invoked after deployment, or the deployed agent invocation failed. - # ── Foundry migration and re-host checks ── - name: "Migration - OpenAI Agents SDK to Foundry" environment: files: @@ -532,7 +1146,7 @@ stimuli: area: migrate prompt: | This project is our existing Python customer-support agent built using OpenAI Agents SDK and self-hosted as a container on our internal platform. Re-host it on Microsoft Foundry with the minimum code changes necessary, preserving its existing architecture and behavior. Run it locally to make sure it works, create a new Foundry project with Foundry models and deploy the agent there, then invoke the deployed agent to make sure it works after deployment. - + Foundry model: gpt-5.4-nano Region: eastus graders: diff --git a/tests/microsoft-foundry/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index ea5827f0d..000000000 --- a/tests/microsoft-foundry/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/finetuning/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/finetuning/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index f5b1a0f04..000000000 --- a/tests/microsoft-foundry/finetuning/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`finetuning - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`finetuning - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/finetuning/integration.test.ts b/tests/microsoft-foundry/finetuning/integration.test.ts deleted file mode 100644 index 8d25f0292..000000000 --- a/tests/microsoft-foundry/finetuning/integration.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Integration Tests for finetuning sub-skill - * - * Tests skill behavior with a real Copilot agent session. - * Requires Copilot CLI to be installed and authenticated. - */ - -import { - useAgentRunner, - doesAssistantMessageIncludeKeyword, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../utils/agent-runner"; -import { isSkillInvoked, withTestResult } from "../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_finetuning - Integration Tests`, () => { - const agent = useAgentRunner({ - useJest: true, - isTest: true - }); - - test("invokes skill for fine-tuning prompt", () => - withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Help me fine-tune gpt-4.1-mini on my dataset", - shouldEarlyTerminate: (metadata) => - isSkillInvoked(metadata, SKILL_NAME) || - doesAssistantMessageIncludeKeyword(metadata, "fine-tun") || - doesAssistantMessageIncludeKeyword(metadata, "training"), - }); - - // Skill should be invoked OR response should mention fine-tuning - const skillInvoked = isSkillInvoked(agentMetadata, SKILL_NAME); - const mentionsFT = doesAssistantMessageIncludeKeyword(agentMetadata, "fine-tun") || - doesAssistantMessageIncludeKeyword(agentMetadata, "training"); - expect(skillInvoked || mentionsFT).toBe(true); - })); - - test("response mentions fine-tuning concepts", () => - withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Help me fine-tune gpt-4.1-mini on my dataset", - shouldEarlyTerminate: (metadata) => - isSkillInvoked(metadata, SKILL_NAME) && - doesAssistantMessageIncludeKeyword(metadata, "training"), - }); - - expect( - doesAssistantMessageIncludeKeyword(agentMetadata, "training") - ).toBe(true); - })); - - test("invokes skill for RFT grader prompt", () => - withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: - "Submit a reinforcement fine-tuning job with a Python grader", - shouldEarlyTerminate: (metadata) => - isSkillInvoked(metadata, SKILL_NAME), - }); - - expect(isSkillInvoked(agentMetadata, SKILL_NAME)).toBe(true); - })); - - test("invokes skill for SFT distillation prompt", () => - withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Distill gpt-4.1-mini into nano using supervised fine-tuning", - shouldEarlyTerminate: (metadata) => - isSkillInvoked(metadata, SKILL_NAME), - }); - - expect(isSkillInvoked(agentMetadata, SKILL_NAME)).toBe(true); - })); -}); diff --git a/tests/microsoft-foundry/finetuning/triggers.test.ts b/tests/microsoft-foundry/finetuning/triggers.test.ts deleted file mode 100644 index 0cd2a5450..000000000 --- a/tests/microsoft-foundry/finetuning/triggers.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Trigger Tests for finetuning sub-skill - * - * Tests that verify the parent microsoft-foundry skill triggers - * on fine-tuning related prompts and routes to the finetuning sub-skill. - */ - -import { TriggerMatcher } from "../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("finetuning - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Fine-tune gpt-4.1-mini on my dataset", - "I want to do supervised fine-tuning on Microsoft Foundry", - "How do I create training data for fine-tuning?", - "Submit a reinforcement fine-tuning job with a Python grader", - "I need to calibrate my RFT grader for fine-tuning", - "Deploy my fine-tuned model", - "Train a custom model on my JSONL dataset", - "Distill gpt-4.1-mini into nano using fine-tuning on Foundry", - "Check my fine-tuning training job status", - "My fine-tuning training job is not working", - "Upload a large training file for fine-tuning", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - } - ); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - // Removed: deploy matches parent skill - "Set up a Kubernetes cluster", - // Removed: Azure matches parent skill - "What is the capital of France?", - "How do I cook pasta?" - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords(), - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "fine-tune ".repeat(1000); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive", () => { - const result1 = triggerMatcher.shouldTrigger("fine-tune my model"); - const result2 = triggerMatcher.shouldTrigger("FINE-TUNE MY MODEL"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/create/toolbox-paths.unit.test.ts b/tests/microsoft-foundry/foundry-agent/create/toolbox-paths.unit.test.ts deleted file mode 100644 index ef542b4de..000000000 --- a/tests/microsoft-foundry/foundry-agent/create/toolbox-paths.unit.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Unit tests for hosted-agent toolbox reference paths. - * - * These tests lock down sample/doc paths that have moved in foundry-samples. - */ - -import { readFile } from "fs/promises"; -import path from "path"; - -const SKILL_NAME = "microsoft-foundry"; - -const readSkillFile = (relativePath: string) => - readFile(path.join(OUTPUT_PATH, "azure-skills", "skills", SKILL_NAME, relativePath), "utf-8"); - -describe("foundry-agent create toolbox reference paths", () => { - test("uses current toolbox sample and docs paths", async () => { - const reference = await readSkillFile("foundry-agent/create/references/use-toolbox-in-hosted-agent.md"); - - expect(reference).toContain("samples/python/hosted-agents/agent-framework/responses/04-foundry-toolbox"); - expect(reference).toContain("samples/csharp/hosted-agents/agent-framework/foundry-toolbox-server-side"); - expect(reference).toContain("learn.microsoft.com/azure/foundry/agents/how-to/tools/toolbox#configure-tools"); - }); - - test("does not reference removed toolbox sample paths", async () => { - const reference = await readSkillFile("foundry-agent/create/references/use-toolbox-in-hosted-agent.md"); - - expect(reference).not.toContain("samples/python/toolbox/maf"); - expect(reference).not.toContain("samples/python/toolbox/copilot-sdk"); - expect(reference).not.toContain("samples/python/toolbox/SUPPORTED_TOOLBOX_TOOLS.md"); - }); - - test("does not describe a declarative toolbox lifecycle", async () => { - const create = await readSkillFile("foundry-agent/create/create-hosted.md"); - const usage = await readSkillFile("foundry-agent/create/references/use-toolbox-in-hosted-agent.md"); - const deploy = await readSkillFile("foundry-agent/deploy/deploy.md"); - const cli = await readSkillFile("foundry-agent/azd-guidance/references/azd-ai-cli.md"); - const guidance = `${create}\n${usage}\n${deploy}\n${cli}`; - - expect(guidance).not.toContain("Does not create a toolbox"); - expect(guidance).not.toContain("host: azure.ai.toolbox"); - expect(guidance).not.toContain("kind: toolbox"); - expect(guidance).not.toContain("declaring the toolbox in `azure.yaml`"); - expect(guidance).not.toContain("connection/toolbox services"); - }); -}); - -describe("foundry-agent toolbox sub-skill", () => { - const TOOLBOX_ENTRY = "foundry-agent/toolbox/toolbox.md"; - - test("lists the supported tool types", async () => { - const toolbox = await readSkillFile(TOOLBOX_ENTRY); - - for (const type of [ - "mcp", - "web_search", - "azure_ai_search", - "code_interpreter", - "file_search", - "openapi", - "a2a_preview", - "toolbox_search_preview", - ]) { - expect(toolbox).toContain(`\`${type}\``); - } - }); - - test("every reference file linked from toolbox.md exists", async () => { - const toolbox = await readSkillFile(TOOLBOX_ENTRY); - - const linked = [...toolbox.matchAll(/\]\((references\/[^)#]+\.md)/g)].map( - (m) => m[1], - ); - expect(linked.length).toBeGreaterThan(0); - - const unique = [...new Set(linked)]; - for (const relative of unique) { - await expect( - readSkillFile(`foundry-agent/toolbox/${relative}`), - ).resolves.toBeTruthy(); - } - }); - - test("locks down the MCP endpoint facts", async () => { - const toolbox = await readSkillFile(TOOLBOX_ENTRY); - - expect(toolbox).toContain("?api-version=v1"); - expect(toolbox).toContain("/toolboxes/{toolbox_name}/mcp?api-version=v1"); - expect(toolbox).toContain( - "/toolboxes/{toolbox_name}/versions/{version}/mcp?api-version=v1", - ); - expect(toolbox).toContain("https://ai.azure.com/.default"); - }); - - test("locks down the tool-naming facts in mcp-protocol.md", async () => { - const mcpProtocol = await readSkillFile( - "foundry-agent/toolbox/references/mcp-protocol.md", - ); - - expect(mcpProtocol).toContain("{server_label}___{tool_name}"); - expect(mcpProtocol).toContain("three underscores"); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/direct-code.unit.test.ts b/tests/microsoft-foundry/foundry-agent/direct-code.unit.test.ts deleted file mode 100644 index 8603fb7c1..000000000 --- a/tests/microsoft-foundry/foundry-agent/direct-code.unit.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Unit tests for direct-code Foundry agent workflow documentation. - * - * These tests lock down service-specific constraints that are easy to regress: - * direct-code deployment is now the preferred azd path for standard hosted - * agents, while container/ACR deployment remains available only when needed. - */ - -import { readFile } from "fs/promises"; -import path from "path"; - -const SKILL_NAME = "microsoft-foundry"; - -const readSkillFile = (relativePath: string) => - readFile(path.join(OUTPUT_PATH, "azure-skills", "skills", SKILL_NAME, relativePath), "utf-8"); - -describe("foundry-agent direct-code workflow docs", () => { - test("create and deploy workflows prefer azd direct-code deployment", async () => { - const createHosted = await readSkillFile("foundry-agent/create/create-hosted.md"); - const quickStart = await readSkillFile("foundry-agent/create/quick-start-hosted.md"); - const deploy = await readSkillFile("foundry-agent/deploy/deploy.md"); - - expect(createHosted).toContain("Pass `--deploy-mode code` by default to use the direct code deployment."); - expect(createHosted).toContain("--deploy-mode code"); - expect(createHosted).toContain("--runtime python_3_13"); - expect(createHosted).toContain("--entry-point main.py"); - expect(quickStart).toContain("| Deploy mode | `code`"); - expect(quickStart).toContain("azd ai agent init --no-prompt"); - expect(quickStart).toContain("--deploy-mode code"); - expect(deploy).toContain("Prefer **direct code deployment through azd**"); - expect(deploy).toContain("`codeConfiguration:` present | **Direct code deploy** through `azd deploy`; no Docker/ACR build."); - expect(deploy).toContain("No `codeConfiguration:` | **Container/ACR deploy** through `azd deploy`"); - expect(deploy).toContain("Default to direct code for standard hosted-agent code."); - }); - - test("legacy manual REST direct-code reference is removed from the workflow", async () => { - const createHosted = await readSkillFile("foundry-agent/create/create-hosted.md"); - const deploy = await readSkillFile("foundry-agent/deploy/deploy.md"); - - expect(deploy).not.toContain("references/direct-code-deployment.md"); - await expect(readSkillFile("foundry-agent/deploy/references/direct-code-deployment.md")) - .rejects - .toThrow(/ENOENT/); - expect(deploy).not.toContain("Foundry-Features: CodeAgents=V1Preview"); - expect(createHosted).not.toContain("POST /agents//versions"); - }); - - test("model deployments stay in azure.yaml for the azd golden path", async () => { - const createHosted = await readSkillFile("foundry-agent/create/create-hosted.md"); - const quickStart = await readSkillFile("foundry-agent/create/quick-start-hosted.md"); - const deployModel = await readSkillFile("models/deploy-model/SKILL.md"); - - expect(createHosted).toContain("`azure.yaml services.ai-project.deployments[]` is the **single source of truth**"); - expect(createHosted).toContain("`azd env set AI_PROJECT_DEPLOYMENTS '[...]'`"); - expect(createHosted).toContain("`az cognitiveservices account deployment create ...`"); - expect(quickStart).toContain("Never `azd env set AI_PROJECT_DEPLOYMENTS '[...]'`"); - expect(quickStart).toContain("Never `az cognitiveservices account deployment create`"); - expect(deployModel).toContain("For azd-managed Foundry projects"); - expect(deployModel).toContain("declare deployments in `azure.yaml services.ai-project.deployments[]`"); - expect(deployModel).toContain("Use this skill only for: (a) Foundry projects not managed by an azd project"); - }); - - test("direct-code deployment uses normal hosted-agent invoke and troubleshoot paths", async () => { - const deploy = await readSkillFile("foundry-agent/deploy/deploy.md"); - const invoke = await readSkillFile("foundry-agent/invoke/invoke.md"); - const troubleshoot = await readSkillFile("foundry-agent/troubleshoot/troubleshoot.md"); - const quickStart = await readSkillFile("foundry-agent/create/quick-start-hosted.md"); - const azdGuidance = await readSkillFile("foundry-agent/azd-guidance/azd-guidance.md"); - - expect(deploy).toContain("### Step 4 -- Verify and invoke"); - expect(deploy).toContain("azd ai agent invoke \"hello, are you up?\""); - expect(deploy).toContain("Run one remote invocation only unless the user explicitly asked"); - expect(invoke).toContain("## Hosted Agent Workflow with azd"); - expect(invoke).toContain("azd ai agent invoke \"hello, are you up?\""); - expect(invoke).toContain("Do not use MCP invoke, session, or file tools for a Hosted Agent."); - expect(invoke).toContain("## Prompt Agent Workflow with Foundry MCP"); - expect(invoke).not.toContain("Direct Code Invocation"); - expect(troubleshoot).toContain("## Workflow"); - expect(troubleshoot).not.toContain("Direct Code Troubleshooting"); - const invokeGuidance = `${deploy}\n${invoke}\n${quickStart}\n${azdGuidance}`; - expect(invokeGuidance).not.toMatch(/confirmation_required|confirmCommand|changes\[\]|confirmation envelope/i); - }); - - test("hosted-agent sessions and files use azd", async () => { - const invoke = await readSkillFile("foundry-agent/invoke/invoke.md"); - const sessions = await readSkillFile("foundry-agent/invoke/references/session-management.md"); - const files = await readSkillFile("foundry-agent/invoke/references/file-operations.md"); - const troubleshoot = await readSkillFile("foundry-agent/troubleshoot/troubleshoot.md"); - - expect(invoke).toContain("azd ai agent sessions stop "); - expect(sessions).toContain("## Automatic Session Handling"); - expect(sessions).toContain("let the server assign one, capture the returned session ID"); - expect(files).toContain("azd ai agent files upload ./input.csv"); - expect(troubleshoot).toContain("azd ai agent show --output json"); - expect(troubleshoot).toContain("azd ai agent monitor --tail 100"); - expect(troubleshoot).not.toContain("agent_get"); - expect(troubleshoot).not.toMatch(/prompt agent/i); - const hostedOperations = `${invoke}\n${sessions}\n${files}\n${troubleshoot}`; - expect(hostedOperations).not.toContain("session_create"); - expect(hostedOperations).not.toContain("session_file_"); - expect(hostedOperations).not.toContain("az rest --method GET"); - }); - - test("agent metadata contract scopes ACR to Docker hosted-agent deployments", async () => { - const contract = await readSkillFile("references/agent-metadata-contract.md"); - - expect(contract).toContain("azureContainerRegistry"); - expect(contract).toContain("Docker/ACR deploy flow"); - expect(contract).not.toContain("✅ for hosted agents | ACR used for deployment and image refresh"); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/eval-datasets/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/foundry-agent/eval-datasets/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index 403cbd27d..000000000 --- a/tests/microsoft-foundry/foundry-agent/eval-datasets/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`eval-datasets - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`eval-datasets - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/foundry-agent/eval-datasets/triggers.test.ts b/tests/microsoft-foundry/foundry-agent/eval-datasets/triggers.test.ts deleted file mode 100644 index 9e7a4086f..000000000 --- a/tests/microsoft-foundry/foundry-agent/eval-datasets/triggers.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Trigger Tests for eval-datasets - */ - -import { TriggerMatcher } from "../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("eval-datasets - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Create a dataset from my Foundry agent traces", - "Refresh my local Foundry dataset cache", - "Version my evaluation dataset for a Foundry agent", - "Detect regressions using my Foundry test datasets", - "Curate trace candidates into a dataset for Microsoft Foundry", - ]; - - test.each(shouldTriggerPrompts)('triggers on: "%s"', (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - }); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Explain how Kubernetes pods work", - "Build me a React dashboard", - "Set up PostgreSQL backups", - ]; - - test.each(shouldNotTriggerPrompts)('does not trigger on: "%s"', (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - }); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/integration.test.ts b/tests/microsoft-foundry/foundry-agent/integration.test.ts deleted file mode 100644 index 80424832c..000000000 --- a/tests/microsoft-foundry/foundry-agent/integration.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Integration Tests for foundry-agent - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_foundry-agent - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - - describe("skill-invocation", () => { - test("invokes skill for prompt agent creation", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Create a new prompt agent with gpt-4o model in Foundry", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for agent troubleshooting", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Troubleshoot my Foundry agent that is returning errors", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/observe/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/foundry-agent/observe/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index 00cd9171e..000000000 --- a/tests/microsoft-foundry/foundry-agent/observe/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`observe - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`observe - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/foundry-agent/observe/integration.test.ts b/tests/microsoft-foundry/foundry-agent/observe/integration.test.ts deleted file mode 100644 index aed56c83a..000000000 --- a/tests/microsoft-foundry/foundry-agent/observe/integration.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Integration Tests for observe - * - * Tests skill behavior with a real Copilot agent session. - * These tests require Copilot CLI to be installed and authenticated. - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests -} from "../../../utils/agent-runner"; -import { isSkillInvoked, withTestResult } from "../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; - -const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_observe - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - - test("invokes skill for evaluate agent prompt", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Evaluate my Foundry agent and check its quality" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); - - test("invokes skill for agent observability prompt", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Set up monitoring and evaluation for my Foundry agent" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); -}); diff --git a/tests/microsoft-foundry/foundry-agent/observe/triggers.test.ts b/tests/microsoft-foundry/foundry-agent/observe/triggers.test.ts deleted file mode 100644 index 65bf25e40..000000000 --- a/tests/microsoft-foundry/foundry-agent/observe/triggers.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Trigger Tests for observe - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("observe - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Evaluate my Foundry agent", - "Run an eval on my agent in Microsoft Foundry", - "Test my agent quality in Foundry", - "Check agent quality metrics in Foundry", - "Why did my agent eval fail in Foundry", - "Analyze eval results for my Foundry agent", - "Cluster failures from my agent evaluation", - "Improve my Foundry agent quality", - "Optimize my agent prompt in Foundry", - "Compare agent versions in Foundry", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem about clouds", - "Help me with AWS SageMaker", - "How do I configure my PostgreSQL database?", - "Explain how Kubernetes pods work", - "Create a REST API in Python", - "Set up a React application", - "Set up a Jenkins CI pipeline for my Java project", - "Write unit tests for my JavaScript code", - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "evaluate agent Foundry ".repeat(100); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive", () => { - const result1 = triggerMatcher.shouldTrigger("EVALUATE AGENT FOUNDRY"); - const result2 = triggerMatcher.shouldTrigger("evaluate agent foundry"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -}); diff --git a/tests/microsoft-foundry/foundry-agent/trace/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/foundry-agent/trace/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index 31d3e60e1..000000000 --- a/tests/microsoft-foundry/foundry-agent/trace/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`trace - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`trace - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/foundry-agent/trace/integration.test.ts b/tests/microsoft-foundry/foundry-agent/trace/integration.test.ts deleted file mode 100644 index eca1e7a16..000000000 --- a/tests/microsoft-foundry/foundry-agent/trace/integration.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Integration Tests for trace - * - * Tests skill behavior with a real Copilot agent session. - * These tests require Copilot CLI to be installed and authenticated. - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests -} from "../../../utils/agent-runner"; -import { isSkillInvoked, withTestResult } from "../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; - -const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_trace - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - - test("invokes skill for trace analysis prompt", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Analyze traces for my Foundry agent in App Insights" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); - - test("invokes skill for failing traces prompt", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Find failing traces and errors for my Foundry agent" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); -}); diff --git a/tests/microsoft-foundry/foundry-agent/trace/triggers.test.ts b/tests/microsoft-foundry/foundry-agent/trace/triggers.test.ts deleted file mode 100644 index 305dbc985..000000000 --- a/tests/microsoft-foundry/foundry-agent/trace/triggers.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Trigger Tests for trace - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("trace - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Analyze my Foundry agent traces in App Insights", - "Search agent conversations in Foundry", - "Find failing traces for my Foundry agent", - "My Foundry agent is slow, show me the latency", - "Show me the trace for this Foundry agent conversation", - "Why is my Foundry agent returning errors in production", - "Search Foundry agent traces by conversation ID", - "Find slow Foundry agent traces in App Insights", - "Show me GenAI telemetry for my Foundry agent", - "Analyze production errors for my Foundry agent", - ]; - - test.each(shouldTriggerPrompts)( - "triggers on: \"%s\"", - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem about clouds", - "Help me with AWS SageMaker", - "How do I configure my PostgreSQL database?", - "Explain how Kubernetes pods work", - "Create a REST API in Python", - "Set up a React application", - "Set up a Jenkins CI pipeline for my Java project", - "Write unit tests for my JavaScript code", - ]; - - test.each(shouldNotTriggerPrompts)( - "does not trigger on: \"%s\"", - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "analyze traces Foundry agent ".repeat(100); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive", () => { - const result1 = triggerMatcher.shouldTrigger("ANALYZE TRACES FOUNDRY AGENT"); - const result2 = triggerMatcher.shouldTrigger("analyze traces foundry agent"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -}); diff --git a/tests/microsoft-foundry/integration.test.ts b/tests/microsoft-foundry/integration.test.ts deleted file mode 100644 index 95f584bf7..000000000 --- a/tests/microsoft-foundry/integration.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Integration Tests for microsoft-foundry - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -// Check if integration tests should be skipped at module level -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -// Log skip reason if skipping -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - describe("skill-invocation", () => { - test("invokes microsoft-foundry skill for AI model deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "How do I deploy an AI model from the Microsoft Foundry catalog?", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for RAG application prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Build a RAG application with Microsoft Foundry using knowledge indexes", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for RBAC role assignment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Grant a user the Foundry User role on my Foundry project", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for service principal CI/CD prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Create a service principal for my Foundry CI/CD pipeline", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for managed identity roles prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Set up managed identity roles for my Foundry project to access Azure Storage", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for audit role assignments prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Who has access to my Foundry project? List all role assignments", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for developer permissions prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Make Bob a project manager in my Microsoft Foundry", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for validate permissions prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Can I deploy models to my Foundry project? Check my permissions", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for agent lifecycle prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Help me build and deploy a Foundry agent", - shouldEarlyTerminate: (metadata) => - isSkillInvoked(metadata, SKILL_NAME), - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for trace-to-dataset prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Create an evaluation dataset from my Foundry agent traces", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes microsoft-foundry skill for dataset versioning prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Version my Foundry evaluation dataset and compare regressions", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/capacity/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/models/deploy/capacity/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index 5d664d66a..000000000 --- a/tests/microsoft-foundry/models/deploy/capacity/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`capacity - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`capacity - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/models/deploy/capacity/integration.test.ts b/tests/microsoft-foundry/models/deploy/capacity/integration.test.ts deleted file mode 100644 index 371ad6ce0..000000000 --- a/tests/microsoft-foundry/models/deploy/capacity/integration.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Integration Tests for capacity discovery - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../../../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_capacity - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - describe("skill-invocation", () => { - test("invokes skill for capacity discovery prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Find available capacity for gpt-4o across all Azure regions", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for region comparison prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Which Azure regions have gpt-4o available with enough TPM capacity?", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/capacity/triggers.test.ts b/tests/microsoft-foundry/models/deploy/capacity/triggers.test.ts deleted file mode 100644 index 87e8fcab7..000000000 --- a/tests/microsoft-foundry/models/deploy/capacity/triggers.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Trigger Tests for capacity discovery - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("capacity - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Check quota availability for model deployment", - "Capacity discovery for my model", - "Multi-project capacity search for gpt-4o", - "Quota analysis for model deployment", - "Find best region for deploying gpt-4o with capacity", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - "Help me with AWS SageMaker", - "Configure my PostgreSQL database", - "Deploy gpt-4o quickly", - "Help me with Kubernetes pods", - "How do I write Python code?", - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "find capacity ".repeat(100); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive", () => { - const result1 = triggerMatcher.shouldTrigger("CHECK CAPACITY FOR MODEL"); - const result2 = triggerMatcher.shouldTrigger("check capacity for model"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/customize-deployment/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/models/deploy/customize-deployment/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index ea5827f0d..000000000 --- a/tests/microsoft-foundry/models/deploy/customize-deployment/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/models/deploy/customize-deployment/integration.test.ts b/tests/microsoft-foundry/models/deploy/customize-deployment/integration.test.ts deleted file mode 100644 index e1086c720..000000000 --- a/tests/microsoft-foundry/models/deploy/customize-deployment/integration.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Integration Tests for customize (customize-deployment) - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../../../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_customize-deployment - Integration Tests`, () => { - describe("skill-invocation", () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - test("invokes skill for custom deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o with custom SKU and capacity configuration", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for PTU deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o with provisioned throughput PTU in my Foundry project", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/customize-deployment/triggers.test.ts b/tests/microsoft-foundry/models/deploy/customize-deployment/triggers.test.ts deleted file mode 100644 index afb79f0f1..000000000 --- a/tests/microsoft-foundry/models/deploy/customize-deployment/triggers.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Trigger Tests for customize-deployment - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe(`${SKILL_NAME} - Trigger Tests`, () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - // Prompts that SHOULD trigger this skill - const shouldTriggerPrompts: string[] = [ - // Core customization phrases - "I want to customize the deployment for gpt-4o", - "customize model deployment", - "deploy with custom settings for my model", - - // SKU selection - "deploy model with specific SKU", - "select SKU for model deployment", - - // Capacity configuration - "set capacity for deployment", - "deploy with 50K TPM capacity", - - // Advanced options - "deployment with advanced options", - "detailed deployment configuration", - - // PTU deployments - "deploy model with PTU configuration", - "PTU provisioned deployment for model", - "provisioned throughput deployment", - "deploy with provisioned capacity", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - } - ); - }); - - describe("Should NOT Trigger", () => { - // Prompts that should NOT trigger this skill - const shouldNotTriggerPrompts: string[] = [ - // General unrelated - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - - // Wrong cloud provider - "Deploy to AWS Lambda", - "Configure GCP Cloud Functions", - - // Quick deployment scenarios (should use deploy-model-optimal-region) - "Set up virtual network", - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("case insensitive matching", () => { - const result = triggerMatcher.shouldTrigger("CUSTOMIZE DEPLOYMENT FOR GPT-4O"); - expect(result.triggered).toBe(true); - }); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index ea5827f0d..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/integration.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/integration.test.ts deleted file mode 100644 index 0c3fd54e3..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/integration.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Integration Tests for preset (deploy-model-optimal-region) - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../../../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_deploy-model-optimal-region - Integration Tests`, () => { - describe("skill-invocation", () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - test("invokes skill for quick deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o quickly to the optimal region", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for best region deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o to the best available region with high availability", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/triggers.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/triggers.test.ts deleted file mode 100644 index fa0a95705..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/triggers.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Trigger Tests for deploy-model-optimal-region - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe(`${SKILL_NAME} - Trigger Tests`, () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - // Prompts that SHOULD trigger this skill - const shouldTriggerPrompts: string[] = [ - // Quick deployment - "Deploy gpt-4o model", - "quick deployment of gpt-4o", - "fast deployment", - - // Optimal region - "find optimal region for deployment", - - // Automatic region selection - "deploy with automatic region", - - // Multi-region capacity check - "find region with capacity", - - // High availability - "high availability deployment", - "deploy model with HA configuration", - - // Generic deployment (should choose this as default) - "deploy gpt-4o model to the optimal region", - "deploy model to Azure", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - } - ); - }); - - describe("Should NOT Trigger", () => { - // Prompts that should NOT trigger this skill - const shouldNotTriggerPrompts: string[] = [ - // General unrelated - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - - // Wrong cloud provider - "Deploy to AWS Lambda", - "Configure GCP Cloud Functions", - - // Customization scenarios (should use customize-deployment) - "Select specific version", - "Choose model version", - "Configure capacity manually", - "Set custom capacity", - "Select RAI policy", - "Configure content filter", - - // Other Azure AI tasks - "Configure RBAC", - - // Non-deployment tasks - "Set up virtual network", - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("multiple trigger phrases in one prompt", () => { - const result = triggerMatcher.shouldTrigger("Quick model deployment to optimal region with high availability"); - expect(result.triggered).toBe(true); - }); - - test("should prefer this skill over customize-deployment for simple requests", () => { - // This is a design preference - simple "deploy" requests should use the fast path - const simpleDeployPrompt = "Deploy gpt-4o model"; - const result = triggerMatcher.shouldTrigger(simpleDeployPrompt); - expect(result.triggered).toBe(true); - }); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/deploy-model/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/models/deploy/deploy-model/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index ea5827f0d..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,163 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], - "name": "microsoft-foundry", -} -`; - -exports[`microsoft-foundry - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/models/deploy/deploy-model/integration.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model/integration.test.ts deleted file mode 100644 index 767fd55e8..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model/integration.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Integration Tests for deploy-model (router) - * - * Tests skill behavior with a real Copilot agent session. - * Runs prompts multiple times to measure skill invocation rate. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - */ - -import { - useAgentRunner, - shouldSkipIntegrationTests, - getIntegrationSkipReason, -} from "../../../../utils/agent-runner"; -import { softCheckSkill, isSkillInvoked, shouldEarlyTerminateForSkillInvocation, withTestResult } from "../../../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; -const RUNS_PER_PROMPT = 5; -const invocationRateThreshold = 0.8; - -const skipTests = shouldSkipIntegrationTests(); -const skipReason = getIntegrationSkipReason(); - -if (skipTests && skipReason) { - console.log(`⏭️ Skipping integration tests: ${skipReason}`); -} - -const describeIntegration = skipTests ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_deploy-model - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - describe("skill-invocation", () => { - test("invokes skill for simple model deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o model to my Azure project", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for capacity discovery prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Where can I deploy gpt-4o? Check capacity across regions", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - - test("invokes skill for customized deployment prompt", () => withTestResult(async ({ setSkillInvocationRate }) => { - let invocationCount = 0; - for (let i = 0; i < RUNS_PER_PROMPT; i++) { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Deploy gpt-4o with custom SKU and capacity settings", - shouldEarlyTerminate: (metadata) => shouldEarlyTerminateForSkillInvocation(metadata, SKILL_NAME) - }); - - softCheckSkill(agentMetadata, SKILL_NAME); - if (isSkillInvoked(agentMetadata, SKILL_NAME)) { - invocationCount += 1; - } - } - const rate = invocationCount / RUNS_PER_PROMPT; - setSkillInvocationRate(rate); - expect(rate).toBeGreaterThanOrEqual(invocationRateThreshold); - })); - }); -}); diff --git a/tests/microsoft-foundry/models/deploy/deploy-model/triggers.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model/triggers.test.ts deleted file mode 100644 index e119df1a2..000000000 --- a/tests/microsoft-foundry/models/deploy/deploy-model/triggers.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Trigger Tests for deploy-model (router) - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe(`${SKILL_NAME} - Trigger Tests`, () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - const shouldTriggerPrompts: string[] = [ - "Deploy a model to Azure OpenAI", - "Deploy gpt-4o model", - "Create a deployment for gpt-4o", - "Help me with model deployment", - "Deploy an OpenAI model to my project", - "Set up a model in my Foundry project", - "Deploy a model out-of-band to my Foundry project", - "Add an ad-hoc model deployment outside my azd lifecycle", - "Find capacity for model deployment", - "Best region for model deployment", - "Capacity analysis for my model", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should NOT Trigger", () => { - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - "Help me with AWS SageMaker", - "Configure my PostgreSQL database", - "Help me with Kubernetes pods", - "How do I write Python code?", - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "deploy model ".repeat(100); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive", () => { - const result1 = triggerMatcher.shouldTrigger("DEPLOY MODEL TO AZURE"); - const result2 = triggerMatcher.shouldTrigger("deploy model to azure"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -}); diff --git a/tests/microsoft-foundry/quota/integration.test.ts b/tests/microsoft-foundry/quota/integration.test.ts deleted file mode 100644 index eab294bf2..000000000 --- a/tests/microsoft-foundry/quota/integration.test.ts +++ /dev/null @@ -1,528 +0,0 @@ -/** - * Integration Tests for microsoft-foundry-quota - * - * Tests skill behavior with a real Copilot agent session for quota management. - * These tests require Copilot CLI to be installed and authenticated. - * - * Prerequisites: - * 1. npm install -g @github/copilot-cli - * 2. Run `copilot` and authenticate - * 3. Have an Azure subscription with Microsoft Foundry resources - * - * Run with: npm run test:integration -- --testPathPatterns=microsoft-foundry-quota - */ - -import { - useAgentRunner, - doesAssistantMessageIncludeKeyword, - shouldSkipIntegrationTests -} from "../../utils/agent-runner"; -import { - isSkillInvoked, - matchesCommand, - withTestResult, - doesAssistantOrToolsIncludeKeyword, - softCheckSkill, - isMcpToolCalled -} from "../../utils/evaluate"; - -const SKILL_NAME = "microsoft-foundry"; - -// Use centralized skip logic from agent-runner -const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe; - -describeIntegration(`${SKILL_NAME}_quota - Integration Tests`, () => { - const agent = useAgentRunner({ - isTest: true, - useJest: true - }); - - describe("View Quota Usage", () => { - test("invokes skill for quota usage check", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Use the microsoft-foundry skill to show me my current quota usage for Microsoft Foundry resources" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); - - test("response includes quota-related commands", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "How do I check my Microsoft Foundry quota limits?" - }); - - const hasQuotaCommand = doesAssistantMessageIncludeKeyword( - agentMetadata, - "az cognitiveservices" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quota" - ); - expect(hasQuotaCommand).toBe(true); - })); - - test("response mentions TPM (Tokens Per Minute)", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Explain quota in Microsoft Foundry" - }); - - const mentionsTPM = doesAssistantMessageIncludeKeyword( - agentMetadata, - "TPM" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "Tokens Per Minute" - ); - expect(mentionsTPM).toBe(true); - })); - }); - - describe("Quota Before Deployment", () => { - test("provides guidance on checking quota before deployment", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Use the microsoft-foundry skill to check if I have enough quota to deploy GPT-4o to Microsoft Foundry" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const hasGuidance = doesAssistantMessageIncludeKeyword( - agentMetadata, - "capacity" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quota" - ); - expect(hasGuidance).toBe(true); - })); - - test("suggests capacity calculation", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "How much quota do I need for a production Foundry deployment?" - }); - - // Require at least one quota-specific term - const hasQuotaTerm = doesAssistantMessageIncludeKeyword( - agentMetadata, - "TPM" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "PTU" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "capacity" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "tokens per minute" - ); - - // Require at least one calculation verb - const hasCalculationVerb = doesAssistantMessageIncludeKeyword( - agentMetadata, - "calculate" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "estimate" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "calculation" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quantify" - ); - - expect(hasQuotaTerm && hasCalculationVerb).toBe(true); - })); - }); - - describe("Request Quota Increase", () => { - test("explains quota increase process", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Using the microsoft-foundry quota skill, how do I request a quota increase for Microsoft Foundry?" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - // Check in both responses and tool execution data - const mentionsPortal = doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "Azure Portal" - ) || doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "portal" - ); - expect(mentionsPortal).toBe(true); - })); - - test("mentions business justification", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Request more TPM quota for Microsoft Foundry and explain what justification is needed" - }); - - // Check in both responses and tool execution data (e.g., file writes) - const mentionsJustification = doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "justification" - ) || doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "business" - ) || doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "reason" - ) || doesAssistantOrToolsIncludeKeyword( - agentMetadata, - "rationale" - ); - expect(mentionsJustification).toBe(true); - })); - }); - - describe("Monitor Quota Across Deployments", () => { - test("provides monitoring commands", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Use the microsoft-foundry quota skill to monitor quota usage across all my Microsoft Foundry deployments" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const hasMonitoring = doesAssistantMessageIncludeKeyword( - agentMetadata, - "deployment" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "usage" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quota" - ); - expect(hasMonitoring).toBe(true); - })); - - test("explains capacity by model tracking", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Show me quota allocation by model in Microsoft Foundry" - }); - - const hasModelTracking = doesAssistantMessageIncludeKeyword( - agentMetadata, - "model" - ) && (doesAssistantMessageIncludeKeyword( - agentMetadata, - "capacity" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quota" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "allocation" - )); - expect(hasModelTracking).toBe(true); - })); - }); - - describe("Troubleshoot Quota Errors", () => { - test("troubleshoots QuotaExceeded error", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "My Microsoft Foundry deployment failed with QuotaExceeded error. Help me fix it." - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const hasTroubleshooting = doesAssistantMessageIncludeKeyword( - agentMetadata, - "QuotaExceeded" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quota" - ); - expect(hasTroubleshooting).toBe(true); - })); - - test("troubleshoots InsufficientQuota error", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "I'm getting an InsufficientQuota error when deploying gpt-4o to eastus in Microsoft Foundry. Use the microsoft-foundry skill to help me troubleshoot and fix this." - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - })); - - test("troubleshoots DeploymentLimitReached error", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "DeploymentLimitReached error in Microsoft Foundry, what should I do?" - }); - - const providesResolution = doesAssistantMessageIncludeKeyword( - agentMetadata, - "delete" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "deployment" - ); - expect(providesResolution).toBe(true); - })); - - test("addresses 429 rate limit errors", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Getting 429 rate limit errors from my Foundry deployment" - }); - - const addresses429 = doesAssistantMessageIncludeKeyword( - agentMetadata, - "429" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "rate limit" - ); - expect(addresses429).toBe(true); - })); - }); - - describe("Capacity Planning", () => { - test("helps with production capacity planning", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Help me plan capacity for production Microsoft Foundry deployment with 1M requests per day" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - // Require at least one quota-specific term - const hasQuotaTerm = doesAssistantMessageIncludeKeyword( - agentMetadata, - "TPM" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "PTU" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "capacity" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "tokens per minute" - ); - - // Require at least one calculation verb - const hasCalculationVerb = doesAssistantMessageIncludeKeyword( - agentMetadata, - "calculate" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "estimate" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "calculation" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "quantify" - ); - - expect(hasQuotaTerm && hasCalculationVerb).toBe(true); - })); - - test("provides best practices", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "What are best practices for quota management in Microsoft Foundry?" - }); - - const hasBestPractices = doesAssistantMessageIncludeKeyword( - agentMetadata, - "best practice" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "optimize" - ); - expect(hasBestPractices).toBe(true); - })); - }); - - describe("Deployment Listing", () => { - test("lists deployments using MCP tools or CLI", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Use the microsoft-foundry skill to list all my Microsoft Foundry model deployments and their capacity" - }); - - // Soft check for skill invocation - agent should use the skill when explicitly asked - softCheckSkill(agentMetadata, SKILL_NAME); - - // Check if agent used Azure MCP tool for deployments (model_deployment_get from azure server) - const usedAzureMcp = isMcpToolCalled(agentMetadata, "azure", /model_deployment/); - - // Check if agent used Azure CLI commands for deployments - const usedCli = matchesCommand(agentMetadata, /az\s+(cognitiveservices|rest|ai)\s+.*?(deployment|model|capacity|quota)/i); - - // Check if Azure CLI commands are mentioned in responses or tool execution data - const mentionsAzCli = doesAssistantOrToolsIncludeKeyword(agentMetadata, "az cognitiveservices") || - doesAssistantOrToolsIncludeKeyword(agentMetadata, "az rest") || - doesAssistantOrToolsIncludeKeyword(agentMetadata, "az ai"); - - // Pass if agent used Azure MCP tools, CLI, or mentioned CLI commands in response/reasoning - expect(usedAzureMcp || usedCli || mentionsAzCli).toBe(true); - })); - }); - - describe("Regional Capacity", () => { - test("explains regional quota distribution", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Using the microsoft-foundry quota skill, explain how quota works across different Azure regions for Foundry" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const mentionsRegion = doesAssistantMessageIncludeKeyword( - agentMetadata, - "region" - ); - expect(mentionsRegion).toBe(true); - })); - - test("suggests deploying to different region when quota exhausted", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "I ran out of quota in East US for Microsoft Foundry. What are my options?" - }); - - const suggestsRegion = doesAssistantMessageIncludeKeyword( - agentMetadata, - "region" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "location" - ); - expect(suggestsRegion).toBe(true); - })); - }); - - describe("Quota Optimization", () => { - test("provides optimization guidance", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "How can I optimize my Microsoft Foundry quota allocation?" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const hasOptimization = doesAssistantMessageIncludeKeyword( - agentMetadata, - "optimize" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "consolidate" - ); - expect(hasOptimization).toBe(true); - })); - - test("suggests deleting unused deployments", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "I need to free up quota in Microsoft Foundry" - }); - - const suggestsDelete = doesAssistantMessageIncludeKeyword( - agentMetadata, - "delete" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "unused" - ); - expect(suggestsDelete).toBe(true); - })); - }); - - describe("Command Output Explanation", () => { - test("explains how to interpret quota usage output", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "What does the quota usage output mean in Microsoft Foundry?" - }); - - const hasExplanation = doesAssistantMessageIncludeKeyword( - agentMetadata, - "currentValue" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "limit" - ); - expect(hasExplanation).toBe(true); - })); - - test("explains TPM concept", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "What is TPM in the context of Microsoft Foundry quotas?" - }); - - const explainTPM = doesAssistantMessageIncludeKeyword( - agentMetadata, - "Tokens Per Minute" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "TPM" - ); - expect(explainTPM).toBe(true); - })); - }); - - describe("Error Resolution Steps", () => { - test("provides step-by-step resolution for quota errors", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "Walk me through fixing a quota error in Microsoft Foundry deployment" - }); - - const isSkillUsed = isSkillInvoked(agentMetadata, SKILL_NAME); - expect(isSkillUsed).toBe(true); - - const hasSteps = doesAssistantMessageIncludeKeyword( - agentMetadata, - "step" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "check" - ); - expect(hasSteps).toBe(true); - })); - - test("offers multiple resolution options", () => withTestResult(async () => { - const agentMetadata = await agent.run({ - requiredSkills: [{ pluginDirname: "azure-skills", name: SKILL_NAME }], - prompt: "What are my options when I hit quota limits in Microsoft Foundry?" - }); - - const hasOptions = doesAssistantMessageIncludeKeyword( - agentMetadata, - "option" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "reduce" - ) || doesAssistantMessageIncludeKeyword( - agentMetadata, - "increase" - ); - expect(hasOptions).toBe(true); - })); - }); -}); diff --git a/tests/microsoft-foundry/resource/create/__snapshots__/triggers.test.ts.snap b/tests/microsoft-foundry/resource/create/__snapshots__/triggers.test.ts.snap deleted file mode 100644 index 743b5a7d5..000000000 --- a/tests/microsoft-foundry/resource/create/__snapshots__/triggers.test.ts.snap +++ /dev/null @@ -1,162 +0,0 @@ -// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing - -exports[`microsoft-foundry:resource/create - Trigger Tests Trigger Keywords Snapshot skill description triggers match snapshot 1`] = ` -{ - "description": "Deploy, evaluate, fine-tune, and manage Foundry agents end-to-end with azd: hosted agent scaffold/run/deploy, prompt agent create, batch eval, continuous eval, prompt optimizer, Agent Optimizer scaffold, agent.yaml, dataset curation from traces, model fine-tuning (SFT/DPO/RFT). USE FOR: azd ai agent, azd provision/deploy, deploy agent, hosted agent, create agent, add tool to agent, invoke agent, evaluate agent, continuous eval, continuous monitoring, agent CI/CD, optimize prompt, improve prompt, optimize agent instructions, agent optimizer, deploy model, Foundry project, RBAC, role assignment, permissions, quota, capacity, region, troubleshoot agent, deployment failure, AI Services, create Foundry resource, provision, knowledge index, customize deployment, onboard, availability, fine-tune, SFT, DPO, RFT, training-data, grader, distillation, fine-tuned model, large file upload. DO NOT USE FOR: Azure Functions, App Service, general Azure deploy (use azure-deploy), general Azure prep (use azure-prepare).", - "extractedKeywords": [ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", - ], -} -`; - -exports[`microsoft-foundry:resource/create - Trigger Tests Trigger Keywords Snapshot skill keywords match snapshot 1`] = ` -[ - "agent", - "agents", - "ai", - "assignment", - "authentication", - "availability", - "azure", - "azure-deploy", - "azure-prepare", - "batch", - "capacity", - "cli", - "container", - "continuous", - "cosmos", - "create", - "curation", - "customize", - "dataset", - "deploy", - "deployment", - "distillation", - "end-to-end", - "eval", - "evaluate", - "failure", - "file", - "fine-tune", - "fine-tuned", - "fine-tuning", - "foundry", - "from", - "functions", - "general", - "grader", - "hosted", - "improve", - "index", - "instructions", - "invoke", - "knowledge", - "large", - "manage", - "mcp", - "microsoft", - "model", - "monitor", - "monitoring", - "onboard", - "optimize", - "optimizer", - "permissions", - "prep", - "project", - "prompt", - "provision", - "quota", - "rbac", - "region", - "resource", - "role", - "scaffold", - "service", - "services", - "storage", - "tool", - "traces", - "training-data", - "troubleshoot", - "upload", - "validation", - "vnet", - "with", - "yaml", -] -`; diff --git a/tests/microsoft-foundry/resource/create/integration.test.ts b/tests/microsoft-foundry/resource/create/integration.test.ts deleted file mode 100644 index 00a7f31aa..000000000 --- a/tests/microsoft-foundry/resource/create/integration.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Integration Tests for microsoft-foundry:resource/create - * - * Tests the skill"s behavior when invoked with real scenarios - */ - -import { loadSkill, type LoadedSkill } from "../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe(`${SKILL_NAME}_resource-create - Integration Tests`, () => { - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - }); - - describe("Skill Loading", () => { - test("skill loads successfully", () => { - expect(skill).toBeDefined(); - expect(skill.metadata).toBeDefined(); - expect(skill.content).toBeDefined(); - }); - - test("skill has correct name", () => { - expect(skill.metadata.name).toBe("microsoft-foundry"); - }); - - test("skill content includes resource/create reference", () => { - expect(skill.content).toContain("resource/create"); - }); - }); - - describe("Workflow Documentation", () => { - test("main file contains all 3 workflows inline", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const mainFilePath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/create-foundry-resource.md" - ); - - const mainContent = await fs.readFile(mainFilePath, "utf-8"); - - expect(mainContent).toContain("### 1. Create Resource Group"); - expect(mainContent).toContain("### 2. Create Foundry Resource"); - expect(mainContent).toContain("### 3. Register Resource Provider"); - }); - }); - - describe("Command Validation", () => { - test("skill contains valid Azure CLI commands", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const mainFilePath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/create-foundry-resource.md" - ); - - const mainContent = await fs.readFile(mainFilePath, "utf-8"); - - // Check for key Azure CLI commands - expect(mainContent).toContain("az group create"); - expect(mainContent).toContain("az cognitiveservices account create"); - expect(mainContent).toContain("az provider register"); - expect(mainContent).toContain("--kind AIServices"); - }); - - test("commands include required parameters", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const mainFilePath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/create-foundry-resource.md" - ); - - const mainContent = await fs.readFile(mainFilePath, "utf-8"); - - expect(mainContent).toContain("--resource-group"); - expect(mainContent).toContain("--name"); - expect(mainContent).toContain("--location"); - expect(mainContent).toContain("--sku"); - }); - }); - - describe("References Pattern", () => { - test("main file is under token limit with condensed content", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const mainFilePath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/create-foundry-resource.md" - ); - - const mainContent = await fs.readFile(mainFilePath, "utf-8"); - const lineCount = mainContent.split("\n").length; - - // Main file should be under 200 lines for token optimization - expect(lineCount).toBeLessThan(200); - }); - - test("references directory exists with detailed content", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const referencesPath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/references" - ); - - const referencesExists = await fs.access(referencesPath).then(() => true).catch(() => false); - expect(referencesExists).toBe(true); - - // Check for required reference files - const workflowsPath = path.join(referencesPath, "workflows.md"); - const patternsPath = path.join(referencesPath, "patterns.md"); - const troubleshootingPath = path.join(referencesPath, "troubleshooting.md"); - - expect(await fs.access(workflowsPath).then(() => true).catch(() => false)).toBe(true); - expect(await fs.access(patternsPath).then(() => true).catch(() => false)).toBe(true); - expect(await fs.access(troubleshootingPath).then(() => true).catch(() => false)).toBe(true); - }); - - test("main file links to reference files", async () => { - const fs = await import("fs/promises"); - const path = await import("path"); - - const mainFilePath = path.join( - OUTPUT_PATH, - "azure-skills", - "skills", - "microsoft-foundry/resource/create/create-foundry-resource.md" - ); - - const mainContent = await fs.readFile(mainFilePath, "utf-8"); - - expect(mainContent).toContain("./references/workflows.md"); - expect(mainContent).toContain("./references/patterns.md"); - expect(mainContent).toContain("./references/troubleshooting.md"); - }); - }); -}); diff --git a/tests/microsoft-foundry/resource/create/triggers.test.ts b/tests/microsoft-foundry/resource/create/triggers.test.ts deleted file mode 100644 index c87352e4d..000000000 --- a/tests/microsoft-foundry/resource/create/triggers.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Trigger Tests for microsoft-foundry:resource/create - * - * Tests that the parent skill triggers on resource creation prompts - * since resource/create is a sub-skill of microsoft-foundry. - */ - -import { TriggerMatcher } from "../../../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../../../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe("microsoft-foundry:resource/create - Trigger Tests", () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger - Resource Creation", () => { - const resourceCreatePrompts: string[] = [ - "Create a new Foundry resource", - "Create Azure AI Services resource", - "Provision a multi-service resource", - "Create AIServices kind resource", - "Set up new AI Services account", - "Create a resource group for Foundry", - "Register Cognitive Services provider", - "Create Azure Cognitive Services multi-service", - "Provision AI Services with CLI", - "Create new Microsoft Foundry resource", - "Set up multi-service Cognitive Services resource", - "Create a Foundry project with azd ai starter basic", - "Set up hosted-agent deployment with ENABLE_HOSTED_AGENTS", - ]; - - test.each(resourceCreatePrompts)( - 'triggers on resource creation prompt: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - } - ); - }); - - describe("Should NOT Trigger", () => { - const nonTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write Python code", - "How do I bake a cake?", - "Set up a virtual machine", - "How do I use Docker?", - "Explain quantum computing", - ]; - - test.each(nonTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt with resource creation keywords", () => { - const longPrompt = "I want to create a new Azure AI Services Foundry resource ".repeat(50); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(result.triggered).toBe(true); - }); - - test("is case insensitive", () => { - const upperResult = triggerMatcher.shouldTrigger("CREATE FOUNDRY RESOURCE"); - const lowerResult = triggerMatcher.shouldTrigger("create foundry resource"); - expect(upperResult.triggered).toBe(true); - expect(lowerResult.triggered).toBe(true); - }); - }); -}); diff --git a/tests/microsoft-foundry/triggers.test.ts b/tests/microsoft-foundry/triggers.test.ts deleted file mode 100644 index f4577ea24..000000000 --- a/tests/microsoft-foundry/triggers.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Trigger Tests for microsoft-foundry - * - * Tests that verify the skill triggers on appropriate prompts - * and does NOT trigger on unrelated prompts. - */ - -import { TriggerMatcher } from "../utils/trigger-matcher"; -import { loadSkill, LoadedSkill } from "../utils/skill-loader"; - -const SKILL_NAME = "microsoft-foundry"; - -describe(`${SKILL_NAME} - Trigger Tests`, () => { - let triggerMatcher: TriggerMatcher; - let skill: LoadedSkill; - - beforeAll(async () => { - skill = await loadSkill({ pluginDirname: "azure-skills", name: SKILL_NAME });; - triggerMatcher = new TriggerMatcher(skill); - }); - - describe("Should Trigger", () => { - // Prompts that SHOULD trigger this skill based on frontmatter USE FOR - const shouldTriggerPrompts: string[] = [ - "How do I deploy an AI model from Microsoft Foundry catalog?", - "Build a RAG application with Microsoft Foundry knowledge index", - "Create an AI agent in Microsoft Foundry with web search", - "Add a tool to my Foundry agent", - "Evaluate agent performance using Foundry evaluators", - "Optimize my prompt for a Microsoft Foundry agent", - "Improve my agent instructions in Microsoft Foundry", - "Use a prompt optimizer on my Foundry system prompt", - "Set up agent monitoring and continuous evaluation in Foundry", - "Set up a CI/CD deployment pipeline for my Foundry agent", - "Help me with Microsoft Foundry model deployment", - "How to use knowledge index for RAG in Microsoft Foundry?", - "Create a new Microsoft Foundry project", - "Set up a Foundry project for my AI agents", - "How do I onboard to Microsoft Foundry and create a project?", - "Provision Foundry infrastructure with azd", - "Scaffold and deploy a hosted Foundry agent with azd ai agent", - "Create a hosted Foundry agent quick start and run a remote smoke test", - "I need a new Foundry project to host my models", - ]; - - test.each(shouldTriggerPrompts)( - 'triggers on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - // TriggerMatcher uses >= 2 keywords or 20% confidence - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should Trigger - RBAC Sub-Skill", () => { - // RBAC-specific prompts that SHOULD trigger this skill - const rbacTriggerPrompts: string[] = [ - "Grant Alice role assignment access to my Microsoft Foundry project", - "Assign Foundry User role to a user in Foundry", - "Make Bob a project manager in Microsoft Foundry", - "Who has role assignment access to my Microsoft Foundry resource?", - "Audit role assignments on my Foundry account", - "Can I deploy models to Foundry? Check my permissions", - "Validate my permissions on the Foundry project", - "Set up managed identity for my Foundry project", - "Configure RBAC setup for my Foundry project Storage access", - "Create a service principal for Foundry CI/CD pipeline", - "Set up service principal for Microsoft Foundry automation", - "Set up RBAC for a developer on my Foundry resource", - "List all RBAC assignments on my Foundry resource", - "Setup developer permissions for Foundry", - ]; - - test.each(rbacTriggerPrompts)( - 'triggers on RBAC prompt: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should Trigger - Private Network Sub-Skill", () => { - // Prompts covering private-network sub-skill Q&A + deployment - const vnetTriggerPrompts: string[] = [ - "How does Foundry VNet isolation work?", - "BYO VNet vs managed VNet in Foundry", - "Explain Foundry private endpoints", - "Deploy Foundry in a private VNet", - "Set up network isolation for my Foundry agents", - "Deploy Foundry with managed virtual network", - ]; - - test.each(vnetTriggerPrompts)( - 'triggers on VNet prompt: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(true); - expect(result.matchedKeywords.length).toBeGreaterThanOrEqual(2); - } - ); - }); - - describe("Should NOT Trigger", () => { - // Prompts that should NOT trigger - completely unrelated topics - const shouldNotTriggerPrompts: string[] = [ - "What is the weather today?", - "Help me write a poem", - "Explain quantum computing", - "Help me with AWS SageMaker", // Wrong cloud provider - "Configure my PostgreSQL database", // Unrelated database task - "Help me with Kubernetes pods", // Use azure-aks - "How do I write Python code?", // Generic programming - "How do I configure a timer-based cron job in my web app?", // Use azure-functions - "Host my static website on a cloud platform", // Use azure-create-app - "How do I create a virtual network for my web app?", // Generic Azure networking — no Foundry - "Set up VNet peering between two subscriptions", // Generic Azure networking - "Configure private endpoints for my Azure SQL database", // Private endpoints but not Foundry - ]; - - test.each(shouldNotTriggerPrompts)( - 'does not trigger on: "%s"', - (prompt) => { - const result = triggerMatcher.shouldTrigger(prompt); - expect(result.triggered).toBe(false); - } - ); - }); - - describe("Trigger Keywords Snapshot", () => { - test("skill keywords match snapshot", () => { - expect(triggerMatcher.getKeywords()).toMatchSnapshot(); - }); - - test("skill description triggers match snapshot", () => { - expect({ - name: skill.metadata.name, - description: skill.metadata.description, - extractedKeywords: triggerMatcher.getKeywords() - }).toMatchSnapshot(); - }); - }); - - describe("Edge Cases", () => { - test("handles empty prompt", () => { - const result = triggerMatcher.shouldTrigger(""); - expect(result.triggered).toBe(false); - }); - - test("handles very long prompt", () => { - const longPrompt = "Microsoft Foundry ".repeat(100); - const result = triggerMatcher.shouldTrigger(longPrompt); - expect(typeof result.triggered).toBe("boolean"); - }); - - test("is case insensitive for Foundry mentions", () => { - const result1 = triggerMatcher.shouldTrigger("Help with MICROSOFT FOUNDRY"); - const result2 = triggerMatcher.shouldTrigger("help with microsoft foundry"); - expect(result1.triggered).toBe(result2.triggered); - }); - }); -});