diff --git a/evals/azure-skills/discover-azure-skills/eval.yaml b/evals/azure-skills/discover-azure-skills/eval.yaml new file mode 100644 index 000000000..7e90ee14d --- /dev/null +++ b/evals/azure-skills/discover-azure-skills/eval.yaml @@ -0,0 +1,111 @@ +name: discover-azure-skills-routing-eval +description: | + Integration evaluation for discover-azure-skills routing. + Tests skill invocation for Azure skill discovery and plugin recommendation prompts. + +tags: + type: integration + skill: discover-azure-skills + +defaults: + runs: 5 + timeout: "10m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + +stimuli: + - name: "Find an Azure plugin" + prompt: "Search the Azure skills catalog and tell me which plugin contains a skill for configuring Azure Load Testing." + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: discover-azure-skills + earlyTerminate: '[{"type":"skill-call","skill":"discover-azure-skills"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - discover-azure-skills + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Azure Kusto Graph related skills are in a different plugin + - name: "Find a skill to generate Azure Kusto graph" + prompt: "Find an Azure agent skill that can generate an Azure Kusto graph." + tags: + type: integration + tier: full + cost: llm + area: output + skill: discover-azure-skills + graders: + - type: skill-invocation + config: + required: + - discover-azure-skills + - type: output-contains + config: + substring: "azure-kusto-graph-skills" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + - name: "Trigger discover by a task" + prompt: "Build an azure ai search index from my blob storage. Generate all the todo items and tell me what information I need to provide you to create it. Avoid using azure-ai skill or azure mcp tool. Although they sound related, they lack the knowledge for this task." + tags: + type: integration + tier: full + cost: llm + area: routing + skill: discover-azure-skills + debug: yes + earlyTerminate: '[{"type":"skill-call","skill":"discover-azure-skills"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - discover-azure-skills + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + - name: "How-to question should not trigger skill discovery" + prompt: "How do I set up an Azure Chaos Studio fault-injection experiment against my VM scale set?" + tags: + type: integration + tier: full + cost: llm + area: negative-routing + skill: discover-azure-skills + graders: + - type: skill-invocation + config: + disallowed: + - discover-azure-skills + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # azure plugin has skill/tools related to resource enumeration + - name: "High-level AKS troubleshooting question" + prompt: "Enumerate Azure resources across my subscription and give me the count of resources per type" + tags: + type: integration + tier: full + cost: llm + area: negative-routing + skill: discover-azure-skills + graders: + - type: skill-invocation + config: + disallowed: + - discover-azure-skills + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" diff --git a/gulpfile.ts b/gulpfile.ts index 2d7c250d1..6f8c1d6e1 100644 --- a/gulpfile.ts +++ b/gulpfile.ts @@ -4,13 +4,15 @@ import * as nbgv from "nerdbank-gitversioning"; import * as path from "path"; import log from "fancy-log"; import { execSync } from "child_process"; -import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, cpSync } from "fs"; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, cpSync, existsSync } from "fs"; import Vinyl = require("vinyl"); // Matches top-level skill files like skills/azure-deploy/SKILL.md but not nested ones. const TOP_LEVEL_SKILL_RE = /^skills[\\/][^\\/]+[\\/]SKILL\.md$/; // Matches plugin.json in the .plugin/, .cursor-plugin/, and .claude-plugin/ directories. const PLUGIN_JSON_RE = /^\.(?:plugin|cursor-plugin|claude-plugin)[\\/]plugin\.json$/; +// Hook manifest files that must be merged (not overwritten) between hooks/shared and hooks/. +const HOOK_MANIFEST_FILENAMES = ["copilot-hooks.json", "cursor-hooks.json", "claude-hooks.json"]; /** * Stamps each top-level skill's SKILL.md with a per-skill NBGV version. @@ -119,10 +121,54 @@ function getPluginDirnames(): string[] { .sort(); } -function copyHookScript(pluginDirname: string) { - const src = path.join(__dirname, "hooks"); +/** + * Merges a shared and a plugin-specific hook manifest: all other top-level + * properties come from the plugin manifest (falling back to the shared one + * if the plugin has none), while `hooks` is merged by concatenating the + * arrays for each event key found in either file. + */ +function mergeHookManifests(sharedPath: string, pluginPath: string): Record { + const sharedManifest = existsSync(sharedPath) ? JSON.parse(readFileSync(sharedPath, "utf-8")) : {}; + const pluginManifest = existsSync(pluginPath) ? JSON.parse(readFileSync(pluginPath, "utf-8")) : {}; + + const sharedHooks = sharedManifest.hooks ?? {}; + const pluginHooks = pluginManifest.hooks ?? {}; + + const mergedHooks: Record = {}; + for (const eventName of new Set([...Object.keys(sharedHooks), ...Object.keys(pluginHooks)])) { + mergedHooks[eventName] = [...(sharedHooks[eventName] ?? []), ...(pluginHooks[eventName] ?? [])]; + } + + return { + ...sharedManifest, + ...pluginManifest, + hooks: mergedHooks, + }; +} + +/** + * Merge-copies `hooks/shared` and `hooks/` into the plugin's output + * hooks directory. Hook manifest JSON files are merged at the top-level + * `hooks` property instead of one overwriting the other. + */ +function buildHookScript(pluginDirname: string) { + const sharedDir = path.join(__dirname, "hooks/shared"); + const pluginDir = path.join(__dirname, "hooks", pluginDirname); const dst = path.join(__dirname, `output/${pluginDirname}/hooks`); - cpSync(src, dst, { recursive: true }); + + mkdirSync(dst, { recursive: true }); + cpSync(sharedDir, dst, { recursive: true, filter: (src) => !HOOK_MANIFEST_FILENAMES.includes(path.basename(src)) }); + if (existsSync(pluginDir)) { + cpSync(pluginDir, dst, { recursive: true, filter: (src) => !HOOK_MANIFEST_FILENAMES.includes(path.basename(src)) }); + } + + for (const manifestFilename of HOOK_MANIFEST_FILENAMES) { + const merged = mergeHookManifests( + path.join(sharedDir, manifestFilename), + path.join(pluginDir, manifestFilename) + ); + writeFileSync(path.join(dst, manifestFilename), JSON.stringify(merged, null, 2) + "\n", "utf-8"); + } } function buildPlugin(pluginDirname: string): Promise { @@ -149,7 +195,7 @@ function buildPlugin(pluginDirname: string): Promise { pipeline.on("end", () => { try { generateChangelog(pluginDirname, legacyChangelog); - copyHookScript(pluginDirname); + buildHookScript(pluginDirname); resolve(); } catch (err) { const error = err instanceof Error ? err : new Error(String(err)); diff --git a/hooks/azure-skills/claude-hooks.json b/hooks/azure-skills/claude-hooks.json new file mode 100644 index 000000000..d5df8e47c --- /dev/null +++ b/hooks/azure-skills/claude-hooks.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/discover-skills.sh\"" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/hooks/azure-skills/copilot-hooks.json b/hooks/azure-skills/copilot-hooks.json new file mode 100644 index 000000000..cd2922631 --- /dev/null +++ b/hooks/azure-skills/copilot-hooks.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "SessionStart": [ + { + "type": "command", + "windows": "${PLUGIN_ROOT}/hooks/scripts/discover-skills.ps1", + "osx": "${PLUGIN_ROOT}/hooks/scripts/discover-skills.sh", + "linux": "${PLUGIN_ROOT}/hooks/scripts/discover-skills.sh", + "bash": "${PLUGIN_ROOT}/hooks/scripts/discover-skills.sh", + "powershell": "${PLUGIN_ROOT}/hooks/scripts/discover-skills.ps1" + } + ] + } +} \ No newline at end of file diff --git a/hooks/azure-skills/cursor-hooks.json b/hooks/azure-skills/cursor-hooks.json new file mode 100644 index 000000000..1c6a6f2e5 --- /dev/null +++ b/hooks/azure-skills/cursor-hooks.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [ + { + "type": "command", + "command": "bash ${CURSOR_PLUGIN_ROOT}/hooks/scripts/discover-skills.sh" + } + ] + } +} \ No newline at end of file diff --git a/hooks/azure-skills/scripts/discover-skills.ps1 b/hooks/azure-skills/scripts/discover-skills.ps1 new file mode 100644 index 000000000..323c57a38 --- /dev/null +++ b/hooks/azure-skills/scripts/discover-skills.ps1 @@ -0,0 +1,4 @@ +Write-Output '{"hookSpecificOutput": {"hookEventName": "SessionStart", "additionalContext": "Make sure to use discover-azure-skills once when the user asks you to execute a task related to Azure if there is no relevant skill/mcp tool in the context. Avoid using discover-azure-skills when the user asks informational questions on how to do something."}}' + +# Output success to stdout (required by hooks) +Write-Success \ No newline at end of file diff --git a/hooks/azure-skills/scripts/discover-skills.sh b/hooks/azure-skills/scripts/discover-skills.sh new file mode 100644 index 000000000..620dd6fb5 --- /dev/null +++ b/hooks/azure-skills/scripts/discover-skills.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo '{"hookSpecificOutput": {"hookEventName": "SessionStart", "additionalContext": "Make sure to use discover-azure-skills once when the user asks you to execute a task related to Azure if there is no relevant skill/mcp tool in the context. Avoid using discover-azure-skills when the user asks informational questions on how to do something."}}' \ No newline at end of file diff --git a/hooks/claude-hooks.json b/hooks/shared/claude-hooks.json similarity index 100% rename from hooks/claude-hooks.json rename to hooks/shared/claude-hooks.json diff --git a/hooks/copilot-hooks.json b/hooks/shared/copilot-hooks.json similarity index 99% rename from hooks/copilot-hooks.json rename to hooks/shared/copilot-hooks.json index 39587eb97..6590f2480 100644 --- a/hooks/copilot-hooks.json +++ b/hooks/shared/copilot-hooks.json @@ -3,14 +3,12 @@ "PostToolUse": [ { "type": "command", - "windows": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.ps1", "osx": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", "linux": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", - "bash": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", "powershell": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.ps1" } ] } -} +} \ No newline at end of file diff --git a/hooks/cursor-hooks.json b/hooks/shared/cursor-hooks.json similarity index 100% rename from hooks/cursor-hooks.json rename to hooks/shared/cursor-hooks.json diff --git a/hooks/scripts/track-telemetry.ps1 b/hooks/shared/scripts/track-telemetry.ps1 similarity index 100% rename from hooks/scripts/track-telemetry.ps1 rename to hooks/shared/scripts/track-telemetry.ps1 diff --git a/hooks/scripts/track-telemetry.sh b/hooks/shared/scripts/track-telemetry.sh similarity index 100% rename from hooks/scripts/track-telemetry.sh rename to hooks/shared/scripts/track-telemetry.sh diff --git a/plugins/azure-skills/skills/discover-azure-skills/SKILL.md b/plugins/azure-skills/skills/discover-azure-skills/SKILL.md new file mode 100644 index 000000000..41950a51e --- /dev/null +++ b/plugins/azure-skills/skills/discover-azure-skills/SKILL.md @@ -0,0 +1,42 @@ +--- +name: discover-azure-skills +description: "Searches the Azure skills catalog and recommends installable agent skills by matching an Azure task to skill metadata and plugin installation guidance. WHEN: before starting any task that involves an Azure or Microsoft-cloud service, product, or data source, when no currently loaded skill or tool already covers it." +license: MIT +metadata: + author: Microsoft + version: "0.0.0-placeholder" +--- + +Follow these steps to discover the available azure skill matching the given task description. + +1. List plugins + +List Azure plugin directories from the GitHub Contents API: https://api.github.com/repos/microsoft/azure-skills/contents/.github/plugins?ref=main + +In the result, each entry whose `type` is `dir` is a plugin directory. Skills are organized by plugins. + +2. List skills + +For each plugin, list their skills from the GitHub Contents API: https://api.github.com/repos/microsoft/azure-skills/contents/.github/plugins/{plugin-dirname}/skills?ref=main + +In the result, each entry whose `type` is `dir` is a skill directory. Each skill has a SKILL.md file that explains what this skill should be used for. + +3. Discover relevant skills + +Eliminate the skills that obviously aren't relevant by their names. Then for each remaining skill, read their description from the API: https://raw.githubusercontent.com/microsoft/azure-skills/main/.github/plugins/{plugin-dirname}/skills/{skill-name}/SKILL.md + +Use the descriptions to further eliminate skills that aren't relevant. + +4. Discover the plugin name of the relevant skills + +For each relevant skill, discover their plugin name by reading the `plugin.json` from the GitHub Contents API: https://raw.githubusercontent.com/microsoft/azure-skills/main/.github/plugins/{plugin-dirname}/.plugin/plugin.json + +This is important because a plugin's name may be different from its directory name. The installation commands depend on the plugin's name. + +5. Report the matched skills + +Report the matched skills and offer instructions to install them. Skills can be installed by installing their plugin. Read the installation instructions matching the agent client to offer the installation instructions. + +- [Copilot CLI](./references/install/copilot-cli.md) +- [Claude Code](./references/install/claude-code.md) +- [Other](./references/install/other.md) \ No newline at end of file diff --git a/plugins/azure-skills/skills/discover-azure-skills/references/install/claude-code.md b/plugins/azure-skills/skills/discover-azure-skills/references/install/claude-code.md new file mode 100644 index 000000000..b0b236f9c --- /dev/null +++ b/plugins/azure-skills/skills/discover-azure-skills/references/install/claude-code.md @@ -0,0 +1,19 @@ +# Steps + +1. Add `azure-skills` marketplace + +Run this slash command in Claude Code + +``` +/plugin marketplace add microsoft/azure-skills +``` + +2. Install the target plugin + +Run this slash command in Claude Code + +``` +/plugin install {plugin-name}@azure-skills +``` + +> Note: {plugin-name} is the discovered plugin's name, not its directory name. diff --git a/plugins/azure-skills/skills/discover-azure-skills/references/install/copilot-cli.md b/plugins/azure-skills/skills/discover-azure-skills/references/install/copilot-cli.md new file mode 100644 index 000000000..09fa742fe --- /dev/null +++ b/plugins/azure-skills/skills/discover-azure-skills/references/install/copilot-cli.md @@ -0,0 +1,19 @@ +# Steps + +1. Add `azure-skills` marketplace + +Run this slash command in Copilot CLI + +``` +/plugin marketplace add microsoft/azure-skills +``` + +2. Install the target plugin + +Run this slash command in Copilot CLI + +``` +/plugin install {plugin-name}@azure-skills +``` + +> Note: {plugin-name} is the discovered plugin's name, not its directory name. diff --git a/plugins/azure-skills/skills/discover-azure-skills/references/install/other.md b/plugins/azure-skills/skills/discover-azure-skills/references/install/other.md new file mode 100644 index 000000000..56fc0d233 --- /dev/null +++ b/plugins/azure-skills/skills/discover-azure-skills/references/install/other.md @@ -0,0 +1,9 @@ +# Steps + +1. Install each skill using `skills` package + +``` +npx skills add https://github.com/microsoft/azure-skills/tree/main/.github/plugins/{plugin-dirname}/skills/{skill-name} +``` + +The skills package installs the skill into the `.agents/skills` directory, which is a universal location recognized by many agent clients. \ No newline at end of file diff --git a/plugins/azure-skills/skills/discover-azure-skills/version.json b/plugins/azure-skills/skills/discover-azure-skills/version.json new file mode 100644 index 000000000..e7aca4d60 --- /dev/null +++ b/plugins/azure-skills/skills/discover-azure-skills/version.json @@ -0,0 +1,6 @@ +{ + "version": "1.0", + "pathFilters": [ + "." + ] +} \ No newline at end of file diff --git a/tests/skills.json b/tests/skills.json index 752ef0319..716179d44 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -16,6 +16,7 @@ "azure-cost", "azure-deploy", "azure-diagnostics", + "discover-azure-skills", "azure-enterprise-infra-planner", "azure-kubernetes", "azure-kusto", @@ -36,7 +37,7 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy,azure-app-onboard,azure-app-onboard-prereq" + "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,discover-azure-skills,azure-enterprise-infra-planner,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy,azure-app-onboard,azure-app-onboard-prereq" } }, { diff --git a/tests/utils/agent-runner.ts b/tests/utils/agent-runner.ts index 78250af0f..85471e6d6 100644 --- a/tests/utils/agent-runner.ts +++ b/tests/utils/agent-runner.ts @@ -922,7 +922,15 @@ export function useAgentRunner(agentRunnerConfig: AgentRunnerConfig) { }), systemMessage: runConfig.systemPrompt, // Disable session telemetry so usage of skills and tools by the test agent runner don't end up sending Copilot CLI telemetry. - enableSessionTelemetry: false + enableSessionTelemetry: false, + hooks: { + // Note: keep this in sync with hooks/azure-skills/scripts/discover-skills + onSessionStart: async (_input, _invocation) => { + return { + additionalContext: "Make sure to use discover-azure-skills once when the user asks you to execute a task related to Azure if there is no relevant skill/mcp tool in the context. Avoid using discover-azure-skills when the user asks informational questions on how to do something." + } + } + } }); entry.session = session;