ACESEvals/pyproject.toml at main · microsoft/ACESEvals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
[project]
name = "aces-evals"
version = "0.1.0"
description = "Agent Capability Evaluation Suite (ACES) — security agent benchmarks. SABER is the internal Microsoft codename for this project."
readme = "README.md"
requires-python = ">=3.11,<3.14"
license = {text = "MIT"}
authors = [
    {name = "MSEC AI Benchmarking Team", email = "kyledeprow@microsoft.com"}
]
keywords = [
    "cybersecurity",
    "security",
    "agent-evaluation",
    "benchmarking",
    "penetration-testing",
    "incident-response",
    "mcp",
    "inspect-ai"
]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Security",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]

dependencies = [
    "saber",
    "inspect-ai",
    "openai>=2.40.0",
    "huggingface-hub",
]

[project.optional-dependencies]
copilot = [
    # GitHub Copilot agent support (requires Node.js 22+ and Copilot CLI)
    "github-copilot-sdk>=0.1.16",
]
claude-code = [
    # Claude Code agent support (requires Anthropic API key)
    "claude-code-sdk>=0.0.25",
]
analysis = [
    "scipy>=1.11.0",
    "seaborn>=0.13.0",
]
dev = [
    "pytest>=7.0.0",
    "black>=23.0.0",
    "isort>=5.12.0",
    "mypy>=1.0.0",
    "pre-commit>=3.0.0",
]

[project.urls]
Homepage = "https://github.com/microsoft/ACESEvals"
Documentation = "https://github.com/microsoft/ACESEvals"
Repository = "https://github.com/microsoft/ACESEvals"
Issues = "https://github.com/microsoft/ACESEvals/issues"

[tool.uv]
package = false

[tool.uv.sources]
# saber and inspect-ai are installed from the GitHub repos (the permanent home):
#   saber      -> ACES library repo
#   inspect-ai -> REQUIRED fork, on the ACESEvals `inspect-ai/dev/aces_integration`
#                 branch. It powers the agent harnesses (`-T agent=copilot` and
#                 `-T agent=claude_code`) plus tool_call limits — none of which
#                 exist in upstream inspect_ai. Do NOT switch this to upstream
#                 inspect-ai: the harness / agent-architecture evals (see
#                 notebooks/*agent_architecture_analysis.ipynb) will break. This
#                 branch must continue to be maintained on GitHub.
saber = { git = "https://github.com/microsoft/ACES.git", branch = "main" }
inspect-ai = { git = "https://github.com/microsoft/ACESEvals.git", branch = "inspect-ai/dev/aces_integration" }
#
# For local development against a checkout, swap the matching line above for:
#   saber = { path = "./external/saber", editable = true }
#   inspect-ai = { path = "./external/inspect_ai", editable = true }
#
# NOTE: these use branch refs for convenience, but uv.lock pins the exact commit
# for reproducibility. After the library moves, update the lock:
#   uv lock --upgrade-package saber && uv sync --all-extras

[dependency-groups]
dev = [
    "pytest-asyncio>=1.2.0",
]