diff --git a/dev-notes/architecture/llama-cpp-first-class.md b/dev-notes/architecture/llama-cpp-first-class.md new file mode 100644 index 000000000..1c3d54172 --- /dev/null +++ b/dev-notes/architecture/llama-cpp-first-class.md @@ -0,0 +1,67 @@ +# First-class llama.cpp integration + +## What was added + +Tau now includes a built-in `llama-cpp` provider and dedicated commands: + +```bash +tau llama-cpp setup +tau llama-cpp doctor +``` + +Setup connects to a running llama.cpp server, queries its OpenAI-compatible +`/v1/models` endpoint, and persists the discovered models and selected default. +Doctor checks discovery, streaming chat completions, and tool-call support. + +## Why it exists + +llama.cpp was previously documented as a generic custom endpoint. Users had to +invent a model ID and configure a fake API key even though llama.cpp does not +enforce authentication by default. That made a supported local runtime feel +like an unsupported workaround. + +## Architecture + +The split remains: + +- `tau_ai` supports OpenAI-compatible endpoints whose bearer token is optional. +- `tau_coding.llama_cpp` owns llama.cpp discovery and diagnostics. +- `tau_coding` owns CLI setup and durable provider preferences. +- `tau_agent` remains unaware of llama.cpp. + +There is intentionally no `LlamaCppProvider` adapter. llama.cpp speaks the +OpenAI chat-completions protocol already, so Tau reuses +`OpenAICompatibleProvider`. A dedicated adapter should only be introduced if a +future llama.cpp behavior cannot be represented by compatibility metadata. + +## Provider metadata + +Provider catalog entries now declare: + +- `auth = "required" | "optional" | "none"` +- `model_discovery = "static" | "openai"` + +The built-in llama.cpp entry uses optional auth and OpenAI model discovery. Tau +omits the `Authorization` header when no key is available, but still supports +servers launched with `llama-server --api-key` through `LLAMA_API_KEY` or a +stored credential. + +## Testing + +```bash +uv run pytest tests/test_llama_cpp.py tests/test_provider_catalog.py \ + tests/test_provider_config.py tests/test_provider_runtime.py tests/test_cli.py \ + tests/test_tau_ai.py +uv run ruff check . +uv run ruff format --check . +uv run mypy +``` + +For a live smoke test: + +```bash +llama-server -hf +tau llama-cpp setup +tau llama-cpp doctor +tau --provider llama-cpp -p "Use the bash tool to print the current directory" +``` diff --git a/src/tau_ai/env.py b/src/tau_ai/env.py index 4f8f93c02..c9d911b17 100644 --- a/src/tau_ai/env.py +++ b/src/tau_ai/env.py @@ -19,7 +19,7 @@ class OpenAICompatibleConfig: """Configuration for an OpenAI-compatible chat completions endpoint.""" - api_key: str + api_key: str | None base_url: str = DEFAULT_OPENAI_COMPATIBLE_BASE_URL headers: Mapping[str, str] | None = None timeout_seconds: float = DEFAULT_OPENAI_COMPATIBLE_TIMEOUT_SECONDS diff --git a/src/tau_ai/openai_compatible.py b/src/tau_ai/openai_compatible.py index 8942e27ec..e1bb7a349 100644 --- a/src/tau_ai/openai_compatible.py +++ b/src/tau_ai/openai_compatible.py @@ -170,10 +170,9 @@ def _stream( async def iterator() -> AsyncIterator[ProviderEvent]: client = self._get_client() - headers = { - **(dict(self._config.headers or {})), - "Authorization": f"Bearer {self._config.api_key}", - } + headers = dict(self._config.headers or {}) + if self._config.api_key: + headers["Authorization"] = f"Bearer {self._config.api_key}" attempt = 0 while True: diff --git a/src/tau_coding/catalog_loader.py b/src/tau_coding/catalog_loader.py index cbe411425..83b09b340 100644 --- a/src/tau_coding/catalog_loader.py +++ b/src/tau_coding/catalog_loader.py @@ -26,8 +26,10 @@ from tau_coding.paths import TauPaths from tau_coding.provider_catalog import ( ModelCatalogMetadata, + ModelDiscovery, ModelInput, ProviderApi, + ProviderAuth, ProviderCatalogEntry, ProviderKind, ) @@ -79,6 +81,8 @@ class _CatalogProvider(BaseModel): base_url: _NonEmptyString api_key_env: _NonEmptyString credential_name: _NonEmptyString | None = None + auth: ProviderAuth = "required" + model_discovery: ModelDiscovery = "static" models: _NonEmptyStringTuple default_model: _NonEmptyString docs_url: _NonEmptyString @@ -212,7 +216,10 @@ def _merge_raw_provider(base: dict[str, Any], overlay: dict[str, Any]) -> dict[s base_models = base.get("models", []) overlay_models = overlay.get("models", []) if isinstance(base_models, list) and isinstance(overlay_models, list): - merged["models"] = list(dict.fromkeys([*overlay_models, *base_models])) + if merged.get("model_discovery") == "openai": + merged["models"] = overlay_models + else: + merged["models"] = list(dict.fromkeys([*overlay_models, *base_models])) for key in ("context_windows", "headers", "compat"): base_mapping = base.get(key) overlay_mapping = overlay.get(key) @@ -313,6 +320,8 @@ def _entry_from_provider(provider: _CatalogProvider, *, source: str) -> Provider api_key_env=provider.api_key_env, credential_name=provider.credential_name, models=provider.models, + auth=provider.auth, + model_discovery=provider.model_discovery, default_model=provider.default_model, docs_url=provider.docs_url, api=provider.api, @@ -397,6 +406,8 @@ def _raw_provider_from_entry(entry: ProviderCatalogEntry) -> dict[str, Any]: "base_url": entry.base_url, "api_key_env": entry.api_key_env, "models": list(entry.models), + "auth": entry.auth, + "model_discovery": entry.model_discovery, "default_model": entry.default_model, "docs_url": entry.docs_url, } @@ -470,6 +481,8 @@ def _catalog_to_toml(raw: dict[str, Any]) -> str: "base_url", "api_key_env", "credential_name", + "auth", + "model_discovery", "models", "default_model", "docs_url", diff --git a/src/tau_coding/cli.py b/src/tau_coding/cli.py index aeef2d249..5875c3529 100644 --- a/src/tau_coding/cli.py +++ b/src/tau_coding/cli.py @@ -4,6 +4,8 @@ import contextlib import sys +from dataclasses import replace +from functools import partial from os import environ from pathlib import Path from typing import Annotated @@ -21,6 +23,15 @@ from tau_ai.env import DEFAULT_OPENAI_COMPATIBLE_BASE_URL from tau_coding.catalog_loader import user_catalog_path from tau_coding.credentials import FileCredentialStore +from tau_coding.llama_cpp import ( + LLAMA_CPP_API_KEY_ENV, + LLAMA_CPP_DEFAULT_BASE_URL, + LLAMA_CPP_PROVIDER_NAME, + LlamaCppError, + configured_llama_cpp_provider, + diagnose_llama_cpp, + discover_llama_cpp, +) from tau_coding.provider_config import ( DEFAULT_MODEL, DEFAULT_PROVIDER_NAME, @@ -29,10 +40,13 @@ ProviderConfig, ProviderSettings, load_provider_settings, + provider_config_from_catalog_entry, provider_kind, resolve_provider_selection, + save_default_provider_model, save_provider_settings, upsert_openai_compatible_provider, + upsert_saved_provider, ) from tau_coding.provider_runtime import create_model_provider from tau_coding.rendering import PrintOutputMode, create_event_renderer @@ -97,6 +111,97 @@ def providers_command() -> None: render_provider_settings(load_provider_settings(), credential_reader=FileCredentialStore()) +def llama_cpp_setup_command( + base_url: Annotated[ + str, + typer.Option("--base-url", help="llama.cpp server URL, with or without /v1."), + ] = LLAMA_CPP_DEFAULT_BASE_URL, + model: Annotated[ + str | None, + typer.Option("--model", "-m", help="Discovered model to use by default."), + ] = None, + api_key: Annotated[ + str | None, + typer.Option("--api-key", help="API key for a protected llama.cpp server."), + ] = None, + set_default: Annotated[ + bool, + typer.Option("--set-default/--no-set-default", help="Make llama.cpp the default."), + ] = True, +) -> None: + """Discover a running llama.cpp server and save it as a Tau provider.""" + if api_key is not None: + typer.echo( + "Warning: --api-key can be exposed in shell history; prefer LLAMA_API_KEY.", + err=True, + ) + resolved_api_key = ( + api_key + or FileCredentialStore().get(LLAMA_CPP_PROVIDER_NAME) + or environ.get(LLAMA_CPP_API_KEY_ENV) + ) + try: + server = anyio.run(partial(discover_llama_cpp, base_url, api_key=resolved_api_key)) + builtin = provider_config_from_catalog_entry(LLAMA_CPP_PROVIDER_NAME) + if not isinstance(builtin, OpenAICompatibleProviderConfig): + raise LlamaCppError("Built-in llama.cpp provider is not OpenAI-compatible") + provider = configured_llama_cpp_provider(builtin, server, model=model) + if api_key and provider.credential_name: + FileCredentialStore().set(provider.credential_name, api_key) + updated = upsert_saved_provider(provider, set_default=set_default) + save_default_provider_model( + provider_name=provider.name, + model=provider.default_model, + fallback_settings=updated, + ) + except (LlamaCppError, RuntimeError) as exc: + raise typer.BadParameter(str(exc)) from exc + typer.echo(f"Found llama.cpp at {provider.base_url}") + typer.echo(f"Discovered models: {', '.join(provider.models)}") + typer.echo(f"Saved provider '{provider.name}' with default model '{provider.default_model}'.") + + +def llama_cpp_doctor_command( + base_url: Annotated[ + str | None, + typer.Option("--base-url", help="Override the configured llama.cpp server URL."), + ] = None, + model: Annotated[ + str | None, + typer.Option("--model", "-m", help="Model to probe."), + ] = None, + api_key: Annotated[ + str | None, + typer.Option("--api-key", help="API key for a protected llama.cpp server."), + ] = None, +) -> None: + """Check llama.cpp discovery, streaming, and coding-tool support.""" + if api_key is not None: + typer.echo( + "Warning: --api-key can be exposed in shell history; prefer LLAMA_API_KEY.", + err=True, + ) + configured = load_provider_settings().get_provider(LLAMA_CPP_PROVIDER_NAME) + if not isinstance(configured, OpenAICompatibleProviderConfig): + raise typer.BadParameter("Configured llama.cpp provider is not OpenAI-compatible") + provider = configured + if base_url is not None: + provider = replace(provider, base_url=base_url) + if model is not None: + models = provider.models if model in provider.models else (*provider.models, model) + provider = replace(provider, models=models, default_model=model) + resolved_api_key = api_key + if resolved_api_key is None and provider.credential_name: + resolved_api_key = FileCredentialStore().get(provider.credential_name) + resolved_api_key = resolved_api_key or environ.get(provider.api_key_env) + report = anyio.run(partial(diagnose_llama_cpp, provider, api_key=resolved_api_key)) + icons = {"ok": "✓", "warning": "!", "error": "✗"} + for diagnostic in report.diagnostics: + typer.echo(f"{icons[diagnostic.status]} {diagnostic.message}") + if not report.ok: + raise typer.Exit(1) + + def setup_command( *, provider_name: str = DEFAULT_PROVIDER_NAME, @@ -247,6 +352,23 @@ def main( providers_command() raise typer.Exit() + if prompt_option is None and command == "llama-cpp": + llama_command, llama_options = _parse_llama_cpp_cli_args(positional_args[1:]) + if llama_command == "setup": + llama_cpp_setup_command( + base_url=str(llama_options.get("base_url", LLAMA_CPP_DEFAULT_BASE_URL)), + model=_optional_cli_string(llama_options.get("model")), + api_key=_optional_cli_string(llama_options.get("api_key")), + set_default=bool(llama_options.get("set_default", True)), + ) + else: + llama_cpp_doctor_command( + base_url=_optional_cli_string(llama_options.get("base_url")), + model=_optional_cli_string(llama_options.get("model")), + api_key=_optional_cli_string(llama_options.get("api_key")), + ) + raise typer.Exit() + if prompt_option is None and command == "setup" and len(positional_args) == 1: setup_command( provider_name=provider or DEFAULT_PROVIDER_NAME, @@ -367,6 +489,49 @@ async def export_session_command( ) +def _optional_cli_string(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _parse_llama_cpp_cli_args(args: list[str]) -> tuple[str, dict[str, object]]: + usage = ( + "Usage: tau llama-cpp setup|doctor [--base-url URL] [--model MODEL] " + "[--api-key KEY] [--no-set-default]" + ) + if not args or args[0] not in {"setup", "doctor"}: + raise typer.BadParameter(usage) + command = args[0] + options: dict[str, object] = {} + index = 1 + while index < len(args): + arg = args[index] + if arg == "--no-set-default" and command == "setup": + options["set_default"] = False + elif arg == "--set-default" and command == "setup": + options["set_default"] = True + elif arg in {"--base-url", "--model", "-m", "--api-key"}: + index += 1 + if index >= len(args): + raise typer.BadParameter(usage) + key = { + "--base-url": "base_url", + "--model": "model", + "-m": "model", + "--api-key": "api_key", + }[arg] + options[key] = args[index] + elif arg.startswith("--base-url="): + options["base_url"] = arg.partition("=")[2] + elif arg.startswith("--model="): + options["model"] = arg.partition("=")[2] + elif arg.startswith("--api-key="): + options["api_key"] = arg.partition("=")[2] + else: + raise typer.BadParameter(usage) + index += 1 + return command, options + + def _parse_export_cli_args(args: list[str]) -> tuple[str, Path | None, str | None]: if not args: raise RuntimeError("Usage: tau export [--format html|jsonl] [output]") @@ -470,6 +635,11 @@ def _provider_credential_status( return f"stored:{provider.credential_name}" if environ.get(provider.api_key_env): return f"env:{provider.api_key_env}" + if isinstance(provider, OpenAICompatibleProviderConfig) and provider.auth in { + "optional", + "none", + }: + return "optional" return "missing" diff --git a/src/tau_coding/data/catalog.toml b/src/tau_coding/data/catalog.toml index 65d2f8327..53e47f0aa 100644 --- a/src/tau_coding/data/catalog.toml +++ b/src/tau_coding/data/catalog.toml @@ -6170,3 +6170,18 @@ context_window = 1048576 max_tokens = 131072 compat = { requiresReasoningContentOnAssistantMessages = true, thinkingFormat = "deepseek" } cost = { input = 1, output = 3, cacheRead = 0.2, cacheWrite = 0 } + +[[providers]] +name = "llama-cpp" +display_name = "llama.cpp" +kind = "openai-compatible" +base_url = "http://127.0.0.1:8080/v1" +api_key_env = "LLAMA_API_KEY" +credential_name = "llama-cpp" +auth = "optional" +model_discovery = "openai" +models = ["local"] +default_model = "local" +docs_url = "https://github.com/ggml-org/llama.cpp/tree/master/tools/server" +api = "openai-completions" +compat = { supportsStore = false, supportsUsageInStreaming = false } diff --git a/src/tau_coding/llama_cpp.py b/src/tau_coding/llama_cpp.py new file mode 100644 index 000000000..e0f277dfa --- /dev/null +++ b/src/tau_coding/llama_cpp.py @@ -0,0 +1,277 @@ +"""llama.cpp server discovery, setup, and diagnostics.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import Literal + +import httpx + +from tau_agent.messages import UserMessage +from tau_agent.tools import AgentTool, AgentToolResult, ToolCancellationToken +from tau_agent.types import JSONValue +from tau_ai import OpenAICompatibleConfig, OpenAICompatibleProvider +from tau_ai.events import ProviderErrorEvent, ProviderResponseStartEvent, ProviderToolCallEvent +from tau_ai.http import create_async_client +from tau_coding.provider_config import OpenAICompatibleProviderConfig + +LLAMA_CPP_PROVIDER_NAME = "llama-cpp" +LLAMA_CPP_DEFAULT_BASE_URL = "http://127.0.0.1:8080/v1" +LLAMA_CPP_API_KEY_ENV = "LLAMA_API_KEY" + + +class LlamaCppError(RuntimeError): + """Raised when Tau cannot connect to or inspect a llama.cpp server.""" + + +@dataclass(frozen=True, slots=True) +class LlamaCppServerInfo: + """Models discovered from a running llama.cpp server.""" + + base_url: str + models: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class LlamaCppDiagnostic: + """One setup/doctor check and its human-readable result.""" + + check: str + status: Literal["ok", "warning", "error"] + message: str + + +@dataclass(frozen=True, slots=True) +class LlamaCppDoctorReport: + """Complete diagnostic report for a llama.cpp provider.""" + + diagnostics: tuple[LlamaCppDiagnostic, ...] + + @property + def ok(self) -> bool: + """Return whether every required check passed.""" + return all(item.status != "error" for item in self.diagnostics) + + +def normalize_llama_cpp_base_url(base_url: str) -> str: + """Return a normalized OpenAI-compatible llama.cpp base URL.""" + normalized = base_url.strip().rstrip("/") + if not normalized: + raise LlamaCppError("llama.cpp base URL cannot be empty") + if normalized.endswith("/v1"): + return normalized + return f"{normalized}/v1" + + +async def discover_llama_cpp( + base_url: str = LLAMA_CPP_DEFAULT_BASE_URL, + *, + api_key: str | None = None, + timeout_seconds: float = 5.0, + client: httpx.AsyncClient | None = None, +) -> LlamaCppServerInfo: + """Connect to llama.cpp and return model IDs from its OpenAI-compatible API.""" + normalized = normalize_llama_cpp_base_url(base_url) + owned_client = client is None + resolved_client = client or create_async_client(timeout=timeout_seconds) + try: + response = await resolved_client.get( + f"{normalized}/models", + headers=_authorization_headers(api_key), + ) + if response.status_code in {401, 403}: + raise LlamaCppError( + "llama.cpp requires an API key. Set LLAMA_API_KEY or pass --api-key." + ) + response.raise_for_status() + try: + payload = response.json() + except ValueError as exc: + raise LlamaCppError("llama.cpp /v1/models returned invalid JSON") from exc + models = _model_ids(payload) + if not models: + raise LlamaCppError( + "llama.cpp returned no models. Wait for the model to finish loading and retry." + ) + return LlamaCppServerInfo(base_url=normalized, models=models) + except LlamaCppError: + raise + except httpx.HTTPStatusError as exc: + raise LlamaCppError( + f"llama.cpp /v1/models returned HTTP {exc.response.status_code}: " + f"{exc.response.text[:200]}" + ) from exc + except httpx.HTTPError as exc: + raise LlamaCppError( + f"Could not connect to llama.cpp at {normalized}. " + "Start llama-server (port 8080 by default) and retry. " + f"{exc}" + ) from exc + finally: + if owned_client: + await resolved_client.aclose() + + +def configured_llama_cpp_provider( + provider: OpenAICompatibleProviderConfig, + server: LlamaCppServerInfo, + *, + model: str | None = None, +) -> OpenAICompatibleProviderConfig: + """Return the built-in llama.cpp provider updated with discovered models.""" + selected = model or server.models[0] + if selected not in server.models: + available = ", ".join(server.models) + raise LlamaCppError( + f"Model {selected!r} is not served by llama.cpp. Available models: {available}" + ) + return replace( + provider, + base_url=server.base_url, + models=server.models, + default_model=selected, + ) + + +async def diagnose_llama_cpp( + provider: OpenAICompatibleProviderConfig, + *, + api_key: str | None = None, + timeout_seconds: float = 10.0, + client: httpx.AsyncClient | None = None, +) -> LlamaCppDoctorReport: + """Check connectivity, discovery, streaming, and tool-call support.""" + diagnostics: list[LlamaCppDiagnostic] = [] + try: + server = await discover_llama_cpp( + provider.base_url, + api_key=api_key, + timeout_seconds=timeout_seconds, + client=client, + ) + except LlamaCppError as exc: + return LlamaCppDoctorReport((LlamaCppDiagnostic("server", "error", str(exc)),)) + + diagnostics.extend( + ( + LlamaCppDiagnostic("server", "ok", f"Found llama.cpp at {server.base_url}"), + LlamaCppDiagnostic("models", "ok", f"Models: {', '.join(server.models)}"), + ) + ) + selected = ( + provider.default_model if provider.default_model in server.models else server.models[0] + ) + runtime = OpenAICompatibleProvider( + OpenAICompatibleConfig( + api_key=api_key, + base_url=server.base_url, + provider_name=LLAMA_CPP_PROVIDER_NAME, + timeout_seconds=timeout_seconds, + max_retries=0, + compat={"supportsStore": False, "supportsUsageInStreaming": False}, + ), + client=client, + ) + try: + stream_ok, stream_error = await _probe_stream(runtime, selected) + diagnostics.append( + LlamaCppDiagnostic( + "streaming", + "ok" if stream_ok else "error", + "Streaming chat completions accepted" + if stream_ok + else f"Streaming chat completion failed: {stream_error}", + ) + ) + if stream_ok: + tools_ok, tools_message = await _probe_tools(runtime, selected) + diagnostics.append( + LlamaCppDiagnostic( + "tools", + "ok" if tools_ok else "warning", + "Tool calls supported" if tools_ok else tools_message, + ) + ) + finally: + if client is None: + await runtime.aclose() + return LlamaCppDoctorReport(tuple(diagnostics)) + + +async def _probe_stream( + provider: OpenAICompatibleProvider, + model: str, +) -> tuple[bool, str | None]: + error: str | None = None + accepted = False + async for event in provider.stream_response( + model=model, + system="Reply briefly.", + messages=[UserMessage(content="Reply with exactly: OK")], + tools=[], + ): + if isinstance(event, ProviderResponseStartEvent): + accepted = True + elif isinstance(event, ProviderErrorEvent): + error = event.message + return accepted and error is None, error + + +async def _probe_tools( + provider: OpenAICompatibleProvider, + model: str, +) -> tuple[bool, str]: + async def executor( + arguments: Mapping[str, JSONValue], + signal: ToolCancellationToken | None = None, + ) -> AgentToolResult: + del arguments, signal + return AgentToolResult(tool_call_id="probe", name="tau_probe", ok=True, content="ok") + + tool = AgentTool( + name="tau_probe", + description="Call this tool to verify tool calling. Always call it.", + input_schema={"type": "object", "properties": {}}, + executor=executor, + ) + error: str | None = None + called = False + async for event in provider.stream_response( + model=model, + system="You must call the tau_probe tool.", + messages=[UserMessage(content="Call tau_probe now.")], + tools=[tool], + ): + if isinstance(event, ProviderToolCallEvent): + called = True + elif isinstance(event, ProviderErrorEvent): + error = event.message + if called: + return True, "Tool calls supported" + if error: + return False, f"Tool-call probe failed: {error}" + return False, ( + "The model did not call the probe tool. Use a tool-capable instruct GGUF and a " + "compatible llama.cpp chat template." + ) + + +def _authorization_headers(api_key: str | None) -> dict[str, str]: + return {"Authorization": f"Bearer {api_key}"} if api_key else {} + + +def _model_ids(payload: object) -> tuple[str, ...]: + if not isinstance(payload, dict): + return () + data = payload.get("data") + if not isinstance(data, list): + return () + models: list[str] = [] + for item in data: + if not isinstance(item, dict): + continue + model_id = item.get("id") + if isinstance(model_id, str) and model_id.strip(): + models.append(model_id.strip()) + return tuple(dict.fromkeys(models)) diff --git a/src/tau_coding/provider_catalog.py b/src/tau_coding/provider_catalog.py index d3a1c5731..d111fa8e4 100644 --- a/src/tau_coding/provider_catalog.py +++ b/src/tau_coding/provider_catalog.py @@ -15,6 +15,8 @@ "google-generative-ai", "mistral-conversations", ] +ProviderAuth = Literal["required", "optional", "none"] +ModelDiscovery = Literal["static", "openai"] ProviderApi = Literal[ "openai-completions", "openai-responses", @@ -57,6 +59,8 @@ class ProviderCatalogEntry: models: tuple[str, ...] default_model: str docs_url: str + auth: ProviderAuth = "required" + model_discovery: ModelDiscovery = "static" api: ProviderApi | None = None context_windows: dict[str, int] | None = None headers: dict[str, str] = field(default_factory=dict) diff --git a/src/tau_coding/provider_config.py b/src/tau_coding/provider_config.py index dbcc11c20..7cb0c0a08 100644 --- a/src/tau_coding/provider_config.py +++ b/src/tau_coding/provider_config.py @@ -27,7 +27,9 @@ from tau_coding.provider_catalog import ( BUILTIN_PROVIDER_CATALOG, ModelCatalogMetadata, + ModelDiscovery, ProviderApi, + ProviderAuth, ProviderCatalogEntry, ProviderKind, ) @@ -97,6 +99,8 @@ class OpenAICompatibleProviderConfig: api: ProviderApi = "openai-completions" api_key_env: str = "OPENAI_API_KEY" credential_name: str | None = None + auth: ProviderAuth = "required" + model_discovery: ModelDiscovery = "static" models: tuple[str, ...] = (DEFAULT_MODEL,) default_model: str = DEFAULT_MODEL context_windows: dict[str, int] = field(default_factory=dict) @@ -138,6 +142,8 @@ def to_json(self) -> dict[str, Any]: "api": self.api, "api_key_env": self.api_key_env, "credential_name": self.credential_name, + "auth": self.auth, + "model_discovery": self.model_discovery, "models": list(self.models), "default_model": self.default_model, "context_windows": dict(self.context_windows), @@ -411,6 +417,8 @@ def provider_config_from_entry(entry: ProviderCatalogEntry) -> ProviderConfig: api=entry.api or _default_api_for_kind(entry.kind), api_key_env=entry.api_key_env, credential_name=entry.credential_name, + auth=entry.auth, + model_discovery=entry.model_discovery, models=entry.models, default_model=entry.default_model, context_windows=context_windows, @@ -779,7 +787,11 @@ def _merge_openai_compatible_provider( existing: OpenAICompatibleProviderConfig, incoming: OpenAICompatibleProviderConfig, ) -> OpenAICompatibleProviderConfig: - models = _unique_strings((*incoming.models, *existing.models)) + models = ( + incoming.models + if incoming.model_discovery == "openai" + else _unique_strings((*incoming.models, *existing.models)) + ) return replace( incoming, models=models, @@ -961,6 +973,13 @@ def _provider_definition_differs_from_catalog( return True if provider.credential_name != entry.credential_name: return True + if isinstance(provider, OpenAICompatibleProviderConfig) and provider.auth != entry.auth: + return True + if ( + isinstance(provider, OpenAICompatibleProviderConfig) + and provider.model_discovery != entry.model_discovery + ): + return True if provider.models != entry.models: return True if getattr(provider, "api", None) != entry.api and entry.api is not None: @@ -997,6 +1016,14 @@ def _catalog_entry_from_provider( api=getattr(provider, "api", None), credential_name=provider.credential_name, models=provider.models, + auth=( + provider.auth if isinstance(provider, OpenAICompatibleProviderConfig) else "required" + ), + model_discovery=( + provider.model_discovery + if isinstance(provider, OpenAICompatibleProviderConfig) + else "static" + ), default_model=( existing.default_model if existing is not None and existing.default_model in provider.models @@ -1470,6 +1497,8 @@ def anthropic_config_from_provider( ) -> AnthropicConfig: """Build Anthropic runtime config from durable settings.""" api_key = _api_key_from_provider(provider, credential_reader=credential_reader) + if api_key is None: + raise RuntimeError(f"Missing provider API key for {provider.name}.") selected_model = model or provider.default_model thinking_budget_tokens = _anthropic_thinking_budget_from_provider( provider, @@ -1514,6 +1543,11 @@ def provider_has_usable_credentials( credential_reader: CredentialReader | None = None, ) -> bool: """Return whether Tau can attempt calls for this provider without prompting setup.""" + if isinstance(provider, OpenAICompatibleProviderConfig) and provider.auth in { + "optional", + "none", + }: + return True if provider.credential_name and credential_reader is not None: if isinstance(provider, OpenAICodexProviderConfig): get_oauth = getattr(credential_reader, "get_oauth", None) @@ -1680,6 +1714,10 @@ def _provider_from_json(data: object) -> ProviderConfig: credential_name = _optional_string( data.get("credential_name"), f"providers[{name}].credential_name" ) + auth = _provider_auth(data.get("auth", "required"), f"providers[{name}].auth") + model_discovery = _model_discovery( + data.get("model_discovery", "static"), f"providers[{name}].model_discovery" + ) models = _string_tuple(data.get("models"), f"providers[{name}].models") default_model = _string(data.get("default_model"), f"providers[{name}].default_model") context_windows = _context_window_dict( @@ -1772,6 +1810,8 @@ def _provider_from_json(data: object) -> ProviderConfig: api=api or _default_api_for_kind(provider_type), api_key_env=api_key_env, credential_name=credential_name, + auth=auth, + model_discovery=model_discovery, models=models, default_model=default_model, context_windows=context_windows, @@ -1793,7 +1833,9 @@ def _api_key_from_provider( provider: ProviderConfig, *, credential_reader: CredentialReader | None, -) -> str: +) -> str | None: + if isinstance(provider, OpenAICompatibleProviderConfig) and provider.auth == "none": + return None if provider.credential_name and credential_reader is not None: credential = credential_reader.get(provider.credential_name) if credential: @@ -1802,6 +1844,11 @@ def _api_key_from_provider( api_key = environ.get(provider.api_key_env) if api_key: return api_key + if isinstance(provider, OpenAICompatibleProviderConfig) and provider.auth in { + "optional", + "none", + }: + return None credential_hint = f" or run /login {provider.name}" if provider.credential_name else "" raise RuntimeError(f"Missing provider API key. Set {provider.api_key_env}{credential_hint}.") @@ -1955,6 +2002,20 @@ def _reject_unimplemented_thinking_config( raise ProviderConfigError(f"{provider_type} thinking controls are not implemented yet") +def _model_discovery(value: object, field_name: str) -> ModelDiscovery: + if value not in {"static", "openai"}: + raise ProviderConfigError(f"Provider field must be static or openai: {field_name}") + return cast(ModelDiscovery, value) + + +def _provider_auth(value: object, field_name: str) -> ProviderAuth: + if value not in {"required", "optional", "none"}: + raise ProviderConfigError( + f"Provider field must be required, optional, or none: {field_name}" + ) + return cast(ProviderAuth, value) + + def _optional_provider_api(value: object, field_name: str) -> ProviderApi | None: if value is None: return None diff --git a/src/tau_coding/tui/app.py b/src/tau_coding/tui/app.py index 4ba902c63..d37a0ebe9 100644 --- a/src/tau_coding/tui/app.py +++ b/src/tau_coding/tui/app.py @@ -3849,7 +3849,11 @@ def _subscription_login_providers( def _api_key_login_providers( providers: Sequence[ProviderCatalogEntry], ) -> tuple[ProviderCatalogEntry, ...]: - return tuple(provider for provider in providers if provider.kind != "openai-codex") + return tuple( + provider + for provider in providers + if provider.kind != "openai-codex" and provider.auth == "required" + ) def _stored_credential_providers( diff --git a/tests/test_cli.py b/tests/test_cli.py index 677e0ba60..7cf4ab074 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1000,6 +1000,78 @@ def get(self, name: str) -> str | None: assert "\tMISSING_API_KEY\tmissing\t" in output +def test_llama_cpp_setup_discovers_and_saves_provider( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + isolate_home(monkeypatch, tmp_path) + monkeypatch.delenv("LLAMA_API_KEY", raising=False) + + async def fake_discover( + base_url: str, + *, + api_key: str | None = None, + ) -> object: + from tau_coding.llama_cpp import LlamaCppServerInfo + + assert base_url == "http://localhost:9090" + assert api_key is None + return LlamaCppServerInfo( + base_url="http://localhost:9090/v1", + models=("qwen.gguf", "coder.gguf"), + ) + + monkeypatch.setattr(cli, "discover_llama_cpp", fake_discover) + + result = CliRunner().invoke( + app, + [ + "llama-cpp", + "setup", + "--base-url", + "http://localhost:9090", + "--model", + "coder.gguf", + ], + ) + + provider = load_provider_settings(TauPaths(home=tmp_path / ".tau")).get_provider("llama-cpp") + assert result.exit_code == 0 + assert "Discovered models: qwen.gguf, coder.gguf" in result.stdout + assert provider.base_url == "http://localhost:9090/v1" + assert provider.models == ("qwen.gguf", "coder.gguf") + assert provider.default_model == "coder.gguf" + assert provider.auth == "optional" + + +def test_llama_cpp_doctor_reports_checks( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + isolate_home(monkeypatch, tmp_path) + + async def fake_diagnose(*_args: object, **_kwargs: object) -> object: + from tau_coding.llama_cpp import ( + LlamaCppDiagnostic, + LlamaCppDoctorReport, + ) + + return LlamaCppDoctorReport( + ( + LlamaCppDiagnostic("server", "ok", "Found llama.cpp"), + LlamaCppDiagnostic("tools", "warning", "Tool probe was inconclusive"), + ) + ) + + monkeypatch.setattr(cli, "diagnose_llama_cpp", fake_diagnose) + + result = CliRunner().invoke(app, ["llama-cpp", "doctor"]) + + assert result.exit_code == 0 + assert "✓ Found llama.cpp" in result.stdout + assert "! Tool probe was inconclusive" in result.stdout + + def test_setup_command_writes_provider_settings( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/test_llama_cpp.py b/tests/test_llama_cpp.py new file mode 100644 index 000000000..2138fea7b --- /dev/null +++ b/tests/test_llama_cpp.py @@ -0,0 +1,122 @@ +"""Tests for Tau's first-class llama.cpp integration.""" + +from __future__ import annotations + +from json import loads + +import httpx +import pytest + +from tau_coding.llama_cpp import ( + LlamaCppError, + configured_llama_cpp_provider, + diagnose_llama_cpp, + discover_llama_cpp, + normalize_llama_cpp_base_url, +) +from tau_coding.provider_config import OpenAICompatibleProviderConfig + + +def test_normalize_llama_cpp_base_url_adds_v1() -> None: + assert normalize_llama_cpp_base_url("http://127.0.0.1:8080/") == ("http://127.0.0.1:8080/v1") + assert normalize_llama_cpp_base_url("http://127.0.0.1:8080/v1/") == ("http://127.0.0.1:8080/v1") + + +@pytest.mark.anyio +async def test_discover_llama_cpp_models_without_auth() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={"object": "list", "data": [{"id": "qwen.gguf"}, {"id": "coder.gguf"}]}, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + info = await discover_llama_cpp("http://localhost:8080", client=client) + + assert info.base_url == "http://localhost:8080/v1" + assert info.models == ("qwen.gguf", "coder.gguf") + assert requests[0].url == "http://localhost:8080/v1/models" + assert "authorization" not in requests[0].headers + + +@pytest.mark.anyio +async def test_discover_llama_cpp_uses_api_key() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] == "Bearer secret" + return httpx.Response(200, json={"data": [{"id": "local-model"}]}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + await discover_llama_cpp("http://localhost:8080/v1", api_key="secret", client=client) + + +@pytest.mark.anyio +async def test_discover_llama_cpp_explains_auth_failure() -> None: + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda _request: httpx.Response(401)) + ) as client: + with pytest.raises(LlamaCppError, match="LLAMA_API_KEY"): + await discover_llama_cpp("http://localhost:8080", client=client) + + +def test_configured_llama_cpp_provider_uses_discovered_models() -> None: + from tau_coding.llama_cpp import LlamaCppServerInfo + + provider = configured_llama_cpp_provider( + OpenAICompatibleProviderConfig( + name="llama-cpp", + auth="optional", + model_discovery="openai", + ), + LlamaCppServerInfo( + base_url="http://localhost:8080/v1", + models=("qwen.gguf", "coder.gguf"), + ), + model="coder.gguf", + ) + + assert provider.models == ("qwen.gguf", "coder.gguf") + assert provider.default_model == "coder.gguf" + + +@pytest.mark.anyio +async def test_diagnose_llama_cpp_checks_streaming_and_tools() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.path == "/v1/models": + return httpx.Response(200, json={"data": [{"id": "qwen.gguf"}]}) + payload = loads(request.content) + if payload.get("tools"): + body = ( + 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-1",' + '"function":{"name":"tau_probe","arguments":"{}"}}]},' + '"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n' + ) + else: + body = ( + 'data: {"choices":[{"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" + ) + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + + provider = OpenAICompatibleProviderConfig( + name="llama-cpp", + base_url="http://localhost:8080/v1", + auth="optional", + model_discovery="openai", + models=("qwen.gguf",), + default_model="qwen.gguf", + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + report = await diagnose_llama_cpp(provider, client=client) + + assert report.ok + assert [item.status for item in report.diagnostics] == ["ok", "ok", "ok", "ok"] + assert len(requests) == 3 + chat_payload = loads(requests[1].content) + assert "store" not in chat_payload + assert "stream_options" not in chat_payload diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py index c87205258..3c641c54c 100644 --- a/tests/test_provider_catalog.py +++ b/tests/test_provider_catalog.py @@ -72,9 +72,25 @@ def test_builtin_catalog_matches_expected_providers() -> None: "xiaomi-token-plan-cn", "xiaomi-token-plan-ams", "xiaomi-token-plan-sgp", + "llama-cpp", ] +def test_builtin_catalog_golden_llama_cpp_entry() -> None: + entry = builtin_provider_entry("llama-cpp") + assert entry is not None + assert entry.display_name == "llama.cpp" + assert entry.base_url == "http://127.0.0.1:8080/v1" + assert entry.auth == "optional" + assert entry.model_discovery == "openai" + assert entry.models == ("local",) + assert entry.default_model == "local" + assert entry.compat == { + "supportsStore": False, + "supportsUsageInStreaming": False, + } + + def test_builtin_catalog_golden_anthropic_entry() -> None: entry = builtin_provider_entry("anthropic") assert entry is not None diff --git a/tests/test_provider_config.py b/tests/test_provider_config.py index 2cdd6868b..8cb6f7883 100644 --- a/tests/test_provider_config.py +++ b/tests/test_provider_config.py @@ -58,8 +58,9 @@ def test_load_provider_settings_missing_file_uses_openai_default(tmp_path: Path) "xiaomi-token-plan-cn", "xiaomi-token-plan-ams", "xiaomi-token-plan-sgp", + "llama-cpp", ] - assert settings.providers[0].default_model == DEFAULT_MODEL + assert settings.get_provider("openai").default_model == DEFAULT_MODEL assert settings.get_provider("anthropic").api_key_env == "ANTHROPIC_API_KEY" assert settings.get_provider("openrouter").api_key_env == "OPENROUTER_API_KEY" assert settings.get_provider("huggingface").api_key_env == "HF_TOKEN" @@ -256,7 +257,10 @@ def test_save_and_load_provider_settings_round_trip(tmp_path: Path) -> None: loaded = load_provider_settings(paths) assert path == tmp_path / ".tau" / "providers.json" - assert loaded == settings + assert loaded.get_provider("local") == settings.get_provider("local") + assert loaded.default_provider == settings.default_provider + assert loaded.scoped_models == settings.scoped_models + assert loaded.get_provider("llama-cpp").auth == "optional" def test_provider_settings_parses_scoped_models() -> None: @@ -412,6 +416,54 @@ def test_openai_compatible_config_from_provider_uses_configured_env_var( assert config.max_retry_delay_seconds == 1.0 +def test_dynamic_provider_merge_replaces_url_and_models() -> None: + existing = OpenAICompatibleProviderConfig( + name="llama-cpp", + base_url="http://localhost:9090/v1", + auth="optional", + model_discovery="openai", + models=("qwen.gguf", "coder.gguf"), + default_model="coder.gguf", + ) + incoming = OpenAICompatibleProviderConfig( + name="llama-cpp", + base_url="http://127.0.0.1:8080/v1", + auth="optional", + model_discovery="openai", + models=("qwen.gguf", "coder.gguf"), + default_model="qwen.gguf", + ) + + updated = upsert_openai_compatible_provider( + ProviderSettings(default_provider="llama-cpp", providers=(existing,)), + incoming, + ) + provider = updated.get_provider("llama-cpp") + + assert provider.base_url == "http://127.0.0.1:8080/v1" + assert provider.models == ("qwen.gguf", "coder.gguf") + assert provider.default_model == "coder.gguf" + + +def test_openai_compatible_config_from_provider_allows_optional_auth( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LLAMA_API_KEY", raising=False) + provider = OpenAICompatibleProviderConfig( + name="llama-cpp", + base_url="http://127.0.0.1:8080/v1", + api_key_env="LLAMA_API_KEY", + auth="optional", + models=("local",), + default_model="local", + ) + + config = openai_compatible_config_from_provider(provider) + + assert config.api_key is None + assert provider_has_usable_credentials(provider) + + def test_openai_compatible_config_from_provider_preserves_openai_base_url_env( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -930,6 +982,7 @@ def test_load_provider_settings_restores_builtin_providers_with_stored_credentia "local", "openai-codex", "openrouter", + "llama-cpp", ] assert settings.default_provider == "local" assert settings.get_provider("openrouter").credential_name == "openrouter" diff --git a/tests/test_provider_runtime.py b/tests/test_provider_runtime.py index e7845cae1..952dfb702 100644 --- a/tests/test_provider_runtime.py +++ b/tests/test_provider_runtime.py @@ -1,6 +1,6 @@ import pytest -from tau_ai import OpenAICodexProvider +from tau_ai import OpenAICodexProvider, OpenAICompatibleProvider from tau_coding import provider_runtime from tau_coding.credentials import FileCredentialStore, OAuthCredential from tau_coding.provider_config import ( @@ -22,6 +22,28 @@ def test_create_model_provider_returns_openai_codex_provider(tmp_path) -> None: assert isinstance(provider, OpenAICodexProvider) +def test_create_model_provider_supports_optional_auth( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + monkeypatch.delenv("LLAMA_API_KEY", raising=False) + store = FileCredentialStore(tmp_path / "credentials.json") + + provider = create_model_provider( + OpenAICompatibleProviderConfig( + name="llama-cpp", + api_key_env="LLAMA_API_KEY", + auth="optional", + models=("local",), + default_model="local", + ), + credential_store=store, + ) + + assert isinstance(provider, OpenAICompatibleProvider) + assert provider._config.api_key is None + + def test_create_model_provider_rejects_model_not_declared_for_provider(tmp_path) -> None: store = FileCredentialStore(tmp_path / "credentials.json") provider_config = OpenAICompatibleProviderConfig( diff --git a/tests/test_tau_ai.py b/tests/test_tau_ai.py index 627d34759..b0e038f4e 100644 --- a/tests/test_tau_ai.py +++ b/tests/test_tau_ai.py @@ -176,6 +176,35 @@ def handler(request: httpx.Request) -> httpx.Response: ] +@pytest.mark.anyio +async def test_openai_compatible_provider_omits_authorization_without_api_key() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + text='data: {"choices":[{"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n', + headers={"content-type": "text/event-stream"}, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + provider = OpenAICompatibleProvider( + OpenAICompatibleConfig(api_key=None, base_url="http://localhost:8080/v1"), + client=client, + ) + await _collect( + provider.stream_response( + model="local", + system="You are Tau.", + messages=[UserMessage(content="Say ok")], + tools=[], + ) + ) + + assert "authorization" not in requests[0].headers + + @pytest.mark.anyio async def test_openai_compatible_provider_includes_configured_reasoning_effort() -> None: requests: list[httpx.Request] = [] diff --git a/website/content/guides/providers-and-models.md b/website/content/guides/providers-and-models.md index ca62a4f12..63f4dd540 100644 --- a/website/content/guides/providers-and-models.md +++ b/website/content/guides/providers-and-models.md @@ -97,16 +97,27 @@ Some installs expose the same server as `llama serve`: llama serve -hf ggml-org/Qwen3.6-35B-A3B-GGUF:Q8_0 ``` -Then register it with Tau: +Then let Tau discover the server and its model IDs: ```bash -export LLAMA_API_KEY=local # any non-empty value unless you started llama.cpp with --api-key +tau llama-cpp setup +``` -tau --provider llama-cpp \ - --base-url http://localhost:8080/v1 \ - --api-key-env LLAMA_API_KEY \ - --model local \ - setup +Tau connects to `http://127.0.0.1:8080` by default, queries `/v1/models`, saves +all discovered model IDs, and selects the first model. Select another model or +server explicitly when needed: + +```bash +tau llama-cpp setup --base-url http://localhost:9090 --model my-model +``` + +No fake API key is required. If you started `llama-server` with `--api-key`, set +`LLAMA_API_KEY` (preferred) or pass `--api-key` during setup. Command-line keys +may be retained in shell history: + +```bash +export LLAMA_API_KEY=secret +tau llama-cpp setup ``` Run Tau against the local model: @@ -118,7 +129,15 @@ tau --provider llama-cpp -p "summarize this project" # one-shot print mode ``` `llama-server` listens on port `8080` by default and only enforces the bearer -token if you launch it with `--api-key`. +token if you launch it with `--api-key`. Diagnose connectivity, streaming, and +tool-call support with: + +```bash +tau llama-cpp doctor +``` + +A warning from the tool probe usually means the GGUF is not a tool-capable +instruct model or its llama.cpp chat template does not support tool calls. For scripted or one-off setup with another OpenAI-compatible server, use the same `tau setup` flow. For example, Ollama's OpenAI-compatible endpoint usually diff --git a/website/content/reference/configuration.md b/website/content/reference/configuration.md index 9ce1dd066..8468770dd 100644 --- a/website/content/reference/configuration.md +++ b/website/content/reference/configuration.md @@ -142,10 +142,14 @@ Provider preferences live in `~/.tau/providers.json`: defaults to `2`; `max_retry_delay_seconds` defaults to `1` (both ≥ 0). - API keys and OAuth credentials are **not** stored here — they live in `~/.tau/credentials.json`. Resolution order: stored credential, then the env - var named by `api_key_env`. + var named by `api_key_env`. Catalog providers may set `auth = "optional"` or + `auth = "none"`; Tau then permits calls without a key and omits the bearer + header. The built-in `llama-cpp` provider uses optional authentication. - The selected model must be present in that provider's `models` list. Add custom or local model names to `models` before using them as defaults, - CLI/TUI selections, or scoped models. + CLI/TUI selections, or scoped models. Providers with + `model_discovery = "openai"` can populate that list from `/v1/models`; run + `tau llama-cpp setup` for the built-in llama.cpp provider. - `scoped_models` are favorites for the **Ctrl+P** quick-cycle. - Older `providers.json` files that contain full `providers` entries are still accepted for compatibility. When Tau saves settings again, provider definitions