feat: add mlx-agent production specialists

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add `mlx-agent` production specialists that run on the local model served by an mlx-agent gateway, with no remote consent or credentials required.
 - Publish `@cavi-ai/ableton-mcp` to npm from each GitHub release with trusted publishing and provenance.
 
 ## 0.4.0

README.md

@@ -63,7 +63,7 @@ For a smaller initial tool catalog, set `ABLETON_MCP_TOOL_PROFILE=core` in the M
 
 ## What it does
 
-For separate creative and functional model contexts, configure optional [production specialists](docs/ableton-mcp/source/pages/guides/production-specialists.md). `ABLETON_MCP_TOOL_PROFILE=producer` exposes twelve tools, with detailed context and durable session history stored in private artifacts. Each role can use an installed Ollama model, a configured ACP agent or an explicitly enabled API client for Claude, Codex, OpenClaw, Hermes, OpenCode or AGY; Live changes retain their existing confirmation guards.
+For separate creative and functional model contexts, configure optional [production specialists](docs/ableton-mcp/source/pages/guides/production-specialists.md). `ABLETON_MCP_TOOL_PROFILE=producer` exposes twelve tools, with detailed context and durable session history stored in private artifacts. Each role can use the local model served by an mlx-agent gateway, an installed Ollama model, a configured ACP agent or an explicitly enabled API client for Claude, Codex, OpenClaw, Hermes, OpenCode or AGY; Live changes retain their existing confirmation guards.
 
 - **Reads**: transport, tempo, key and scale, quantization, grooves, cue points, tracks, scenes, clips, notes, clip envelopes, devices and parameters, mixer and routing, rack hierarchies, and the Live browser.
 - **Guarded mutations**: track, scene and clip lifecycle, device loading and parameters, MIDI note editing, mixing and routing, undo and redo, and panic.

apps/ableton-mcp/src/producer-api-client.mjs

@@ -38,12 +38,29 @@ async function runCodex(options, body, signal, onEvent) {
   }
 }
 
-export async function generateApiClientResponse(config, messages, { signal, fetch, env, cwd }) {
+// Chat Completions carries the chat template's thinking switch; Messages requests reason before answering.
+// The turn's response schema goes to the gateway as a json_schema response_format.
+async function generateMlxAgentResponse(config, messages, { apiKey, fetch, signal, format }) {
+  const response = await fetch(new URL("/v1/chat/completions", config.endpoint), {
+    method: "POST", signal,
+    headers: { "Content-Type": "application/json", ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
+    body: JSON.stringify({ model: config.model || "default_model", messages, max_tokens: 2048, stream: false,
+      chat_template_kwargs: { enable_thinking: false },
+      ...(format ? { response_format: { type: "json_schema", json_schema: { name: "producer_response", schema: format } } } : {}) })
+  });
+  if (!response.ok) throw new Error("specialist API request failed");
+  const content = (await response.json()).choices?.[0]?.message?.content;
+  if (typeof content !== "string" || !content.trim()) throw new Error("specialist API run did not complete");
+  return content;
+}
+
+export async function generateApiClientResponse(config, messages, { signal, fetch, env, cwd, format }) {
   signal.throwIfAborted();
   const keyName = config.apiKeyEnv || keyNames[config.provider];
   const apiKey = env[keyName]?.trim();
-  const optionalPassword = config.provider === "opencode" && !config.apiKeyEnv &&
-    ["localhost", "127.0.0.1", "[::1]"].includes(new URL(config.endpoint).hostname);
+  const optionalPassword = (config.provider === "mlx-agent" && !config.apiKeyEnv) ||
+    (config.provider === "opencode" && !config.apiKeyEnv &&
+      ["localhost", "127.0.0.1", "[::1]"].includes(new URL(config.endpoint).hostname));
   if (!apiKey && !optionalPassword) throw new Error(`specialist API key is missing from ${keyName}`);
   let failure;
   const boundedFetch = async (url, init) => {
@@ -91,7 +108,14 @@ export async function generateApiClientResponse(config, messages, { signal, fetc
   };
   try {
     const body = { instructions: messages[0].content, input: messages.slice(1) };
-    if (["openclaw", "hermes"].includes(config.provider)) {
+    if (config.provider === "mlx-agent") {
+      output = await generateMlxAgentResponse(config, messages, { apiKey, fetch: boundedFetch, signal, format });
+      if (Buffer.byteLength(output) > 16000) {
+        failure = new Error("specialist output limit exceeded");
+        throw failure;
+      }
+      outcome = "completed";
+    } else if (["openclaw", "hermes"].includes(config.provider)) {
       await generateGatewayResponse(config, messages, { apiKey, fetch: boundedFetch, signal, handlers });
     } else if (config.provider === "codex") {
       await runCodex(options, body, signal, handlers.onEvent);

apps/ableton-mcp/src/producer-runtime.mjs

@@ -60,7 +60,7 @@ function responseFormat(packet) {
 }
 const runtimeProperties = {
   type: { enum: ["ollama", "acp", "api-client"] }, model: { type: "string", minLength: 1, maxLength: 200 },
-  provider: { enum: ["claude", "codex", "openclaw", "hermes", "opencode", "agy"] },
+  provider: { enum: ["claude", "codex", "openclaw", "hermes", "opencode", "agy", "mlx-agent"] },
   apiKeyEnv: { type: "string", pattern: "^[A-Za-z_][A-Za-z0-9_]*$", maxLength: 200 },
   agentId: { type: "string", pattern: "^[A-Za-z0-9_-]+$", maxLength: 100 },
   directory: { type: "string", minLength: 1, maxLength: 2000 }, username: { type: "string", minLength: 1, maxLength: 100 },
@@ -94,11 +94,14 @@ export function parseProducerConfig(environment) {
     if (runtime.type === "api-client") {
       const gateway = ["openclaw", "hermes"].includes(runtime.provider);
       const cloud = ["claude", "codex"].includes(runtime.provider);
-      if (!runtime.provider || runtime.allowRemote !== true ||
+      // An mlx-agent gateway serves only its own local model, so it needs no remote consent or model name.
+      const mlx = runtime.provider === "mlx-agent";
+      const model = mlx ? runtime.model === undefined || runtime.model.trim() : runtime.model?.trim();
+      if (!runtime.provider || (runtime.allowRemote !== true && !mlx) ||
           ["command", "args", "env"].some(key => Object.hasOwn(runtime, key)) ||
-          (gateway ? !runtime.agentId || runtime.model !== undefined : !runtime.model?.trim() || runtime.agentId !== undefined) ||
+          (gateway ? !runtime.agentId || runtime.model !== undefined : !model || runtime.agentId !== undefined) ||
           (runtime.provider !== "opencode" && (runtime.directory !== undefined || runtime.username !== undefined)))
-        throw new Error("API-client config requires provider and allowRemote: true, model for model APIs or agentId for gateways; process settings are not supported");
+        throw new Error("API-client config requires provider and allowRemote: true (except mlx-agent), model for model APIs or agentId for gateways; process settings are not supported");
       if (cloud) {
         if (runtime.endpoint !== undefined) throw new Error("cloud API-client config uses the official endpoint");
       } else {
@@ -109,6 +112,7 @@ export function parseProducerConfig(environment) {
         if (url.username || url.password || url.search || url.hash || url.pathname !== "/" ||
             !protocols.includes(url.protocol) || (!local && !["https:", "wss:"].includes(url.protocol)))
           throw new Error("API-client config endpoint must be a loopback origin or a remote TLS origin without URL credentials");
+        if (mlx && !local) throw new Error("mlx-agent config endpoint must be a loopback origin");
       }
       if (runtime.provider === "opencode" && (!/^[^/\s]+\/\S+$/.test(runtime.model) || (runtime.directory !== undefined && !runtime.directory.trim())))
         throw new Error("OpenCode config requires providerID/modelID and a nonblank directory when supplied");
@@ -178,7 +182,7 @@ export function createProducerRuntime(config, dependencies = {}) {
       if (config.type === "api-client") {
         const { generateApiClientResponse } = await import("./producer-api-client.mjs");
         return parseProducerResponse(await generateApiClientResponse(config, messages, {
-          signal: activeSignal, fetch, env: dependencies.env || process.env, cwd
+          signal: activeSignal, fetch, env: dependencies.env || process.env, cwd, format: responseFormat(packet)
         }));
       }
       const endpoint = endpointFor(config);

apps/ableton-mcp/test/producer-api-client.test.mjs

@@ -31,6 +31,45 @@ test("API-client configuration requires an explicit provider, model and remote c
   }
 });
 
+test("mlx-agent specialists use a loopback gateway without remote consent or credentials", () => {
+  const parse = creative => parseProducerConfig({ ABLETON_MCP_PRODUCER_CONFIG: JSON.stringify({ creative }) });
+  const local = { type: "api-client", provider: "mlx-agent", endpoint: "http://127.0.0.1:19080" };
+  assert.deepEqual(parse(local), { creative: local });
+  assert.equal(parse({ ...local, model: "served-model" }).creative.model, "served-model");
+  for (const invalid of [{ endpoint: undefined }, { endpoint: "https://example.com" }, { endpoint: "http://127.0.0.1:19080/v1" },
+    { model: " " }, { agentId: "specialist" }, { directory: "/tmp" }, { command: "agent" }]) {
+    assert.throws(() => parse({ ...local, ...invalid }), /config/);
+  }
+});
+
+test("mlx-agent specialists ask the gateway's served model for an answer without reasoning", { timeout: 5000 }, async t => {
+  const requests = [];
+  const server = createServer(async (request, response) => {
+    let body = "";
+    for await (const chunk of request) body += chunk;
+    requests.push({ path: request.url, authorization: request.headers.authorization, body: JSON.parse(body) });
+    response.writeHead(200, { "Content-Type": "application/json" });
+    response.end(JSON.stringify({ choices: [{ message: { role: "assistant", content: JSON.stringify(reply) } }] }));
+  });
+  t.after(() => { server.closeAllConnections(); server.close(); });
+  server.listen(0, "127.0.0.1");
+  await once(server, "listening");
+  const local = { type: "api-client", provider: "mlx-agent", endpoint: `http://127.0.0.1:${server.address().port}` };
+  assert.deepEqual(await createProducerRuntime(local, { env: {} }).generate(packet), reply);
+  assert.equal(requests.length, 1);
+  assert.equal(requests[0].path, "/v1/chat/completions");
+  assert.equal(requests[0].authorization, undefined);
+  assert.equal(requests[0].body.model, "default_model");
+  assert.deepEqual(requests[0].body.chat_template_kwargs, { enable_thinking: false });
+  assert.equal(requests[0].body.stream, false);
+  assert.equal(requests[0].body.tools, undefined);
+  assert.equal(requests[0].body.messages[0].role, "system");
+  const format = requests[0].body.response_format;
+  assert.equal(format.type, "json_schema");
+  assert.equal(format.json_schema.schema.properties.reads.maxItems, 4);
+  assert.equal(format.json_schema.schema.properties.operations.maxItems, 0);
+});
+
 test("the published client sends bounded history to the official API with no native tools or redirects", async () => {
   let calls = 0;
   const generate = runtime(async (url, init) => {

docs/ableton-mcp/source/pages/guides/production-specialists.md

@@ -6,6 +6,17 @@ Production specialists keep detailed musical and operational context outside the
 
 Specialists are optional. Set `ABLETON_MCP_PRODUCER_CONFIG` to a JSON object with a `creative` entry, a `functional` entry, or both. Each role has its own runtime configuration. Nothing starts until `run_production_task` requests that role.
 
+For the local model served by an [mlx-agent](https://github.com/cavi-ai/mlx-agent) gateway:
+
+```json
+{
+  "creative": { "type": "api-client", "provider": "mlx-agent", "endpoint": "http://127.0.0.1:8080" },
+  "functional": { "type": "api-client", "provider": "mlx-agent", "endpoint": "http://127.0.0.1:8080" }
+}
+```
+
+The endpoint must be the gateway's loopback origin. Requests use its Chat Completions API at `/v1/chat/completions` with the model `default_model`, which the gateway resolves to the model it serves; an optional `model` names that model explicitly. Each request sets `chat_template_kwargs.enable_thinking` to `false`, so reasoning models answer directly within the 2,048-token budget. The gateway serves only its configured local model, so `allowRemote` and credentials are not required; an optional `apiKeyEnv` sends that variable's value as a bearer token. Each request also sends the turn's response schema as a `json_schema` `response_format`. The output is validated locally, as for other API clients. The configured gateway is a trusted boundary.
+
 For an installed local Ollama model:
 
 ```json

docs/ableton-mcp/source/pages/reference/configuration.md

@@ -13,7 +13,7 @@ All configuration is environment variables. Each one is optional.
 | `ABLETON_MCP_SNAPSHOT_DIR` | `~/.cavi/ableton-mcp/snapshots` | Private named track-state snapshots; device-chain and MIDI-feel templates use separate subdirectories. |
 | `ABLETON_MCP_FIXTURE` | unset | `1` makes `npm start` serve fixture data without Live. `ableton-mcp serve` ignores it. |
 | `ABLETON_MCP_TOOL_PROFILE` | `all` | `core` advertises 63 common tools; `producer` advertises the compact local specialist workflow; `directory` advertises individual native tools without producer coordinators; `all` advertises the full catalog. Changing this requires a server restart. CLI `call` is unaffected. |
-| `ABLETON_MCP_PRODUCER_CONFIG` | unset | JSON with independent optional `creative` and `functional` runtimes. Supports direct Ollama and ACP v1 agents. See [Production specialists](../guides/production-specialists.md). |
+| `ABLETON_MCP_PRODUCER_CONFIG` | unset | JSON with independent optional `creative` and `functional` runtimes. Supports mlx-agent gateways, direct Ollama, ACP v1 agents and API clients. See [Production specialists](../guides/production-specialists.md). |
 | `ABLETON_MCP_PRODUCER_DIR` | `~/.cavi/ableton-mcp/producer` | Private project context, proposals, intent, feedback and execution receipts. Artifacts persist until removed. |
 | `ABLETON_MCP_LIVE_APP_PATH` | Detected from the running bridge | Absolute Ableton Live `.app` bundle path. Required for `open_live_set` with `mode: "launch"` while Live is closed; optional for open/save when detection from a running bridge succeeds. |