diff --git a/.github/workflows/ingest.yml b/.github/workflows/ingest.yml index dc63535a..06802458 100644 --- a/.github/workflows/ingest.yml +++ b/.github/workflows/ingest.yml @@ -57,7 +57,7 @@ jobs: } >> "$GITHUB_OUTPUT" - run: npm run ingest:webhook env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} MESSAGE: ${{ steps.ingest-payload.outputs.message }} - run: npm run data:validate - name: Check for changes diff --git a/.github/workflows/llm-evals.yml b/.github/workflows/llm-evals.yml index eebde904..a38726fb 100644 --- a/.github/workflows/llm-evals.yml +++ b/.github/workflows/llm-evals.yml @@ -54,10 +54,10 @@ jobs: run: npm run test:eval:changed -- "origin/${BASE_REF}" env: BASE_REF: ${{ github.base_ref }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - name: Run changed LLM function evals if: ${{ env.CAN_RUN_EVALS == 'true' && github.event_name == 'workflow_dispatch' }} run: npm run test:eval:changed -- origin/main env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} diff --git a/package-lock.json b/package-lock.json index 5f021f41..ab6db053 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,9 @@ "workspaces": [ "packages/*" ], + "dependencies": { + "@google/genai": "^2.8.0" + }, "devDependencies": { "@biomejs/biome": "2.5.0", "@changesets/cli": "^2.31.0", @@ -926,6 +929,30 @@ "node": ">=18" } }, + "node_modules/@google/genai": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-2.8.0.tgz", + "integrity": "sha512-pc2ayxqO5+O7AvnHBqpNHIk7PAZkHZgL31tbyx0gJZBSS9qPYiQoqwK7oYOw/ePmG6QY4EMSu+304vD5QlhXAw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, "node_modules/@inquirer/external-editor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", @@ -1126,6 +1153,63 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", @@ -1579,12 +1663,17 @@ "version": "24.13.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz", "integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" } }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -1730,6 +1819,15 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ansi-colors": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", @@ -1777,6 +1875,26 @@ "node": ">=12" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/better-path-resolve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/better-path-resolve/-/better-path-resolve-1.0.0.tgz", @@ -1790,6 +1908,15 @@ "node": ">=4" } }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -1803,6 +1930,12 @@ "node": ">=8" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/ccount": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", @@ -1892,6 +2025,15 @@ "node": ">= 8" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1989,6 +2131,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/enquirer": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", @@ -2098,6 +2249,12 @@ "node": ">=12.0.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/extendable-error": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/extendable-error/-/extendable-error-0.1.7.tgz", @@ -2150,6 +2307,29 @@ } } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -2177,6 +2357,18 @@ "node": ">=8" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fs-extra": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", @@ -2216,6 +2408,34 @@ "node": ">=10" } }, + "node_modules/gaxios": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.5.tgz", + "integrity": "sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -2250,6 +2470,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/google-auth-library": { + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.7.0.tgz", + "integrity": "sha512-QpTAbNJ36TliZLx3TTtahR8HG0hN9RllL1e3FymOvQSIKK8JmgV58H924ub2wa2DsS3ANjjP1Aw1N+Ramc8hqQ==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -2500,6 +2746,19 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/human-id": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/human-id/-/human-id-4.1.3.tgz", @@ -2619,6 +2878,15 @@ "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", "license": "Apache-2.0" }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-nd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-nd/-/json-nd-1.0.0.tgz", @@ -2635,6 +2903,27 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -2928,6 +3217,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -3691,6 +3986,44 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/obug": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", @@ -3705,24 +4038,6 @@ "node": ">=12.20.0" } }, - "node_modules/openai": { - "version": "6.42.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.42.0.tgz", - "integrity": "sha512-1WFEt/uXMXOLhYRNkgJWo08Y2YNvNwpVU72K7ibrWgWpNOXd4VojXLbe6SQ4bLiUQ3Y8jz4IiyVkylJCL1DtZg==", - "license": "Apache-2.0", - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, "node_modules/outdent": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.5.0.tgz", @@ -3782,6 +4097,19 @@ "node": ">=6" } }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -3948,6 +4276,29 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/quansync": { "version": "0.2.11", "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", @@ -4050,6 +4401,15 @@ "node": ">=8" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -4128,6 +4488,26 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -4452,7 +4832,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/unist-util-find-after": { @@ -4795,6 +5174,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4828,6 +5216,27 @@ "node": ">=8" } }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/zod": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", @@ -4924,6 +5333,7 @@ "version": "2.0.0-alpha.32", "license": "UNLICENSED", "dependencies": { + "@google/genai": "^2.8.0", "@mrtdown/core": "2.0.0-alpha.26", "@mrtdown/fs": "2.0.0-alpha.26", "@mrtdown/ingest-contracts": "2.0.0-alpha.29", @@ -4933,7 +5343,6 @@ "luxon": "^3.7.2", "mdast-util-gfm": "^3.1.0", "mdast-util-to-markdown": "^2.1.2", - "openai": "^6.42.0", "zod": "^4.4.3" }, "devDependencies": { diff --git a/package.json b/package.json index 2a254adb..cdfeb0a5 100644 --- a/package.json +++ b/package.json @@ -55,5 +55,8 @@ }, "overrides": { "zod": "^4.0.14" + }, + "dependencies": { + "@google/genai": "^2.8.0" } } diff --git a/packages/triage/README.md b/packages/triage/README.md index 89607a2f..cb73c362 100644 --- a/packages/triage/README.md +++ b/packages/triage/README.md @@ -38,14 +38,14 @@ implicitly through `npm test`, `npm run test:packages`, or CI harness commands. Required environment variables: -- `OPENAI_API_KEY`: OpenAI API key used by eval cases. +- `GEMINI_API_KEY`: Gemini API key used by eval cases. Model dependency: -- `extractClaimsFromNewEvidence` uses `gpt-5.4-mini`. -- `triageNewEvidence` uses `gpt-5.4-mini`. -- title/slug generation uses `gpt-5.4-nano`. -- translation uses `gpt-5.4-nano`. +- `extractClaimsFromNewEvidence` uses `gemini-3-flash-preview`. +- `triageNewEvidence` uses `gemini-3-flash-preview`. +- title/slug generation uses `gemini-3.1-flash-lite`. +- translation uses `gemini-3.1-flash-lite`. Expected cost before running the checked-in eval set: less than USD 1 with the current short fixtures. Revisit this estimate in the same PR when adding or diff --git a/packages/triage/package.json b/packages/triage/package.json index 2aea370e..01d840f5 100644 --- a/packages/triage/package.json +++ b/packages/triage/package.json @@ -33,6 +33,7 @@ "test:eval": "node ../../scripts/run-fixture-command.mjs -- npm exec -- vitest --run --config vitest.eval.config.ts" }, "dependencies": { + "@google/genai": "^2.8.0", "@mrtdown/core": "2.0.0-alpha.26", "@mrtdown/fs": "2.0.0-alpha.26", "@mrtdown/ingest-contracts": "2.0.0-alpha.29", @@ -42,7 +43,6 @@ "luxon": "^3.7.2", "mdast-util-gfm": "^3.1.0", "mdast-util-to-markdown": "^2.1.2", - "openai": "^6.42.0", "zod": "^4.4.3" }, "devDependencies": { diff --git a/packages/triage/src/helpers/estimateOpenAICost.test.ts b/packages/triage/src/helpers/estimateOpenAICost.test.ts deleted file mode 100644 index 46898f69..00000000 --- a/packages/triage/src/helpers/estimateOpenAICost.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - estimateOpenAICostFromUsage, - OpenAIUsageCostTracker, -} from './estimateOpenAICost.js'; - -describe('estimateOpenAICostFromUsage', () => { - it('prices gpt-5.4 mini usage with cached input tokens', () => { - const estimate = estimateOpenAICostFromUsage({ - model: 'gpt-5.4-mini', - usage: { - inputTokens: 1000, - cachedInputTokens: 100, - outputTokens: 2000, - totalTokens: 3000, - }, - }); - - expect(estimate?.estimatedCostUsd).toBeCloseTo(0.0096825); - }); - - it('prices gpt-5.4 nano usage', () => { - const estimate = estimateOpenAICostFromUsage({ - model: 'gpt-5.4-nano', - usage: { - inputTokens: 1000, - cachedInputTokens: 100, - outputTokens: 2000, - totalTokens: 3000, - }, - }); - - expect(estimate?.estimatedCostUsd).toBeCloseTo(0.002682); - }); -}); - -describe('OpenAIUsageCostTracker', () => { - it('sums usage and cost across multiple responses', () => { - const tracker = new OpenAIUsageCostTracker(); - - tracker.add({ - model: 'gpt-5.4-mini', - usage: { - inputTokens: 1000, - cachedInputTokens: 100, - outputTokens: 2000, - totalTokens: 3000, - }, - }); - tracker.add({ - model: 'gpt-5.4-mini', - usage: { - inputTokens: 3000, - cachedInputTokens: 200, - outputTokens: 4000, - totalTokens: 7000, - }, - }); - - const summary = tracker.summary(); - - expect(summary.estimatedCostUsd).toBeCloseTo(0.0291975); - expect(summary).toEqual({ - usage: { - inputTokens: 4000, - cachedInputTokens: 300, - outputTokens: 6000, - totalTokens: 10000, - }, - estimatedCostUsd: summary.estimatedCostUsd, - modelsWithoutPricing: [], - }); - }); - - it('tracks models without configured pricing', () => { - const tracker = new OpenAIUsageCostTracker(); - - tracker.add({ - model: 'unknown-model', - usage: { - inputTokens: 1000, - cachedInputTokens: 0, - outputTokens: 1000, - totalTokens: 2000, - }, - }); - - expect(tracker.summary()).toEqual({ - usage: { - inputTokens: 1000, - cachedInputTokens: 0, - outputTokens: 1000, - totalTokens: 2000, - }, - estimatedCostUsd: null, - modelsWithoutPricing: ['unknown-model'], - }); - }); -}); diff --git a/packages/triage/src/helpers/estimateOpenAICost.ts b/packages/triage/src/helpers/estimateOpenAICost.ts deleted file mode 100644 index 0349a945..00000000 --- a/packages/triage/src/helpers/estimateOpenAICost.ts +++ /dev/null @@ -1,209 +0,0 @@ -type ResponsesUsageLike = - | { - input_tokens: number; - output_tokens: number; - total_tokens: number; - input_tokens_details?: { - cached_tokens?: number; - } | null; - } - | null - | undefined; - -type ChatCompletionUsageLike = - | { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; - prompt_tokens_details?: { - cached_tokens?: number; - } | null; - } - | null - | undefined; - -export type OpenAITokenUsage = { - inputTokens: number; - cachedInputTokens: number; - outputTokens: number; - totalTokens: number; -}; - -export type OpenAIModelPricing = { - inputUsdPer1MTokens: number; - cachedInputUsdPer1MTokens: number; - outputUsdPer1MTokens: number; -}; - -export type OpenAIUsageCostSummary = { - usage: OpenAITokenUsage | null; - estimatedCostUsd: number | null; - modelsWithoutPricing: string[]; -}; - -export const OPENAI_MODEL_PRICING: Record = { - 'gpt-5.4': { - inputUsdPer1MTokens: 2.5, - cachedInputUsdPer1MTokens: 0.25, - outputUsdPer1MTokens: 15, - }, - 'gpt-5.4-mini': { - inputUsdPer1MTokens: 0.75, - cachedInputUsdPer1MTokens: 0.075, - outputUsdPer1MTokens: 4.5, - }, - 'gpt-5.4-nano': { - inputUsdPer1MTokens: 0.2, - cachedInputUsdPer1MTokens: 0.02, - outputUsdPer1MTokens: 1.25, - }, -}; - -export function normalizeOpenAIResponsesUsage( - usage: ResponsesUsageLike, -): OpenAITokenUsage | null { - if (usage == null) { - return null; - } - - return { - inputTokens: usage.input_tokens, - cachedInputTokens: usage.input_tokens_details?.cached_tokens ?? 0, - outputTokens: usage.output_tokens, - totalTokens: usage.total_tokens, - }; -} - -export function normalizeOpenAIChatCompletionUsage( - usage: ChatCompletionUsageLike, -): OpenAITokenUsage | null { - if (usage == null) { - return null; - } - - return { - inputTokens: usage.prompt_tokens, - cachedInputTokens: usage.prompt_tokens_details?.cached_tokens ?? 0, - outputTokens: usage.completion_tokens, - totalTokens: usage.total_tokens, - }; -} - -export function estimateOpenAICostFromUsage({ - model, - usage, - pricingByModel = OPENAI_MODEL_PRICING, -}: { - model: string; - usage: OpenAITokenUsage | null; - pricingByModel?: Record; -}) { - if (usage == null) { - return null; - } - - const pricing = pricingByModel[model]; - if (pricing == null) { - return null; - } - - const cachedInputTokens = Math.max(usage.cachedInputTokens, 0); - const uncachedInputTokens = Math.max( - usage.inputTokens - cachedInputTokens, - 0, - ); - const outputTokens = Math.max(usage.outputTokens, 0); - - const estimatedCostUsd = - (uncachedInputTokens / 1_000_000) * pricing.inputUsdPer1MTokens + - (cachedInputTokens / 1_000_000) * pricing.cachedInputUsdPer1MTokens + - (outputTokens / 1_000_000) * pricing.outputUsdPer1MTokens; - - return { - estimatedCostUsd, - usage, - pricing, - }; -} - -export function sumOpenAITokenUsage( - left: OpenAITokenUsage | null, - right: OpenAITokenUsage | null, -): OpenAITokenUsage | null { - if (left == null) { - return right; - } - if (right == null) { - return left; - } - - return { - inputTokens: left.inputTokens + right.inputTokens, - cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, - outputTokens: left.outputTokens + right.outputTokens, - totalTokens: left.totalTokens + right.totalTokens, - }; -} - -export class OpenAIUsageCostTracker { - private usage: OpenAITokenUsage | null = null; - private estimatedCostUsd = 0; - private readonly modelsWithoutPricing = new Set(); - - add({ model, usage }: { model: string; usage: OpenAITokenUsage | null }) { - if (usage == null) { - return; - } - - this.usage = sumOpenAITokenUsage(this.usage, usage); - - const estimate = estimateOpenAICostFromUsage({ model, usage }); - if (estimate == null) { - this.modelsWithoutPricing.add(model); - return; - } - - this.estimatedCostUsd += estimate.estimatedCostUsd; - } - - summary(): OpenAIUsageCostSummary { - return { - usage: this.usage, - estimatedCostUsd: - this.modelsWithoutPricing.size === 0 ? this.estimatedCostUsd : null, - modelsWithoutPricing: [...this.modelsWithoutPricing].sort(), - }; - } -} - -export function logOpenAIUsageCostSummary({ - label, - summary, -}: { - label: string; - summary: OpenAIUsageCostSummary; -}) { - if (summary.usage == null) { - console.log(`[${label}] Usage is unavailable`); - return; - } - - console.log(`[${label}] Total usage:`, { - inputTokens: summary.usage.inputTokens, - cachedInputTokens: summary.usage.cachedInputTokens, - outputTokens: summary.usage.outputTokens, - totalTokens: summary.usage.totalTokens, - }); - - if (summary.estimatedCostUsd != null) { - console.log( - `[${label}] Total estimated cost (USD):`, - summary.estimatedCostUsd.toFixed(8), - ); - return; - } - - console.log( - `[${label}] No pricing configured for model(s): ${summary.modelsWithoutPricing.join(', ')}`, - ); -} diff --git a/packages/triage/src/helpers/geminiUsage.test.ts b/packages/triage/src/helpers/geminiUsage.test.ts new file mode 100644 index 00000000..5303af2d --- /dev/null +++ b/packages/triage/src/helpers/geminiUsage.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { GeminiUsageTracker, normalizeGeminiUsage } from './geminiUsage.js'; + +describe('normalizeGeminiUsage', () => { + it('normalizes Gemini usage metadata', () => { + expect( + normalizeGeminiUsage({ + promptTokenCount: 1000, + cachedContentTokenCount: 100, + candidatesTokenCount: 2000, + thoughtsTokenCount: 300, + totalTokenCount: 3300, + }), + ).toEqual({ + inputTokens: 1000, + cachedInputTokens: 100, + outputTokens: 2000, + thoughtTokens: 300, + totalTokens: 3300, + }); + }); + + it('falls back to summing token counts when total is absent', () => { + expect( + normalizeGeminiUsage({ + promptTokenCount: 10, + candidatesTokenCount: 20, + thoughtsTokenCount: 3, + }), + ).toEqual({ + inputTokens: 10, + cachedInputTokens: 0, + outputTokens: 20, + thoughtTokens: 3, + totalTokens: 33, + }); + }); +}); + +describe('GeminiUsageTracker', () => { + it('sums usage across multiple responses', () => { + const tracker = new GeminiUsageTracker(); + + tracker.add({ + inputTokens: 1000, + cachedInputTokens: 100, + outputTokens: 2000, + thoughtTokens: 300, + totalTokens: 3300, + }); + tracker.add({ + inputTokens: 3000, + cachedInputTokens: 200, + outputTokens: 4000, + thoughtTokens: 500, + totalTokens: 7500, + }); + + expect(tracker.summary()).toEqual({ + usage: { + inputTokens: 4000, + cachedInputTokens: 300, + outputTokens: 6000, + thoughtTokens: 800, + totalTokens: 10800, + }, + }); + }); +}); diff --git a/packages/triage/src/helpers/geminiUsage.ts b/packages/triage/src/helpers/geminiUsage.ts new file mode 100644 index 00000000..54188adf --- /dev/null +++ b/packages/triage/src/helpers/geminiUsage.ts @@ -0,0 +1,98 @@ +type GeminiUsageLike = + | { + promptTokenCount?: number; + cachedContentTokenCount?: number; + candidatesTokenCount?: number; + thoughtsTokenCount?: number; + totalTokenCount?: number; + } + | null + | undefined; + +export type GeminiTokenUsage = { + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + thoughtTokens: number; + totalTokens: number; +}; + +export type GeminiUsageSummary = { + usage: GeminiTokenUsage | null; +}; + +export function normalizeGeminiUsage( + usage: GeminiUsageLike, +): GeminiTokenUsage | null { + if (usage == null) { + return null; + } + + const inputTokens = usage.promptTokenCount ?? 0; + const outputTokens = usage.candidatesTokenCount ?? 0; + const thoughtTokens = usage.thoughtsTokenCount ?? 0; + + return { + inputTokens, + cachedInputTokens: usage.cachedContentTokenCount ?? 0, + outputTokens, + thoughtTokens, + totalTokens: + usage.totalTokenCount ?? inputTokens + outputTokens + thoughtTokens, + }; +} + +export function sumGeminiTokenUsage( + left: GeminiTokenUsage | null, + right: GeminiTokenUsage | null, +): GeminiTokenUsage | null { + if (left == null) { + return right; + } + if (right == null) { + return left; + } + + return { + inputTokens: left.inputTokens + right.inputTokens, + cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, + outputTokens: left.outputTokens + right.outputTokens, + thoughtTokens: left.thoughtTokens + right.thoughtTokens, + totalTokens: left.totalTokens + right.totalTokens, + }; +} + +export class GeminiUsageTracker { + private usage: GeminiTokenUsage | null = null; + + add(usage: GeminiTokenUsage | null) { + this.usage = sumGeminiTokenUsage(this.usage, usage); + } + + summary(): GeminiUsageSummary { + return { + usage: this.usage, + }; + } +} + +export function logGeminiUsageSummary({ + label, + summary, +}: { + label: string; + summary: GeminiUsageSummary; +}) { + if (summary.usage == null) { + console.log(`[${label}] Gemini usage is unavailable`); + return; + } + + console.log(`[${label}] Gemini total usage:`, { + inputTokens: summary.usage.inputTokens, + cachedInputTokens: summary.usage.cachedInputTokens, + outputTokens: summary.usage.outputTokens, + thoughtTokens: summary.usage.thoughtTokens, + totalTokens: summary.usage.totalTokens, + }); +} diff --git a/packages/triage/src/llm/client.test.ts b/packages/triage/src/llm/client.test.ts index edc8f123..3236865f 100644 --- a/packages/triage/src/llm/client.test.ts +++ b/packages/triage/src/llm/client.test.ts @@ -1,103 +1,31 @@ -import { describe, expect, it, vi } from 'vitest'; -import { isRetryableOpenAIError, runOpenAIRequestWithRetry } from './client.js'; - -describe('isRetryableOpenAIError', () => { - it('treats transient statuses as retryable', () => { - expect(isRetryableOpenAIError({ status: 408 })).toBe(true); - expect(isRetryableOpenAIError({ status: 409 })).toBe(true); - expect(isRetryableOpenAIError({ status: 429 })).toBe(true); - expect(isRetryableOpenAIError({ status: 500 })).toBe(true); - }); - - it('does not retry client validation errors', () => { - expect(isRetryableOpenAIError({ status: 400 })).toBe(false); - expect(isRetryableOpenAIError({ status: 422 })).toBe(false); - }); +import { GoogleGenAI } from '@google/genai'; +import { afterEach, describe, expect, it } from 'vitest'; +import { getGeminiClient } from './client.js'; + +const originalApiKey = process.env.GEMINI_API_KEY; + +afterEach(() => { + if (originalApiKey == null) { + delete process.env.GEMINI_API_KEY; + } else { + process.env.GEMINI_API_KEY = originalApiKey; + } }); -describe('runOpenAIRequestWithRetry', () => { - it('retries a transient failure and returns the successful result', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const request = vi - .fn<() => Promise>() - .mockRejectedValueOnce({ status: 500 }) - .mockResolvedValueOnce('ok'); - const sleep = vi.fn<() => Promise>().mockResolvedValue(undefined); - - await expect( - runOpenAIRequestWithRetry(request, { - label: 'testRequest', - initialDelayMs: 5, - sleep, - }), - ).resolves.toBe('ok'); - - expect(request).toHaveBeenCalledTimes(2); - expect(sleep).toHaveBeenCalledWith(5); - warnSpy.mockRestore(); - }); - - it('uses retry-after-ms headers when available', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const request = vi - .fn<() => Promise>() - .mockRejectedValueOnce({ - status: 429, - headers: { 'retry-after-ms': '123' }, - }) - .mockResolvedValueOnce('ok'); - const sleep = vi.fn<() => Promise>().mockResolvedValue(undefined); - - await expect( - runOpenAIRequestWithRetry(request, { - label: 'testRequest', - sleep, - }), - ).resolves.toBe('ok'); - - expect(sleep).toHaveBeenCalledWith(123); - warnSpy.mockRestore(); - }); - - it('caps exponential retry delays', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const request = vi - .fn<() => Promise>() - .mockRejectedValueOnce({ status: 500 }) - .mockRejectedValueOnce({ status: 500 }) - .mockRejectedValueOnce({ status: 500 }) - .mockResolvedValueOnce('ok'); - const sleep = vi.fn<() => Promise>().mockResolvedValue(undefined); - - await expect( - runOpenAIRequestWithRetry(request, { - label: 'testRequest', - initialDelayMs: 500, - maxDelayMs: 750, - sleep, - }), - ).resolves.toBe('ok'); +describe('getGeminiClient', () => { + it('requires a Gemini API key', () => { + delete process.env.GEMINI_API_KEY; - expect(sleep).toHaveBeenNthCalledWith(1, 500); - expect(sleep).toHaveBeenNthCalledWith(2, 750); - expect(sleep).toHaveBeenNthCalledWith(3, 750); - warnSpy.mockRestore(); + expect(() => getGeminiClient()).toThrow( + 'GEMINI_API_KEY must be set before creating Gemini client', + ); }); - it('does not retry non-retryable failures', async () => { - const request = vi.fn<() => Promise>().mockRejectedValue({ - status: 400, - }); - const sleep = vi.fn<() => Promise>().mockResolvedValue(undefined); + it('creates a Gemini SDK client', () => { + process.env.GEMINI_API_KEY = 'test-key'; - await expect( - runOpenAIRequestWithRetry(request, { - label: 'testRequest', - sleep, - }), - ).rejects.toEqual({ status: 400 }); + const client = getGeminiClient(); - expect(request).toHaveBeenCalledTimes(1); - expect(sleep).not.toHaveBeenCalled(); + expect(client).toBeInstanceOf(GoogleGenAI); }); }); diff --git a/packages/triage/src/llm/client.ts b/packages/triage/src/llm/client.ts index 16124950..bbf07cd7 100644 --- a/packages/triage/src/llm/client.ts +++ b/packages/triage/src/llm/client.ts @@ -1,161 +1,10 @@ -import OpenAI from 'openai'; +import { GoogleGenAI } from '@google/genai'; -export type OpenAIRetryOptions = { - label: string; - maxAttempts?: number; - initialDelayMs?: number; - maxDelayMs?: number; - sleep?: (ms: number) => Promise; -}; - -export function getOpenAiClient() { - const apiKey = process.env.OPENAI_API_KEY; +export function getGeminiClient() { + const apiKey = process.env.GEMINI_API_KEY; if (apiKey == null || apiKey.trim() === '') { - throw new Error('OPENAI_API_KEY must be set before creating OpenAI client'); - } - - return new OpenAI({ - apiKey, - }); -} - -export async function runOpenAIRequestWithRetry( - request: () => Promise, - { - label, - maxAttempts = 6, - initialDelayMs = 1_000, - maxDelayMs = 30_000, - sleep = sleepMs, - }: OpenAIRetryOptions, -): Promise { - let attempt = 1; - - while (true) { - try { - return await request(); - } catch (error) { - if (attempt >= maxAttempts || !isRetryableOpenAIError(error)) { - throw error; - } - - const delayMs = - getRetryAfterDelayMs(error, maxDelayMs) ?? - Math.min(maxDelayMs, initialDelayMs * 2 ** (attempt - 1)); - console.warn( - `${label}: OpenAI request failed with a retryable error; retrying attempt ${attempt + 1}/${maxAttempts} in ${delayMs}ms.`, - ); - - await sleep(delayMs); - attempt++; - } - } -} - -export function isRetryableOpenAIError(error: unknown): boolean { - if (error == null || typeof error !== 'object') { - return false; - } - - const status = getNumericProperty(error, 'status'); - if (status != null) { - return status === 408 || status === 409 || status === 429 || status >= 500; - } - - const code = getStringProperty(error, 'code'); - return ( - code === 'server_error' || - code === 'rate_limit_exceeded' || - code === 'timeout' - ); -} - -function getNumericProperty(value: object, key: string): number | null { - if (!(key in value)) { - return null; - } - - const property = value[key as keyof typeof value]; - return typeof property === 'number' ? property : null; -} - -function getStringProperty(value: object, key: string): string | null { - if (!(key in value)) { - return null; - } - - const property = value[key as keyof typeof value]; - return typeof property === 'string' ? property : null; -} - -function getRetryAfterDelayMs( - error: unknown, - maxDelayMs: number, -): number | null { - if (error == null || typeof error !== 'object' || !('headers' in error)) { - return null; - } - - const headers = error.headers; - const retryAfterMs = parseNumericHeader( - getHeaderValue(headers, 'retry-after-ms'), - ); - if (retryAfterMs != null) { - return Math.min(maxDelayMs, retryAfterMs); - } - - const retryAfter = getHeaderValue(headers, 'retry-after'); - if (retryAfter == null) { - return null; - } - - const retryAfterSeconds = Number(retryAfter); - if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) { - return Math.min(maxDelayMs, retryAfterSeconds * 1_000); - } - - const retryAfterDate = Date.parse(retryAfter); - if (Number.isNaN(retryAfterDate)) { - return null; - } - - return Math.min(maxDelayMs, Math.max(0, retryAfterDate - Date.now())); -} - -function getHeaderValue(headers: unknown, key: string): string | null { - if (headers == null || typeof headers !== 'object') { - return null; - } - - if ('get' in headers && typeof headers.get === 'function') { - const value = headers.get(key); - return typeof value === 'string' ? value : null; - } - - for (const [headerKey, headerValue] of Object.entries(headers)) { - if (headerKey.toLowerCase() !== key) { - continue; - } - - if (typeof headerValue === 'string') { - return headerValue; - } + throw new Error('GEMINI_API_KEY must be set before creating Gemini client'); } - return null; -} - -function parseNumericHeader(value: string | null): number | null { - if (value == null) { - return null; - } - - const parsed = Number(value); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; -} - -function sleepMs(ms: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); + return new GoogleGenAI({ apiKey }); } diff --git a/packages/triage/src/llm/common/jsonSchema.test.ts b/packages/triage/src/llm/common/gemini.test.ts similarity index 77% rename from packages/triage/src/llm/common/jsonSchema.test.ts rename to packages/triage/src/llm/common/gemini.test.ts index 35258e54..357e294f 100644 --- a/packages/triage/src/llm/common/jsonSchema.test.ts +++ b/packages/triage/src/llm/common/gemini.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; -import { toOpenAiJsonSchema } from './jsonSchema.js'; +import { toGeminiJsonSchema } from './gemini.js'; function findSchemaKey(value: unknown, key: string): boolean { if (Array.isArray(value)) { @@ -17,8 +17,8 @@ function findSchemaKey(value: unknown, key: string): boolean { ); } -describe('toOpenAiJsonSchema', () => { - it('rewrites Zod discriminated-union oneOf entries to anyOf', () => { +describe('toGeminiJsonSchema', () => { + it('rewrites Zod discriminated-union oneOf entries to Gemini-supported anyOf', () => { const schema = z.object({ result: z.discriminatedUnion('kind', [ z.object({ kind: z.literal('existing'), id: z.string() }), @@ -26,7 +26,7 @@ describe('toOpenAiJsonSchema', () => { ]), }); - const jsonSchema = toOpenAiJsonSchema(schema); + const jsonSchema = toGeminiJsonSchema(schema); expect(findSchemaKey(jsonSchema, 'oneOf')).toBe(false); expect(findSchemaKey(jsonSchema, 'anyOf')).toBe(true); diff --git a/packages/triage/src/llm/common/gemini.ts b/packages/triage/src/llm/common/gemini.ts new file mode 100644 index 00000000..da5bef2f --- /dev/null +++ b/packages/triage/src/llm/common/gemini.ts @@ -0,0 +1,144 @@ +import type { + Content, + FunctionCall, + FunctionDeclaration, + GenerateContentConfig, + GenerateContentResponse, +} from '@google/genai'; +import { FunctionCallingConfigMode, ThinkingLevel } from '@google/genai'; +import { type ZodType, z } from 'zod'; +import { assert } from '../../util/assert.js'; +import type { ToolRegistry } from './tool.js'; + +type JsonObject = { [key: string]: unknown }; + +export function buildGeminiJsonConfig({ + systemPrompt, + responseSchema, + toolRegistry, +}: { + systemPrompt: string; + responseSchema: ZodType; + toolRegistry?: ToolRegistry; +}): GenerateContentConfig { + const toolDeclarations = + toolRegistry == null ? [] : toGeminiToolDeclarations(toolRegistry); + + return { + systemInstruction: systemPrompt, + httpOptions: { + timeout: 45_000, + retryOptions: { + attempts: 2, + }, + }, + responseMimeType: 'application/json', + responseJsonSchema: toGeminiJsonSchema(responseSchema), + thinkingConfig: { + thinkingLevel: ThinkingLevel.LOW, + }, + tools: + toolDeclarations.length > 0 + ? [{ functionDeclarations: toolDeclarations }] + : undefined, + toolConfig: + toolDeclarations.length > 0 + ? { + functionCallingConfig: { + mode: FunctionCallingConfigMode.AUTO, + }, + } + : undefined, + }; +} + +export function parseGeminiJsonResponse( + response: GenerateContentResponse, + schema: ZodType, +): T { + const text = response.text; + assert(text != null && text.trim() !== '', 'Gemini response text is empty'); + + let parsedJson: unknown; + try { + parsedJson = JSON.parse(text); + } catch (error) { + throw new Error( + `Gemini response text is not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + return schema.parse(parsedJson); +} + +export function toGeminiJsonSchema(schema: ZodType): JsonObject { + return replaceOneOfWithAnyOf(z.toJSONSchema(schema)) as JsonObject; +} + +export function getGeminiFunctionCalls( + response: GenerateContentResponse, +): FunctionCall[] { + return response.functionCalls ?? []; +} + +export function getGeminiModelContent( + response: GenerateContentResponse, +): Content | null { + return response.candidates?.[0]?.content ?? null; +} + +export function toGeminiFunctionResponseContent({ + functionCall, + output, +}: { + functionCall: FunctionCall; + output: string; +}): Content { + assert( + functionCall.name != null && functionCall.name.trim() !== '', + 'Gemini function call name is missing', + ); + + return { + role: 'user', + parts: [ + { + functionResponse: { + id: functionCall.id, + name: functionCall.name, + response: { output }, + }, + }, + ], + }; +} + +function toGeminiToolDeclarations( + toolRegistry: ToolRegistry, +): FunctionDeclaration[] { + return Object.values(toolRegistry).map((tool) => { + return { + name: tool.name, + description: tool.description, + parametersJsonSchema: tool.paramsSchema, + }; + }); +} + +function replaceOneOfWithAnyOf(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(replaceOneOfWithAnyOf); + } + + if (value == null || typeof value !== 'object') { + return value; + } + + const output: JsonObject = {}; + for (const [key, child] of Object.entries(value)) { + output[key === 'oneOf' ? 'anyOf' : key] = replaceOneOfWithAnyOf(child); + } + return output; +} diff --git a/packages/triage/src/llm/common/jsonSchema.ts b/packages/triage/src/llm/common/jsonSchema.ts deleted file mode 100644 index 8ec79331..00000000 --- a/packages/triage/src/llm/common/jsonSchema.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { z } from 'zod'; - -type JsonObject = { [key: string]: unknown }; - -function replaceOneOfWithAnyOf(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(replaceOneOfWithAnyOf); - } - - if (value == null || typeof value !== 'object') { - return value; - } - - const output: JsonObject = {}; - for (const [key, child] of Object.entries(value)) { - output[key === 'oneOf' ? 'anyOf' : key] = replaceOneOfWithAnyOf(child); - } - return output; -} - -export function toOpenAiJsonSchema(schema: z.ZodType): JsonObject { - return replaceOneOfWithAnyOf(z.toJSONSchema(schema)) as JsonObject; -} diff --git a/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/index.ts b/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/index.ts index 06402406..ce5951be 100644 --- a/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/index.ts +++ b/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/index.ts @@ -1,20 +1,25 @@ +import type { GenerateContentResponse } from '@google/genai'; +import { type Content, createUserContent } from '@google/genai'; import { type Claim, ClaimSchema } from '@mrtdown/core'; import type { MRTDownRepository } from '@mrtdown/fs'; import { DateTime } from 'luxon'; -import type { - ParsedResponse, - ResponseInputItem, -} from 'openai/resources/responses/responses.js'; import z from 'zod'; import { - logOpenAIUsageCostSummary, - normalizeOpenAIResponsesUsage, - OpenAIUsageCostTracker, -} from '../../../helpers/estimateOpenAICost.js'; + GeminiUsageTracker, + logGeminiUsageSummary, + normalizeGeminiUsage, +} from '../../../helpers/geminiUsage.js'; import { assert } from '../../../util/assert.js'; -import { getOpenAiClient, runOpenAIRequestWithRetry } from '../../client.js'; -import { toOpenAiJsonSchema } from '../../common/jsonSchema.js'; +import { getGeminiClient } from '../../client.js'; +import { + buildGeminiJsonConfig, + getGeminiFunctionCalls, + getGeminiModelContent, + parseGeminiJsonResponse, + toGeminiFunctionResponseContent, +} from '../../common/gemini.js'; import type { ToolRegistry } from '../../common/tool.js'; +import { GEMINI_TRIAGE_MODEL } from '../../models.js'; import { normalizeClaimsForEvidence } from './normalizeClaimsForEvidence.js'; import { buildSystemPrompt } from './prompt.js'; import { FindLinesTool } from './tools/FindLinesTool.js'; @@ -22,7 +27,7 @@ import { FindServicesTool } from './tools/FindServicesTool.js'; import { FindStationsTool } from './tools/FindStationsTool.js'; import { ResolveRelativeDateTool } from './tools/ResolveRelativeDateTool.js'; -const TOOL_CALL_LIMIT = 5; +const TOOL_CALL_LIMIT = 8; const ResponseSchema = z.object({ claims: z.array(ClaimSchema), }); @@ -63,176 +68,145 @@ export async function extractClaimsFromNewEvidence( [resolveRelativeDateTool.name]: resolveRelativeDateTool, }; - const context: ResponseInputItem[] = [ - { - role: 'user', - content: ` + const context: Content[] = [ + createUserContent( + ` Evidence: ${params.newEvidence.text} Timestamp: ${evidenceTs.toISO({ includeOffset: true, suppressMilliseconds: true })} `.trim(), - }, + ), ]; const systemPrompt = buildSystemPrompt(); - const model = 'gpt-5.4-mini'; - const usageCostTracker = new OpenAIUsageCostTracker(); + const model = GEMINI_TRIAGE_MODEL; + const usageTracker = new GeminiUsageTracker(); let toolCallCount = 0; let reachedToolCallLimit = false; - let response: ParsedResponse>; + let response: GenerateContentResponse; do { - response = await runOpenAIRequestWithRetry( - () => - getOpenAiClient().responses.parse({ - model, - instructions: systemPrompt, - input: context, - reasoning: { - effort: 'medium', - }, - text: { - format: { - type: 'json_schema', - name: 'Response', - strict: true, - schema: toOpenAiJsonSchema(ResponseSchema), - }, - }, - tools: Object.values(toolRegistry).map((tool) => { - return { - type: 'function', - name: tool.name, - description: tool.description, - parameters: tool.paramsSchema, - strict: true, - }; + response = await getGeminiClient().models.generateContent({ + model, + contents: context, + config: buildGeminiJsonConfig({ + systemPrompt, + responseSchema: ResponseSchema, + toolRegistry, + }), + }); + + const modelContent = getGeminiModelContent(response); + if (modelContent != null) { + context.push(modelContent); + } + + const functionCalls = getGeminiFunctionCalls(response); + for (const functionCall of functionCalls) { + assert( + functionCall.name != null && functionCall.name.trim() !== '', + 'Gemini function call name is missing', + ); + + toolCallCount++; + + if (toolCallCount > TOOL_CALL_LIMIT) { + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: 'Ran out of tool calls. Stop calling.', }), + ); + console.log( + 'Forced short-circuit, returning error message in tool call result.', + ); + reachedToolCallLimit = true; + break; + } - // Don't persist conversation with OpenAI, but include reasoning content to - // continue the thread with the same reasoning. - store: false, - include: ['reasoning.encrypted_content'], - }), - { - label: 'extractClaimsFromNewEvidence', - }, - ); - - for (const item of response.output) { - switch (item.type) { - case 'function_call': { - /** - * Prevent the `parsed_arguments` field from being included - * https://github.com/openai/openai-python/issues/2374 - */ - context.push({ - type: 'function_call', - id: item.id, - call_id: item.call_id, - name: item.name, - arguments: item.arguments, - }); - toolCallCount++; - - if (toolCallCount > TOOL_CALL_LIMIT) { - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: 'Ran out of tool calls. Stop Calling.', - }); - console.log( - 'Forced short-circuit, returning error message in tool call result.', - ); - reachedToolCallLimit = true; - break; - } - - if (item.name in toolRegistry) { - const tool = toolRegistry[item.name]; - - let params: unknown; - - try { - params = tool.parseParams(JSON.parse(item.arguments)); - } catch (e) { - console.error( - `[extractClaimsFromNewEvidence] Error parsing parameters for tool "${item.name}" with arguments "${item.arguments}":`, - e, - ); - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Invalid parameters for tool "${item.name}". Please try again.`, - }); - continue; - } - - let result: string; - - try { - result = await tool.runner(params); - } catch (e) { - console.error( - `[extractClaimsFromNewEvidence] Error running tool "${item.name}":`, - e, - ); - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Tool "${item.name}" failed. Please continue without it or try a different call.`, - }); - continue; - } - - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: result, - }); - } else { - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Unknown tool "${item.name}". Please use one of the available tools.`, - }); - } - - break; + if (functionCall.name in toolRegistry) { + const tool = toolRegistry[functionCall.name]; + + let parsedParams: unknown; + + try { + parsedParams = tool.parseParams(functionCall.args ?? {}); + } catch (e) { + console.error( + `[extractClaimsFromNewEvidence] Error parsing parameters for tool "${functionCall.name}" with arguments "${JSON.stringify(functionCall.args ?? {})}":`, + e, + ); + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Invalid parameters for tool "${functionCall.name}". Please try again.`, + }), + ); + continue; } - default: { - context.push(item as ResponseInputItem); - break; + + let result: string; + + try { + result = await tool.runner(parsedParams); + } catch (e) { + console.error( + `[extractClaimsFromNewEvidence] Error running tool "${functionCall.name}":`, + e, + ); + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Tool "${functionCall.name}" failed. Please continue without it or try a different call.`, + }), + ); + continue; } - } - if (reachedToolCallLimit) { - break; + context.push( + toGeminiFunctionResponseContent({ functionCall, output: result }), + ); + } else { + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Unknown tool "${functionCall.name}". Please use one of the available tools.`, + }), + ); } } - const usage = normalizeOpenAIResponsesUsage(response.usage); - usageCostTracker.add({ model, usage }); + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); } while ( !reachedToolCallLimit && - response.output.some((item) => item.type === 'function_call') + getGeminiFunctionCalls(response).length > 0 ); - logOpenAIUsageCostSummary({ - label: 'extractClaimsFromNewEvidence', - summary: usageCostTracker.summary(), - }); - if (reachedToolCallLimit) { - throw new Error(`Exceeded tool call limit of ${TOOL_CALL_LIMIT}`); + response = await getGeminiClient().models.generateContent({ + model, + contents: context, + config: buildGeminiJsonConfig({ + systemPrompt: `${systemPrompt} + +Tool-call budget is exhausted. Do not call more tools. Return the best schema-conforming final claim JSON using only the evidence and tool results already provided.`, + responseSchema: ResponseSchema, + }), + }); + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); } - assert(response.output_parsed != null, 'Response output parsed is null'); + logGeminiUsageSummary({ + label: 'extractClaimsFromNewEvidence', + summary: usageTracker.summary(), + }); + + const parsed = parseGeminiJsonResponse(response, ResponseSchema); return { claims: normalizeClaimsForEvidence({ - claims: response.output_parsed.claims, + claims: parsed.claims, evidenceTs: params.newEvidence.ts, repo: params.repo, }), diff --git a/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/prompt.ts b/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/prompt.ts index f4a6caa7..4dfbb7ff 100644 --- a/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/prompt.ts +++ b/packages/triage/src/llm/functions/extractClaimsFromNewEvidence/prompt.ts @@ -19,6 +19,9 @@ Extract only operational impact claims (service or station facility impact) that Use tools whenever an ID or station mapping is uncertain. Prefer correctness over guessing. Use resolveRelativeDate whenever the evidence uses relative weekday language and you need concrete timestamps. +Do not use resolveRelativeDate for explicit calendar dates or date ranges that +already name the day/month/year, even when weekday names are present (for +example, "Sat & Sun from 7 to 8 February 2026" is explicit, not relative). For phrase mapping: - "this Saturday" -> weekday=SA, weekOffset=0 - "next Saturday" -> weekday=SA, weekOffset=1 @@ -28,6 +31,10 @@ For phrase mapping: Relative-weekday tool-call policy (strict): - If evidence mentions relative weekday language ("this Sat", "next Sunday", "this weekend", "next weekend", "this/next Mon..Fri"), you MUST call resolveRelativeDate before returning output. - If "weekend" is mentioned, you MUST call resolveRelativeDate for both SA and SU (same weekOffset as implied by phrase). +- Bare weekday names with explicit calendar dates are not relative weekday + language. Do not call resolveRelativeDate for explicit dates such as + "Saturday 7 February 2026", "Sat & Sun from 7 to 8 February 2026", or + "on 18 Feb". - Do not guess or hand-construct ISO dates for relative weekday phrases when resolveRelativeDate can be used. - Apply this policy even if you think the final result may be claims: []. @@ -81,8 +88,14 @@ Service-hours adjustment impact-window rule (strict): - Example: "services will start later at 6.30am and end at 9pm daily from 1 to 8 February 2026" -> the impact window is 21:00:00 to 06:30:00, not 06:30:00 to 21:00:00. Effect disambiguation: -- "Longer waits", "headways adjusted", "additional travel time", "single-loop operation", shuttle/train bridging, or similar degraded-but-running service language is NOT "no-service". +- "Longer waits", "waits of up to N minutes", "headways adjusted", + "additional travel time", "single-loop operation", shuttle/train bridging, or + similar degraded-but-running service language is NOT "no-service". - Use "reduced-service" for degraded planned operations unless the evidence explicitly says trains are suspended, service is closed, or no trains are running. +- For planned maintenance/testing/integrated-systems evidence, "waits of up to + N minutes" or "longer waits of up to N minutes" means reduced-service. Do not + encode N as service.kind = "delay" duration; the number describes the + degraded frequency/headway, not an incident delay duration. - If evidence mentions a future planned suspension in broad terms ("planned", "expected", "first half of 2026") without a concrete service suspension window, do not convert that into a present or fixed future "no-service" claim. Prefer the concrete degraded service claim that is explicitly stated. - "Only one platform in use/may be used" at a terminal or station during a future window should generally map to a planned degraded operation claim (usually "reduced-service"), even if the post also says "service as usual" at announcement time. - Deterministic mapping: if evidence says one platform is closed/temporarily closed while trains continue to run (for example, "other platform operates as usual", shuttle provided, longer waits), you MUST emit planned service claims with service effect "reduced-service" for affected services. Do not return claims: [] for this pattern. @@ -192,6 +205,8 @@ Field guidance: - If relative weekday language appears, confirm required resolveRelativeDate call(s) were made (weekend => SA and SU). - If evidence has no unambiguous one-direction qualifier, ensure claims include all directional services for the affected service/line. - Only return a single directional service claim when explicit direction-only wording is present. + - For planned testing/maintenance that says "longer waits" or "waits of up + to N minutes", confirm service effect is reduced-service, not delay. - For "service will start at X" or "start later at X" service-hours adjustments, confirm the fixed impact window ends at X and starts at 00:00 on the same date. Reject X-to-midnight windows. - For every fixed period: confirm endAt > startAt. If not, switch to start-only. - For every recurring period: confirm endAt > startAt. If not, extend endAt to one year after startAt. diff --git a/packages/triage/src/llm/functions/generateIssueTitleAndSlug/index.ts b/packages/triage/src/llm/functions/generateIssueTitleAndSlug/index.ts index fb20a342..1b7136cc 100644 --- a/packages/triage/src/llm/functions/generateIssueTitleAndSlug/index.ts +++ b/packages/triage/src/llm/functions/generateIssueTitleAndSlug/index.ts @@ -1,13 +1,15 @@ -import type { ResponseInputItem } from 'openai/resources/responses/responses.mjs'; import { z } from 'zod'; import { - logOpenAIUsageCostSummary, - normalizeOpenAIResponsesUsage, - OpenAIUsageCostTracker, -} from '../../../helpers/estimateOpenAICost.js'; -import { assert } from '../../../util/assert.js'; -import { getOpenAiClient, runOpenAIRequestWithRetry } from '../../client.js'; -import { toOpenAiJsonSchema } from '../../common/jsonSchema.js'; + GeminiUsageTracker, + logGeminiUsageSummary, + normalizeGeminiUsage, +} from '../../../helpers/geminiUsage.js'; +import { getGeminiClient } from '../../client.js'; +import { + buildGeminiJsonConfig, + parseGeminiJsonResponse, +} from '../../common/gemini.js'; +import { GEMINI_FAST_MODEL } from '../../models.js'; import { buildSystemPrompt } from './prompt.js'; export const ResponseSchema = z.object({ @@ -33,47 +35,25 @@ export async function generateIssueTitleAndSlug( params: GenerateIssueTitleAndSlugParams, ): Promise { const systemPrompt = buildSystemPrompt(); - const model = 'gpt-5.4-nano'; - const usageCostTracker = new OpenAIUsageCostTracker(); + const model = GEMINI_FAST_MODEL; + const usageTracker = new GeminiUsageTracker(); - const context: ResponseInputItem[] = [ - { - role: 'user', - content: ` + const response = await getGeminiClient().models.generateContent({ + model, + contents: ` Text: ${params.text} `.trim(), - }, - ]; - - const response = await runOpenAIRequestWithRetry( - () => - getOpenAiClient().responses.parse({ - model, - input: context, - instructions: systemPrompt, - text: { - format: { - type: 'json_schema', - name: 'Response', - strict: true, - schema: toOpenAiJsonSchema(ResponseSchema), - }, - }, - store: false, - }), - { - label: 'generateIssueTitleAndSlug', - }, - ); + config: buildGeminiJsonConfig({ + systemPrompt, + responseSchema: ResponseSchema, + }), + }); - const usage = normalizeOpenAIResponsesUsage(response.usage); - usageCostTracker.add({ model, usage }); - logOpenAIUsageCostSummary({ + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); + logGeminiUsageSummary({ label: 'generateIssueTitleAndSlug', - summary: usageCostTracker.summary(), + summary: usageTracker.summary(), }); - assert(response.output_parsed != null, 'Response output parsed is null'); - - return response.output_parsed; + return parseGeminiJsonResponse(response, ResponseSchema); } diff --git a/packages/triage/src/llm/functions/translate/index.ts b/packages/triage/src/llm/functions/translate/index.ts index c1d4bca7..66877a04 100644 --- a/packages/triage/src/llm/functions/translate/index.ts +++ b/packages/triage/src/llm/functions/translate/index.ts @@ -1,28 +1,27 @@ import { TranslationsSchema } from '@mrtdown/core'; -import type { ResponseInputItem } from 'openai/resources/responses/responses.js'; import { - logOpenAIUsageCostSummary, - normalizeOpenAIResponsesUsage, - OpenAIUsageCostTracker, -} from '../../../helpers/estimateOpenAICost.js'; -import { assert } from '../../../util/assert.js'; -import { getOpenAiClient, runOpenAIRequestWithRetry } from '../../client.js'; -import { toOpenAiJsonSchema } from '../../common/jsonSchema.js'; + GeminiUsageTracker, + logGeminiUsageSummary, + normalizeGeminiUsage, +} from '../../../helpers/geminiUsage.js'; +import { getGeminiClient } from '../../client.js'; +import { + buildGeminiJsonConfig, + parseGeminiJsonResponse, +} from '../../common/gemini.js'; import { TRANSLATE_MODEL } from './model.js'; export async function translate(text: string) { const model = TRANSLATE_MODEL; - const usageCostTracker = new OpenAIUsageCostTracker(); - - const context: ResponseInputItem[] = [{ role: 'user', content: text }]; + const usageTracker = new GeminiUsageTracker(); - const response = await runOpenAIRequestWithRetry( - () => - getOpenAiClient().responses.parse({ - model, - input: context, - instructions: - `You are a helpful assistant that translates text to the following languages: + const response = await getGeminiClient().models.generateContent({ + model, + contents: text, + config: buildGeminiJsonConfig({ + responseSchema: TranslationsSchema, + systemPrompt: + `You are a helpful assistant that translates text to the following languages: - English - Chinese (Simplified) - Malay @@ -33,35 +32,14 @@ Line names, station names, service IDs, station codes, operator names, road name Copy those proper nouns exactly as written in the source text into every locale. Do not translate, transliterate, localize, shorten, or abbreviate those proper nouns. `.trim(), - text: { - format: { - type: 'json_schema', - name: 'Translation', - strict: true, - schema: toOpenAiJsonSchema(TranslationsSchema), - }, - }, - reasoning: { - effort: 'low', - summary: 'concise', - }, - store: false, - include: ['reasoning.encrypted_content'], - }), - { - label: 'translate', - }, - ); + }), + }); - const usage = normalizeOpenAIResponsesUsage(response.usage); - usageCostTracker.add({ model, usage }); - logOpenAIUsageCostSummary({ + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); + logGeminiUsageSummary({ label: 'translate', - summary: usageCostTracker.summary(), + summary: usageTracker.summary(), }); - const parsed = response.output_parsed; - assert(parsed != null, 'Response output parsed is null'); - - return parsed; + return parseGeminiJsonResponse(response, TranslationsSchema); } diff --git a/packages/triage/src/llm/functions/translate/model.ts b/packages/triage/src/llm/functions/translate/model.ts index 0e976269..6e252d01 100644 --- a/packages/triage/src/llm/functions/translate/model.ts +++ b/packages/triage/src/llm/functions/translate/model.ts @@ -1,2 +1,4 @@ -export const TRANSLATE_MODEL = 'gpt-5.4-nano'; -export const TRANSLATE_MODEL_SOURCE = `@openai/${TRANSLATE_MODEL}`; +import { GEMINI_FAST_MODEL, modelSource } from '../../models.js'; + +export const TRANSLATE_MODEL = GEMINI_FAST_MODEL; +export const TRANSLATE_MODEL_SOURCE = modelSource(TRANSLATE_MODEL); diff --git a/packages/triage/src/llm/functions/triageNewEvidence/index.ts b/packages/triage/src/llm/functions/triageNewEvidence/index.ts index 43786e06..e72717c0 100644 --- a/packages/triage/src/llm/functions/triageNewEvidence/index.ts +++ b/packages/triage/src/llm/functions/triageNewEvidence/index.ts @@ -1,20 +1,24 @@ +import type { GenerateContentResponse } from '@google/genai'; +import { type Content, createUserContent } from '@google/genai'; import { IssueIdSchema, IssueTypeSchema } from '@mrtdown/core'; import type { MRTDownRepository } from '@mrtdown/fs'; import { DateTime } from 'luxon'; -import type { - ParsedResponse, - ResponseInputItem, -} from 'openai/resources/responses/responses.mjs'; import z from 'zod'; import { - logOpenAIUsageCostSummary, - normalizeOpenAIResponsesUsage, - OpenAIUsageCostTracker, -} from '../../../helpers/estimateOpenAICost.js'; + GeminiUsageTracker, + logGeminiUsageSummary, + normalizeGeminiUsage, +} from '../../../helpers/geminiUsage.js'; import { assert } from '../../../util/assert.js'; -import { getOpenAiClient, runOpenAIRequestWithRetry } from '../../client.js'; -import { toOpenAiJsonSchema } from '../../common/jsonSchema.js'; +import { getGeminiClient } from '../../client.js'; +import { + buildGeminiJsonConfig, + getGeminiFunctionCalls, + getGeminiModelContent, + toGeminiFunctionResponseContent, +} from '../../common/gemini.js'; import type { ToolRegistry } from '../../common/tool.js'; +import { GEMINI_TRIAGE_MODEL } from '../../models.js'; import { buildSystemPrompt } from './prompt.js'; import { FindIssuesByDateRangeTool } from './tools/FindIssuesByDateRangeTool.js'; import { FindIssuesTool } from './tools/FindIssuesTool.js'; @@ -65,169 +69,195 @@ export async function triageNewEvidence(params: TriageNewEvidenceParams) { const systemPrompt = buildSystemPrompt(); - const context: ResponseInputItem[] = [ - { - role: 'user', - content: ` + const context: Content[] = [ + createUserContent( + ` Evidence: ${params.newEvidence.text} Timestamp: ${evidenceTs.toISO({ includeOffset: true, suppressMilliseconds: true })} `.trim(), - }, + ), ]; let toolCallCount = 0; let reachedToolCallLimit = false; - const model = 'gpt-5.4-mini'; - const usageCostTracker = new OpenAIUsageCostTracker(); + const model = GEMINI_TRIAGE_MODEL; + const usageTracker = new GeminiUsageTracker(); - let response: ParsedResponse>; + let response: GenerateContentResponse; do { - response = await runOpenAIRequestWithRetry( - () => - getOpenAiClient().responses.parse({ - model, - input: context, - instructions: systemPrompt, - reasoning: { - effort: 'low', - }, - text: { - format: { - type: 'json_schema', - name: 'Response', - strict: true, - schema: toOpenAiJsonSchema(ResponseSchema), - }, - }, - tools: Object.values(toolRegistry).map((tool) => { - return { - type: 'function', - name: tool.name, - description: tool.description, - parameters: tool.paramsSchema, - strict: true, - }; + response = await getGeminiClient().models.generateContent({ + model, + contents: context, + config: buildGeminiJsonConfig({ + systemPrompt, + responseSchema: ResponseSchema, + toolRegistry, + }), + }); + + const modelContent = getGeminiModelContent(response); + if (modelContent != null) { + context.push(modelContent); + } + + const functionCalls = getGeminiFunctionCalls(response); + for (const functionCall of functionCalls) { + assert( + functionCall.name != null && functionCall.name.trim() !== '', + 'Gemini function call name is missing', + ); + + toolCallCount++; + + if (toolCallCount > TOOL_CALL_LIMIT) { + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: 'Ran out of tool calls. Stop calling.', }), - // Don't persist conversation with OpenAI, but include reasoning content to - // continue the thread with the same reasoning. - store: false, - include: ['reasoning.encrypted_content'], - }), - { - label: 'triageNewEvidence', - }, - ); + ); + console.log( + 'Forced short-circuit, returning error message in tool call result.', + ); + reachedToolCallLimit = true; + break; + } + + if (functionCall.name in toolRegistry) { + const tool = toolRegistry[functionCall.name]; + + let parsedParams: unknown; - for (const item of response.output) { - switch (item.type) { - case 'function_call': { - /** - * Prevent the `parsed_arguments` field from being included - * https://github.com/openai/openai-python/issues/2374 - */ - context.push({ - type: 'function_call', - id: item.id, - call_id: item.call_id, - name: item.name, - arguments: item.arguments, - }); - toolCallCount++; - - if (toolCallCount > TOOL_CALL_LIMIT) { - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: 'Ran out of tool calls. Stop Calling.', - }); - console.log( - 'Forced short-circuit, returning error message in tool call result.', - ); - reachedToolCallLimit = true; - break; - } - - if (item.name in toolRegistry) { - const tool = toolRegistry[item.name]; - - let params: unknown; - - try { - params = tool.parseParams(JSON.parse(item.arguments)); - } catch (e) { - console.error( - `[triageNewEvidence] Error parsing parameters for tool "${item.name}" with arguments "${item.arguments}":`, - e, - ); - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Invalid parameters for tool "${item.name}". Please try again.`, - }); - continue; - } - - let result: string; - - try { - result = await tool.runner(params); - } catch (e) { - console.error( - `[triageNewEvidence] Error running tool "${item.name}":`, - e, - ); - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Tool "${item.name}" failed. Please continue without it or try a different call.`, - }); - continue; - } - - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: result, - }); - } else { - context.push({ - type: 'function_call_output', - call_id: item.call_id, - output: `Unknown tool "${item.name}". Please use one of the available tools.`, - }); - } - - break; + try { + parsedParams = tool.parseParams(functionCall.args ?? {}); + } catch (e) { + console.error( + `[triageNewEvidence] Error parsing parameters for tool "${functionCall.name}" with arguments "${JSON.stringify(functionCall.args ?? {})}":`, + e, + ); + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Invalid parameters for tool "${functionCall.name}". Please try again.`, + }), + ); + continue; } - default: { - context.push(item as ResponseInputItem); - break; + + let result: string; + + try { + result = await tool.runner(parsedParams); + } catch (e) { + console.error( + `[triageNewEvidence] Error running tool "${functionCall.name}":`, + e, + ); + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Tool "${functionCall.name}" failed. Please continue without it or try a different call.`, + }), + ); + continue; } - } - if (reachedToolCallLimit) { - break; + context.push( + toGeminiFunctionResponseContent({ functionCall, output: result }), + ); + } else { + context.push( + toGeminiFunctionResponseContent({ + functionCall, + output: `Unknown tool "${functionCall.name}". Please use one of the available tools.`, + }), + ); } } - const usage = normalizeOpenAIResponsesUsage(response.usage); - usageCostTracker.add({ model, usage }); + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); } while ( !reachedToolCallLimit && - response.output.some((item) => item.type === 'function_call') + getGeminiFunctionCalls(response).length > 0 ); - logOpenAIUsageCostSummary({ + if (reachedToolCallLimit) { + response = await getGeminiClient().models.generateContent({ + model, + contents: context, + config: buildGeminiJsonConfig({ + systemPrompt: `${systemPrompt} + +Tool-call budget is exhausted. Do not call more tools. Return the best schema-conforming final triage JSON using only the evidence and tool results already provided.`, + responseSchema: ResponseSchema, + }), + }); + usageTracker.add(normalizeGeminiUsage(response.usageMetadata)); + } + + logGeminiUsageSummary({ label: 'triageNewEvidence', - summary: usageCostTracker.summary(), + summary: usageTracker.summary(), }); - if (reachedToolCallLimit) { - throw new Error(`Exceeded tool call limit of ${TOOL_CALL_LIMIT}`); + return parseTriageResponse(response); +} + +function parseTriageResponse(response: GenerateContentResponse) { + const text = response.text; + assert(text != null && text.trim() !== '', 'Gemini response text is empty'); + + let parsedJson: unknown; + try { + parsedJson = JSON.parse(text); + } catch (error) { + throw new Error( + `Gemini response text is not valid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + return ResponseSchema.parse(normalizeTriageResponse(parsedJson)); +} + +function normalizeTriageResponse(value: unknown): unknown { + if (value == null || typeof value !== 'object') { + return value; + } + + const result = (value as { result?: unknown }).result; + if (result == null || typeof result !== 'object') { + return value; + } + + const kind = (result as { kind?: unknown }).kind; + if (typeof kind !== 'string') { + return value; } - assert(response.output_parsed != null, 'Response output parsed is null'); + const normalizedKind = + { + existing: 'part-of-existing-issue', + existing_issue: 'part-of-existing-issue', + 'existing-issue': 'part-of-existing-issue', + new: 'part-of-new-issue', + new_issue: 'part-of-new-issue', + 'new-issue': 'part-of-new-issue', + irrelevant: 'irrelevant-content', + irrelevant_content: 'irrelevant-content', + }[kind] ?? kind; + + if (normalizedKind === kind) { + return value; + } - return response.output_parsed; + return { + ...value, + result: { + ...result, + kind: normalizedKind, + }, + }; } diff --git a/packages/triage/src/llm/functions/triageNewEvidence/prompt.ts b/packages/triage/src/llm/functions/triageNewEvidence/prompt.ts index ac354777..96584405 100644 --- a/packages/triage/src/llm/functions/triageNewEvidence/prompt.ts +++ b/packages/triage/src/llm/functions/triageNewEvidence/prompt.ts @@ -33,6 +33,11 @@ TOOL USE GUIDANCE: - findIssuesByDateRange(startAt, endAt): search by issue date, evidence timestamps, and active impact periods. Use this before creating a new issue when evidence references a prior incident date such as "on May 18" or only gives generic wording like "fell in front of an oncoming train". - getIssue(issueId): inspect full evidence and current impact state for a candidate. - Once findIssues returns a plausible candidate, call getIssue for that candidate. Do not spend more than two findIssues calls on the same evidence before inspecting a candidate. +- Do not call getIssue for the same issue ID more than once. After inspecting a + candidate, immediately decide whether it overlaps. If the inspected issue has + a different disruption segment, wider/narrower unmatched scope, different + station pair, or non-overlapping maintenance window, return part-of-new-issue + without re-inspecting the same issue. - For follow-up news articles about investigations, deaths, causes, incident response, safety measures, lawsuits, or later official comments, search around the incident date and attach to the existing operational issue when the event matches. CLASSIFICATION RULES: @@ -77,6 +82,9 @@ DISRUPTIONS: - A fault "between B and C" on the same line is a different incident, even if it shares one endpoint station - A fault "between A and C" is a different incident from an existing issue "between A and B" because the new evidence extends beyond the existing scope - Example: existing issue "between Kennedy Town and Admiralty" does NOT match new evidence "between Admiralty and Causeway Bay"; return a new disruption issue. +- For the example above, after inspecting the Kennedy Town-Admiralty issue once, + return part-of-new-issue immediately. Do not call getIssue again for the same + non-overlapping issue. MAINTENANCE: - Service-level planned works that affect operating hours or service availability diff --git a/packages/triage/src/llm/models.ts b/packages/triage/src/llm/models.ts new file mode 100644 index 00000000..e85a6de4 --- /dev/null +++ b/packages/triage/src/llm/models.ts @@ -0,0 +1,6 @@ +export const GEMINI_TRIAGE_MODEL = 'gemini-3-flash-preview'; +export const GEMINI_FAST_MODEL = 'gemini-3.1-flash-lite'; + +export function modelSource(model: string) { + return `@google/${model}`; +} diff --git a/packages/triage/src/util/ingestContent/index.test.ts b/packages/triage/src/util/ingestContent/index.test.ts index 23ec6a94..07ba713c 100644 --- a/packages/triage/src/util/ingestContent/index.test.ts +++ b/packages/triage/src/util/ingestContent/index.test.ts @@ -133,7 +133,7 @@ describe('ingestContent', () => { 'en-SG': 'Downtown Line delay near Beauty World', }, titleMeta: { - source: '@openai/gpt-5.4-nano', + source: '@google/gemini-3.1-flash-lite', }, }); expect(bundle.evidence).toHaveLength(1); @@ -153,7 +153,7 @@ describe('ingestContent', () => { 'Report count: 4', ].join('\n\n'), render: { - source: '@openai/gpt-5.4-nano', + source: '@google/gemini-3.1-flash-lite', }, }); expect(bundle.impactEvents).toEqual( diff --git a/scripts/check-package-boundaries.mjs b/scripts/check-package-boundaries.mjs index 193e460e..c58363e3 100644 --- a/scripts/check-package-boundaries.mjs +++ b/scripts/check-package-boundaries.mjs @@ -48,7 +48,12 @@ for (const file of files) { continue; } - const text = readFileSync(join(repoRoot, file), 'utf8'); + const filePath = join(repoRoot, file); + if (!existsSync(filePath)) { + continue; + } + + const text = readFileSync(filePath, 'utf8'); for (const match of text.matchAll(importPattern)) { const specifier = match[1]; const dependency = specifier.match(/^@mrtdown\/([^/]+)/)?.[1];