From 79e5b819d197ccd99520e9379b9661520beb6d4b Mon Sep 17 00:00:00 2001 From: Brendan Ryan <1572504+brendanjryan@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:21:25 +0000 Subject: [PATCH 1/2] fix: improve unpaid service health coverage Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- README.md | 15 ++++++++ package.json | 2 +- schemas/services.ts | 22 ++++++++++- scripts/service-health/check.test.ts | 56 +++++++++++++++++++++++++++- scripts/service-health/http.test.ts | 5 ++- scripts/service-health/report.ts | 41 ++++++++++++++++++++ scripts/service-health/run.ts | 9 ++++- 7 files changed, 144 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3f20832d..54a7d350 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,21 @@ docker compose down Set `TEMPO_LOCALNET_IMAGE` to pin a release tag or image digest. Set `TEMPO_RPC_URL` to run the same test against an existing localnet. +Audit the service directory without payment or credentials: + +```bash +pnpm check:services # All listed services +pnpm check:services --service=codex # One service +``` + +The report includes link availability, endpoint probe results, and per-service +endpoint coverage. A reachable homepage is not evidence of a working paid API. +POST routes require an explicitly safe `healthCheck` fixture in +`schemas/services.ts`; routes with side effects remain skipped. These probes +verify payment Challenges, not payment or fulfillment. The complete JSON report +includes a `services` summary alongside individual results. +The checker respects configured HTTP proxies and system certificate authorities. + ### Publish a blog post 1. Copy [`templates/blog-post.mdx`](templates/blog-post.mdx) to `src/pages/blog/.mdx`. diff --git a/package.json b/package.json index 709f7bae..ca60d4be 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "check:ci": "biome check", "check:markdown": "node scripts/check-markdown-components.mjs", "check:generated": "pnpm generate:pages && git diff --exit-code -- src/pages.gen.ts", - "check:services": "node scripts/service-health/run.ts", + "check:services": "NODE_USE_ENV_PROXY=1 node --use-system-ca scripts/service-health/run.ts", "check:sdk-drift": "tsx .github/actions/sdk-drift-check/check-sdk-drift.ts", "check:types": "tsc --noEmit", "dev": "node scripts/generate-discovery.ts && node scripts/copy-vocs-theme.ts && vite dev", diff --git a/schemas/services.ts b/schemas/services.ts index 96582457..08c79bc8 100644 --- a/schemas/services.ts +++ b/schemas/services.ts @@ -1423,7 +1423,12 @@ export const services: ServiceDef[] = [ { route: "POST /v1/scrape", desc: "Scrape a URL", amount: "2000" }, { route: "POST /v1/crawl", desc: "Crawl a website", amount: "5000" }, { route: "POST /v1/map", desc: "Map website URLs", amount: "2000" }, - { route: "POST /v1/search", desc: "Search the web", amount: "4000" }, + { + healthCheck: { body: { query: "Machine Payments Protocol", limit: 1 } }, + route: "POST /v1/search", + desc: "Search the web", + amount: "4000", + }, { route: "POST /v1/extract", desc: "Extract structured data", @@ -4548,6 +4553,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { domain: "example.com" } }, route: "POST /abstract-company-enrichment/lookup", desc: "Lookup", amount: "6000", @@ -4828,6 +4834,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { content: "Machine Payments Protocol" } }, route: "POST /diffbot-nl/analyze", desc: "Analyze Text", dynamic: true, @@ -4934,6 +4941,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { email: "test@example.com" } }, route: "POST /abstract-email-reputation/check", desc: "Check", amount: "6000", @@ -4967,6 +4975,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { base: "USD" } }, route: "POST /abstract-exchange-rates/live", desc: "Live Rates", amount: "6000", @@ -5114,6 +5123,9 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { + body: { country: "US", year: "2026", month: "1", day: "1" }, + }, route: "POST /abstract-holidays/lookup", desc: "Lookup", amount: "6000", @@ -5219,6 +5231,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { iban: "DE89370400440532013000" } }, route: "POST /abstract-iban-validation/validate", desc: "Validate", amount: "6000", @@ -5251,6 +5264,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { ip_address: "8.8.8.8" } }, route: "POST /abstract-ip-intelligence/lookup", desc: "Lookup", amount: "6000", @@ -6189,6 +6203,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { location: "San Francisco, CA" } }, route: "POST /abstract-timezone/current-time", desc: "Current Time", amount: "6000", @@ -6240,6 +6255,7 @@ export const services: ServiceDef[] = [ unitType: "request", }, { + healthCheck: { body: {} }, route: "POST /abstract-vat/categories", desc: "Categories", amount: "6000", @@ -6272,6 +6288,7 @@ export const services: ServiceDef[] = [ payments: [TEMPO_PAYMENT], endpoints: [ { + healthCheck: { body: { url: "https://example.com" } }, route: "POST /abstract-web-scraping/scrape", desc: "Scrape", amount: "6000", @@ -7601,6 +7618,9 @@ export const services: ServiceDef[] = [ dynamic: true, }, { + healthCheck: { + body: { query: "Machine Payments Protocol", max_results: 1 }, + }, route: "POST /search", desc: "Execute a search query using Tavily Search.", dynamic: true, diff --git a/scripts/service-health/check.test.ts b/scripts/service-health/check.test.ts index ae498c88..37d4ed1c 100644 --- a/scripts/service-health/check.test.ts +++ b/scripts/service-health/check.test.ts @@ -14,7 +14,7 @@ import { } from "./checks.ts"; import { checkProbe, type Result } from "./http.ts"; import { type Issue, syncIssue } from "./issues.ts"; -import { renderReport } from "./report.ts"; +import { renderReport, summarizeServices } from "./report.ts"; const service: ServiceDef = { categories: [], @@ -363,6 +363,60 @@ describe("execution", () => { }); describe("reporting", () => { + it("does not count link checks as endpoint coverage", () => { + const results: Result[] = [ + { ...result, service: "link-only", target: "link", outcome: "pass" }, + { ...result, service: "skipped-only", outcome: "skipped", attempts: 0 }, + { ...result, service: "blocked", outcome: "inconclusive" }, + { ...result, service: "verified", outcome: "pass" }, + ]; + expect(summarizeServices(results)).toEqual([ + { + service: "link-only", + endpoints: 0, + attempted: 0, + passed: 0, + failed: 0, + inconclusive: 0, + skipped: 0, + }, + { + service: "skipped-only", + endpoints: 1, + attempted: 0, + passed: 0, + failed: 0, + inconclusive: 0, + skipped: 1, + }, + { + service: "blocked", + endpoints: 1, + attempted: 1, + passed: 0, + failed: 0, + inconclusive: 1, + skipped: 0, + }, + { + service: "verified", + endpoints: 1, + attempted: 1, + passed: 1, + failed: 0, + inconclusive: 0, + skipped: 0, + }, + ]); + const report = renderReport(results, "today", "run"); + expect(report).toContain( + "2/4 services have an attempted endpoint probe; 1/4 have at least one passing endpoint probe", + ); + expect(report).toContain( + "2 services have no attempted endpoint probes: link-only, skipped-only.", + ); + }); + it("renders findings as a bounded Markdown table with escaped cells", () => { const body = renderReport( [ diff --git a/scripts/service-health/http.test.ts b/scripts/service-health/http.test.ts index 7b7eb10f..3de9808b 100644 --- a/scripts/service-health/http.test.ts +++ b/scripts/service-health/http.test.ts @@ -152,7 +152,10 @@ describe("actionability", () => { "run", ); expect(report).toContain("1 skipped"); - expect(report).not.toContain("fixture-needed"); + expect(report).toContain("no attempted endpoint probes: fixture-needed."); + expect(report).not.toContain( + "| Service | Endpoint or link | HTTP | Finding | URL |", + ); }); }); diff --git a/scripts/service-health/report.ts b/scripts/service-health/report.ts index 9321ef9d..dcb64c34 100644 --- a/scripts/service-health/report.ts +++ b/scripts/service-health/report.ts @@ -1,6 +1,31 @@ import type { Result } from "./http.ts"; export const issueMarker = ""; + +export function summarizeServices(results: Result[]) { + const services = new Map(); + for (const result of results) { + const findings = services.get(result.service) ?? []; + findings.push(result); + services.set(result.service, findings); + } + return [...services].map(([service, findings]) => { + const endpoints = findings.filter((result) => result.target !== "link"); + return { + service, + endpoints: endpoints.length, + attempted: endpoints.filter((result) => result.attempts > 0).length, + passed: endpoints.filter((result) => result.outcome === "pass").length, + failed: endpoints.filter((result) => result.outcome === "fail").length, + inconclusive: endpoints.filter( + (result) => result.outcome === "inconclusive", + ).length, + skipped: endpoints.filter((result) => result.outcome === "skipped") + .length, + }; + }); +} + export function renderReport( results: Result[], checkedAt: string, @@ -12,6 +37,9 @@ export function renderReport( results.filter((result) => result.outcome === outcome).length, ]), ); + const summaries = summarizeServices(results); + const untested = summaries.filter((service) => service.attempted === 0); + const verified = summaries.filter((service) => service.passed > 0); const lines = [ issueMarker, "## Weekly service health", @@ -21,6 +49,19 @@ export function renderReport( "", `${counts.pass} passed; ${counts.fail} failed; ${counts.inconclusive} inconclusive; ${counts.skipped} skipped.`, "", + `Endpoint coverage: ${summaries.length - untested.length}/${summaries.length} services have an attempted endpoint probe; ${verified.length}/${summaries.length} have at least one passing endpoint probe. Link checks do not count as endpoint coverage.`, + "", + ...(untested.length + ? [ + `${untested.length} services have no attempted endpoint probes: ${untested + .slice(0, 100) + .map((service) => service.service.replace(/[^a-zA-Z0-9-]/g, "")) + .join( + ", ", + )}${untested.length > 100 ? ", … (see the JSON report)" : ""}.`, + "", + ] + : []), "These checks do not pay or verify fulfillment. Skipped and inconclusive checks are coverage gaps, not evidence of downtime. GET/HEAD redirects follow up to three HTTPS hops. Each unique request is made once per run.", "", "Update broken catalog URLs, contact the provider for repeat failures, or add safe endpoint fixtures in schemas/services.ts. Do not remove services based only on a failed API root URL.", diff --git a/scripts/service-health/run.ts b/scripts/service-health/run.ts index b33adac5..5f124d53 100644 --- a/scripts/service-health/run.ts +++ b/scripts/service-health/run.ts @@ -6,7 +6,7 @@ import { services } from "../../schemas/services.ts"; import { checkServices } from "./check.ts"; import { checkMpp, checkUrls } from "./checks.ts"; import { type Issue, syncIssue } from "./issues.ts"; -import { renderReport } from "./report.ts"; +import { renderReport, summarizeServices } from "./report.ts"; const args = process.argv.slice(2); if (args.some((arg) => arg !== "--issue" && !/^--service=.+$/.test(arg))) @@ -39,7 +39,12 @@ const body = renderReport(results, checkedAt, runUrl); writeFileSync( join(outputDirectory, "results.json"), JSON.stringify( - { checkedAt, results, revision: process.env.GITHUB_SHA }, + { + checkedAt, + results, + revision: process.env.GITHUB_SHA, + services: summarizeServices(results), + }, null, 2, ), From 3a39e1c356b35a6532101c24088416056b5f6d3f Mon Sep 17 00:00:00 2001 From: Brendan Ryan <1572504+brendanjryan@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:39:04 +0000 Subject: [PATCH 2/2] docs: remove service health README notes Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/README.md b/README.md index 54a7d350..3f20832d 100644 --- a/README.md +++ b/README.md @@ -53,21 +53,6 @@ docker compose down Set `TEMPO_LOCALNET_IMAGE` to pin a release tag or image digest. Set `TEMPO_RPC_URL` to run the same test against an existing localnet. -Audit the service directory without payment or credentials: - -```bash -pnpm check:services # All listed services -pnpm check:services --service=codex # One service -``` - -The report includes link availability, endpoint probe results, and per-service -endpoint coverage. A reachable homepage is not evidence of a working paid API. -POST routes require an explicitly safe `healthCheck` fixture in -`schemas/services.ts`; routes with side effects remain skipped. These probes -verify payment Challenges, not payment or fulfillment. The complete JSON report -includes a `services` summary alongside individual results. -The checker respects configured HTTP proxies and system certificate authorities. - ### Publish a blog post 1. Copy [`templates/blog-post.mdx`](templates/blog-post.mdx) to `src/pages/blog/.mdx`.