diff --git a/.dockerignore b/.dockerignore index 891c644f7..2cc11f57e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,7 @@ databases .env.* .git .github -docs +docs/* src/test *.md *.log diff --git a/.env.example b/.env.example index 5b151e342..70c43f011 100644 --- a/.env.example +++ b/.env.example @@ -51,7 +51,6 @@ export P2P_ipV6BindTcpPort= export P2P_ipV6BindWsPort= export P2P_ANNOUNCE_ADDRESSES= export P2P_ANNOUNCE_PRIVATE= -export P2P_pubsubPeerDiscoveryInterval= export P2P_dhtMaxInboundStreams= export P2P_dhtMaxOutboundStreams= export P2P_mDNSInterval= @@ -64,14 +63,173 @@ export P2P_ENABLE_CIRCUIT_RELAY_CLIENT= export P2P_BOOTSTRAP_NODES= export P2P_FILTER_ANNOUNCED_ADDRESSES= +## p2p keys that are mapped but were undocumented here +# +# Commented out with their real defaults, for the same reason as the timeout +# budgets below: a bare `export KEY=` is NOT "unset". It reaches the config as an +# empty string, and `z.coerce.number()` turns that into 0. Measured against +# `OceanNodeP2PConfigSchema`, a blank value silently replaces the default with 0 +# (or with "") for every key in this block except P2P_DHT_FILTER, +# P2P_CIRCUIT_RELAYS and P2P_ENABLE_NETWORK_STATS - so a blank +# P2P_MAX_CONNECTIONS would cap the node at zero connections. To keep a default, +# leave the line commented out. + +# How long to wait before discovering bootstrap nodes, in ms. Default 10000. +# export P2P_BOOTSTRAP_TIMEOUT=10000 +# Tag applied to a bootstrap peer before discovering it. Default bootstrap. +# export P2P_BOOTSTRAP_TAGNAME=bootstrap +# Value of that bootstrap peer tag. Default 50. +# export P2P_BOOTSTRAP_TAGVALUE=50 +# Remove the bootstrap peer tag after this many ms. This node sets no default of +# its own - left unset, libp2p's own bootstrap TTL applies. Example: 120000. +# export P2P_BOOTSTRAP_TTL=120000 +# Port used for IPv4 secure-websocket (wss) connections. Default 9005. +# export P2P_ipV4BindWssPort=9005 +# Address filtering in the DHT: filterNone (or 0) = none, filterPrivate (or 1) +# = filter private addresses (default), filterPublic (or 2) = filter public +# addresses. An unrecognised value falls back to the default with a warning. +# export P2P_DHT_FILTER=filterPrivate +# Force this node's kad-dht into server mode instead of letting it auto-switch +# based on whether it currently has a public address. Only set this if the +# operator already knows the node is reachable. Default false. +# export P2P_DHT_FORCE_SERVER=false +# Number of circuit relay servers to reserve a slot on. Default 0. +# export P2P_CIRCUIT_RELAYS=0 +# Connection count below which libp2p dials peers from the peer book; 0 disables +# that behaviour. Default 1. +# export P2P_MIN_CONNECTIONS=1 +# Connection count above which libp2p starts pruning connections. Default 300. +# export P2P_MAX_CONNECTIONS=300 +# After a failed dial, do not auto-dial that peer again for this many ms. +# Default 120000 (2 minutes). +# export P2P_AUTODIALPEERRETRYTHRESHOLD=120000 +# How many peers from the peer book are added to the dial queue at once. +# Default 5. +# export P2P_AUTODIALCONCURRENCY=5 +# Maximum number of addresses tried for one peer before giving up. Default 30 - +# a single bootstrap peer now has around 6 addresses (tcp/ws/wss x v4/v6), and this +# budget is spent per address before the transport check runs. +# export P2P_MAXPEERADDRSTODIAL=30 +# Maximum number of dials that may be queued at once. Default 500 (libp2p's own default). +# export P2P_MAXDIALQUEUELENGTH=500 +# Time between closing a connection and opening the next auto-dialled one, in +# ms. Default 5000. +# export P2P_AUTODIALINTERVAL=5000 +# Exposes the getP2pNetworkStats HTTP endpoint. It reports private information +# such as your addresses, so it is off by default. Default false. +# export P2P_ENABLE_NETWORK_STATS=false + +## p2p timeout / attempt budgets +# +# Deliberately commented out rather than listed in the blank `export KEY=` style +# used above, for two reasons: +# +# 1. A blank value is not a default. `export P2P_SENDTO_DIAL_MS=` was measured +# reaching `OceanNodeP2PConfigSchema` as `''`, coercing to `0` and landing in +# the config as a 0 ms - i.e. instantly expired - budget, while the code used +# 15000. That is now fixed: `normalizeP2pBudget` (shared by the schema and the +# `P2P_TIMEOUTS` getters) drops a blank, non-numeric, zero or negative value +# and falls back to the default shown below. But a blank line still tells a +# reader nothing, and it was wrong for long enough to be worth avoiding. +# 2. Commented-out lines carry the actual default, so this file documents the +# value you are overriding. +# +# Rules for every key here: a positive integer, milliseconds unless stated +# otherwise. A malformed value (`15s`, `0`, `-1`, blank) is IGNORED with a +# fallback to the default - it is an operator typo, not a reason to refuse to +# boot. Values are floored to an integer. Every millisecond budget also has a +# floor of 50ms: below that a budget expires before the round trip it is meant +# to bound can finish, so such a value is ignored the same way. The two keys +# that are counts rather than durations - P2P_SENDTO_MAX_ATTEMPTS and +# P2P_COMMAND_MAX_INBOUND_STREAMS - keep a floor of 1. + +# How long a Kademlia findPeer walk may run. Default 20000 (20s). +# export P2P_FINDPEER_TIMEOUT_MS=20000 +# How long a findProviders query may run; without it kad-dht falls through to 180s. Default 20000. +# export P2P_FINDPROVIDERS_TIMEOUT_MS=20000 +# Per-frame idle budget when reading a response stream. Default 60000 (60s). +# export P2P_STREAM_IDLE_TIMEOUT_MS=60000 +# Ceiling on a WHOLE response body, measured from the first frame the caller reads. The +# idle budget above rearms on every frame, so it bounds a stall, not a transfer; this one +# bounds the transfer. Raise it if your peers legitimately stream for longer than an hour. +# Default 3600000 (60 min). +# export P2P_STREAM_BODY_TIMEOUT_MS=3600000 +# sendTo stage 1 - address resolution. Default 20000 (20s). +# export P2P_SENDTO_RESOLVE_MS=20000 +# sendTo stage 2 - dial. Default 15000 (15s). +# export P2P_SENDTO_DIAL_MS=15000 +# sendTo stage 3 - stream open + command write + status read. Default 10000 (10s). +# export P2P_SENDTO_STREAM_MS=10000 +# Overall deadline for one sendTo SETUP phase, across all attempts; does not bound +# the response body. Default 45000 (45s) = resolve + dial + stream. +# export P2P_SENDTO_TOTAL_MS=45000 +# Number of sendTo attempts, each with fresh per-stage signals. Default 2, hard cap 5 +# (a larger value is clamped to 5, not ignored). +# export P2P_SENDTO_MAX_ATTEMPTS=2 +# Budget for a contentRouting.provide() (DDO / C2D capability advertise). Default 20000 (20s). +# export P2P_ADVERTISE_TIMEOUT_MS=20000 +# Budget for a local peerStore lookup. Default 3000 (3s). +# export P2P_PEERSTORE_GET_MS=3000 +# Budget for the opportunistic dial of a newly discovered peer. Default 10000 (10s). +# export P2P_DISCOVERY_DIAL_MS=10000 +# maxInboundStreams for the ocean command protocol handler. Not a timeout. Default 32. +# export P2P_COMMAND_MAX_INBOUND_STREAMS=32 +# Overall FindDDO deadline across all providers. Default 60000 (60s). +# export P2P_FINDDDO_TIMEOUT_MS=60000 +# Budget for asking ONE provider for a DDO. Providers are queried concurrently and the +# first legitimate answer wins, so this bounds one branch rather than dividing the +# deadline above. Default 10000 (10s). +# export P2P_FINDDDO_PROVIDER_TIMEOUT_MS=10000 +# How long FindDDO remembers that a DDO id was found nowhere, to blunt a hot re-query +# loop. Consulted after the local database lookup, never before, so a DDO this node +# holds is always returned. Default 30000 (30s). +# export P2P_DDO_NOT_FOUND_CACHE_MS=30000 +# Lifetime of the app-level "these are the peer's addresses" cache. Invalidated whenever +# a dial against cached addresses fails, so a stale entry corrects itself rather than +# making a peer unreachable. Default 45000 (45s). +# export P2P_RESOLVE_CACHE_MS=45000 +# Lifetime of the negative half of that cache - "this peer resolved to nothing". Short +# on purpose: a peer that was offline comes back on its own schedule and must become +# reachable again without a restart. Default 15000 (15s). +# export P2P_RESOLVE_NEGATIVE_CACHE_MS=15000 +# Ceiling on concurrent outbound sendTo calls, so a provider fan-out or the indexer's +# decrypt loop cannot starve the dial queue. A count, not a timeout; half of +# P2P_connectionsMaxParallelDials by default, hard-capped at 200. Default 25. +# export P2P_SENDTO_MAX_CONCURRENCY=25 +# How many peers the DHT routing table must hold before the node reports its P2P +# interface as ready in the status endpoint. A count, not a timeout. Default 4 - the +# number of disjoint lookup paths one DHT query is configured to run, i.e. the smallest +# table at which a query can use the fan-out it is set up for. +# export P2P_READY_MIN_ROUTING_PEERS=4 +# Delay before kad-dht runs its first self-query, the query that populates the routing +# table. Raised above kad-dht's own 1000 because at one second a fresh node has no DHT +# peers yet - bootstrap peers are discovered after P2P_BOOTSTRAP_TIMEOUT and still have +# to be dialled - so the query failed against an empty table and was not retried for +# five minutes. Default 20000 (20s) = bootstrap timeout + discovery dial budget; raise +# it in step if you raise either of those. +# export P2P_INITIAL_QUERY_SELF_MS=20000 +# How long the peer store keeps a peer's addresses before treating them as expired. +# libp2p's own default is 3600000 (1 hour), but a DHT provider record is valid for 48 +# hours, so with the library default the network kept handing us providers whose +# addresses we had already dropped. Both this and the peer-record lifetime below match +# the provider-record lifetime instead. Lower it only if your peers change address more +# often than they re-announce. Default 172800000 (48 hours). +# export P2P_PEERSTORE_MAX_ADDRESS_AGE_MS=172800000 +# How long a peer record with no addresses survives before eviction (libp2p default +# 21600000, 6 hours). Held to at least the address lifetime above - a lower value would +# evict the record while its addresses were still valid, undoing the setting above. +# Default 172800000 (48 hours). +# export P2P_PEERSTORE_MAX_PEER_AGE_MS=172800000 + ## compute -# Each environment defines its own resources (CPU, RAM, disk, GPUs) with full configuration. -# CPU, RAM, and disk are per-env exclusive: inUse tracked only within the environment where the job runs. -# A global check ensures the aggregate usage across all environments does not exceed physical capacity. -# GPUs are shared-exclusive: if a job on envA uses gpu0, it shows as in-use on envB too. -# CPU cores are automatically partitioned across environments based on each env's cpu.total. -# CPU and RAM defaults are auto-detected from the system when not configured. -# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","environments":[{"id":"envA","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":16,"max":16,"min":1,"type":"ram"},{"id":"disk","total":500,"max":500,"min":10,"type":"disk"},{"id":"gpu0","total":1,"max":1,"min":0,"type":"gpu","init":{"deviceRequests":{"Driver":"nvidia","DeviceIDs":["0"],"Capabilities":[["gpu"]]}}}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01},{"id":"gpu0","price":5}]}]}}]}]' +# Resources are defined at the Docker-connection level (socketPath) and shared across all environments. +# cpu, ram, and disk are auto-detected from the host — omit them to use all available capacity, +# or include them to cap/reserve (e.g. limit an 8-core host to 6 cores for compute). +# GPUs and other hardware go in the connection-level "resources" array with kind:"discrete". +# Each environment references pool resources by id using lightweight refs {id, total?, min?, max?}. +# Dual-gate tracking for fungible resources: per-env ceiling (Gate 1) + engine-wide pool (Gate 2). +# Discrete resources (GPUs) are tracked globally — a GPU in use on envA shows as in-use on envB too. +# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":500},{"id":"gpu0","kind":"discrete","type":"gpu","total":1,"description":"NVIDIA A100","platform":"nvidia","driverVersion":"570.195.03","init":{"deviceRequests":{"Driver":"nvidia","DeviceIDs":["GPU-uuid-a"],"Capabilities":[["gpu"]]}}}],"environments":[{"id":"envA","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu"},{"id":"ram"},{"id":"disk","max":500},{"id":"gpu0"}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01},{"id":"gpu0","price":5}]}]}}]}]' export DOCKER_COMPUTE_ENVIRONMENTS= diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 6fe0c7a2e..000000000 --- a/.eslintignore +++ /dev/null @@ -1,4 +0,0 @@ -*.js -dist/ -.eslintrc - diff --git a/.eslintrc b/.eslintrc deleted file mode 100644 index 7f439ccba..000000000 --- a/.eslintrc +++ /dev/null @@ -1,30 +0,0 @@ -{ - "parser": "@typescript-eslint/parser", - "parserOptions": { - "sourceType": "module", - "ecmaFeatures": { - "jsx": false - } - }, - "extends": ["oceanprotocol", "plugin:prettier/recommended"], - "plugins": ["@typescript-eslint"], - "rules": { - "no-empty": ["error", { "allowEmptyCatch": true }], - "prefer-destructuring": ["warn", { "object": true, "array": false }], - "no-dupe-class-members": ["warn"], - "no-useless-constructor": ["warn"], - "constructor-super": ["warn"], - "require-await": "error", - "no-unused-vars": ["error"] - }, - "env": { - "es6": true, - "browser": true, - "mocha": true, - "node": true, - "jest": true - }, - "globals": { - "NodeJS": true - } -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05b8495f5..0fddf2548 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: '24.19.0' - name: Cache node_modules uses: actions/cache@v3 env: @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - node: ['18.20.4', 'v20.19.0', 'v22.15.0'] + node: ['24.19.0'] steps: - uses: actions/checkout@v4 @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: '24.19.0' - name: Cache node_modules uses: actions/cache@v3 env: @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: '24.19.0' - name: Cache node_modules uses: actions/cache@v3 env: @@ -118,6 +118,7 @@ jobs: with: repository: 'oceanprotocol/barge' path: 'barge' + ref: '43cfdfd21154a2bae00770b779e7c39390ff5043' - name: Login to Docker Hub if: ${{ env.DOCKERHUB_PASSWORDNONO && env.DOCKERHUB_USERNAMENONO }} run: | @@ -129,6 +130,8 @@ jobs: working-directory: ${{ github.workspace }}/barge run: | bash -x start_ocean.sh --no-node --with-typesense 2>&1 > start_ocean.log & + env: + CONTRACTS_VERSION: '2.9.0' - run: npm ci - run: npm run build - run: docker image ls @@ -194,7 +197,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: '24.19.0' - name: Cache node_modules uses: actions/cache@v3 @@ -213,6 +216,7 @@ jobs: with: repository: 'oceanprotocol/barge' path: 'barge' + ref: feature/node-v4 - name: Login to Docker Hub if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }} run: | @@ -240,7 +244,6 @@ jobs: with: repository: 'oceanprotocol/ocean.js' path: 'ocean.js' - ref: main - name: Build ocean-js working-directory: ${{ github.workspace }}/ocean.js run: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9a81f4571..9e56541c3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,8 @@ on: pull_request: branches: - 'main' + - 'next-4' + - 'deps/node_v24' env: DOCKERHUB_IMAGE: ${{ 'oceanprotocol/ocean-node' }} diff --git a/.nvmrc b/.nvmrc index 42126c054..60ade1ae0 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1,2 +1 @@ -22 - +24.19.0 diff --git a/CLAUDE.md b/CLAUDE.md old mode 100755 new mode 100644 index 036f57125..0fa68b217 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,30 +2,43 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - Ocean Node is the all-in-one backend for the Ocean Protocol stack. A single Node process replaces three legacy components: **Provider** (data access / encryption / compute), **Aquarius** (metadata cache) and the **subgraph** (on-chain event indexing). It is a -TypeScript ESM project (Node 22) that exposes an HTTP API and a libp2p P2P interface, both +TypeScript ESM project (Node 24) that exposes an HTTP API and a libp2p P2P interface, both of which dispatch to the same set of command handlers. --- ## 1. Environment & tooling prerequisites -- **Node.js 22 is required** (`.nvmrc` pins `22`). Always run `nvm use` (or - `source ~/.nvm/nvm.sh && nvm use`) before any `npm`, build, or test command. The wrong - Node version fails with errors like `Unexpected token 'with'` or missing `GLIBC_2.38`. +- **Node.js 24 is required** (`.nvmrc` pins `24.19.0` — the current Node 24 LTS "Krypton" — + matching the Dockerfile and CI; `package.json` `engines` requires `>=24`, so Node 22 is + no longer a supported runtime). The Dockerfile additionally pins each base + image by digest, so bumping the version means updating the `sha256:` alongside the tag — + resolve it with `docker buildx imagetools inspect node:-trixie`, not from the tag + alone. Always run `nvm use` (or `source ~/.nvm/nvm.sh && nvm use`) before + any `npm`, build, or test command. The wrong Node version fails with errors like + `Unexpected token 'with'`, missing `GLIBC_2.38`, or — since the SQLite layer uses the + built-in `node:sqlite` module — `ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite` on Node < 22.13. This is enforced by `.cursor/rules/tests-nvm.mdc` and the in-repo `CLAUDE.md`. -- If `sqlite3` native bindings break after switching to Node 22, rebuild from source: - `npm_config_build_from_source=true npm rebuild sqlite3`. -- **`postinstall` runs `scripts/fix-libp2p-http-utils.js`** — a patch applied to a libp2p - dependency. Expect it to run on every `npm install`; don't remove it. +- **There is no `postinstall` step.** `npm install` is plain. (Historically a `postinstall` + ran `scripts/fix-libp2p-http-utils.js` to default a missing URL port to 443/80 in + `@libp2p/http-utils`; upstream shipped that fix in `2.0.3`, so both the hook and the + script were removed.) - **Docker + docker-compose** are needed for the metadata database (Typesense or Elasticsearch) and for C2D (Compute-to-Data) via the local Docker socket. -- TypeScript config: ESM (`module: esnext`, `target: ES2022`, `moduleResolution: node`), - `experimentalDecorators` + `emitDecoratorMetadata` enabled, `rootDir: ./src`, - `outDir: ./dist`. All local imports use the `.js` extension (compiled ESM convention). +- TypeScript config: **TypeScript 6**, ESM (`module: esnext`, `target: ES2022`, + `moduleResolution: node`), `experimentalDecorators` + `emitDecoratorMetadata` enabled, + `rootDir: ./src`, `outDir: ./dist`. Most local imports use the `.js` extension + (compiled ESM convention), though ~31 relative imports still omit it. + - `strict` is **on**, with two documented exceptions: `strictNullChecks: false` and + `useUnknownInCatchVariables: false`. Turning either on is a real project (~1324 and + ~389 errors); do it subsystem by subsystem, not in passing. + - Two things must be done before TypeScript 7: `moduleResolution` has to move off + `node10` (`ignoreDeprecations: "6.0"` only defers it, and `nodenext` currently costs + ~123 errors), and `typescript-eslint` has to support TS 7 — its peer range is + `>=4.8.4 <6.1.0`, so it pins us below 7 today. ### Only-mandatory config: `PRIVATE_KEY` @@ -64,19 +77,27 @@ The Dockerfile uses the identical CMD. The compiled entry point is `dist/index.j ### Lint / format ```bash -npm run lint # eslint (.ts,.tsx) + type-check +npm run lint # eslint + type-check npm run lint:fix # eslint --fix npm run format # prettier --write '**/*.{js,jsx,ts,tsx}' ``` -ESLint extends `oceanprotocol` + `prettier/recommended`. Notable rules: `require-await` -is an **error**, `no-unused-vars` is an **error**, empty catch blocks are allowed. Prettier: +ESLint 10 with flat config in `eslint.config.js` (there is no `.eslintrc`/`.eslintignore`). +`eslint-config-oceanprotocol` is **not** used — it is pinned to eslint ^8 and cannot follow +eslint to flat config, so the preset is composed in-repo from `@eslint/js`, +`typescript-eslint`, `eslint-plugin-security`, `eslint-plugin-promise` and +`eslint-plugin-prettier`. Only `**/*.ts` is linted; `.js` is ignored as build output. +Notable rules: `require-await` is an **error**, `no-unused-vars` is an **error** (with +`args: 'none'`, `caughtErrors: 'none'`), empty catch blocks are allowed. `@typescript-eslint`'s +`recommended` set is deliberately *not* extended, and a few rules that eslint 9/10 and the +newer plugins added are switched off — see the comments in `eslint.config.js`. Prettier: no semicolons, single quotes, `printWidth: 90`, no trailing commas, 2-space tabs. ### Tests (important build quirk) **Tests run against compiled JS in `dist/test/`, not the TypeScript source.** The `test:*` scripts all call `npm run build-tests` first, which: + - compiles `src/` (incl. `src/test`) into `dist/`, - copies `src/test/.env.test` and `.env.test2` into `dist/test`, - copies `src/test/config.json` to `$HOME/config.json`. @@ -189,7 +210,7 @@ the command, looks up the handler by `task.command`, and calls `handler.handle(t Two front doors, one dispatcher: - **HTTP `POST /directCommand`** (`src/components/httpRoutes/commands.ts`): validates the - body, then decides *local vs remote*. If the command targets this node (or no P2P), it + body, then decides _local vs remote_. If the command targets this node (or no P2P), it calls `oceanNode.handleDirectProtocolCommand(...)`. If it targets another peer and P2P is enabled, it forwards via `oceanNode.getP2PNode().sendTo(node, msg, multiAddrs)`. Responses are streamed back to the client (binary or text). @@ -225,6 +246,7 @@ register it in the `CoreHandlersRegistry` constructor; add param validation; opt REST route in `src/components/httpRoutes/` and mount it in `httpRoutes/index.ts`. Handler source is grouped under `src/components/core/`: + - `handler/` — general handlers (ddo, download, encrypt, fees, nonce, query, status, p2p, auth, accessList, escrow, fileInfo, persistentStorage, policyServer, getJobs). - `compute/` — C2D command handlers: `initialize`, `startCompute` (paid), `freeStartCompute`, @@ -263,13 +285,14 @@ Handler source is grouped under `src/components/core/`: `getComputeResult` (+ `getComputeStreamableLogs`) → `stopCompute`. Paid compute settles via the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). - See `docs/compute-pricing.md`, `docs/GPU.md`. + See `docs/compute.md`. - **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, indexer state, logs, orders, ddoState, access lists, escrow events — behind the `Abstract*Database` interfaces in `BaseDatabase.ts`. **SQLite** is always used for the nonce DB, config DB, C2D job DB, and auth-token DB (works even with no metadata DB - configured). See `docs/database.md`. + configured) — via Node's built-in `node:sqlite` module (no native addon), wrapped by + `SqliteClient` in `src/components/database/sqliteClient.ts`. See `docs/database.md`. - **KeyManager/** — provider-abstraction over the node key (`docs/KeyManager.md`). Currently `RawPrivateKeyProvider` (from `PRIVATE_KEY`); derives the libp2p peerId/keys and the EVM address, and caches the ethers signer. Designed to add KMS providers (GCP/AWS) later. @@ -292,8 +315,9 @@ Handler source is grouped under `src/components/core/`: `accessList.ts`, `asset.ts`, `attestation.ts`. - Runtime data dirs: `databases/` (SQLite files + libp2p LevelDB store), `c2d_storage/` (C2D job working data), `logs/`, `schemas/` (SHACL DDO validation schemas — shipped into the - Docker image), `docs/serviceTemplates/` (operator service-on-demand templates, referenced by - `SERVICE_TEMPLATES_PATH`). + Docker image). Service-on-demand templates are not shipped: the node reads + them from the folder given by `serviceTemplatesPath` / `SERVICE_TEMPLATES_PATH`, which the + operator mounts in at run time. - `tsoa.json` configures OpenAPI spec generation from `src/components/httpRoutes/**`; the actual routing is plain Express routers, not tsoa-generated. @@ -301,9 +325,9 @@ Handler source is grouped under `src/components/core/`: ## 5. Docker & deployment -Multi-stage `Dockerfile` (builder + slim runner) on `node:22`. The runner ships only -`dist/`, `node_modules`, `schemas/`, `config.json`, and `docs/serviceTemplates/` -(`.dockerignore` excludes the rest of `docs/`). It exposes P2P ports `9000-9003,9005` and +Multi-stage `Dockerfile` (builder + slim runner) on `node:24`. The runner ships only +`dist/`, `node_modules`, `schemas/`, and `config.json` (`.dockerignore` excludes all of +`docs/`) — no service templates. It exposes P2P ports `9000-9003,9005` and HTTP `8000`. `docker-entrypoint.sh` handles Docker socket group membership at runtime so C2D can talk to `/var/run/docker.sock`. Deployment options (Docker, local Docker build via `quickstart`, PM2, plain npm) are in `README.md`; production deployment details in @@ -316,5 +340,5 @@ can talk to `/var/run/docker.sock`. Deployment options (Docker, local Docker bui `Arhitecture.md` (note the spelling), `API.md` (full HTTP API reference — very large, plus a Postman collection), `env.md` (authoritative env-var reference), `database.md`, `Storage.md` / `persistentStorage.md`, `KeyManager.md`, `PolicyServer.md`, `services.md` -(Service-on-Demand), `compute-pricing.md` / `GPU.md` (C2D), `networking.md`, `Logs.md`, +(Service-on-Demand), `compute.md` (C2D configuration: resources, GPUs, constraints, pricing), `networking.md`, `Logs.md`, `Publishing.md`, `testing.md`, `dockerDeployment.md`. diff --git a/Dockerfile b/Dockerfile index 1567fa7e6..fe6f98041 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22.22.2-trixie@sha256:17ccc50fade521c62e2acefd0c975bf5eb2a09632b8717fa7f8b1c2b4e967a07 AS builder +FROM node:24.19.0-trixie@sha256:66bb8d36ae1ddd72199ed235a089904874ca4079ee517936ca3adb80506a75c1 AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ python3 \ build-essential \ @@ -14,7 +14,7 @@ COPY . . RUN npm run build && npm prune --omit=dev -FROM node:22.22.2-trixie-slim@sha256:76043ed3132293c26b960ede4358d3c8ba424ee64662cd2d56318b76fcc51c4c AS runner +FROM node:24.19.0-trixie-slim@sha256:0711b541c1c33a8a530ac4f0d391baa9a15b3d804695b1b24a47daa5fb60e74d AS runner RUN apt-get update && apt-get install -y --no-install-recommends \ dumb-init \ gosu \ @@ -44,10 +44,22 @@ COPY --chown=node:node --from=builder /usr/src/app/schemas ./schemas COPY --chown=node:node --from=builder /usr/src/app/package.json ./ COPY --chown=node:node --from=builder /usr/src/app/config.json ./ -RUN mkdir -p databases c2d_storage logs +# `databases` holds everything the node must not lose across a restart: the SQLite files +# (nonce, config, C2D jobs, auth tokens) and `databases/p2p-store`, the LevelDatastore that +# backs libp2p. kad-dht's reprovider refreshes this node's provider records from that +# datastore, so if it starts empty the node stops answering for content it still holds, and the +# records expire out of the network without anything noticing. The directory is created and +# handed to the unprivileged `node` user here, and VOLUME declares the mount point so a named +# volume or bind mount can be attached; docker-entrypoint.sh re-applies ownership at runtime +# for the bind-mount case, where the host directory arrives owned by whoever created it. +# A persistent mount here is REQUIRED, not optional - see "Persistent node data" in +# docs/dockerDeployment.md. +RUN mkdir -p databases/p2p-store c2d_storage logs \ + && chown -R node:node databases c2d_storage logs +VOLUME ["/usr/src/app/databases"] COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN chmod +x /usr/local/bin/docker-entrypoint.sh ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] -CMD ["node", "--max-old-space-size=28784", "--trace-warnings", "--experimental-specifier-resolution=node", "dist/index.js"] +CMD ["node", "--import", "./dist/telemetry/otel.js", "--max-old-space-size=28784", "--trace-warnings", "--experimental-specifier-resolution=node", "dist/index.js"] diff --git a/README.md b/README.md index c3d1d33ba..6a5233b23 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ This command will run you through the process of setting up the environmental va > [!NOTE] > The quickstart script attempts to automatically detect GPUs (NVIDIA via `nvidia-smi`, others via `lspci`) and appends them to your `DOCKER_COMPUTE_ENVIRONMENTS`. +> Detected GPUs are added to `DOCKER_COMPUTE_ENVIRONMENTS[0].resources` (the connection-level resource pool), and a lightweight ref is added to `DOCKER_COMPUTE_ENVIRONMENTS[0].environments[0].resources`. > If you choose to manually configure `DOCKER_COMPUTE_ENVIRONMENTS` before running the script (e.g. via environment variable), be aware that auto-detected GPUs will be **merged** into your configuration, which could lead to duplication if you already manually defined them. > For most users, it is recommended to let the script handle GPU detection automatically. @@ -153,5 +154,5 @@ Your node is now running. To start additional nodes, repeat these steps in a new - [Network Configuration](docs/networking.md) - [Logging & accessing logs](docs/networking.md) - [Docker Deployment Guide](docs/dockerDeployment.md) -- [C2D GPU Guide](docs/GPU.md) -- [Compute pricing](docs/compute-pricing.md) +- [Compute (C2D) Configuration — resources, GPUs, constraints, pricing](docs/compute.md) +- [Services (Service-on-Demand)](docs/services.md) diff --git a/config.json b/config.json index 42d364183..8bd75acda 100644 --- a/config.json +++ b/config.json @@ -24,13 +24,12 @@ "ipV6BindTcpPort": 9002, "ipV6BindWsPort": 9003, "announceAddresses": [], - "pubsubPeerDiscoveryInterval": 10000, "dhtMaxInboundStreams": 500, "dhtMaxOutboundStreams": 500, - "dhtFilter": null, + "dhtFilter": "filterPrivate", "mDNSInterval": 20000, - "connectionsMaxParallelDials": 15, - "connectionsDialTimeout": 30000, + "connectionsMaxParallelDials": 50, + "connectionsDialTimeout": 15000, "upnp": true, "autoNat": true, "enableCircuitRelayServer": false, @@ -55,7 +54,7 @@ "maxConnections": 300, "autoDialPeerRetryThreshold": 7200000, "autoDialConcurrency": 5, - "maxPeerAddrsToDial": 5, + "maxPeerAddrsToDial": 30, "autoDialInterval": 5000, "enableNetworkStats": false }, @@ -94,18 +93,31 @@ "validateUnsignedDDO": true, "jwtSecret": "ocean-node-secret", "enableBenchmark": false, + "serviceTemplatesPath": "databases/serviceTemplates/", "dockerComputeEnvironments": [ { "socketPath": "/var/run/docker.sock", + "resources": [ + { + "id": "disk", + "total": 1 + } + ], "environments": [ { "storageExpiry": 604800, "maxJobDuration": 3600, "minJobDuration": 60, "resources": [ + { + "id": "cpu" + }, + { + "id": "ram" + }, { "id": "disk", - "total": 1 + "max": 1 } ], "access": { diff --git a/deploy/telemetry/docker-compose.telemetry.yml b/deploy/telemetry/docker-compose.telemetry.yml new file mode 100644 index 000000000..5df7a136c --- /dev/null +++ b/deploy/telemetry/docker-compose.telemetry.yml @@ -0,0 +1,45 @@ +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1 + command: ["--config=/etc/otel-collector.yaml"] + volumes: ["./otel-collector.yaml:/etc/otel-collector.yaml:ro"] + # Bound to loopback only: this dev stack has an open OTLP receiver and an anonymous-admin + # Grafana, neither of which should be reachable on all host interfaces by default. + ports: ["127.0.0.1:4318:4318", "127.0.0.1:4317:4317"] + depends_on: [prometheus, tempo] + prometheus: + image: prom/prometheus:v3.1.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./rules.yml:/etc/prometheus/rules.yml:ro + ports: ["127.0.0.1:9090:9090"] + tempo-init: + image: grafana/tempo:2.7.0 + user: root + entrypoint: ["chown", "-R", "10001:10001", "/var/tempo"] + volumes: ["tempo-data:/var/tempo"] + tempo: + image: grafana/tempo:2.7.0 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./tempo.yaml:/etc/tempo.yaml:ro + - tempo-data:/var/tempo + ports: ["127.0.0.1:3200:3200"] + depends_on: [tempo-init] + grafana: + image: grafana/grafana:11.5.1 + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + GF_AUTH_DISABLE_LOGIN_FORM: "true" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: ["127.0.0.1:3001:3000"] + depends_on: [prometheus, tempo] +volumes: + tempo-data: diff --git a/deploy/telemetry/grafana/dashboards/ocean-node-compute.json b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json new file mode 100644 index 000000000..64ef3d84b --- /dev/null +++ b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json @@ -0,0 +1,1433 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "type": "row", + "title": "CPU & memory", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Total CPU usage %", + "description": "sum(ocean_compute_cpu_usage_percent) — Σ container CPU usage across all engines.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_cpu_usage_percent{service_instance_id=~\"$instance\"})", + "legendFormat": "cpu %", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 3, + "type": "timeseries", + "title": "CPU used ratio (vs host cores)", + "description": "Recording rule ocean_node:compute_cpu_used_ratio = Σ cpu_usage_percent / (Σ host_cores × 100).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_node:compute_cpu_used_ratio", + "legendFormat": "used ratio", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Cores allocated vs host cores", + "description": "Declared allocation vs physical capacity, per engine cluster.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_cpu_cores_allocated{service_instance_id=~\"$instance\"})", + "legendFormat": "allocated", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_cpu_host_cores{service_instance_id=~\"$instance\"})", + "legendFormat": "host cores", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 5, + "type": "stat", + "title": "Total RAM used", + "description": "sum(ocean_compute_memory_used_bytes) across all engines.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_memory_used_bytes{service_instance_id=~\"$instance\"})", + "legendFormat": "ram used", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 6, + "type": "timeseries", + "title": "RAM used ratio", + "description": "Recording rule ocean_node:compute_mem_used_ratio = Σ used_bytes / Σ limit_bytes.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_node:compute_mem_used_ratio", + "legendFormat": "used ratio", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Disk used", + "description": "sum(ocean_compute_disk_used_bytes) across all engines.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_disk_used_bytes{service_instance_id=~\"$instance\"})", + "legendFormat": "disk used", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "row", + "title": "Network & jobs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "panels": [] + }, + { + "id": 9, + "type": "timeseries", + "title": "Network (current aggregate)", + "description": "Point-in-time snapshot gauge, NOT a rate — container network counters are cumulative per ephemeral container, so a fleet sum cannot be rate()’d meaningfully. See docs/telemetry.md.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_network_rx_bytes{service_instance_id=~\"$instance\"})", + "legendFormat": "rx", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_network_tx_bytes{service_instance_id=~\"$instance\"})", + "legendFormat": "tx", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Running vs queued jobs", + "description": "ocean_compute_jobs_running / ocean_compute_jobs_queued by free (true/false).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (free) (ocean_compute_jobs_running{service_instance_id=~\"$instance\"})", + "legendFormat": "running free={{free}}", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (free) (ocean_compute_jobs_queued{service_instance_id=~\"$instance\"})", + "legendFormat": "queued free={{free}}", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Jobs started / finished rate", + "description": "ocean_compute_jobs_started_total vs ocean_compute_jobs_finished_total by status.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(ocean_compute_jobs_started_total{service_instance_id=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "started", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status) (rate(ocean_compute_jobs_finished_total{service_instance_id=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "finished {{status}}", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "row", + "title": "GPU", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "panels": [] + }, + { + "id": 13, + "type": "gauge", + "title": "GPU total utilization", + "description": "avg(ocean_compute_gpu_utilization_percent) across all GPUs / engines.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(ocean_compute_gpu_utilization_percent{service_instance_id=~\"$instance\"})", + "legendFormat": "gpu util", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Per-GPU utilization", + "description": "ocean_compute_gpu_utilization_percent, one series per device.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_gpu_utilization_percent{service_instance_id=~\"$instance\"}", + "legendFormat": "{{gpu}} {{vendor}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "GPU memory used / total", + "description": "Per-GPU ocean_compute_gpu_memory_used_bytes vs ocean_compute_gpu_memory_total_bytes; ratio via ocean_node:compute_gpu_mem_used_ratio.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_gpu_memory_used_bytes{service_instance_id=~\"$instance\"}", + "legendFormat": "used {{gpu}} {{vendor}}", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_gpu_memory_total_bytes{service_instance_id=~\"$instance\"}", + "legendFormat": "total {{gpu}} {{vendor}}", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "stat", + "title": "GPU memory used ratio", + "description": "Recording rule ocean_node:compute_gpu_mem_used_ratio.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 35 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_node:compute_gpu_mem_used_ratio", + "legendFormat": "used ratio", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 17, + "type": "timeseries", + "title": "GPU temperature", + "description": "ocean_compute_gpu_temperature_celsius, one series per device.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 35 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_gpu_temperature_celsius{service_instance_id=~\"$instance\"}", + "legendFormat": "{{gpu}} {{vendor}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "GPU power draw", + "description": "ocean_compute_gpu_power_watts, one series per device.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 35 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_gpu_power_watts{service_instance_id=~\"$instance\"}", + "legendFormat": "{{gpu}} {{vendor}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 19, + "type": "row", + "title": "Pools, throttling & data freshness", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "panels": [] + }, + { + "id": 20, + "type": "stat", + "title": "GPU devices in use", + "description": "sum(ocean_compute_gpu_devices_in_use) — deduped by resourceId across shared jobs.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 44 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_gpu_devices_in_use{service_instance_id=~\"$instance\"})", + "legendFormat": "devices in use", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 21, + "type": "stat", + "title": "CPU throttled containers", + "description": "sum(ocean_compute_cpu_throttled_containers) — containers with throttledPeriods > 0.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 44 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(ocean_compute_cpu_throttled_containers{service_instance_id=~\"$instance\"})", + "legendFormat": "throttled", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 22, + "type": "stat", + "title": "Sample freshness guard", + "description": "max(ocean_compute_oldest_sample_age_seconds). Alert if it exceeds 2× C2D_METRICS_INTERVAL_SECONDS — a stale value silently going flat looks identical to zero load otherwise.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 44 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(ocean_compute_oldest_sample_age_seconds{service_instance_id=~\"$instance\"})", + "legendFormat": "oldest sample age", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 23, + "type": "table", + "title": "Per-env pool utilization", + "description": "ocean_compute_env_resource_inuse / ocean_compute_env_resource_total by (env, resource).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 9, + "w": 18, + "x": 0, + "y": 52 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_compute_env_resource_inuse{service_instance_id=~\"$instance\"} / clamp_min(ocean_compute_env_resource_total{service_instance_id=~\"$instance\"}, 0.0001)", + "legendFormat": "{{env}} — {{resource}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "align": "auto" + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true, + "instance": true + } + } + } + ] + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "ocean", + "ocean-node", + "compute" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(ocean_compute_cpu_host_cores, service_instance_id)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(ocean_compute_cpu_host_cores, service_instance_id)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Ocean Node — Compute Resources", + "uid": "ocean-node-compute", + "version": 1, + "weekStart": "" +} diff --git a/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json b/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json new file mode 100644 index 000000000..82218edf9 --- /dev/null +++ b/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json @@ -0,0 +1,1303 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "type": "row", + "title": "Connections & DHT", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "timeseries", + "title": "Connected peers", + "description": "Active libp2p connections by direction.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (direction) (ocean_p2p_connections{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"})", + "legendFormat": "{{direction}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "DHT routing-table peers", + "description": "Size of the local Kademlia routing table.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_p2p_dht_routing_table_peers{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}", + "legendFormat": "{{service_instance_id}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 4, + "type": "stat", + "title": "DHT mode", + "description": "1 = server, 0 = client (from dht.getMode()).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_p2p_dht_mode{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}", + "legendFormat": "dht mode", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 5, + "type": "row", + "title": "P2P health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "panels": [] + }, + { + "id": 6, + "type": "stat", + "title": "P2P ready", + "description": "getP2PStatus().ready per node (0/1).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 10 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_p2p_ready{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}", + "legendFormat": "{{service_instance_id}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 7, + "type": "timeseries", + "title": "sendTo rate by outcome", + "description": "ocean_p2p_sendto_total split by outcome (ok/fail).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 10 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(ocean_p2p_sendto_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "sendTo failure reasons", + "description": "The 7 bounded SENDTO_FAIL_REASONS.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 10 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (reason) (rate(ocean_p2p_sendto_total{outcome=\"fail\", service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{reason}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "stat", + "title": "sendTo failure rate", + "description": "Recording rule ocean_node:p2p_sendto_fail_rate — 5m window.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_node:p2p_sendto_fail_rate", + "legendFormat": "fail rate", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Resolve hit/miss", + "description": "ocean_p2p_resolve_total by result (peerstore_hit/dht_hit/miss).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (result) (rate(ocean_p2p_resolve_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{result}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "stat", + "title": "Resolve miss rate", + "description": "Recording rule ocean_node:p2p_resolve_miss_rate — 5m window.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 18 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_node:p2p_resolve_miss_rate", + "legendFormat": "miss rate", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 12, + "type": "row", + "title": "Dial queue, relays & peer churn", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "panels": [] + }, + { + "id": 13, + "type": "timeseries", + "title": "Dial queue", + "description": "ocean_p2p_dial_queue by status (queued/active).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status) (ocean_p2p_dial_queue{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"})", + "legendFormat": "{{status}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Relay reservations", + "description": "Active circuit-relay reservations held by this node.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ocean_p2p_relay_reservations{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}", + "legendFormat": "{{service_instance_id}} {{kind}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Peer connect / disconnect rate", + "description": "ocean_p2p_peer_connect_total vs ocean_p2p_peer_disconnect_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(ocean_p2p_peer_connect_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "connect", + "range": true, + "instant": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(ocean_p2p_peer_disconnect_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "disconnect", + "range": true, + "instant": false, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "row", + "title": "DHT provide / find", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "panels": [] + }, + { + "id": 17, + "type": "timeseries", + "title": "DHT provide outcomes", + "description": "ocean_p2p_dht_provide_total by outcome.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(ocean_p2p_dht_provide_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "DHT find outcomes", + "description": "ocean_p2p_dht_find_total by kind (providers/peer) and outcome.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (kind, outcome) (rate(ocean_p2p_dht_find_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__rate_interval]))", + "legendFormat": "{{kind}} — {{outcome}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 19, + "type": "row", + "title": "Certificates, commands & fleet", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "panels": [] + }, + { + "id": 20, + "type": "stat", + "title": "autoTLS cert expiry", + "description": "Seconds until the AutoTLS certificate expires. Red below 7 days (604800s).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "min(ocean_p2p_autotls_cert_expiry_seconds{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"})", + "legendFormat": "expiry", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 259200 + }, + { + "color": "green", + "value": 604800 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + }, + { + "id": 21, + "type": "barchart", + "title": "Inbound command rate by command (top 15)", + "description": "ocean_p2p_command_total over the dashboard range, bounded to SUPPORTED_PROTOCOL_COMMANDS.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(15, sum by (command) (rate(ocean_p2p_command_total{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"}[$__range])))", + "legendFormat": "{{command}}", + "range": false, + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 1, + "fillOpacity": 80 + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "xTickLabelRotation": 0, + "showValue": "auto", + "legend": { + "showLegend": false, + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 22, + "type": "stat", + "title": "Fleet: nodes reporting / ready share", + "description": "Only meaningful in Ocean-internal (fleet) mode. count() = nodes emitting p2p.ready; avg() = share ready.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 45 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "count(ocean_p2p_ready{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"})", + "legendFormat": "nodes reporting", + "range": false, + "instant": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(ocean_p2p_ready{service_instance_id=~\"$instance\", ocean_node_role=~\"$role\"})", + "legendFormat": "ready share", + "range": false, + "instant": true, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + } + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "ocean", + "ocean-node", + "p2p" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(ocean_p2p_ready, service_instance_id)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(ocean_p2p_ready, service_instance_id)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(ocean_p2p_ready, ocean_node_role)", + "hide": 0, + "includeAll": true, + "label": "Role", + "multi": true, + "name": "role", + "options": [], + "query": "label_values(ocean_p2p_ready, ocean_node_role)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Ocean Node — P2P", + "uid": "ocean-node-p2p", + "version": 1, + "weekStart": "" +} diff --git a/deploy/telemetry/grafana/provisioning/dashboards/dashboards.yaml b/deploy/telemetry/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 000000000..9d00fe95e --- /dev/null +++ b/deploy/telemetry/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: ocean-node + folder: Ocean Node + type: file + allowUiUpdates: false + updateIntervalSeconds: 30 + options: { path: /var/lib/grafana/dashboards } diff --git a/deploy/telemetry/grafana/provisioning/datasources/datasources.yaml b/deploy/telemetry/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 000000000..2e8b6ab8f --- /dev/null +++ b/deploy/telemetry/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: ocean-node-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: { httpMethod: POST, exemplarTraceIdDestinations: [{ name: trace_id, datasourceUid: ocean-node-tempo }] } + - name: Tempo + uid: ocean-node-tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: { tracesToMetrics: { datasourceUid: ocean-node-prometheus }, nodeGraph: { enabled: true } } diff --git a/deploy/telemetry/otel-collector.internal.yaml b/deploy/telemetry/otel-collector.internal.yaml new file mode 100644 index 000000000..f68f3f47f --- /dev/null +++ b/deploy/telemetry/otel-collector.internal.yaml @@ -0,0 +1,63 @@ +# Ocean-internal collector variant. +# +# The Ocean-internal deployment (4 bootstraps + relay tier + core nodes) is a bounded, +# dozens-scale fleet, so this uses the SAME pipeline shape as otel-collector.yaml +# (operator-local) — resource_to_telemetry_conversion stays enabled and +# `service.instance.id` (the node's peerId) is kept on every metric series, giving +# per-node dashboards. See plan §3 / §8.3. +# +# What actually differs from the operator-local file, before this is used for real: +# 1. Auth on the OTLP receiver. This file's `otlp` receiver is the open dev receiver — +# fine on localhost, not fine on a shared endpoint. Put a token/header check in front +# of it (e.g. an `headers_setter`/auth extension validating the header the pushing +# nodes send via `OTEL_EXPORTER_OTLP_HEADERS`), or terminate it behind a reverse proxy +# that enforces the token before traffic reaches the collector. +# 2. Persistent storage + retention sized for the fleet. The `prometheusremotewrite` / +# `otlp/tempo` exporters below point at the same local Prometheus/Tempo used for +# operator-local dev; a real deployment swaps these for durable storage (self-hosted +# Prometheus with a persistent volume + real retention, or Grafana Cloud Mimir/Tempo — +# swap `prometheusremotewrite.endpoint` for the Mimir `/api/v1/push` URL + auth headers, +# and `otlp/tempo.endpoint` for the Tempo `:443` endpoint, per on-mcp's telemetry README). +# +# Both are open decisions (plan §12, items 1-2) — this file is the starting point, not a +# finished hardened config. + +receivers: + otlp: + protocols: + http: { endpoint: 0.0.0.0:4318 } + grpc: { endpoint: 0.0.0.0:4317 } +processors: + memory_limiter: { check_interval: 1s, limit_mib: 512, spike_limit_mib: 128 } + batch: { timeout: 10s, send_batch_size: 8192 } + attributes/scrub: # defence-in-depth; the app never sets these on metrics + actions: + - { key: privateKey, action: delete } + - { key: private_key, action: delete } + - { key: PRIVATE_KEY, action: delete } + - { key: consumerAddress, action: delete } + - { key: did, action: delete } + - { key: jobId, action: delete } + - { key: nonce, action: delete } + - { key: signature, action: delete } + # Fallback only — if the core fleet ever grows to thousands of instances, drop instance + # identity from METRICS (keep it on traces for drill-down) with this processor. Not enabled + # by default: uncomment this block, add `transform/strip_instance` to the metrics pipeline + # below, and set `prometheusremotewrite.resource_to_telemetry_conversion.enabled: false`. + # + # transform/strip_instance: + # metric_statements: + # - context: resource + # statements: [ 'delete_key(attributes, "service.instance.id")' ] +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: { insecure: true } + resource_to_telemetry_conversion: { enabled: true } # bounded fleet: keep instance id + otlp/tempo: + endpoint: tempo:4317 + tls: { insecure: true } +service: + pipelines: + metrics: { receivers: [otlp], processors: [memory_limiter, attributes/scrub, batch], exporters: [prometheusremotewrite] } + traces: { receivers: [otlp], processors: [memory_limiter, attributes/scrub, batch], exporters: [otlp/tempo] } diff --git a/deploy/telemetry/otel-collector.yaml b/deploy/telemetry/otel-collector.yaml new file mode 100644 index 000000000..987308682 --- /dev/null +++ b/deploy/telemetry/otel-collector.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + http: { endpoint: 0.0.0.0:4318 } + grpc: { endpoint: 0.0.0.0:4317 } +processors: + memory_limiter: { check_interval: 1s, limit_mib: 512, spike_limit_mib: 128 } + batch: { timeout: 10s, send_batch_size: 8192 } + attributes/scrub: # defence-in-depth; the app never sets these on metrics + actions: + - { key: privateKey, action: delete } + - { key: private_key, action: delete } + - { key: PRIVATE_KEY, action: delete } + - { key: consumerAddress, action: delete } + - { key: did, action: delete } + - { key: jobId, action: delete } + - { key: nonce, action: delete } + - { key: signature, action: delete } +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: { insecure: true } + resource_to_telemetry_conversion: { enabled: true } # operator-local: keep instance id + otlp/tempo: + endpoint: tempo:4317 + tls: { insecure: true } +service: + pipelines: + metrics: { receivers: [otlp], processors: [memory_limiter, attributes/scrub, batch], exporters: [prometheusremotewrite] } + traces: { receivers: [otlp], processors: [memory_limiter, attributes/scrub, batch], exporters: [otlp/tempo] } diff --git a/deploy/telemetry/prometheus.yml b/deploy/telemetry/prometheus.yml new file mode 100644 index 000000000..ac2fb2d10 --- /dev/null +++ b/deploy/telemetry/prometheus.yml @@ -0,0 +1,5 @@ +global: { scrape_interval: 15s } +rule_files: [/etc/prometheus/rules.yml] +scrape_configs: + - job_name: prometheus + static_configs: [{ targets: ['localhost:9090'] }] diff --git a/deploy/telemetry/rules.yml b/deploy/telemetry/rules.yml new file mode 100644 index 000000000..89344eb9d --- /dev/null +++ b/deploy/telemetry/rules.yml @@ -0,0 +1,15 @@ +groups: + - name: ocean-node-compute + rules: + - record: ocean_node:compute_cpu_used_ratio + expr: sum(ocean_compute_cpu_usage_percent) / clamp_min(sum(ocean_compute_cpu_host_cores) * 100, 0.0001) + - record: ocean_node:compute_mem_used_ratio + expr: sum(ocean_compute_memory_used_bytes) / clamp_min(sum(ocean_compute_memory_limit_bytes), 0.0001) + - record: ocean_node:compute_gpu_mem_used_ratio + expr: sum(ocean_compute_gpu_memory_used_bytes) / clamp_min(sum(ocean_compute_gpu_memory_total_bytes), 0.0001) + - name: ocean-node-p2p + rules: + - record: ocean_node:p2p_sendto_fail_rate + expr: sum(rate(ocean_p2p_sendto_total{outcome="fail"}[5m])) / clamp_min(sum(rate(ocean_p2p_sendto_total[5m])), 0.0001) + - record: ocean_node:p2p_resolve_miss_rate + expr: sum(rate(ocean_p2p_resolve_total{result="miss"}[5m])) / clamp_min(sum(rate(ocean_p2p_resolve_total[5m])), 0.0001) diff --git a/deploy/telemetry/scripts/import-dashboard.sh b/deploy/telemetry/scripts/import-dashboard.sh new file mode 100755 index 000000000..11b9e9a62 --- /dev/null +++ b/deploy/telemetry/scripts/import-dashboard.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# Upload an ocean-node telemetry dashboard to a running Grafana via the HTTP API. +# +# Only needed when you are NOT using file provisioning (the local docker-compose stack already +# provisions both dashboards on boot). Use this for Grafana Cloud or an existing shared Grafana. +# +# GRAFANA_URL=https://your-org.grafana.net \ +# GRAFANA_TOKEN=glsa_xxx \ +# ./deploy/telemetry/scripts/import-dashboard.sh [p2p|compute] +# +# Minting a token: Grafana -> Administration -> Users and access -> Service accounts -> +# Add service account -> role "Editor" -> Add service account token. Copy it into GRAFANA_TOKEN. +# +# Optional: +# GRAFANA_FOLDER_UID target folder (default: the "General" folder) +# DASHBOARD_FILE path to the dashboard JSON (overrides the p2p|compute shorthand) + +set -euo pipefail + +cd "$(dirname "$0")/../../.." + +WHICH="${1:-p2p}" +case "$WHICH" in + p2p) DEFAULT_FILE="deploy/telemetry/grafana/dashboards/ocean-node-p2p.json" ;; + compute) DEFAULT_FILE="deploy/telemetry/grafana/dashboards/ocean-node-compute.json" ;; + *) DEFAULT_FILE="$WHICH" ;; # allow passing a path directly +esac + +# Exported, not just assigned: the final `node -e` block reads it from `process.env`. +export GRAFANA_URL="${GRAFANA_URL:-http://localhost:3001}" +DASHBOARD_FILE="${DASHBOARD_FILE:-$DEFAULT_FILE}" + +if [ ! -f "$DASHBOARD_FILE" ]; then + echo "dashboard JSON not found: $DASHBOARD_FILE" >&2 + echo "usage: $0 [p2p|compute|]" >&2 + exit 1 +fi + +if [ -z "${GRAFANA_TOKEN:-}" ]; then + echo "GRAFANA_TOKEN is required (service-account token with Editor rights)." >&2 + echo "See the header of this script for how to mint one." >&2 + exit 1 +fi + +# The dashboards declare a DS_PROMETHEUS datasource variable. Resolve it to the datasource uid on +# the target Grafana so the imported copy is immediately usable. +PROM_UID="${PROM_UID:-}" +[ -z "$PROM_UID" ] && PROM_UID=$( + curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" \ + | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{try{const j=JSON.parse(s);const m=j.find(d=>d.type==='prometheus');console.log(m?m.uid:'')}catch{console.log('')}})" +) + +if [ -z "$PROM_UID" ]; then + echo "No Prometheus datasource found on $GRAFANA_URL — add one first, or set PROM_UID." >&2 + exit 1 +fi + +echo "Importing $DASHBOARD_FILE -> $GRAFANA_URL" +echo " Prometheus datasource: $PROM_UID" + +PAYLOAD=$( + DASHBOARD_FILE="$DASHBOARD_FILE" \ + PROM_UID="$PROM_UID" \ + FOLDER_UID="${GRAFANA_FOLDER_UID:-}" \ + node -e ' + const fs = require("fs") + const dashboard = JSON.parse(fs.readFileSync(process.env.DASHBOARD_FILE, "utf8")) + // Strip the id so Grafana creates or updates by uid rather than rejecting a stale id. + dashboard.id = null + const inputs = [ + { name: "DS_PROMETHEUS", type: "datasource", pluginId: "prometheus", value: process.env.PROM_UID } + ] + const body = { dashboard, overwrite: true, inputs } + if (process.env.FOLDER_UID) body.folderUid = process.env.FOLDER_UID + process.stdout.write(JSON.stringify(body)) + ' +) + +RESPONSE=$( + curl -fsS -X POST "$GRAFANA_URL/api/dashboards/import" \ + -H "Authorization: Bearer $GRAFANA_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "$PAYLOAD" +) + +echo "$RESPONSE" | node -e ' + let s = "" + process.stdin.on("data", (d) => (s += d)).on("end", () => { + try { + const j = JSON.parse(s) + console.log(`\nImported: ${j.title ?? "dashboard"}`) + console.log(`URL: ${process.env.GRAFANA_URL}${j.importedUrl ?? ""}`) + } catch { + console.log(s) + } + }) +' diff --git a/deploy/telemetry/scripts/verify-telemetry.sh b/deploy/telemetry/scripts/verify-telemetry.sh new file mode 100755 index 000000000..b5189814f --- /dev/null +++ b/deploy/telemetry/scripts/verify-telemetry.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# End-to-end telemetry check for ocean-node: brings up the local collector/Prometheus/Tempo/ +# Grafana stack, and — if the environment is already configured to run a node (PRIVATE_KEY set, +# etc., per docs/env.md) — starts one with telemetry enabled and asserts that P2P metrics land in +# Prometheus and the dashboards are provisioned. +# +# ./deploy/telemetry/scripts/verify-telemetry.sh +# +# Exits non-zero with a summary if any check fails. Safe to re-run; it cleans up what it starts. +# +# Flags: +# --keep leave the stack (and the node, if started) running afterwards +# --no-up assume the stack is already running +# --no-node don't attempt to start a node — just verify the stack + provisioning +# +# Starting a real ocean-node requires a working node config (PRIVATE_KEY, RPCS, etc. — see +# docs/env.md). This script does not fabricate one: if PRIVATE_KEY isn't already set in the +# environment, it verifies the telemetry stack only and tells you how to drive traffic yourself. + +set -uo pipefail + +# Guarded: with `set -uo pipefail` (no -e) a failed cd would silently run every check against the +# caller's directory. +cd "$(dirname "$0")/../../.." || { echo "cannot cd to repo root" >&2; exit 2; } + +COMPOSE_FILE="deploy/telemetry/docker-compose.telemetry.yml" +PROM_URL="${PROM_URL:-http://localhost:9090}" +TEMPO_URL="${TEMPO_URL:-http://localhost:3200}" +GRAFANA_URL="${GRAFANA_URL:-http://localhost:3001}" + +KEEP=0 +DO_UP=1 +DO_NODE=1 +for arg in "$@"; do + case "$arg" in + --keep) KEEP=1 ;; + --no-up) DO_UP=0 ;; + --no-node) DO_NODE=0 ;; + *) echo "unknown flag: $arg" >&2; exit 2 ;; + esac +done + +PASS=0 +FAIL=0 +RESULTS=() + +ok() { PASS=$((PASS+1)); RESULTS+=(" PASS $1"); printf ' \033[32mPASS\033[0m %s\n' "$1"; } +bad() { FAIL=$((FAIL+1)); RESULTS+=(" FAIL $1"); printf ' \033[31mFAIL\033[0m %s\n' "$1"; } +step() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } + +BUILD_LOG=$(mktemp -t ocean-node-build.XXXXXX) +SERVER_LOG=$(mktemp -t ocean-node-server.XXXXXX) + +SERVER_PID="" +cleanup() { + rm -f "$BUILD_LOG" "$SERVER_LOG" + if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + if [ "$KEEP" -eq 0 ] && [ "$DO_UP" -eq 1 ]; then + docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +need() { command -v "$1" >/dev/null 2>&1 || { echo "missing required command: $1" >&2; exit 2; }; } +need curl +need docker +need node + +# ── 1. stack ──────────────────────────────────────────────────────────────────────────────── +if [ "$DO_UP" -eq 1 ]; then + step "Starting telemetry stack" + docker compose -f "$COMPOSE_FILE" up -d +fi + +wait_for() { # url, label, attempts + local url="$1" label="$2" tries="${3:-60}" + for _ in $(seq 1 "$tries"); do + if curl -fsS -o /dev/null "$url" 2>/dev/null; then ok "$label is up"; return 0; fi + sleep 2 + done + bad "$label did not become ready at $url" + return 1 +} + +step "Waiting for backends" +wait_for "$PROM_URL/-/ready" "Prometheus" +wait_for "$TEMPO_URL/ready" "Tempo" +wait_for "$GRAFANA_URL/api/health" "Grafana" + +# ── 2. dashboards provisioned ─────────────────────────────────────────────────────────────── +step "Checking Grafana provisioning" +if curl -fsS "$GRAFANA_URL/api/dashboards/uid/ocean-node-p2p" >/dev/null 2>&1; then + ok 'dashboard ocean-node-p2p is provisioned' +else + bad 'dashboard ocean-node-p2p was not found in Grafana' +fi +if curl -fsS "$GRAFANA_URL/api/dashboards/uid/ocean-node-compute" >/dev/null 2>&1; then + ok 'dashboard ocean-node-compute is provisioned' +else + bad 'dashboard ocean-node-compute was not found in Grafana' +fi + +# ── 3. build + run a node (best-effort — needs a real node config) ───────────────────────── +if [ "$DO_NODE" -eq 1 ] && [ -n "${PRIVATE_KEY:-}" ]; then + step "Building ocean-node" + if npm run build >"$BUILD_LOG" 2>&1; then + ok "build succeeded" + else + bad "build failed:" + tail -20 "$BUILD_LOG" >&2 || true + exit 1 + fi + + step "Starting ocean-node with telemetry enabled" + export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" + export OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-ocean-node}" + export DEPLOYMENT_ENVIRONMENT=verify + # Export fast so the script does not wait a full minute for the first metric flush. + export OTEL_METRIC_EXPORT_INTERVAL=5000 + + node --import ./dist/telemetry/otel.js dist/index.js >"$SERVER_LOG" 2>&1 & + SERVER_PID=$! + + for _ in $(seq 1 45); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + bad "node exited during startup — see log below" + tail -30 "$SERVER_LOG" || true + exit 1 + fi + if grep -q '\[telemetry\] enabled' "$SERVER_LOG" 2>/dev/null; then break; fi + sleep 2 + done + + if kill -0 "$SERVER_PID" 2>/dev/null; then + ok "node is running (pid $SERVER_PID)" + else + bad "node is not running" + fi + + if grep -q '\[telemetry\] enabled' "$SERVER_LOG" 2>/dev/null; then + ok "telemetry reported itself enabled" + else + bad "no telemetry-enabled line seen in node output — telemetry did not start" + fi + + step "Waiting for export (metric interval ${OTEL_METRIC_EXPORT_INTERVAL}ms + collector batch)" + sleep 25 + + step "Checking Prometheus" + + prom_value() { # promql -> scalar (empty when no series) + curl -fsS --get "$PROM_URL/api/v1/query" --data-urlencode "query=$1" 2>/dev/null \ + | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const j=JSON.parse(s);const r=j.data?.result??[];console.log(r.length?r[0].value[1]:"")}catch{console.log("")}})' + } + + check_present() { # promql, label + if [ -n "$(prom_value "$1")" ]; then + ok "$2" + else + bad "$2 — no data for: $1" + fi + } + + check_present 'ocean_p2p_ready' 'ocean_p2p_ready is present' + check_present 'ocean_p2p_connections' 'ocean_p2p_connections is present' + check_present 'ocean_p2p_dht_routing_table_peers' 'ocean_p2p_dht_routing_table_peers is present' + + # Cardinality guard: peerId/did/jobId must never appear as metric labels. + # A selector matches `