diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9028c33 --- /dev/null +++ b/.env.example @@ -0,0 +1,35 @@ +# Replay executor configuration. Copy to `.env` and adjust. +# Real environment variables always override values in `.env`. + +# --- Redis --------------------------------------------------------------- + +# Which client to benchmark: phpredis | relay | predis (predis not yet supported) +REPLAY_REDIS_CLIENT=phpredis + +REPLAY_REDIS_HOST=127.0.0.1 +REPLAY_REDIS_PORT=6379 +REPLAY_REDIS_PASSWORD= +REPLAY_REDIS_DB=0 +REPLAY_REDIS_TLS=0 +# Reuse one connection per worker (recommended, matches production drop-ins) +REPLAY_REDIS_PERSISTENT=1 +REPLAY_REDIS_TIMEOUT=2.0 + +# --- Corpus -------------------------------------------------------------- + +# Directory of captured traces (NNNNN.json + manifest.json) +REPLAY_CORPUS=./corpus + +# --- SQL ----------------------------------------------------------------- + +# sleep = reproduce each query's captured duration as a usleep (no DB needed) +# execute = run the captured queries against MySQL (needs a matching schema/data) +REPLAY_SQL_MODE=sleep + +# Only used when REPLAY_SQL_MODE=execute +REPLAY_DB_HOST=127.0.0.1 +REPLAY_DB_PORT=3306 +REPLAY_DB_NAME= +REPLAY_DB_USER= +REPLAY_DB_PASSWORD= +REPLAY_DB_CHARSET=utf8mb4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8776bd6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Local secrets / config +.env + +# Captured corpus working dir (keep the committed example traces) +corpus/raw/ diff --git a/CAPTURE.md b/CAPTURE.md new file mode 100644 index 0000000..e007d60 --- /dev/null +++ b/CAPTURE.md @@ -0,0 +1,153 @@ +# Building the request-capture mechanism + +> Handoff instructions for a future Claude session. Read this top to bottom, +> then read `corpus/00000.json` and `corpus/manifest.json` — those are the +> canonical, hand-authored examples of the exact output you must produce. + +## Goal + +Capture real WordPress request data from a live site so it can be **replayed** +later without WordPress or MySQL (see `TODO.md` and `replay.js`). For each of +~10–100 sampled requests, record the interleaved timeline of **PHP compute +gaps**, **Redis commands** (reads + writes, verbatim), and **SQL queries** +(timing + full text), and write one JSON trace file per request into `corpus/`. + +The replay side already exists in skeleton form (`stubs/replay-server.php` +serves traces by `?id=N`; `replay.js` drives it). Your job is the **capture** +side and a small **manifest build** step. Do **not** change the replay/metrics +behavior of `replay.js` or `lib/metrics.js`. + +## Output format (authoritative) + +The format is fully specified by the examples in this repo: +- `corpus/00000.json` — one captured request (a homepage hit: alloptions read, + an options miss → SQL → SET, a main-query SQL, a pipelined write, a pipelined + read). **Match this schema exactly.** +- `corpus/manifest.json` — the corpus index. + +Key rules (all illustrated in the example): +- **One file per request**, zero-padded id (`00000.json`), id matches `?id=N`. +- `timeline` is an **ordered, interleaved** list of events. Durations there are + **integer microseconds** (`us`). `summary` uses **float milliseconds** + (`ms_*`) and field names that match the existing metric vocabulary in + `lib/metrics.js` (`hit_ratio`, `bytes`, `sql_queries`, `redis_reads`/`writes` + ↔ `store-reads`/`store-writes`, `ms_total` ↔ `ms-total`). +- Event types: `php` (compute gap → replay `usleep`), `redis` (one network + round-trip carrying 1+ commands), `sql` (query → replay `usleep`, text kept + for analysis). +- `redis.mode` ∈ `single` | `pipeline` | `multi` — preserves round-trip + structure (round trips dominate latency; do not flatten pipelines). +- **Verbatim Redis values**, **binary-safe**: each element of `cmd.args` is a + plain JSON string when it is valid UTF-8, otherwise the object + `{ "b64": "" }`. Keys stay readable; serialized/compressed payloads + (igbinary, lz4/zstd) are base64. Also record `value_bytes`/`ttl` on writes and + `reply_bytes`/`hit` on reads. + +Consider also emitting a `corpus/schema.json` (JSON Schema) and validating +output against it. + +## What to build + +1. A capture **mu-plugin** modeled on `stubs/k6-metrics.php` (same coding + style, same `wp_object_cache` detection). Suggested: + `stubs/k6-capture.php`. +2. A small **manifest build** step that scans the written trace files and emits + `corpus/manifest.json`. Suggested: `bin/build-corpus.php` (or `.sh`). Keep + capture and manifest separate so capture stays race-free (see below). + +## Where the data comes from + +### Redis commands + per-op timing — investigate first, then wrap + +There is no portable per-command hook in PhpRedis. Options, in order of +preference: +1. **Check OCP first.** Object Cache Pro may already expose command logging / + an analytics log you can tap (the user maintains OCP — confirm the current + API rather than guessing). If it yields ordered commands with timing, use it. +2. **Decorate the client.** Otherwise wrap the drop-in's Redis client + (`$wp_object_cache->redis_instance()` for Redis Object Cache; OCP's client + accessor) in a logging proxy that, for every command, records + `{ cmd, args, t_start, t_end }` using `hrtime(true)`. + - For **pipelines/MULTI**, wrap the client's `pipeline()` / `multi()` so + commands buffered until `exec()` are grouped into one `redis` event with + the right `mode`; the round-trip `us` is measured across the `exec()`. + - Capture `hit` from read replies (non-null/found), `reply_bytes` from the + raw reply length, `value_bytes`/`ttl` from write args. + +Do **not** use Redis `MONITOR` — it drops under load, hides pipelining, and +skews timing. + +### SQL — `$wpdb->queries` + +- Ensure `define('SAVEQUERIES', true)` for capture runs (document this; it adds + overhead so it must be capture-only). +- After the request, read `$wpdb->queries`; each entry is + `[0] => SQL, [1] => seconds (float), [2] => caller`. Convert seconds → `us`. + `rows` is not in `$wpdb->queries` — leave it out or best-effort. + +### Timing totals & PHP gaps + +- `summary.ms_total` from `$timestart`/the OCP footnote (`ms-total`). +- **PHP gaps are derived, not measured directly:** order all Redis + SQL ops by + their `hrtime` start, then a `php` event's `us` = gap between the previous + op's end and the next op's start. Prepend the bootstrap gap (request start → + first op) and append the tail gap (last op end → response). The sum of all + `php` + `redis` + `sql` durations should ≈ `ms_total`. + +## Sampling / triggering + +Capture must be **opt-in per request** so you control exactly which URLs become +traces and avoid capturing admin/cron/REST noise: +- Trigger on a request header/cookie/query flag (e.g. `X-K6-Capture: 1` or + `?k6_capture=1`), set only by the operator driving the capture (curl/k6 over a + fixed URL list). Bail early otherwise. +- Reuse the bail-out guards from `stubs/k6-metrics.php` (skip WP-CLI, cron, + AJAX, REST, JSON, robots, etc.). +- Write the trace on the `shutdown` hook (after the response, like + `k6-metrics.php`'s `maybePrint`). + +## Writing files & id assignment (avoid races) + +Multiple PHP-FPM workers may capture concurrently — do **not** assign sequential +ids at write time. Instead: +- Capture writes to a raw dir with a collision-free name, e.g. + `corpus/raw/--.json`, written atomically (temp file + + `rename()`). +- `bin/build-corpus.php` later sorts raw files (by capture time or a provided + URL order), assigns sequential ids `0..N-1`, writes `corpus/NNNNN.json`, and + emits `corpus/manifest.json`. This is also where you can dedupe by URL or cap + the count. + +## Gotchas + +- **Binary values**: OCP serializes (igbinary) and may compress (lz4/zstd) — + values are binary; you MUST base64 them per the encoding rule. Verify a + round-trip (`base64_decode` reproduces the exact bytes). +- **Persistent connections / object cache state**: capture reflects whatever was + warm at request time; the `hit`/`miss` flags record that. Replay's warm-vs-cold + handling is a separate concern (`TODO.md`). +- **SAVEQUERIES overhead**: capture-only; never enable on the benchmarked SUT. +- **Don't let capture itself pollute the trace**: exclude the plugin's own + Redis/file I/O from the recorded ops. +- Keep file sizes sane: a homepage `alloptions` blob can be large; verbatim + + base64 inflates ~33%. Fine for ~100 requests; gzip (`*.json.gz`) only if + needed. + +## Verification + +1. Stand up (or point at) a WordPress site with an object cache drop-in. +2. Drive a known URL with the capture trigger; confirm a raw file appears. +3. Run `bin/build-corpus.php`; confirm `corpus/NNNNN.json` + `manifest.json` + are produced and that a produced trace **validates against the example + schema** (same keys/shape as `corpus/00000.json`). +4. Round-trip a `{b64}` value: `base64_decode` equals the original bytes. +5. Sanity-check timing: `sum(php+redis+sql us) ≈ ms_total * 1000`. +6. End-to-end: once `stubs/replay-server.php` is updated (later phase) to read + `corpus/`, confirm `k6 run replay.js` drives the real corpus. + +## Out of scope (later phases) + +- The real replay executor (firing the captured commands via Relay/PhpRedis in + `stubs/replay-server.php`). +- The PhpBench subject (separate track in `TODO.md`). +- Do **not** modify `replay.js` or `lib/metrics.js`. diff --git a/README.md b/README.md index d55d822..2d4b2f6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,83 @@ k6 run woo-customer.js --env SITE_URL=https://example.com --env BYPASS_CACHE=1 This script requires [seeded users](#seeding-users). +### `replay.js` + +Replays a fixed corpus of captured requests ("traces") against a stateless +replay endpoint, instead of driving a real WordPress site. Every run executes +the *identical* workload — the same traces in the same per-VU order — so +results are comparable across runs and across horizontally scaled load +generators. k6 owns which trace runs when; the server just executes whatever +`id` it is handed. + +By default it fires 100 unique traces, each repeated until 10,000 requests are +reached, as fast as the system under test allows (closed-loop). + +```bash +k6 run replay.js --env SITE_URL=http://localhost:8080 +k6 run replay.js --env SITE_URL=http://localhost:8080 --env VUS=200 +``` + +The load profile is selectable via `--env SCENARIO=`: + +```bash +# fixed (default): exactly TOTAL requests, each trace run equally often +k6 run replay.js --env SITE_URL=http://localhost:8080 --env SCENARIO=fixed --env TOTAL=10000 + +# ramping: ramp concurrency up/down to find the saturation point +k6 run replay.js --env SITE_URL=http://localhost:8080 --env SCENARIO=ramping \ + --env STAGES="30s:50,1m:200,2m:200,30s:0" + +# constant: hold VUS for a duration +k6 run replay.js --env SITE_URL=http://localhost:8080 --env SCENARIO=constant --env VUS=100 --env DURATION=2m +``` + +Tunable via env vars: `SCENARIO` (`fixed`/`ramping`/`constant`/`shared`), +`TRACES` (unique traces, default 100), `TOTAL` (total requests, default 10000), +`VUS` (concurrency, default 100), `STAGES`, `DURATION`, `START_VUS`, +`MAX_DURATION`, `REPLAY_PATH` (endpoint path, default `/render`). + +Metrics are collected the same way as the other scripts — from the [Object +Cache Pro footnote](lib/metrics.js) comment in the response body (enable OCP's +`analytics.footnote` on the real SUT). + +To scale across multiple load-generator machines, use k6 execution segments — +the trace mapping is segment-safe, so coverage stays complete and reproducible: + +```bash +# machine 1 of 4 +k6 run replay.js --env SITE_URL=http://lb \ + --execution-segment "0:1/4" \ + --execution-segment-sequence "0,1/4,2/4,3/4,1" +``` + +There are two endpoints to drive with `replay.js`: + +**Real executor — [`replay/index.php`](./replay/index.php)** replays captured +traces (`corpus/NNNNN.json`) against a real Redis using the configured client +(phpredis/relay), and emits a measured Object Cache Pro-style footnote so the +metrics above are real. Configure it with environment variables — copy +[`.env.example`](./.env.example) to `.env` and adjust: + +```bash +cp .env.example .env # set REPLAY_REDIS_* (and REPLAY_REDIS_CLIENT) +php -S 0.0.0.0:8080 replay/index.php +k6 run replay.js --env SITE_URL=http://localhost:8080 +``` + +SQL is reproduced as a sleep by default (`REPLAY_SQL_MODE=sleep`, no database +needed); set `REPLAY_SQL_MODE=execute` with `REPLAY_DB_*` to run the captured +queries against MySQL instead. For real benchmarking, serve via nginx + +PHP-FPM so the corpus and connections are preloaded/persisted per worker. + +**Dummy — [`stubs/replay-server.php`](./stubs/replay-server.php)** needs no +Redis: it fakes the work and prints a fake footnote, useful for exercising +`replay.js` and the metrics pipeline offline. + +```bash +php -S 0.0.0.0:8080 stubs/replay-server.php +``` + ## Reset WooCommerce ``` diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..7cbf008 --- /dev/null +++ b/TODO.md @@ -0,0 +1,99 @@ +# WordPress benchmark simulation — TODO + +## Goal + +Benchmark a WordPress site's cache/Redis layer **without** standing up +WordPress, MySQL or doing real page rendering. We capture a small corpus of +real requests once, then *replay* them: sleep for the captured PHP compute +time and fire the captured Redis commands. This isolates the Redis/Relay layer +and lets every benchmark run execute an **identical, reproducible workload** so +systems (Relay vs PhpRedis vs Predis, Redis vs Valkey/KeyDB, single vs +clustered, etc.) can be compared apples-to-apples — including horizontally +scaled setups. + +## Architecture + +``` + (1) capture (2) corpus (3) replay endpoint (4) driver + ─────────── ────────── ─────────────────── ────────── + real WP request → N traces of → stateless PHP-FPM ←── k6 (replay.js) + (one-off) Redis cmds + endpoint: id in, fires ?id=N, + compute time + replay trace, return owns ordering + TTLs + batches metrics & determinism +``` + +- **The server is dumb / stateless.** k6 decides which trace runs and in what + order via `?id=N`; the server just executes trace N. All determinism lives in + the driver, which keeps runs comparable and lets us shard across machines. +- **Closed-loop** by default (run as fast as the SUT allows) for capability + comparison; ramping/constant scenarios available for saturation testing. + +## What's done + +- [x] `replay.js` — k6 driver. Deterministic, segment-safe trace selection; + selectable scenarios (`fixed` / `ramping` / `constant` / `shared`) via + `--env SCENARIO=`; configurable `VUS`, `TOTAL`, `TRACES`, `STAGES`, etc. +- [x] `stubs/replay-server.php` — dummy endpoint that fakes the work and returns + the metric JSON shape `replay.js` parses. Placeholder for the real + executor. +- [x] README section documenting usage. + +## What needs to be done + +### Capture (not the hard part — do later) +- [ ] Instrument the real Redis client (wrap Relay/PhpRedis as used by the + object-cache drop-in) to log per request: ordered commands + args, + **pipeline/MULTI batch boundaries**, TTLs, response sizes, and the + **per-batch timing gaps** (not just one total). Tag each with a request id. +- [ ] Decide capture mechanism — client wrapper preferred over Redis `MONITOR` + (MONITOR drops under load, hides pipelining, skews timing). +- [ ] Capture 100 representative requests → corpus file(s) (one per id). +- [ ] Decide corpus format (JSON/MessagePack) and how the server preloads it + (static array / APCu) so request handling is pure CPU + Redis. + +### Replay executor (replace the dummy) +- [ ] Implement the real replay in `stubs/replay-server.php` (or a small app): + load trace by id, `usleep` the captured compute/gap times, fire the + captured Redis commands via the **production client**, preserving batch + structure & TTLs, and **consume responses** (so client-side + deserialization / Relay's in-memory layer is exercised). +- [ ] Run behind real **nginx + PHP-FPM** (not `php -S`) for a realistic + process model. Pin `pm.max_children` (sleeping workers hold slots) and + use persistent Redis connections to match production. +- [ ] Decide warm vs cold cache: pre-warm pass, or discard first N iterations. +- [ ] Emit real metrics in the response (total_ms, redis_ms, cmd_count, hits, + misses, bytes) — `replay.js` already parses these. + +### Driver / orchestration +- [ ] Validate distributed runs with execution segments across >1 machine. +- [ ] Optional `bin/` wrapper for a concurrency sweep (e.g. VUS=50/100/200/400) + to produce a capability curve. +- [ ] Decide metrics sink: k6 JSON summary to start, Prometheus/Grafana later. + +## Separate track: PhpBench micro-benchmark of the Redis fake workload + +In addition to the k6 / HTTP path above, we want to run the **same captured +Redis fake workload under [PhpBench](https://phpbench.readthedocs.io)**, as a +standalone, single-process micro-benchmark. This gives us tight statistical +rigor (revs/iterations, mean/mode/stdev, retry until stable) on the *pure* +replay cost — no HTTP, nginx or FPM in the path — which complements the k6 +capability/concurrency numbers. + +- [ ] Add a PhpBench `*Bench.php` subject that loads a trace and replays its + Redis commands via the production client (reuse the same replay code as + the endpoint so both paths exercise identical logic). +- [ ] Parameterize over the corpus (`@ParamProviders`) so each trace is its own + benchmarked subject; report per-trace and aggregate. +- [ ] Pin revs/iterations/warmup; commit a `phpbench.json` config. +- [ ] Compare clients/backends (Relay / PhpRedis / Predis) as separate runs. +- [ ] Keep this isolated-Redis benchmark distinct from the k6 end-to-end one — + they answer different questions (raw call cost vs system capability). + +## Open decisions + +- [ ] Corpus multiplicity: 100 unique traces × 100 repeats = 10,000 (current + default). Confirm or adjust per real capture. +- [ ] CPU contention: `sleep` yields the CPU (conservative, isolates Redis). + Switch to a calibrated busy-loop only if PHP CPU cost must be modeled. +- [ ] Key namespacing: replay captured keys verbatim (realistic shared-cache + contention) vs per-worker namespacing. Default: verbatim. diff --git a/corpus/00000.json b/corpus/00000.json new file mode 100644 index 0000000..906cf88 --- /dev/null +++ b/corpus/00000.json @@ -0,0 +1,85 @@ +{ + "schema": 1, + "id": 0, + "request": { + "method": "GET", + "url": "/", + "status": 200, + "captured_at": "2026-06-02T12:00:00Z" + }, + "summary": { + "ms_total": 11.54, + "ms_php": 8.60, + "ms_redis": 1.02, + "ms_sql": 1.92, + "redis_reads": 4, + "redis_writes": 3, + "redis_hits": 3, + "redis_misses": 1, + "hit_ratio": 75.0, + "sql_queries": 2, + "bytes": 61040 + }, + "timeline": [ + { "op": "php", "us": 612 }, + { + "op": "redis", + "us": 310, + "mode": "single", + "cmds": [ + { "cmd": "GET", "args": ["wp:ocp:1:options:alloptions"], "reply_bytes": 48213, "hit": true } + ] + }, + { "op": "php", "us": 1840 }, + { + "op": "redis", + "us": 60, + "mode": "single", + "cmds": [ + { "cmd": "GET", "args": ["wp:ocp:1:options:rewrite_rules"], "reply_bytes": 0, "hit": false } + ] + }, + { + "op": "sql", + "us": 742, + "query": "SELECT option_value FROM wp_options WHERE option_name = 'rewrite_rules' LIMIT 1", + "caller": "require('wp-blog-header.php'), wp(), WP->main, WP->query_posts, ..., get_option", + "rows": 1 + }, + { + "op": "redis", + "us": 95, + "mode": "single", + "cmds": [ + { "cmd": "SET", "args": ["wp:ocp:1:options:rewrite_rules", { "b64": "TzozMzoiQUNQX1Jld3JpdGVfUnVsZXMiOjI6e3M6Nzoi..." }], "value_bytes": 2048, "ttl": 0 } + ] + }, + { "op": "php", "us": 920 }, + { + "op": "sql", + "us": 1180, + "query": "SELECT wp_posts.ID FROM wp_posts WHERE wp_posts.post_type = 'post' AND wp_posts.post_status = 'publish' ORDER BY wp_posts.post_date DESC LIMIT 0, 10", + "caller": "require('wp-blog-header.php'), wp(), WP->main, WP->query_posts, WP_Query->query, WP_Query->get_posts", + "rows": 10 + }, + { + "op": "redis", + "us": 150, + "mode": "pipeline", + "cmds": [ + { "cmd": "SETEX", "args": ["wp:ocp:1:posts:101", "300", { "b64": "TzU6IldQX1Bvc3QiOjIzOntzOjI6IklEIjtpOjEwMTs..." }], "value_bytes": 5120, "ttl": 300 }, + { "cmd": "SETEX", "args": ["wp:ocp:1:posts:102", "300", { "b64": "TzU6IldQX1Bvc3QiOjIzOntzOjI6IklEIjtpOjEwMjs..." }], "value_bytes": 4880, "ttl": 300 } + ] + }, + { + "op": "redis", + "us": 405, + "mode": "pipeline", + "cmds": [ + { "cmd": "GET", "args": ["wp:ocp:1:post_meta:101"], "reply_bytes": 1320, "hit": true }, + { "cmd": "GET", "args": ["wp:ocp:1:post_meta:102"], "reply_bytes": 1290, "hit": true } + ] + }, + { "op": "php", "us": 5230 } + ] +} diff --git a/corpus/manifest.json b/corpus/manifest.json new file mode 100644 index 0000000..4380df5 --- /dev/null +++ b/corpus/manifest.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "generator": "k6-capture-mu-plugin/1.0", + "captured_at": "2026-06-02T12:00:00Z", + "site": "https://example.com", + "cache": "object-cache-pro", + "client": "relay", + "count": 3, + "traces": [ + { "id": 0, "file": "00000.json", "method": "GET", "url": "/", "ms_total": 11.54, "redis_ops": 7, "sql_queries": 2 }, + { "id": 1, "file": "00001.json", "method": "GET", "url": "/sample-page/", "ms_total": 9.81, "redis_ops": 5, "sql_queries": 1 }, + { "id": 2, "file": "00002.json", "method": "GET", "url": "/?p=42", "ms_total": 14.27, "redis_ops": 12, "sql_queries": 4 } + ] +} diff --git a/replay.js b/replay.js new file mode 100644 index 0000000..cf0a092 --- /dev/null +++ b/replay.js @@ -0,0 +1,157 @@ +import http from 'k6/http' +import exec from 'k6/execution' +import { check } from 'k6' +import { Rate } from 'k6/metrics' + +import Metrics from './lib/metrics.js' +import { validateSiteUrl } from './lib/helpers.js' + +/** + * Deterministic trace replay. + * + * Instead of driving a real WordPress site, this fires a fixed corpus of + * captured requests ("traces") against a stateless replay endpoint. Each run + * executes the *identical* workload so results are comparable across runs and + * across horizontally scaled load generators. + * + * The server is dumb: k6 owns which trace runs and in which order, the server + * just executes whatever `id` it is handed. All determinism lives here. + * + * k6 run replay.js --env SITE_URL=http://localhost:8080 + * k6 run replay.js --env SITE_URL=http://localhost:8080 --env SCENARIO=ramping + * + * Scale across N load-generator machines with execution segments (the trace + * mapping below is segment-safe, so coverage stays complete and reproducible): + * + * # machine 1 of 4 + * k6 run replay.js --env SITE_URL=http://lb \ + * --execution-segment "0:1/4" \ + * --execution-segment-sequence "0,1/4,2/4,3/4,1" + */ + +const TRACES = Number(__ENV.TRACES || 100) // number of unique captured traces +const TOTAL = Number(__ENV.TOTAL || 10000) // total requests to fire (count-based scenarios) +const VUS = Number(__ENV.VUS || 100) // concurrency +const SCENARIO = __ENV.SCENARIO || 'fixed' // fixed | ramping | constant | shared +const REPLAY_PATH = __ENV.REPLAY_PATH || '/render' + +export const options = { + summaryTimeUnit: 'ms', + scenarios: { + replay: buildScenario(), + }, + thresholds: { + http_req_failed: ['rate<0.01'], + }, + ext: { + loadimpact: { + name: 'Deterministic trace replay', + note: 'Replay a fixed corpus of captured requests as an identical, reproducible workload.', + projectID: __ENV.PROJECT_ID || null, + }, + }, +} + +/** + * Pick the load profile via `--env SCENARIO=...`: + * + * fixed (default) per-vu-iterations — exactly TOTAL requests, each trace + * run an equal number of times. Most reproducible; best + * for apples-to-apples capability comparison. + * ramping ramping-vus — ramp concurrency up/down over STAGES to + * find the saturation point. Duration-based. + * constant constant-vus — hold VUS for DURATION. + * shared shared-iterations — TOTAL requests pulled from a shared + * pool as fast as possible (set is fixed, per-trace count + * is best-effort rather than exact). + * + * Tunables: VUS, TOTAL, DURATION, START_VUS, STAGES, MAX_DURATION. + * STAGES format: "30s:50,1m:200,2m:200,30s:0" (duration:targetVUs, ...) + */ +function buildScenario () { + switch (SCENARIO) { + case 'ramping': + return { + executor: 'ramping-vus', + startVUs: Number(__ENV.START_VUS || 0), + stages: parseStages(__ENV.STAGES) || [ + { duration: '30s', target: 50 }, + { duration: '1m', target: 200 }, + { duration: '2m', target: 200 }, + { duration: '30s', target: 0 }, + ], + gracefulRampDown: __ENV.GRACEFUL_RAMP_DOWN || '10s', + } + + case 'constant': + return { + executor: 'constant-vus', + vus: VUS, + duration: __ENV.DURATION || '2m', + } + + case 'shared': + return { + executor: 'shared-iterations', + vus: VUS, + iterations: TOTAL, + maxDuration: __ENV.MAX_DURATION || '30m', + } + + case 'fixed': + default: + return { + executor: 'per-vu-iterations', + vus: VUS, + iterations: Math.ceil(TOTAL / VUS), // total = VUS * iterations + maxDuration: __ENV.MAX_DURATION || '30m', + } + } +} + +function parseStages (spec) { + if (! spec) { + return null + } + + return spec.split(',').map(stage => { + const [duration, target] = stage.split(':') + + return { duration: duration.trim(), target: Number(target) } + }) +} + +const errorRate = new Rate('errors') + +// Object cache metrics are parsed from the Object Cache Pro footnote comment in +// the response body — the same shared `Metrics` class the other scripts use. +// For the real SUT, enable OCP's `analytics.footnote`; the dummy endpoint +// prints a fake footnote so this works for local testing. +const metrics = new Metrics() + +export function setup () { + validateSiteUrl(__ENV.SITE_URL) + + console.info(`Replaying ${TRACES} traces using "${SCENARIO}" scenario`) +} + +export default function () { + // Deterministic, segment-safe trace selection. Each VU walks the trace + // corpus in order, offset by its global VU id so concurrent VUs work on + // different traces at any instant (no artificial lockstep on trace 0). + // `idInTest` is stable across VUs and execution segments, and this mapping + // needs no shared state, so it is identical every run and never overlaps + // between load-generator machines — regardless of the chosen scenario. + const traceId = (exec.vu.iterationInScenario + (exec.vu.idInTest - 1)) % TRACES + + const response = http.get(`${__ENV.SITE_URL}${REPLAY_PATH}?id=${traceId}`, { + tags: { trace: String(traceId) }, + }) + + check(response, { + 'response code is 200': response => response.status === 200, + }) + + errorRate.add(response.status >= 400) + metrics.addResponseMetrics(response) +} diff --git a/replay/config.php b/replay/config.php new file mode 100644 index 0000000..493ce7b --- /dev/null +++ b/replay/config.php @@ -0,0 +1,124 @@ +redisClient = strtolower(self::env('REPLAY_REDIS_CLIENT', 'phpredis')); + $c->redisHost = self::env('REPLAY_REDIS_HOST', '127.0.0.1'); + $c->redisPort = (int) self::env('REPLAY_REDIS_PORT', '6379'); + $c->redisPassword = self::env('REPLAY_REDIS_PASSWORD', '') ?: null; + $c->redisDb = (int) self::env('REPLAY_REDIS_DB', '0'); + $c->redisTls = self::bool('REPLAY_REDIS_TLS', false); + $c->redisPersistent = self::bool('REPLAY_REDIS_PERSISTENT', true); + $c->redisTimeout = (float) self::env('REPLAY_REDIS_TIMEOUT', '2.0'); + + $c->corpusDir = rtrim(self::env('REPLAY_CORPUS', __DIR__ . '/../corpus'), '/'); + + $c->sqlMode = strtolower(self::env('REPLAY_SQL_MODE', 'sleep')); + $c->dbHost = self::env('REPLAY_DB_HOST', '127.0.0.1'); + $c->dbPort = (int) self::env('REPLAY_DB_PORT', '3306'); + $c->dbName = self::env('REPLAY_DB_NAME', ''); + $c->dbUser = self::env('REPLAY_DB_USER', ''); + $c->dbPassword = self::env('REPLAY_DB_PASSWORD', ''); + $c->dbCharset = self::env('REPLAY_DB_CHARSET', 'utf8mb4'); + + $c->validate(); + + return $c; + } + + private function validate(): void + { + if (! in_array($this->redisClient, ['phpredis', 'relay', 'predis'], true)) { + throw new RuntimeException('REPLAY_REDIS_CLIENT must be one of: phpredis, relay, predis'); + } + + if (! in_array($this->sqlMode, ['sleep', 'execute'], true)) { + throw new RuntimeException('REPLAY_SQL_MODE must be "sleep" or "execute"'); + } + + if ($this->sqlMode === 'execute' && $this->dbName === '') { + throw new RuntimeException('REPLAY_SQL_MODE=execute requires REPLAY_DB_NAME (and likely REPLAY_DB_USER/PASSWORD)'); + } + } + + private static function env(string $key, string $default): string + { + $value = getenv($key); + + return $value === false ? $default : $value; + } + + private static function bool(string $key, bool $default): bool + { + $value = getenv($key); + + if ($value === false || $value === '') { + return $default; + } + + return in_array(strtolower($value), ['1', 'true', 'yes', 'on'], true); + } + + private static function loadEnvFile(string $path): void + { + if (! is_readable($path)) { + return; + } + + foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) { + $line = trim($line); + + if ($line === '' || $line[0] === '#' || ! str_contains($line, '=')) { + continue; + } + + [$key, $value] = explode('=', $line, 2); + $key = trim($key); + $value = trim($value); + + // Strip matching surrounding quotes. + if (strlen($value) >= 2 && ($value[0] === '"' || $value[0] === "'") && $value[-1] === $value[0]) { + $value = substr($value, 1, -1); + } + + // Real environment variables win over the .env file. + if (getenv($key) === false) { + putenv("{$key}={$value}"); + } + } + } +} diff --git a/replay/index.php b/replay/index.php new file mode 100644 index 0000000..634bb98 --- /dev/null +++ b/replay/index.php @@ -0,0 +1,169 @@ +corpusDir); + +$id = isset($_GET['id']) ? (int) $_GET['id'] : -1; + +if (! isset($corpus[$id])) { + http_response_code(404); + header('Content-Type: text/plain'); + echo "unknown trace id: {$id}"; + return; +} + +$redis = connect_redis($config); +$pdo = $config->sqlMode === 'execute' ? connect_pdo($config) : null; + +$metrics = (new Replayer($redis, $config, $pdo))->replay($corpus[$id]); + +header('Content-Type: text/html; charset=UTF-8'); +echo "\nTrace {$id}\n"; +echo '

Replayed trace ' . $id . "

\n"; +echo build_footnote($config, $redis, $metrics) . "\n"; +echo "\n"; + +/** Load the corpus once per worker (manifest first, else scan NNNNN.json). */ +function load_corpus(string $dir): array +{ + static $corpus = null; + + if ($corpus !== null) { + return $corpus; + } + + $corpus = []; + $manifest = $dir . '/manifest.json'; + + if (is_readable($manifest)) { + $index = json_decode((string) file_get_contents($manifest), true); + + foreach ($index['traces'] ?? [] as $entry) { + $file = $dir . '/' . $entry['file']; + if (is_readable($file)) { + $corpus[(int) $entry['id']] = json_decode((string) file_get_contents($file), true); + } + } + } else { + foreach (glob($dir . '/[0-9]*.json') ?: [] as $file) { + $trace = json_decode((string) file_get_contents($file), true); + if (isset($trace['id'])) { + $corpus[(int) $trace['id']] = $trace; + } + } + } + + return $corpus; +} + +/** Connect once per worker (persistent), reused across requests. */ +function connect_redis(ReplayConfig $config): ReplayRedis +{ + static $redis = null; + + if ($redis === null) { + $redis = new ReplayRedis($config); + } + + return $redis; +} + +function connect_pdo(ReplayConfig $config): PDO +{ + static $pdo = null; + + if ($pdo === null) { + $dsn = sprintf( + 'mysql:host=%s;port=%d;dbname=%s;charset=%s', + $config->dbHost, + $config->dbPort, + $config->dbName, + $config->dbCharset + ); + + $pdo = new PDO($dsn, $config->dbUser, $config->dbPassword, [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_PERSISTENT => true, + ]); + } + + return $pdo; +} + +/** Build an OCP-style footnote from measured + server metrics (parsed by lib/metrics.js). */ +function build_footnote(ReplayConfig $config, ReplayRedis $redis, array $m): string +{ + $parts = [ + 'plugin=replay', + 'client=' . $config->redisClient, + sprintf('metric#hits=%d', $m['hits']), + sprintf('metric#misses=%d', $m['misses']), + sprintf('metric#hit-ratio=%s', $m['hit_ratio']), + sprintf('metric#bytes=%d', $m['bytes']), + sprintf('metric#sql-queries=%d', $m['sql_queries']), + sprintf('metric#ms-total=%s', round($m['ms_total'], 2)), + sprintf('sample#ms-cache=%s', round($m['ms_redis'], 2)), + sprintf('sample#store-reads=%d', $m['reads']), + sprintf('sample#store-writes=%d', $m['writes']), + sprintf('sample#store-hits=%d', $m['hits']), + sprintf('sample#store-misses=%d', $m['misses']), + ]; + + $info = $redis->info(); + + if ($info) { + $kh = (int) ($info['keyspace_hits'] ?? 0); + $km = (int) ($info['keyspace_misses'] ?? 0); + $kt = $kh + $km; + + $parts[] = sprintf('sample#redis-hits=%d', $kh); + $parts[] = sprintf('sample#redis-hit-ratio=%s', $kt > 0 ? round($kh / ($kt / 100), 2) : 100); + $parts[] = sprintf('sample#redis-ops-per-sec=%d', (int) ($info['instantaneous_ops_per_sec'] ?? 0)); + $parts[] = sprintf('sample#redis-used-memory=%d', (int) ($info['used_memory'] ?? 0)); + $parts[] = sprintf('sample#redis-memory-fragmentation-ratio=%s', $info['mem_fragmentation_ratio'] ?? 0); + } + + $stats = $redis->stats(); + + if ($stats) { + $rh = (int) ($stats['stats']['hits'] ?? 0); + $rm = (int) ($stats['stats']['misses'] ?? 0); + $rt = $rh + $rm; + $used = (int) ($stats['memory']['used'] ?? 0); + $total = (int) ($stats['memory']['total'] ?? 0); + + $parts[] = sprintf('sample#relay-hit-ratio=%s', $rt > 0 ? round($rh / ($rt / 100), 2) : 100); + $parts[] = sprintf('sample#relay-ops-per-sec=%d', (int) ($stats['stats']['ops_per_sec'] ?? 0)); + $parts[] = sprintf('sample#relay-memory-active=%d', $used); + $parts[] = sprintf('sample#relay-memory-ratio=%s', $total > 0 ? round($used / $total * 100, 2) : 0); + } + + return ''; +} diff --git a/replay/redis.php b/replay/redis.php new file mode 100644 index 0000000..d9ea4e3 --- /dev/null +++ b/replay/redis.php @@ -0,0 +1,105 @@ +redisClient === 'predis') { + throw new RuntimeException('Predis is not supported by the executor yet; use phpredis or relay.'); + } + + $this->isRelay = $config->redisClient === 'relay'; + + if ($this->isRelay && ! class_exists('\Relay\Relay')) { + throw new RuntimeException('REPLAY_REDIS_CLIENT=relay but the Relay extension is not installed.'); + } + + if (! $this->isRelay && ! class_exists('\Redis')) { + throw new RuntimeException('REPLAY_REDIS_CLIENT=phpredis but the phpredis extension is not installed.'); + } + + $this->redis = $this->isRelay ? new \Relay\Relay() : new \Redis(); + + $host = ($config->redisTls ? 'tls://' : '') . $config->redisHost; + + $connected = $config->redisPersistent + ? $this->redis->pconnect($host, $config->redisPort, $config->redisTimeout) + : $this->redis->connect($host, $config->redisPort, $config->redisTimeout); + + if (! $connected) { + throw new RuntimeException("Could not connect to Redis at {$config->redisHost}:{$config->redisPort}"); + } + + if ($config->redisPassword !== null) { + $this->redis->auth($config->redisPassword); + } + + if ($config->redisDb !== 0) { + $this->redis->select($config->redisDb); + } + } + + /** Execute a single command in its own round-trip; returns the raw reply. */ + public function command(string $cmd, array $args) + { + return $this->redis->rawCommand($cmd, ...$args); + } + + /** + * Execute a batch of [cmd, args] in one pipelined round-trip. + * Returns replies aligned with the batch order. + */ + public function pipeline(array $batch): array + { + $pipe = $this->redis->pipeline(); + + foreach ($batch as [$cmd, $args]) { + $pipe->rawCommand($cmd, ...$args); + } + + return (array) $pipe->exec(); + } + + /** Execute a batch of [cmd, args] inside MULTI/EXEC. */ + public function transaction(array $batch): array + { + $multi = $this->redis->multi(); + + foreach ($batch as [$cmd, $args]) { + $multi->rawCommand($cmd, ...$args); + } + + return (array) $multi->exec(); + } + + public function info(): array + { + return (array) $this->redis->info(); + } + + /** Relay in-memory cache stats, or null for phpredis. */ + public function stats(): ?array + { + if ($this->isRelay && method_exists($this->redis, 'stats')) { + return $this->redis->stats(); + } + + return null; + } +} diff --git a/replay/replayer.php b/replay/replayer.php new file mode 100644 index 0000000..a3e5abf --- /dev/null +++ b/replay/replayer.php @@ -0,0 +1,175 @@ + measured metrics for one trace */ + public function replay(array $trace): array + { + $reads = $writes = $hits = $misses = $bytes = $sqlCount = 0; + $redisNs = $sqlNs = $phpNs = 0; + + $started = hrtime(true); + + foreach ($trace['timeline'] ?? [] as $event) { + switch ($event['op'] ?? '') { + case 'php': + $us = (int) ($event['us'] ?? 0); + if ($us > 0) { + usleep($us); + } + $phpNs += $us * 1000; + break; + + case 'sql': + $sqlCount++; + $t0 = hrtime(true); + $this->replaySql($event); + $sqlNs += hrtime(true) - $t0; + break; + + case 'redis': + $t0 = hrtime(true); + [$r, $w, $h, $m, $b] = $this->replayRedis($event); + $redisNs += hrtime(true) - $t0; + $reads += $r; + $writes += $w; + $hits += $h; + $misses += $m; + $bytes += $b; + break; + } + } + + $totalNs = hrtime(true) - $started; + $lookups = $hits + $misses; + + return [ + 'ms_total' => $totalNs / 1e6, + 'ms_redis' => $redisNs / 1e6, + 'ms_sql' => $sqlNs / 1e6, + 'ms_php' => $phpNs / 1e6, + 'reads' => $reads, + 'writes' => $writes, + 'hits' => $hits, + 'misses' => $misses, + 'hit_ratio' => $lookups > 0 ? round($hits / ($lookups / 100), 1) : 100.0, + 'bytes' => $bytes, + 'sql_queries' => $sqlCount, + ]; + } + + /** @return array{0:int,1:int,2:int,3:int,4:int} [reads, writes, hits, misses, bytes] */ + private function replayRedis(array $event): array + { + $cmds = $event['cmds'] ?? []; + + $batch = []; + foreach ($cmds as $c) { + $batch[] = [strtoupper($c['cmd']), $this->decodeArgs($c['args'] ?? [])]; + } + + switch ($event['mode'] ?? 'single') { + case 'pipeline': + $replies = $this->redis->pipeline($batch); + break; + case 'multi': + $replies = $this->redis->transaction($batch); + break; + default: + $replies = []; + foreach ($batch as [$cmd, $args]) { + $replies[] = $this->redis->command($cmd, $args); + } + } + + $reads = $writes = $hits = $misses = $bytes = 0; + + foreach ($cmds as $i => $c) { + $cmd = strtoupper($c['cmd']); + $reply = $replies[$i] ?? null; + + if (in_array($cmd, self::WRITE_COMMANDS, true)) { + $writes++; + $bytes += (int) ($c['value_bytes'] ?? 0); + } else { + $reads++; + if ($this->replyIsHit($reply)) { + $hits++; + $bytes += is_string($reply) ? strlen($reply) : (int) ($c['reply_bytes'] ?? 0); + } else { + $misses++; + } + } + } + + return [$reads, $writes, $hits, $misses, $bytes]; + } + + private function replyIsHit($reply): bool + { + return $reply !== false && $reply !== null && $reply !== []; + } + + /** @return list args with `{b64}` elements decoded to raw bytes */ + private function decodeArgs(array $args): array + { + return array_map(static function ($arg) { + if (is_array($arg) && isset($arg['b64'])) { + return (string) base64_decode((string) $arg['b64'], true); + } + + return (string) $arg; + }, $args); + } + + private function replaySql(array $event): void + { + if ($this->config->sqlMode === 'execute' && $this->pdo !== null) { + try { + $this->pdo->query((string) ($event['query'] ?? ''))->fetchAll(); + } catch (\Throwable $e) { + // The replay DB may not match the captured schema/data; the goal + // is to exercise the query cost, so swallow result errors. + } + + return; + } + + $us = (int) ($event['us'] ?? 0); + if ($us > 0) { + usleep($us); + } + } +} diff --git a/stubs/replay-server.php b/stubs/replay-server.php new file mode 100644 index 0000000..7e4b572 --- /dev/null +++ b/stubs/replay-server.php @@ -0,0 +1,126 @@ + 20 + ($i % 30), // pretend N Redis commands + 'compute_us' => 500 + ($i % 50) * 10, // pretend PHP compute time (µs) + 'hit_ratio' => ($i % 100) / 100.0, + ]; + } + + return $corpus; +} + +$corpus = load_corpus(); + +$id = isset($_GET['id']) ? (int) $_GET['id'] : -1; + +if (! isset($corpus[$id])) { + http_response_code(400); + header('Content-Type: application/json'); + echo json_encode(['error' => 'unknown trace id', 'id' => $id]); + return; +} + +$trace = $corpus[$id]; + +/** + * Simulate the request. Replace this block with the real payload executor: + * sleep for the captured compute time, then fire the captured Redis commands + * (preserving pipelining/MULTI batches and TTLs) and consume the responses. + */ +$start = hrtime(true); +usleep((int) $trace['compute_us']); // fake PHP compute / think time +$redisStart = hrtime(true); +usleep((int) ($trace['commands'] * 5)); // fake Redis round-trip time +$redisEnd = hrtime(true); + +$hits = (int) round($trace['commands'] * $trace['hit_ratio']); +$misses = $trace['commands'] - $hits; +$ratio = $trace['commands'] > 0 ? round($hits / ($trace['commands'] / 100), 1) : 100; +$totalMs = round(($redisEnd - $start) / 1e6, 2); + +$client = isset($_GET['client']) ? preg_replace('/[^a-z]/', '', strtolower($_GET['client'])) : 'relay'; +$client = $client !== '' ? $client : 'relay'; + +// Fake Object Cache Pro footnote — same format the real mu-plugin prints +// (see k6-metrics.php) and that lib/metrics.js parses out of the response body. +$footnote = sprintf( + '', + $client, + $hits, + $misses, + $ratio, + 1024 * $trace['commands'], + $trace['commands'], + $totalMs, + sys_getloadavg()[0] ?? 0, + $hits * 3, + $ratio, + 50000 + $id * 10, + 8 * 1024 * 1024 + $id * 1024, + 1000 + $id, + min(100, $ratio + 5), + 60000 + $id * 10, + 900 + $id, + 32 * 1024 * 1024, + 42.5 +); + +// Mimic a real WordPress page: HTML body with the footnote near the end. +header('Content-Type: text/html; charset=UTF-8'); +echo "\nTrace {$id}\n"; +echo "

Replayed trace {$id} ({$trace['commands']} commands)

\n"; +echo $footnote . "\n"; +echo "\n";