From fb713609f801e08ced9261b4dbb7994f7fb4f771 Mon Sep 17 00:00:00 2001 From: Peng Yong Date: Mon, 29 Jun 2026 17:34:09 +0800 Subject: [PATCH] fix(watch): guard against nil http_code on gRPC stream errors A watch over a gRPC stream can report an error with no HTTP status, e.g. `{"error":{"grpc_code":14,"message":"...EOF"}}` on a transport / stream error. The check `body.error.http_code >= 500` then evaluated `nil >= 500`, raising "attempt to compare nil with number" inside the watch coroutine. That crashes the watch read, skips cancel_watch, and leaks the watcher on the etcd side; a caller that immediately restarts the watch with no backoff (e.g. APISIX config_etcd run_watch) turns this into a tight watch-recreate loop that can drive etcd mvcc_watcher_total into the millions and OOM it. Parse http_code defensively and treat a missing code (transport / stream error) or any 5xx as an endpoint failure: report_failure and return a graceful error so the connection is closed and rebuilt and the watcher is cancelled properly. Errors that do carry an http_code (e.g. 4xx) keep falling through as before. Fixes #222. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Peng Yong --- lib/resty/etcd/v3.lua | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/resty/etcd/v3.lua b/lib/resty/etcd/v3.lua index 8c9b257..25c1ceb 100644 --- a/lib/resty/etcd/v3.lua +++ b/lib/resty/etcd/v3.lua @@ -911,11 +911,25 @@ local function request_chunk(self, method, path, opts, timeout) body, err = decode_json(chunk) if not body then return nil, "failed to decode json body: " .. (err or " unknown") - elseif body.error and body.error.http_code >= 500 then - -- health_check retry should do nothing here - -- and let connection closed to create a new one - health_check.report_failure(endpoint.http_host) - return nil, endpoint.http_host .. ": " .. body.error.http_status + elseif body.error then + -- A watch over a gRPC stream can report an error with no HTTP status, + -- e.g. {"error":{"grpc_code":14,"message":"...EOF"}} on a transport / + -- stream error. The previous `body.error.http_code >= 500` then compared + -- nil with a number ("attempt to compare nil with number"), crashing the + -- watch coroutine, skipping cancel_watch and leaking watchers on etcd. + -- Parse http_code defensively; treat a missing code (transport / stream + -- error) or any 5xx as an endpoint failure -> report_failure and let the + -- connection close so a fresh one is created. Other errors (e.g. a 4xx + -- with an http_code) fall through to the caller as before. + local raw = body.error.http_code + local http_code = (type(raw) == "number" and raw) + or (type(raw) == "string" and tonumber(raw)) or nil + if http_code == nil or http_code >= 500 then + health_check.report_failure(endpoint.http_host) + return nil, endpoint.http_host .. ": " + .. (body.error.http_status or body.error.message + or "watch stream error") + end end if body.result and body.result.events then