Skip to content

Commit 932137c

Browse files
dimsdims
committed
ate-setup: setting for the router's ext_proc circuit breaker
ATE_ATENET_EXTPROC_MAX_REQUESTS (atenet.router.extprocMaxRequests, --atenet-extproc-max-requests) passes --extproc-max-requests to the router the way the route timeout and parked budget settings pass theirs. On a dense host 1536 synchronized execs overflowed the derived breaker of 2048 and Envoy answered 503 "reset reason: overflow"; the only way to raise it was a patch on the deployment.
1 parent 69d07e0 commit 932137c

12 files changed

Lines changed: 116 additions & 36 deletions

File tree

‎cmd/ate-setup/commands.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ a pre-scan pass, so they may appear anywhere on its command line.
2727
| `--atenet-dataplane envoy\|agentgateway` | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) |
2828
| `--atenet-route-timeout DURATION` | `ATE_ATENET_ROUTE_TIMEOUT=DURATION` | New. The router's `--route-timeout` (default `5m`). Envoy dataplane only |
2929
| `--atenet-parked-request-budget DURATION` | `ATE_ATENET_PARKED_REQUEST_BUDGET=DURATION` | New. The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only |
30+
| `--atenet-extproc-max-requests N` | `ATE_ATENET_EXTPROC_MAX_REQUESTS=N` | New. The router's `--extproc-max-requests`, the circuit breaker on Envoy's ext_proc cluster (default: twice `--parked-request-max`, at least `1024`). Envoy dataplane only |
3031
| `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed |
3132
| `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer |
3233
| `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent |

‎cmd/ate-setup/internal/config/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,9 @@ type Config struct {
162162
// --route-timeout and --parked-request-budget; zero keeps the router default.
163163
RouterRouteTimeout time.Duration
164164
RouterParkedRequestBudget time.Duration
165+
// RouterExtProcMaxRequests is the router's --extproc-max-requests; zero
166+
// keeps the router's derivation from --parked-request-max.
167+
RouterExtProcMaxRequests int
165168
// The read/write and owner connections can use different login identities.
166169
// With one configured connection, both pools use it. Both empty selects
167170
// bundled PostgreSQL.

‎cmd/ate-setup/internal/config/config_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ func loadEnv(t *testing.T) {
6060
"ATE_API_POSTGRES_SCHEMA",
6161
"ATE_API_POSTGRES_SERVER_CA_FILE",
6262
"ATE_ATENET_DATAPLANE",
63+
"ATE_ATENET_EXTPROC_MAX_REQUESTS",
6364
"ATE_ATENET_PARKED_REQUEST_BUDGET",
6465
"ATE_ATENET_ROUTE_TIMEOUT",
6566
"ATE_CREDENTIAL_PROVIDER",

‎cmd/ate-setup/internal/config/load.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro
247247
Router: r.String("atenet.dataplane"),
248248
RouterRouteTimeout: r.Duration("atenet.router.routeTimeout"),
249249
RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"),
250+
RouterExtProcMaxRequests: r.Int("atenet.router.extprocMaxRequests"),
250251
PostgresSchema: r.String("ateapi.postgres.schema"),
251252
PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"),
252253
PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"),
@@ -402,13 +403,20 @@ func validateResolved(cfg *Config, r *Resolved) error {
402403
}
403404

404405
// Only the envoy dataplane runs the atenet-router container the flags go on.
405-
for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget"} {
406+
for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget", "atenet.router.extprocMaxRequests"} {
406407
v, _ := r.Value(key)
407408
if v.Raw == "" {
408409
continue
409410
}
410-
if r.Duration(key) <= 0 {
411-
return &InvalidError{Value: v, Want: "a positive duration"}
411+
switch v.Setting.Kind {
412+
case KindDuration:
413+
if r.Duration(key) <= 0 {
414+
return &InvalidError{Value: v, Want: "a positive duration"}
415+
}
416+
case KindInt:
417+
if r.Int(key) <= 0 {
418+
return &InvalidError{Value: v, Want: "a positive integer"}
419+
}
412420
}
413421
if cfg.Router != RouterEnvoy {
414422
return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"}

‎cmd/ate-setup/internal/config/load_test.go‎

Lines changed: 42 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ func TestLoadFlagsRouterSettings(t *testing.T) {
230230
loadEnv(t)
231231
t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m")
232232
t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m")
233+
t.Setenv("ATE_ATENET_EXTPROC_MAX_REQUESTS", "4096")
233234

234235
fs, err := optionsFlagSet(Options{})
235236
if err != nil {
@@ -245,37 +246,55 @@ func TestLoadFlagsRouterSettings(t *testing.T) {
245246
if cfg.RouterParkedRequestBudget != 2*time.Minute {
246247
t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget)
247248
}
249+
if cfg.RouterExtProcMaxRequests != 4096 {
250+
t.Errorf("RouterExtProcMaxRequests = %d, want 4096", cfg.RouterExtProcMaxRequests)
251+
}
248252
}
249253

250254
func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) {
251-
loadEnv(t)
252-
t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway)
253-
t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m")
255+
for env, value := range map[string]string{
256+
"ATE_ATENET_PARKED_REQUEST_BUDGET": "2m",
257+
"ATE_ATENET_EXTPROC_MAX_REQUESTS": "4096",
258+
} {
259+
t.Run(env, func(t *testing.T) {
260+
loadEnv(t)
261+
t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway)
262+
t.Setenv(env, value)
254263

255-
fs, err := optionsFlagSet(Options{})
256-
if err != nil {
257-
t.Fatal(err)
258-
}
259-
_, err = LoadFlags(fs, LoadOptions{})
260-
if err == nil {
261-
t.Fatal("LoadFlags() = nil error, want a dataplane conflict")
262-
}
263-
for _, want := range []string{"ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_DATAPLANE"} {
264-
if !strings.Contains(err.Error(), want) {
265-
t.Errorf("error %q does not name %s", err, want)
266-
}
264+
fs, err := optionsFlagSet(Options{})
265+
if err != nil {
266+
t.Fatal(err)
267+
}
268+
_, err = LoadFlags(fs, LoadOptions{})
269+
if err == nil {
270+
t.Fatal("LoadFlags() = nil error, want a dataplane conflict")
271+
}
272+
for _, want := range []string{env, "ATE_ATENET_DATAPLANE"} {
273+
if !strings.Contains(err.Error(), want) {
274+
t.Errorf("error %q does not name %s", err, want)
275+
}
276+
}
277+
})
267278
}
268279
}
269280

270281
func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) {
271-
loadEnv(t)
272-
t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "0s")
282+
for _, tc := range []struct{ env, value, want string }{
283+
{"ATE_ATENET_ROUTE_TIMEOUT", "0s", "positive duration"},
284+
{"ATE_ATENET_EXTPROC_MAX_REQUESTS", "0", "positive integer"},
285+
{"ATE_ATENET_EXTPROC_MAX_REQUESTS", "-1", "positive integer"},
286+
} {
287+
t.Run(tc.env+"="+tc.value, func(t *testing.T) {
288+
loadEnv(t)
289+
t.Setenv(tc.env, tc.value)
273290

274-
fs, err := optionsFlagSet(Options{})
275-
if err != nil {
276-
t.Fatal(err)
277-
}
278-
if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), "positive duration") {
279-
t.Errorf("LoadFlags() error = %v, want a positive-duration error", err)
291+
fs, err := optionsFlagSet(Options{})
292+
if err != nil {
293+
t.Fatal(err)
294+
}
295+
if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), tc.want) {
296+
t.Errorf("LoadFlags() error = %v, want a %s error", err, tc.want)
297+
}
298+
})
280299
}
281300
}

‎cmd/ate-setup/internal/config/setting.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,11 @@ var Registry = []Setting{
174174
Kind: KindDuration,
175175
Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)",
176176
},
177+
{
178+
Key: "atenet.router.extprocMaxRequests", Env: "ATE_ATENET_EXTPROC_MAX_REQUESTS", Flag: "atenet-extproc-max-requests",
179+
Kind: KindInt,
180+
Usage: "The router's --extproc-max-requests, the circuit breaker on Envoy's ext_proc cluster; unset keeps the router's derivation (envoy dataplane only)",
181+
},
177182
{
178183
Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE",
179184
Flag: "experimental-additional-egress-extproc-service", Kind: KindString,

‎cmd/ate-setup/internal/steps/router.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package steps
1717
import (
1818
"fmt"
1919
"math"
20+
"strconv"
2021
"strings"
2122
"time"
2223

@@ -30,12 +31,13 @@ import (
3031
// terminationGracePeriodSeconds is sized for: the router default.
3132
const manifestParkedRequestBudget = 5 * time.Second
3233

33-
// configureAtenetRouter sets --route-timeout and --parked-request-budget on
34-
// the atenet-router container and grows terminationGracePeriodSeconds with the
35-
// budget, since the router's drain deadline follows it. A no-op when unset.
34+
// configureAtenetRouter sets --route-timeout, --parked-request-budget and
35+
// --extproc-max-requests on the atenet-router container and grows
36+
// terminationGracePeriodSeconds with the budget, since the router's drain
37+
// deadline follows it. A no-op when unset.
3638
func (e *Env) configureAtenetRouter(objs []*unstructured.Unstructured) error {
37-
routeTimeout, budget := e.Cfg.RouterRouteTimeout, e.Cfg.RouterParkedRequestBudget
38-
if routeTimeout <= 0 && budget <= 0 {
39+
routeTimeout, budget, maxRequests := e.Cfg.RouterRouteTimeout, e.Cfg.RouterParkedRequestBudget, e.Cfg.RouterExtProcMaxRequests
40+
if routeTimeout <= 0 && budget <= 0 && maxRequests <= 0 {
3941
return nil
4042
}
4143
var dep *unstructured.Unstructured
@@ -87,6 +89,10 @@ func (e *Env) configureAtenetRouter(objs []*unstructured.Unstructured) error {
8789
}
8890
}
8991
}
92+
if maxRequests > 0 {
93+
log.Infof("Setting the router's --extproc-max-requests to %d", maxRequests)
94+
args = setFlag(args, "--extproc-max-requests", strconv.Itoa(maxRequests))
95+
}
9096
container["args"] = args
9197
containers[idx] = container
9298
return unstructured.SetNestedSlice(dep.Object, containers, "spec", "template", "spec", "containers")

‎cmd/ate-setup/internal/steps/router_test.go‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func flagValues(args []any, flag string) []string {
7373
}
7474

7575
func TestConfigureAtenetRouter(t *testing.T) {
76-
objs, e := routerObjects(t, &config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute})
76+
objs, e := routerObjects(t, &config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute, RouterExtProcMaxRequests: 4096})
7777
_, grace := routerPodSpec(t, objs)
7878
if err := e.configureAtenetRouter(objs); err != nil {
7979
t.Fatalf("configureAtenetRouter: %v", err)
@@ -85,16 +85,40 @@ func TestConfigureAtenetRouter(t *testing.T) {
8585
if v := flagValues(args, "--parked-request-budget"); len(v) != 1 || v[0] != "2m0s" {
8686
t.Errorf("--parked-request-budget = %v, want [2m0s]", v)
8787
}
88+
if v := flagValues(args, "--extproc-max-requests"); len(v) != 1 || v[0] != "4096" {
89+
t.Errorf("--extproc-max-requests = %v, want [4096]", v)
90+
}
8891
if want := grace + 115; got != want {
8992
t.Errorf("terminationGracePeriodSeconds = %d, want %d", got, want)
9093
}
9194
}
9295

96+
// The breaker alone must still reach the container, with the grace period untouched.
97+
func TestConfigureAtenetRouterExtProcMaxRequestsAlone(t *testing.T) {
98+
objs, e := routerObjects(t, &config.Config{RouterExtProcMaxRequests: 4096})
99+
_, grace := routerPodSpec(t, objs)
100+
if err := e.configureAtenetRouter(objs); err != nil {
101+
t.Fatalf("configureAtenetRouter: %v", err)
102+
}
103+
args, got := routerPodSpec(t, objs)
104+
if v := flagValues(args, "--extproc-max-requests"); len(v) != 1 || v[0] != "4096" {
105+
t.Errorf("--extproc-max-requests = %v, want [4096]", v)
106+
}
107+
for _, flag := range []string{"--route-timeout", "--parked-request-budget"} {
108+
if v := flagValues(args, flag); len(v) != 0 {
109+
t.Errorf("%s = %v, want unset", flag, v)
110+
}
111+
}
112+
if got != grace {
113+
t.Errorf("terminationGracePeriodSeconds = %d, want %d unchanged", got, grace)
114+
}
115+
}
116+
93117
// Unset settings keep the manifest as checked in, which must not set the flags itself.
94118
func TestConfigureAtenetRouterUnsetLeavesManifestAlone(t *testing.T) {
95119
objs, e := routerObjects(t, &config.Config{})
96120
before, grace := routerPodSpec(t, objs)
97-
for _, flag := range []string{"--route-timeout", "--parked-request-budget"} {
121+
for _, flag := range []string{"--route-timeout", "--parked-request-budget", "--extproc-max-requests"} {
98122
if v := flagValues(before, flag); len(v) != 0 {
99123
t.Errorf("the manifest sets %s=%v; the settings expect it unset", flag, v)
100124
}

‎docs/examples/substrate-install.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,14 @@ rolloutTimeout: 1m0s
3939
# Ingress and egress dataplane: envoy or agentgateway.
4040
atenet:
4141
dataplane: envoy
42-
# Router timeouts for agent workloads; unset keeps the router defaults (5s, 5m). Envoy only.
42+
# Router settings for agent workloads; unset keeps the router defaults. Envoy only.
4343
router:
44+
# Park budget and route timeout; the router defaults are 5s and 5m.
4445
parkedRequestBudget: 2m
4546
routeTimeout: 30m
47+
# Circuit breaker on Envoy's ext_proc cluster; the router derives twice the
48+
# parking lot (at least 1024) when unset.
49+
extprocMaxRequests: 4096
4650

4751
# Prebuilt images. Both halves are required together: a tag with no repository
4852
# would be dropped, leaving a build from source that looks like the release the

‎docs/operator-install.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the
294294
| `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) |
295295
| `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) |
296296
| `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) |
297+
| `atenet.router.extprocMaxRequests` | `ATE_ATENET_EXTPROC_MAX_REQUESTS` | `--atenet-extproc-max-requests` | — | (global) |
297298
| `atenet.router.parkedRequestBudget` | `ATE_ATENET_PARKED_REQUEST_BUDGET` | `--atenet-parked-request-budget` | — | (global) |
298299
| `atenet.router.routeTimeout` | `ATE_ATENET_ROUTE_TIMEOUT` | `--atenet-route-timeout` | — | (global) |
299300
| `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) |

0 commit comments

Comments
 (0)