From d8717b36c3083ed6286edc322913f9b59a6626d3 Mon Sep 17 00:00:00 2001 From: dashashutosh80 Date: Fri, 29 May 2026 09:26:41 +0530 Subject: [PATCH 001/164] =?UTF-8?q?fix:=20auto-heal=20Completed=20CHIs=20w?= =?UTF-8?q?ith=20sustained-NotReady=20hosts=20Pre-fix,=20a=20ClickHouse=20?= =?UTF-8?q?pod=20that=20regressed=20to=20Ready=3DFalse=20while=20its=20CHI?= =?UTF-8?q?=20was=20Status=3DCompleted=20could=20stay=20stuck=20indefinite?= =?UTF-8?q?ly:=20the=20CHI=20generation=20never=20changes,=20the=20operato?= =?UTF-8?q?r's=20ReconcileUpdate=20path=20had=20no=20handler=20for=20Ready?= =?UTF-8?q?=E2=86=92NotReady=20on=20Completed=20CHIs,=20and=20shouldForceR?= =?UTF-8?q?estartHost=20had=20no=20case=20for=20sustained=20Ready=3DFalse.?= =?UTF-8?q?=20Field=20observation:=2026-hour=20stalls=20of=20otherwise-hea?= =?UTF-8?q?lthy=20clusters=20until=20an=20external=20trigger=20(operator?= =?UTF-8?q?=20pod=20restart)=20bumped=20the=20reconcile=20loop.=20Close=20?= =?UTF-8?q?both=20gaps:=20=20=20Signal=20side=20-=20worker-pod-retry.go=20?= =?UTF-8?q?adds=20=20=20recoverCompletedReconcileOnPodNotReady,=20wired=20?= =?UTF-8?q?into=20processReconcilePod=20=20=20alongside=20the=20existing?= =?UTF-8?q?=20recoverAbortedReconcileOnPodReady.=20Schedules=20a=20=20=20d?= =?UTF-8?q?elayed=20re-enqueue=20(time.AfterFunc)=20so=20the=20decision-si?= =?UTF-8?q?de=20gets=20to=20=20=20evaluate=20sustain=20rather=20than=20fir?= =?UTF-8?q?e=20on=20every=20flap.=20=20=20Decision=20side=20-=20worker.go?= =?UTF-8?q?=20adds=20a=20new=20case=20to=20shouldForceRestartHost=20=20=20?= =?UTF-8?q?gated=20on=20ShouldRecoverCompletedOnPodNotReady()=20and=20a=20?= =?UTF-8?q?freshly-fetched=20=20=20isPoustainedNotReady()=20check=20that?= =?UTF-8?q?=20reads=20=20=20Pod.Status.Conditions[PodReady].LastTransition?= =?UTF-8?q?Time=20-=20the=20same=20signal=20=20=20kube-proxy=20uses=20for?= =?UTF-8?q?=20EndpointSlice=20membership.=20Config:=20reconcile.recovery.f?= =?UTF-8?q?rom.completed.{onPodNotReady,=20onPodNotReadyThreshold},=20defa?= =?UTF-8?q?ult=20retry=20/=205m.=20Default-on;=20the=20cost=20of=20not=20a?= =?UTF-8?q?cting=20by=20default=20is=20multi-hour=20outage,=20the=20cost?= =?UTF-8?q?=20of=20acting=20is=20at=20most=20one=20StatefulSet=20restart?= =?UTF-8?q?=20per=20host=20per=20threshold=20window.=20Observability:=20tw?= =?UTF-8?q?o=20new=20EventReasons=20(StuckHostRecoveryTriggered,=20HostStu?= =?UTF-8?q?ckNotReady),=20split=20so=20a=20debounced=20flap=20leaves=20onl?= =?UTF-8?q?y=20the=20schedule=20event=20behind=20while=20a=20real=20outage?= =?UTF-8?q?=20leaves=20both.=20No=20new=20Prometheus=20counters=20-=20even?= =?UTF-8?q?ts=20on=20the=20CHI=20provide=20the=20observability=20hook=20wi?= =?UTF-8?q?thout=20expanding=20the=20operator's=20/metrics=20surface.=20Bo?= =?UTF-8?q?th=20signal=20and=20decision=20sides=20short-circuit=20when=20t?= =?UTF-8?q?he=20pod=20is=20in=20a=20kubelet-driven=20failure=20mode=20(Ima?= =?UTF-8?q?gePullBackOff,=20CrashLoopBackOff,=20Pending,=20etc.)=20so=20th?= =?UTF-8?q?e=20operator=20does=20not=20race=20kubelet=20on=20its=20own=20r?= =?UTF-8?q?ecovery.=20The=20signal=20side=20also=20skips=20pods=20with=20a?= =?UTF-8?q?=20non-zero=20DeletionTimestamp=20(graceful=20shutdown=20is=20n?= =?UTF-8?q?ot=20a=20regression).=20Validated=20end-to-end=20on=20kind=201.?= =?UTF-8?q?30=20with=20a=2065%=20bidirectional=20iptables=20drop=20on=20th?= =?UTF-8?q?e=20pod's=208123/tcp:=20self-heal=20in=20~10=20min=20vs.=20inde?= =?UTF-8?q?finite=20stall=20pre-fix.=20New=20unit=20tests=20for=20each=20n?= =?UTF-8?q?ew=20function=20(pure=20predicates=20and=20accessors),=20all=20?= =?UTF-8?q?table-driven.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: dashashutosh80 --- .../v1/type_configuration_chop.go | 52 ++++- .../type_configuration_chop_recovery_test.go | 64 ++++++ pkg/controller/chi/worker-boilerplate.go | 2 + pkg/controller/chi/worker-pod-retry.go | 110 ++++++++++ pkg/controller/chi/worker-pod-retry_test.go | 183 ++++++++++++++++ pkg/controller/chi/worker-status-helpers.go | 90 ++++++++ .../chi/worker-status-helpers_test.go | 201 ++++++++++++++++++ pkg/controller/chi/worker.go | 12 ++ .../common/announcer/event-emitter.go | 9 + 9 files changed, 722 insertions(+), 1 deletion(-) create mode 100644 pkg/controller/chi/worker-status-helpers_test.go diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index d1ffbfb6f..2c88de135 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -58,11 +58,15 @@ const ( // OnConfigurationChangeRestart means exit the process so the pod restarts with the new config. OnConfigurationChangeRestart = "restart" - // RecoveryActionNone means do nothing, CHI stays Aborted. + // RecoveryActionNone means do nothing, CHI stays in its current state. RecoveryActionNone = "none" // RecoveryActionRetry means re-enqueue CHI for reconcile (default). RecoveryActionRetry = "retry" + // defaultCompletedOnPodNotReadyThreshold is the minimum time a pod must remain in + // Ready=False before the operator considers the host stuck and re-enqueues a reconcile + defaultCompletedOnPodNotReadyThreshold = 5 * time.Minute + // Default values for ClickHouse user configuration // 1. user/profile // 2. user/quota @@ -578,6 +582,8 @@ type OperatorConfigReconcileRecovery struct { type OperatorConfigReconcileRecoveryFrom struct { // Aborted scope — recovery from Status=Aborted. Aborted OperatorConfigReconcileRecoveryScope `json:"aborted,omitempty" yaml:"aborted,omitempty"` + // Completed scope — recovery from Status=Completed when a child pod regresses to Ready=False + Completed OperatorConfigReconcileRecoveryCompletedScope `json:"completed,omitempty" yaml:"completed,omitempty"` // Future: Failed, Broken, etc. } @@ -592,6 +598,21 @@ type OperatorConfigReconcileRecoveryScope struct { // Future: OnKeeperReady, OnOperatorRestart. } +// OperatorConfigReconcileRecoveryCompletedScope holds the event→action mappings for the +// Completed scope. +type OperatorConfigReconcileRecoveryCompletedScope struct { + // OnPodNotReady controls reaction when a pod belonging to a Completed CHI flips + // Ready=True → Ready=False and stays NotReady for at least OnPodNotReadyThreshold: + // nil / "retry" (default) — re-enqueue CHI for reconcile so shouldForceRestartHost + // can decide whether to restart the host + // "none" — do nothing, host stays Ready=False until external action + OnPodNotReady *types.String `json:"onPodNotReady,omitempty" yaml:"onPodNotReady,omitempty"` + // OnPodNotReadyThreshold is the minimum duration a pod must remain in Ready=False + // before this scope fires. Accepts any time.ParseDuration string (default "5m" + // when unset, empty, or unparseable). + OnPodNotReadyThreshold *types.String `json:"onPodNotReadyThreshold,omitempty" yaml:"onPodNotReadyThreshold,omitempty"` +} + type OperatorConfigReconcileRuntime struct { ReconcileCHIsThreadsNumber int `json:"reconcileCHIsThreadsNumber" yaml:"reconcileCHIsThreadsNumber"` ReconcileShardsThreadsNumber int `json:"reconcileShardsThreadsNumber" yaml:"reconcileShardsThreadsNumber"` @@ -1634,6 +1655,35 @@ func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { return value == RecoveryActionRetry } +// ShouldRecoverCompletedOnPodNotReady reports whether the operator should re-enqueue a +// CHI reconcile when a pod belonging to a Completed CHI flips to Ready=False and stays +// there for longer than CompletedOnPodNotReadyThreshold. Default is to retry. +// Backed by reconcile.recovery.from.completed.onPodNotReady config key. +func (c *OperatorConfig) ShouldRecoverCompletedOnPodNotReady() bool { + value := strings.ToLower(c.Reconcile.Recovery.From.Completed.OnPodNotReady.String()) + if value == "" { + // Default behavior — retry + return true + } + return value == RecoveryActionRetry +} + +// CompletedOnPodNotReadyThreshold returns the minimum duration a pod must remain in +// Ready=False before the Completed recovery scope fires. Falls back to the package +// default (5m) if the config value is unset, empty, or unparseable. +// Backed by reconcile.recovery.from.completed.onPodNotReadyThreshold config key. +func (c *OperatorConfig) CompletedOnPodNotReadyThreshold() time.Duration { + raw := strings.TrimSpace(c.Reconcile.Recovery.From.Completed.OnPodNotReadyThreshold.String()) + if raw == "" { + return defaultCompletedOnPodNotReadyThreshold + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return defaultCompletedOnPodNotReadyThreshold + } + return d +} + // IsNamespaceWatched returns whether specified namespace is in a list of watched // TODO unify with GetInformerNamespace func (c *OperatorConfig) IsNamespaceWatched(namespace string) bool { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go index 1efc717f2..f5c1b8be2 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go @@ -16,6 +16,7 @@ package v1 import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -57,3 +58,66 @@ func TestRecoveryActionConstants(t *testing.T) { require.Equal(t, "none", RecoveryActionNone) require.Equal(t, "retry", RecoveryActionRetry) } + +// TestShouldRecoverCompletedOnPodNotReady verifies the accessor's behavior across the +// full matrix of possible values for reconcile.recovery.from.completed.onPodNotReady. +// Mirrors TestShouldRecoverAbortedOnPodReady so symmetric config keys behave identically. +func TestShouldRecoverCompletedOnPodNotReady(t *testing.T) { + tests := []struct { + name string + onPodNotRdy *types.String + expected bool + }{ + {"nil defaults to retry (close the gap by default)", nil, true}, + {"empty string defaults to retry", types.NewString(""), true}, + {"retry lowercase", types.NewString("retry"), true}, + {"Retry mixed case", types.NewString("Retry"), true}, + {"RETRY upper case", types.NewString("RETRY"), true}, + {"none lowercase — opt-out", types.NewString("none"), false}, + {"None mixed case", types.NewString("None"), false}, + {"NONE upper case", types.NewString("NONE"), false}, + {"unknown value treated as no-retry (fail safe)", types.NewString("bogus"), false}, + {"whitespace-only treated as no-retry", types.NewString(" "), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &OperatorConfig{} + c.Reconcile.Recovery.From.Completed.OnPodNotReady = tc.onPodNotRdy + require.Equal(t, tc.expected, c.ShouldRecoverCompletedOnPodNotReady()) + }) + } +} + +// TestCompletedOnPodNotReadyThreshold verifies the threshold parser. Unparseable, +// empty, and non-positive values must fall back to the package default — operators +// who *really* want to disable the safety net should use onPodNotReady=none, not +// pass a malformed duration. +func TestCompletedOnPodNotReadyThreshold(t *testing.T) { + tests := []struct { + name string + raw *types.String + expected time.Duration + }{ + {"nil falls back to default", nil, defaultCompletedOnPodNotReadyThreshold}, + {"empty falls back to default", types.NewString(""), defaultCompletedOnPodNotReadyThreshold}, + {"whitespace falls back to default", types.NewString(" "), defaultCompletedOnPodNotReadyThreshold}, + {"unparseable falls back to default", types.NewString("five minutes"), defaultCompletedOnPodNotReadyThreshold}, + {"zero falls back to default (don't accidentally disable)", + types.NewString("0s"), defaultCompletedOnPodNotReadyThreshold}, + {"negative falls back to default", types.NewString("-30s"), defaultCompletedOnPodNotReadyThreshold}, + {"30 seconds — aggressive", types.NewString("30s"), 30 * time.Second}, + {"5 minutes — the documented default in string form", + types.NewString("5m"), 5 * time.Minute}, + {"1 hour — conservative", types.NewString("1h"), time.Hour}, + {"complex duration: 1h30m", types.NewString("1h30m"), 90 * time.Minute}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &OperatorConfig{} + c.Reconcile.Recovery.From.Completed.OnPodNotReadyThreshold = tc.raw + require.Equal(t, tc.expected, c.CompletedOnPodNotReadyThreshold()) + }) + } +} diff --git a/pkg/controller/chi/worker-boilerplate.go b/pkg/controller/chi/worker-boilerplate.go index 97b157602..d8b28fde1 100644 --- a/pkg/controller/chi/worker-boilerplate.go +++ b/pkg/controller/chi/worker-boilerplate.go @@ -151,6 +151,8 @@ func (w *worker) processReconcilePod(ctx context.Context, cmd *cmd_queue.Reconci // and re-enqueue the CHI for reconcile. Controlled by config option // reconcile.recovery.from.aborted.onPodReady (default: retry). w.recoverAbortedReconcileOnPodReady(ctx, cmd.Old, cmd.New) + // Symmetric path for Ready→NotReady on Completed CHIs. + w.recoverCompletedReconcileOnPodNotReady(ctx, cmd.Old, cmd.New) return nil case cmd_queue.ReconcileDelete: w.a.V(1).M(cmd.Old).F().Info("Delete Pod. %s/%s", cmd.Old.Namespace, cmd.Old.Name) diff --git a/pkg/controller/chi/worker-pod-retry.go b/pkg/controller/chi/worker-pod-retry.go index 86b901b46..3cb4d2980 100644 --- a/pkg/controller/chi/worker-pod-retry.go +++ b/pkg/controller/chi/worker-pod-retry.go @@ -17,6 +17,7 @@ package chi import ( "context" "strings" + "time" core "k8s.io/api/core/v1" @@ -28,6 +29,14 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/k8s" ) +// stuckHostMinDelay floors the deferred-re-enqueue delay so a 1-second flap +// can't produce an immediate reconcile. +const stuckHostMinDelay = 5 * time.Second + +// stuckHostExtraDelay buffers threshold against apiserver/informer latency +// so the eventual reconcile observes an up-to-date LastTransitionTime. +const stuckHostExtraDelay = 2 * time.Second + // normalizeTimeAbortReasons enumerates Aborted reasons that cannot recover via // pod transitions — the spec itself must be edited. Auto-recovery skips these // to avoid metrics churn on pod-Ready flips that would just re-trigger the same @@ -132,3 +141,104 @@ func isPodNotReadyToReadyTransition(oldPod, newPod *core.Pod) bool { isReadyNow := !k8s.PodHasNotReadyContainers(newPod) return wasNotReady && isReadyNow } + +// recoverCompletedReconcileOnPodNotReady is the symmetric counterpart of +// recoverAbortedReconcileOnPodReady. It inspects a pod update event and schedules a +// delayed CHI reconcile when a child pod of a Completed CHI transitions Ready → NotReady. +// The delay equals the configured threshold (default 5m), so by the time the reconcile +// fires, shouldForceRestartHost can observe a sustained Ready=False and decide whether +// to restart the host. +func (w *worker) recoverCompletedReconcileOnPodNotReady(ctx context.Context, oldPod, newPod *core.Pod) { + if !chop.Config().ShouldRecoverCompletedOnPodNotReady() { + return + } + + if !isPodReadyToNotReadyTransition(oldPod, newPod) { + return + } + + // Skip pods that are terminating — the Ready→NotReady flip is normal + // shutdown bookkeeping, not a host regression. + if newPod.GetDeletionTimestamp() != nil && !newPod.GetDeletionTimestamp().IsZero() { + return + } + + // Skip pods already in a kubelet-driven failure mode (ImagePullBackOff, + // CrashLoopBackOff, Pending, etc.) — kubelet is handling those and an + // operator-driven StatefulSet rollout would just race it. + if podIsInKubeletFailureMode(newPod) { + return + } + + cr, err := w.c.GetCR(&newPod.ObjectMeta) + if err != nil || cr == nil { + return + } + + if !shouldTriggerStuckHostRecovery(cr) { + return + } + + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + delay := stuckHostScheduleDelay(newPod, threshold, time.Now()) + + w.a.V(1).M(cr).F(). + WithEvent(cr, a.EventActionReconcile, a.EventReasonStuckHostRecoveryTriggered). + Info( + "Stuck-host recovery scheduled: pod %s became NotReady while CHI %s/%s is Completed; "+ + "re-enqueue in %s (threshold %s)", + newPod.Name, cr.Namespace, cr.Name, delay.Truncate(time.Second), threshold, + ) + + scheduled := cr + time.AfterFunc(delay, func() { + w.c.enqueueObject(cmd_queue.NewReconcileCHI(cmd_queue.ReconcileAdd, nil, scheduled)) + }) +} + +// shouldTriggerStuckHostRecovery reports whether the given CHI is a valid stuck-host +// recovery target: status is Completed and the CHI is not being deleted. +func shouldTriggerStuckHostRecovery(cr *api.ClickHouseInstallation) bool { + if cr == nil { + return false + } + status := cr.EnsureStatus() + if status.GetStatus() != api.StatusCompleted { + return false + } + if !cr.GetDeletionTimestamp().IsZero() { + return false + } + return true +} + +// isPodReadyToNotReadyTransition reports whether the pod transitioned from "all containers +// ready" to "some container not ready". The dual of isPodNotReadyToReadyTransition. +func isPodReadyToNotReadyTransition(oldPod, newPod *core.Pod) bool { + if oldPod == nil || newPod == nil { + return false + } + wasReady := !k8s.PodHasNotReadyContainers(oldPod) + isNotReadyNow := k8s.PodHasNotReadyContainers(newPod) + return wasReady && isNotReadyNow +} + +// stuckHostScheduleDelay computes how long to wait before firing the stuck-host +// re-enqueue. It returns max(threshold − elapsed + extra, minDelay), clamped to +// non-negative. +func stuckHostScheduleDelay(newPod *core.Pod, threshold time.Duration, now time.Time) time.Duration { + elapsed := time.Duration(0) + if newPod != nil { + for _, cond := range newPod.Status.Conditions { + if cond.Type == core.PodReady && !cond.LastTransitionTime.IsZero() { + elapsed = now.Sub(cond.LastTransitionTime.Time) + break + } + } + } + delay := threshold - elapsed + stuckHostExtraDelay + if delay < stuckHostMinDelay { + delay = stuckHostMinDelay + } + return delay +} diff --git a/pkg/controller/chi/worker-pod-retry_test.go b/pkg/controller/chi/worker-pod-retry_test.go index 39893bb18..829375919 100644 --- a/pkg/controller/chi/worker-pod-retry_test.go +++ b/pkg/controller/chi/worker-pod-retry_test.go @@ -145,3 +145,186 @@ func TestShouldTriggerAutoRecovery(t *testing.T) { }) } } + +// TestIsPodReadyToNotReadyTransition verifies the dual of isPodNotReadyToReadyTransition: +// fires only on Ready→NotReady, mirrors the same nil/edge-case handling. +func TestIsPodReadyToNotReadyTransition(t *testing.T) { + tests := []struct { + name string + old, new *core.Pod + expected bool + }{ + {"nil old", nil, pod(false), false}, + {"nil new", pod(true), nil, false}, + {"both nil", nil, nil, false}, + {"ready → not ready (the target case)", pod(true), pod(false), true}, + {"ready → ready (no transition)", pod(true), pod(true), false}, + {"not ready → ready (wrong direction, handled by sibling)", pod(false), pod(true), false}, + {"not ready → not ready", pod(false), pod(false), false}, + {"multi-container: all ready → one not ready", multiContainerPod(true, true), multiContainerPod(false, true), true}, + {"multi-container: one not ready → all ready", multiContainerPod(true, false), multiContainerPod(true, true), false}, + {"multi-container: all ready → all ready", multiContainerPod(true, true), multiContainerPod(true, true), false}, + {"empty statuses → not ready (fires; empty counts as ready)", + &core.Pod{}, pod(false), true}, + {"12-container pod: last flips to not ready", + multiContainerPod(true, true, true, true, true, true, true, true, true, true, true, true), + multiContainerPod(true, true, true, true, true, true, true, true, true, true, true, false), + true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := isPodReadyToNotReadyTransition(tc.old, tc.new) + require.Equal(t, tc.expected, got) + }) + } +} + +// TestShouldTriggerStuckHostRecovery verifies the CR-state gate used by +// recoverCompletedReconcileOnPodNotReady. +func TestShouldTriggerStuckHostRecovery(t *testing.T) { + makeCR := func(status string, deleting bool) *api.ClickHouseInstallation { + cr := &api.ClickHouseInstallation{ + ObjectMeta: meta.ObjectMeta{Name: "chi", Namespace: "ns"}, + } + cr.EnsureStatus().Status = status + if deleting { + now := meta.NewTime(time.Now()) + cr.ObjectMeta.DeletionTimestamp = &now + } + return cr + } + + tests := []struct { + name string + cr *api.ClickHouseInstallation + expected bool + }{ + {"nil CR — reject", nil, false}, + // The target case: Completed CHI whose host has just regressed. + {"Completed, not deleting — accept (the target case)", makeCR(api.StatusCompleted, false), true}, + // Aborted is the sibling path's responsibility; firing stuck-host recovery on it + // would double-enqueue with recoverAbortedReconcileOnPodReady once the pod + // eventually becomes Ready again. + {"Aborted — reject (handled by sibling recoverAbortedReconcileOnPodReady path)", + makeCR(api.StatusAborted, false), false}, + // InProgress means a reconcile is already in flight; let it observe the pod state + // on its own rather than racing another enqueue. + {"InProgress — reject (reconcile already running)", makeCR(api.StatusInProgress, false), false}, + {"Terminating — reject", makeCR(api.StatusTerminating, false), false}, + {"Completed but being deleted — reject", makeCR(api.StatusCompleted, true), false}, + // Fresh CR with no status field set yet — happens between Create and the first + // status update by the operator. + {"empty status (fresh CR) — reject", makeCR("", false), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, shouldTriggerStuckHostRecovery(tc.cr)) + }) + } +} + +// TestStuckHostScheduleDelay verifies the delay computation for the deferred re-enqueue. +// The helper is pure (clock + threshold injected as args), so we can exercise the +// boundary conditions without time-mocking the rest of the controller. +func TestStuckHostScheduleDelay(t *testing.T) { + now := time.Date(2026, 5, 28, 12, 0, 0, 0, time.UTC) + + // podWithReadyTransition builds a pod whose PodReady condition transitioned at the + // given offset from "now". Negative offset means the transition is in the past. + podWithReadyTransition := func(offset time.Duration) *core.Pod { + return &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(offset))}, + }, + }, + } + } + + tests := []struct { + name string + pod *core.Pod + threshold time.Duration + // expected delay must satisfy lo <= got <= hi (small tolerance for arithmetic). + expectMin time.Duration + expectMax time.Duration + }{ + { + // Fresh transition: full threshold + small extra padding for apiserver + // catch-up. With threshold=5m and elapsed=0, delay should be ~5m02s. + name: "fresh transition: schedule full threshold + extra", + pod: podWithReadyTransition(0), + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay - time.Second, + expectMax: 5*time.Minute + stuckHostExtraDelay + time.Second, + }, + { + // Already half-elapsed: remaining ~2.5m + extra. + name: "half-elapsed: schedule the remainder", + pod: podWithReadyTransition(-150 * time.Second), + threshold: 5 * time.Minute, + expectMin: 150*time.Second + stuckHostExtraDelay - time.Second, + expectMax: 150*time.Second + stuckHostExtraDelay + time.Second, + }, + { + // Threshold already past at schedule time (e.g. operator restart after + // long outage): clamp to stuckHostMinDelay rather than firing instantly, + // so a single quick flap doesn't produce an immediate restart. + name: "threshold already past: clamp to minDelay", + pod: podWithReadyTransition(-10 * time.Minute), + threshold: 5 * time.Minute, + expectMin: stuckHostMinDelay, + expectMax: stuckHostMinDelay, + }, + { + // Nil pod: no LastTransitionTime info → treat as elapsed=0 → full threshold. + name: "nil pod: full threshold", + pod: nil, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Pod has no PodReady condition (very early in lifecycle): elapsed=0. + name: "pod missing PodReady condition: full threshold", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{}}}, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Zero LastTransitionTime (apiserver hasn't stamped it yet): treat as + // elapsed=0, schedule full threshold. + name: "zero LastTransitionTime: full threshold", + pod: &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse}, + }, + }, + }, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Threshold smaller than minDelay: minDelay still floors the result. + name: "tiny threshold: clamp to minDelay", + pod: podWithReadyTransition(0), + threshold: 1 * time.Second, + expectMin: stuckHostMinDelay, + expectMax: stuckHostMinDelay, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := stuckHostScheduleDelay(tc.pod, tc.threshold, now) + require.GreaterOrEqual(t, got, tc.expectMin, "delay below expected minimum") + require.LessOrEqual(t, got, tc.expectMax, "delay above expected maximum") + }) + } +} diff --git a/pkg/controller/chi/worker-status-helpers.go b/pkg/controller/chi/worker-status-helpers.go index 8da4e493d..f04cb3f2d 100644 --- a/pkg/controller/chi/worker-status-helpers.go +++ b/pkg/controller/chi/worker-status-helpers.go @@ -18,6 +18,8 @@ import ( "context" "time" + core "k8s.io/api/core/v1" + log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" @@ -52,6 +54,94 @@ func (w *worker) isPodReady(ctx context.Context, host *api.Host) bool { return false } +// isPodSustainedNotReady reports whether the host's pod is currently Ready=False AND +// has been so for at least `threshold`. Returns false for pods whose failure mode is +// already being handled by kubelet (ImagePullBackOff, CrashLoopBackOff, Pending, etc.) +// so the operator does not race kubelet on its own recovery path. +func (w *worker) isPodSustainedNotReady(ctx context.Context, host *api.Host, threshold time.Duration) bool { + if threshold <= 0 { + // Threshold of 0/negative means "feature disabled" + return false + } + pod, err := w.c.kube.Pod().Get(ctx, host) + if err != nil || pod == nil { + return false + } + if podIsInKubeletFailureMode(pod) { + return false + } + return podIsSustainedNotReady(pod, threshold, time.Now()) +} + +// podIsSustainedNotReady is the pure inner predicate of isPodSustainedNotReady, +// extracted so it can be exercised without a kube client. Returns true iff the pod +// has a PodReady condition that is currently not True and whose LastTransitionTime +// is at least `threshold` in the past relative to `now`. +func podIsSustainedNotReady(pod *core.Pod, threshold time.Duration, now time.Time) bool { + if pod == nil || threshold <= 0 { + return false + } + for _, cond := range pod.Status.Conditions { + if cond.Type != core.PodReady { + continue + } + if cond.Status == core.ConditionTrue { + return false + } + // Status is False or Unknown. Treat both as "not ready" + if cond.LastTransitionTime.IsZero() { + return false + } + return now.Sub(cond.LastTransitionTime.Time) >= threshold + } + // No PodReady condition at all. + return false +} + +// kubeletDrivenWaitingReasons is the set of container Waiting.Reason values that +// indicate kubelet is already actively recovering the pod and a parallel +// operator-driven StatefulSet rollout would just race kubelet. +var kubeletDrivenWaitingReasons = map[string]struct{}{ + "CrashLoopBackOff": {}, + "ImagePullBackOff": {}, + "ErrImagePull": {}, + "InvalidImageName": {}, + "CreateContainerError": {}, + "RunContainerError": {}, + "ContainerCannotRun": {}, + "CreateContainerConfigError": {}, +} + +// podIsInKubeletFailureMode reports whether the pod is in a state where kubelet +// (or the kube-scheduler) is already handling the failure: not yet scheduled, +// in Pending phase, or any container in a kubelet-driven waiting reason. +// In those states an operator-driven reconcile would race kubelet without value. +func podIsInKubeletFailureMode(pod *core.Pod) bool { + if pod == nil { + return false + } + if pod.Status.Phase == core.PodPending { + return true + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting == nil { + continue + } + if _, hit := kubeletDrivenWaitingReasons[cs.State.Waiting.Reason]; hit { + return true + } + } + for _, cs := range pod.Status.InitContainerStatuses { + if cs.State.Waiting == nil { + continue + } + if _, hit := kubeletDrivenWaitingReasons[cs.State.Waiting.Reason]; hit { + return true + } + } + return false +} + func (w *worker) isPodStarted(ctx context.Context, host *api.Host) bool { if pod, err := w.c.kube.Pod().Get(ctx, host); err == nil { return k8s.PodHasAllContainersStarted(pod) diff --git a/pkg/controller/chi/worker-status-helpers_test.go b/pkg/controller/chi/worker-status-helpers_test.go new file mode 100644 index 000000000..cb4d7391b --- /dev/null +++ b/pkg/controller/chi/worker-status-helpers_test.go @@ -0,0 +1,201 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chi + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestPodIsSustainedNotReady covers the pure post-fetch decision used by +// isPodSustainedNotReady. +func TestPodIsSustainedNotReady(t *testing.T) { + now := time.Date(2026, 5, 28, 12, 0, 0, 0, time.UTC) + + withReady := func(status core.ConditionStatus, transitionOffset time.Duration) *core.Pod { + return &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: status, + LastTransitionTime: meta.NewTime(now.Add(transitionOffset))}, + }, + }, + } + } + + tests := []struct { + name string + pod *core.Pod + threshold time.Duration + expected bool + }{ + { + name: "nil pod — never sustained", + pod: nil, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "zero threshold — feature disabled, never fires", + pod: withReady(core.ConditionFalse, -30*time.Minute), + threshold: 0, + expected: false, + }, + { + name: "negative threshold — feature disabled, never fires", + pod: withReady(core.ConditionFalse, -30*time.Minute), + threshold: -1 * time.Second, + expected: false, + }, + { + name: "no PodReady condition — early lifecycle, never sustained", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodInitialized, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + }}}, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=True — not sustained even with old LastTransitionTime", + pod: withReady(core.ConditionTrue, -30*time.Minute), + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=False but only 1m ago — under threshold (transient)", + pod: withReady(core.ConditionFalse, -1*time.Minute), + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=False for exactly the threshold — fires (>= semantics)", + pod: withReady(core.ConditionFalse, -5*time.Minute), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=False for 26h — the production incident, fires", + pod: withReady(core.ConditionFalse, -26*time.Hour), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=Unknown for 10m — treated as not-ready, fires", + pod: withReady(core.ConditionUnknown, -10*time.Minute), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=False but LastTransitionTime is zero — conservative, don't fire", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{{Type: core.PodReady, Status: core.ConditionFalse}}}}, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "multiple PodReady entries — use first match", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + {Type: core.PodReady, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now)}, + }}}, + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodScheduled present alongside PodReady=False — still fires on Ready", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodScheduled, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now.Add(-1 * time.Hour))}, + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + }}}, + threshold: 5 * time.Minute, + expected: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, podIsSustainedNotReady(tc.pod, tc.threshold, now)) + }) + } +} + +// TestPodIsInKubeletFailureMode locks in the kubelet-recovery filter: any pod whose +// failure mode is already being handled by kubelet (image pull errors, crash loops, +// pending, etc.) must NOT trigger the stuck-host recovery path. +func TestPodIsInKubeletFailureMode(t *testing.T) { + waitingContainer := func(reason string) *core.Pod { + return &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + ContainerStatuses: []core.ContainerStatus{ + {Name: "clickhouse", State: core.ContainerState{ + Waiting: &core.ContainerStateWaiting{Reason: reason}, + }}, + }, + }} + } + waitingInit := func(reason string) *core.Pod { + return &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + InitContainerStatuses: []core.ContainerStatus{ + {Name: "init", State: core.ContainerState{ + Waiting: &core.ContainerStateWaiting{Reason: reason}, + }}, + }, + }} + } + + tests := []struct { + name string + pod *core.Pod + expected bool + }{ + {"nil pod", nil, false}, + {"no statuses, running phase", &core.Pod{Status: core.PodStatus{Phase: core.PodRunning}}, false}, + {"Pending phase — scheduler/kubelet handling", &core.Pod{Status: core.PodStatus{Phase: core.PodPending}}, true}, + {"ImagePullBackOff — kubelet handling", waitingContainer("ImagePullBackOff"), true}, + {"ErrImagePull — kubelet handling", waitingContainer("ErrImagePull"), true}, + {"InvalidImageName — kubelet handling", waitingContainer("InvalidImageName"), true}, + {"CrashLoopBackOff — kubelet handling", waitingContainer("CrashLoopBackOff"), true}, + {"CreateContainerError — kubelet handling", waitingContainer("CreateContainerError"), true}, + {"RunContainerError — kubelet handling", waitingContainer("RunContainerError"), true}, + {"ContainerCannotRun — kubelet handling", waitingContainer("ContainerCannotRun"), true}, + {"CreateContainerConfigError — kubelet handling", waitingContainer("CreateContainerConfigError"), true}, + {"init container in ImagePullBackOff — kubelet handling", waitingInit("ImagePullBackOff"), true}, + {"ContainerCreating — transient, not kubelet failure", waitingContainer("ContainerCreating"), false}, + {"PodInitializing — transient, not kubelet failure", waitingContainer("PodInitializing"), false}, + {"running container, no waiting state", &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + ContainerStatuses: []core.ContainerStatus{ + {Name: "clickhouse", Ready: true, State: core.ContainerState{ + Running: &core.ContainerStateRunning{}, + }}, + }, + }}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, podIsInKubeletFailureMode(tc.pod)) + }) + } +} diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index caf251699..77e1eed29 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -27,6 +27,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" + "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/controller/chi/metrics" "github.com/altinity/clickhouse-operator/pkg/controller/common" a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" @@ -193,6 +194,17 @@ func (w *worker) shouldForceRestartHost(ctx context.Context, host *api.Host) boo w.a.V(1).M(host).F().Info("Host with unknown version and in CrashLoopBackOff should be restarted. It most likely is unable to start due to bad config. Host: %s", host.GetName()) return true + case chop.Config().ShouldRecoverCompletedOnPodNotReady() && + w.isPodSustainedNotReady(ctx, host, chop.Config().CompletedOnPodNotReadyThreshold()): + // Closes the gap where Completed CHIs with a sustained-NotReady host + // were left stuck indefinitely. + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + w.a.V(1).M(host).F(). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonHostStuckNotReady). + Info("Host pod has been Ready=False past threshold %s — force restart. Host: %s", + threshold, host.GetName()) + return true + default: w.a.V(1).M(host).F().Info("Host force restart is not required. Host: %s", host.GetName()) return false diff --git a/pkg/controller/common/announcer/event-emitter.go b/pkg/controller/common/announcer/event-emitter.go index f7247b6c5..e71547ad1 100644 --- a/pkg/controller/common/announcer/event-emitter.go +++ b/pkg/controller/common/announcer/event-emitter.go @@ -65,6 +65,15 @@ const ( // reconcile was aborted, on observing a recovery signal (e.g. a pod became Ready). EventReasonAutoRecoveryTriggered = "AutoRecoveryTriggered" + // EventReasonStuckHostRecoveryTriggered fires when the operator re-enqueues a + // Completed CHI for reconcile because one of its hosts has been Ready=False for + // longer than the configured threshold. + EventReasonStuckHostRecoveryTriggered = "StuckHostRecoveryTriggered" + + // EventReasonHostStuckNotReady fires when shouldForceRestartHost decides to force + // a host restart because the pod has been Ready=False past the configured threshold. + EventReasonHostStuckNotReady = "HostStuckNotReady" + // EventReasonKeeperUpdateNoEndpointChange fires when the operator observes a referenced // CHK reconcile completing but decides not to trigger a CHI reconcile because the resolved // zookeeper endpoints have not changed. From 4e127babc3a8d685895f18f04bc5740c0360fb3a Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 5 Jun 2026 14:44:43 +0500 Subject: [PATCH 002/164] 0.27.2 --- release | 2 +- releases | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/release b/release index 83b473049..3edc695dc 100644 --- a/release +++ b/release @@ -1 +1 @@ -0.27.1 +0.27.2 diff --git a/releases b/releases index f2fb57115..1874fa93d 100644 --- a/releases +++ b/releases @@ -1,3 +1,4 @@ +0.27.1 0.27.0 0.26.3 0.26.2 From 3e995c045c0f09a94fe86384550e653723a39cc7 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 5 Jun 2026 14:46:12 +0500 Subject: [PATCH 003/164] env: manifests --- .../clickhouse-operator-install-ansible.yaml | 54 ++++++++--------- ...house-operator-install-bundle-v1beta1.yaml | 58 +++++++++---------- .../clickhouse-operator-install-bundle.yaml | 58 +++++++++---------- ...use-operator-install-template-v1beta1.yaml | 46 +++++++-------- .../clickhouse-operator-install-template.yaml | 46 +++++++-------- .../clickhouse-operator-install-tf.yaml | 54 ++++++++--------- deploy/operator/parts/crd.yaml | 14 ++--- 7 files changed, 165 insertions(+), 165 deletions(-) diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 46a8b2125..cae199914 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1809,14 +1809,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3610,7 +3610,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4303,14 +4303,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5220,7 +5220,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5246,7 +5246,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5478,7 +5478,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5500,7 +5500,7 @@ metadata: name: etc-clickhouse-operator-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6186,7 +6186,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6202,7 +6202,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6301,7 +6301,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6401,7 +6401,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6464,7 +6464,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6480,7 +6480,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6577,7 +6577,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6595,7 +6595,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6603,7 +6603,7 @@ data: # Template parameters available: # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN={{ password }} # @@ -6613,7 +6613,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6624,9 +6624,9 @@ stringData: # # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6637,7 +6637,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6693,7 +6693,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6775,7 +6775,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6872,7 +6872,7 @@ metadata: name: clickhouse-operator-metrics namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index a294e39af..82dd6bb26 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1792,14 +1792,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3581,7 +3581,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4265,14 +4265,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5179,7 +5179,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 # Template Parameters: # @@ -5204,7 +5204,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5424,7 +5424,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5457,7 +5457,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5677,7 +5677,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5699,7 +5699,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6384,7 +6384,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6400,7 +6400,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6494,7 +6494,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6592,7 +6592,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6654,7 +6654,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6670,7 +6670,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6764,7 +6764,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6782,7 +6782,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6790,7 +6790,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6800,7 +6800,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6811,9 +6811,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6824,7 +6824,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6880,7 +6880,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6960,7 +6960,7 @@ spec: - containerPort: 9999 name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7056,7 +7056,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 623d8e115..27908766d 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1802,14 +1802,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3603,7 +3603,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4296,14 +4296,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5213,7 +5213,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5239,7 +5239,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5471,7 +5471,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5505,7 +5505,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5737,7 +5737,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5759,7 +5759,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6445,7 +6445,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6461,7 +6461,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6560,7 +6560,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6660,7 +6660,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6723,7 +6723,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6739,7 +6739,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6836,7 +6836,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6854,7 +6854,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6862,7 +6862,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6872,7 +6872,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6883,9 +6883,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6896,7 +6896,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6952,7 +6952,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7034,7 +7034,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7131,7 +7131,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 4e027223e..7b6198106 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1792,14 +1792,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3581,7 +3581,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4265,14 +4265,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5179,7 +5179,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 # Template Parameters: # @@ -5204,7 +5204,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5424,7 +5424,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5446,7 +5446,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6131,7 +6131,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6147,7 +6147,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6241,7 +6241,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6339,7 +6339,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6401,7 +6401,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6417,7 +6417,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6511,7 +6511,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6529,7 +6529,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6537,7 +6537,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6547,7 +6547,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6571,7 +6571,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6803,7 +6803,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 97bdd7d6f..af7e2fa46 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1802,14 +1802,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3603,7 +3603,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4296,14 +4296,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5213,7 +5213,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5239,7 +5239,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5471,7 +5471,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5493,7 +5493,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6179,7 +6179,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6195,7 +6195,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6294,7 +6294,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6394,7 +6394,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6457,7 +6457,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6473,7 +6473,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6570,7 +6570,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6588,7 +6588,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6596,7 +6596,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6606,7 +6606,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6630,7 +6630,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6865,7 +6865,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 4c1ed23b5..c3eb2f69c 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1809,14 +1809,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3610,7 +3610,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4303,14 +4303,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5220,7 +5220,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5246,7 +5246,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5478,7 +5478,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5500,7 +5500,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -6186,7 +6186,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6202,7 +6202,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6301,7 +6301,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6401,7 +6401,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6464,7 +6464,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6480,7 +6480,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6577,7 +6577,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6595,7 +6595,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6603,7 +6603,7 @@ data: # Template parameters available: # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=${password} # @@ -6613,7 +6613,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6624,9 +6624,9 @@ stringData: # # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6637,7 +6637,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6693,7 +6693,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6775,7 +6775,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6872,7 +6872,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 03b5dbebd..6a9c83cb7 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4027,14 +4027,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -8053,7 +8053,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -8876,14 +8876,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced From 16c69e87387eba72f3a02d1ed0499d01ed7a6d35 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 5 Jun 2026 14:46:12 +0500 Subject: [PATCH 004/164] env: helm chart --- deploy/helm/clickhouse-operator/Chart.yaml | 4 ++-- deploy/helm/clickhouse-operator/README.md | 2 +- ...ition-clickhouseinstallations.clickhouse.altinity.com.yaml | 4 ++-- ...ickhouseinstallationtemplates.clickhouse.altinity.com.yaml | 4 ++-- ...usekeeperinstallations.clickhouse-keeper.altinity.com.yaml | 4 ++-- ...ckhouseoperatorconfigurations.clickhouse.altinity.com.yaml | 2 +- .../templates/generated/Deployment-clickhouse-operator.yaml | 4 ++-- .../templates/generated/Secret-clickhouse-operator.yaml | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deploy/helm/clickhouse-operator/Chart.yaml b/deploy/helm/clickhouse-operator/Chart.yaml index ca82bf48a..a6631c54e 100644 --- a/deploy/helm/clickhouse-operator/Chart.yaml +++ b/deploy/helm/clickhouse-operator/Chart.yaml @@ -17,8 +17,8 @@ description: |- kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ``` type: application -version: 0.27.1 -appVersion: 0.27.1 +version: 0.27.2 +appVersion: 0.27.2 home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg maintainers: diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md index 8e9cfa4ca..25b5070f3 100644 --- a/deploy/helm/clickhouse-operator/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.27.1](https://img.shields.io/badge/Version-0.27.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.27.1](https://img.shields.io/badge/AppVersion-0.27.1-informational?style=flat-square) +![Version: 0.27.2](https://img.shields.io/badge/Version-0.27.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.27.2](https://img.shields.io/badge/AppVersion-0.27.2-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index 267d11645..64a94900a 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index bf5909b09..6bece68a8 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 6ce0865ac..68ebc8ea8 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 257915bde..d26c225f1 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml index 825679095..5cfc08e3c 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator diff --git a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml index 358c495d6..43d4be33f 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # From d20f0d471781cdbc41731c688081eb867b85ea18 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 5 Jun 2026 14:54:08 +0500 Subject: [PATCH 005/164] dev: brancher --- dev/start_new_release_branch.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/start_new_release_branch.sh b/dev/start_new_release_branch.sh index c7decfddd..476858861 100755 --- a/dev/start_new_release_branch.sh +++ b/dev/start_new_release_branch.sh @@ -69,6 +69,11 @@ esac read -p "Press enter to start new release" echo "Starting new release: ${NEW_RELEASE}" +# Pull latest master +git checkout master +git pull +git pull altinity master + # Create release branch git branch "${NEW_RELEASE}" git checkout "${NEW_RELEASE}" From 86492955984e4ff137200153ba10b48d9d02e393 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 5 Jun 2026 14:55:00 +0500 Subject: [PATCH 006/164] dev: releaser --- dev/start_new_release_branch.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/start_new_release_branch.sh b/dev/start_new_release_branch.sh index 476858861..aa8d2cf30 100755 --- a/dev/start_new_release_branch.sh +++ b/dev/start_new_release_branch.sh @@ -101,5 +101,9 @@ git -C "${SRC_ROOT}" commit -m "env: manifests" git -C "${SRC_ROOT}" add deploy/helm/ git -C "${SRC_ROOT}" commit -m "env: helm chart" +# Push new branch to altinity +git push altinity + +# Repository status echo "git status:" git -C "${SRC_ROOT}" status From 77520e72bd25e497e141d767ab639c6461cdc319 Mon Sep 17 00:00:00 2001 From: alz Date: Fri, 5 Jun 2026 14:26:44 +0300 Subject: [PATCH 007/164] Cleanup/improve inter-cluster comm tests --- ...test-039-0-communications-with-secret.yaml | 4 -- ...test-039-1-communications-with-secret.yaml | 4 -- ...test-039-2-communications-with-secret.yaml | 4 -- ...test-039-3-communications-with-secret.yaml | 4 -- ...test-039-4-communications-with-secret.yaml | 72 ------------------- tests/e2e/test_operator.py | 64 ++++------------- 6 files changed, 14 insertions(+), 138 deletions(-) delete mode 100644 tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml diff --git a/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml index 18b0b4a8f..60e2de558 100644 --- a/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" layout: diff --git a/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml index 4bc8ef649..82dcc9a78 100644 --- a/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml index a46bee7d0..64ea5ef03 100644 --- a/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml index 5d7fcc2ac..fc7768c79 100644 --- a/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml deleted file mode 100644 index 8b9be245a..000000000 --- a/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-039-secret-communications" -spec: - useTemplates: - - name: clickhouse-version - configuration: - users: - default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 - clusters: - - name: "default" - secure: "yes" - secret: - auto: "yes" - layout: - shardsCount: 2 - replicasCount: 1 - settings: - tcp_port: 9000 # keep for localhost - tcp_port_secure: 9440 - interserver_http_port: _removed_ - interserver_https_port: 9009 - files: - settings.xml: | - - - - /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt - /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key - /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem - none - - AcceptCertificateHandler - - true - true - sslv2,sslv3 - true - - - true - true - sslv2,sslv3 - true - none - - AcceptCertificateHandler - - - - - server.crt: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: server.crt - server.key: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: server.key - dhparam.pem: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: dhparam.pem - diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 0faac7db8..a0fc5e8f1 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -4368,12 +4368,6 @@ def test_039(self, step=0, delete_chi=0): cluster = "default" manifest = f"manifests/chi/test-039-{step}-communications-with-secret.yaml" chi = yaml_manifest.get_name(util.get_full_path(manifest)) - util.require_keeper(keeper_type=self.context.keeper_type) - - with Given("clickhouse-certs.yaml secret is installed"): - kubectl.apply( - util.get_full_path("manifests/secret/clickhouse-certs.yaml"), - ) with Given("chi exists"): kubectl.create_and_check( @@ -4381,7 +4375,6 @@ def test_039(self, step=0, delete_chi=0): check={ "apply_templates": { current().context.clickhouse_template, - "manifests/secret/test-038-secret.yaml", }, "pod_count": 2, "do_not_delete": 1, @@ -4390,50 +4383,26 @@ def test_039(self, step=0, delete_chi=0): wait_for_cluster(chi, cluster, 2, pwd="qkrq") - with When("I create distributed table that use secure port and insert data into it"): - clickhouse.query( - chi, - "CREATE OR REPLACE TABLE secure on cluster '{cluster}' (a UInt32) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY a", - pwd="qkrq", - ) - clickhouse.query( - chi, - "CREATE OR REPLACE TABLE secure_dist on cluster '{cluster}' as secure ENGINE = Distributed('{cluster}', default, secure, a%2)", - pwd="qkrq", - ) - clickhouse.query( - chi, - "INSERT INTO secure_dist select number as a from numbers(10)", - pwd="qkrq", - ) - if step == 0: with Then("Select in cluster with no secret should fail"): - r = clickhouse.query_with_error(chi, "SELECT count(a) FROM secure_dist", pwd="qkrq") + r = clickhouse.query_with_error(chi, "SELECT * FROM cluster('{cluster}', system.one)", pwd="qkrq") assert "AUTHENTICATION_FAILED" in r with And("Select from all-sharded with no secret should fail"): r = clickhouse.query_with_error(chi, "SELECT * FROM cluster('all-sharded', system.one)", pwd="qkrq") assert "AUTHENTICATION_FAILED" in r if step > 0: with Then("Select in cluster with secret should pass"): - r = clickhouse.query(chi, "SELECT count() FROM secure_dist", pwd="qkrq") - assert r == "10" + r = clickhouse.query(chi, "SELECT * FROM cluster('{cluster}', system.one) limit 1", pwd="qkrq") + assert r == "0" with And("Select from all-sharded with secret should pass"): r = clickhouse.query_with_error(chi, "SELECT * FROM cluster('all-sharded', system.one) limit 1", pwd="qkrq") assert r == "0" - - if step == 4: - with Then("Create replicated table to test interserver_https_port"): - clickhouse.query( - chi, - "CREATE OR REPLACE TABLE secure_repl on cluster 'all-replicated' (a UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{uuid}', '{replica}') PARTITION BY tuple() ORDER BY a", - pwd="qkrq", - ) - clickhouse.query( - chi, - "INSERT INTO secure_repl select number as a from numbers(10)", - pwd="qkrq", - ) + with And("Select from all-clusters with secret should pass"): + r = clickhouse.query_with_error(chi, "SELECT * FROM cluster('all-clusters', system.one) limit 1", pwd="qkrq") + assert r == "0" + with And("Select from all-replicated with secret should pass"): + r = clickhouse.query_with_error(chi, "SELECT * FROM cluster('all-replicated', system.one) limit 1", pwd="qkrq") + assert r == "0" with Finally("I delete namespace"): delete_test_namespace() @@ -4475,17 +4444,12 @@ def test_010039_3(self): """Check clickhouse-operator support inter-cluster communications with k8s secret.""" create_shell_namespace_clickhouse_template() - test_039(step=3) - - -@TestScenario -@Requirements(RQ_SRS_026_ClickHouseOperator_InterClusterCommunicationWithSecret("1.0")) -@Name("test_010039_4. Inter-cluster communications over HTTPS") -def test_010039_4(self): - """Check clickhouse-operator support inter-cluster communications over HTTPS.""" - create_shell_namespace_clickhouse_template() + with Given("test-038-secret.yamlsecret is installed"): + kubectl.apply( + util.get_full_path("manifests/secret/test-038-secret.yaml"), + ) - test_039(step=4, delete_chi=1) + test_039(step=3) @TestScenario From ab162a15d90941eb75f449e42d5ccc6e3734d48c Mon Sep 17 00:00:00 2001 From: alz Date: Fri, 5 Jun 2026 14:48:35 +0300 Subject: [PATCH 008/164] Removed 020013 that is fully covered by 020016 --- .../test-020013-chk-insecure-baseline.yaml | 30 --------- tests/e2e/test_operator.py | 63 ++++--------------- 2 files changed, 11 insertions(+), 82 deletions(-) delete mode 100644 tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml diff --git a/tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml b/tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml deleted file mode 100644 index 1b8e6f4c6..000000000 --- a/tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: "clickhouse-keeper.altinity.com/v1" -kind: "ClickHouseKeeperInstallation" -metadata: - name: test-020013-insecure -spec: - # Legacy CHK shape: no cluster.secure flag. Used by test_020013 as a - # regression sentinel — secure-flag normalizer changes must be dormant here - # (no zk-secure Service port, no 1 in Raft XML). Mirrors - # test-020011 minus the secure flag so any divergence is attributable to - # cluster.secure alone. - defaults: - templates: - podTemplate: default - configuration: - clusters: - - name: "keeper" - layout: - replicasCount: 1 - settings: - logger/level: "information" - logger/console: "true" - listen_host: "0.0.0.0" - keeper_server/four_letter_word_white_list: "*" - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-keeper - image: "clickhouse/clickhouse-keeper:25.8" diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index a0fc5e8f1..b4cf70355 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -7452,58 +7452,6 @@ def test_020011(self): delete_test_namespace() -@TestScenario -@Name("test_020013. CHK without cluster.secure preserves insecure-only Service (back-compat)") -@Requirements(RQ_SRS_026_ClickHouseOperator_Create("1.0")) -def test_020013(self): - """Back-compat regression sentinel. A CHK that does NOT declare - cluster.secure=yes (legacy default) must still reconcile to Completed - when no FIPS hardening is in effect. Proves the secure-flag normalizer - changes are dormant on non-adopters: no zk-secure port on the Service, - no 1 in Raft XML, and crExposesSecureZK()==false keeps - the emitted Service byte-identical to the legacy insecure-only shape. - """ - create_shell_namespace_clickhouse_template() - - chk_manifest = "manifests/chk/test-020013-chk-insecure-baseline.yaml" - chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) - - with When("Apply CHK without cluster.secure (legacy default)"): - kubectl.create_and_check( - manifest=chk_manifest, - kind="chk", - check={ - "pod_count": 1, - "chk_status": "Completed", - "do_not_delete": 1, - }, - ) - - with Then("CR-scope Service exposes only zk:2181 (no zk-secure)"): - svc = kubectl.get("service", f"keeper-{chk}") - port_names = {p["name"] for p in svc["spec"]["ports"]} - assert "zk" in port_names, error( - f"expected plain zk port, got {port_names}" - ) - assert "zk-secure" not in port_names, error( - f"zk-secure port must be absent without cluster.secure=yes; got {port_names}" - ) - - with And("Common ConfigMap raft XML omits 1"): - cm = kubectl.get("configmap", f"chk-{chk}-common-configd") - data = cm.get("data", {}) - assert "chop-generated-raft.xml" in data, error( - f"chop-generated-raft.xml missing; ConfigMap keys: {list(data.keys())}" - ) - raft_xml = data["chop-generated-raft.xml"] - assert "1" not in raft_xml, error( - f"unexpected 1 in back-compat raft xml:\n{raft_xml}" - ) - - with Finally("I clean up"): - delete_test_namespace() - - @TestScenario @Name("test_020014. CHK cluster.insecure=no + secure=yes wires TLS-only listener and pgrep liveness fallback") @Requirements(RQ_SRS_026_ClickHouseOperator_Create("1.0")) @@ -7710,6 +7658,17 @@ def test_020016(self): f"legacy CHK; ConfigMap keys: {list(data.keys())}" ) + with And("Common ConfigMap raft XML omits 1"): + cm = kubectl.get("configmap", f"chk-{chk}-common-configd") + data = cm.get("data", {}) + assert "chop-generated-raft.xml" in data, error( + f"chop-generated-raft.xml missing; ConfigMap keys: {list(data.keys())}" + ) + raft_xml = data["chop-generated-raft.xml"] + assert "1" not in raft_xml, error( + f"unexpected 1 in back-compat raft xml:\n{raft_xml}" + ) + with And("Liveness probe uses bash /dev/tcp ruok (NOT pgrep fallback)"): sts = kubectl.get("statefulset", f"chk-{chk}-{cluster}-{host}") cmd = sts["spec"]["template"]["spec"]["containers"][0]["livenessProbe"]["exec"]["command"] From c996d30d47ccdf62416f9f2a9292a3a4860aa2cb Mon Sep 17 00:00:00 2001 From: alz Date: Fri, 5 Jun 2026 16:01:44 +0300 Subject: [PATCH 009/164] Removed test-020 since multi-volume scenarios are fully tested by 021 --- .../chi/test-020-1-multi-volume.yaml | 51 --------------- .../chi/test-020-2-multi-volume.yaml | 51 --------------- tests/e2e/test_operator.py | 63 ------------------- 3 files changed, 165 deletions(-) delete mode 100644 tests/e2e/manifests/chi/test-020-1-multi-volume.yaml delete mode 100644 tests/e2e/manifests/chi/test-020-2-multi-volume.yaml diff --git a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml deleted file mode 100644 index c5ec3f271..000000000 --- a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-020-1-multi-volume" -spec: - useTemplates: - - name: clickhouse-version - configuration: - clusters: - - name: simple - layout: - shardsCount: 1 - settings: - storage_configuration/disks/disk2/path: /var/lib/clickhouse2/ - storage_configuration/policies/default/volumes/default/disk: default - storage_configuration/policies/default/volumes/disk2/disk: disk2 - defaults: -# storageManagement: -# provisioner: StatefulSet - templates: - podTemplate: default - templates: - volumeClaimTemplates: - - name: disk1 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 300Mi - - name: disk2 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - volumeMounts: - - name: disk1 - mountPath: /var/lib/clickhouse - - name: disk2 - mountPath: /var/lib/clickhouse2 - command: - - /bin/bash - - '-c' - - chown clickhouse /var/lib/clickhouse2 && /entrypoint.sh diff --git a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml deleted file mode 100644 index 5013608ce..000000000 --- a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-020-2-multi-volume" -spec: - useTemplates: - - name: clickhouse-version - configuration: - clusters: - - name: simple - layout: - shardsCount: 1 - settings: - storage_configuration/disks/disk2/path: /var/lib/clickhouse2/ - storage_configuration/policies/default/volumes/default/disk: default - storage_configuration/policies/default/volumes/disk2/disk: disk2 - defaults: - storageManagement: - provisioner: Operator - templates: - podTemplate: default - templates: - volumeClaimTemplates: - - name: disk1 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 300Mi - - name: disk2 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - volumeMounts: - - name: disk1 - mountPath: /var/lib/clickhouse - - name: disk2 - mountPath: /var/lib/clickhouse2 - command: - - /bin/bash - - '-c' - - chown clickhouse /var/lib/clickhouse2 && /entrypoint.sh diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index b4cf70355..ef4d5c46a 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -2479,69 +2479,6 @@ def test_010019_2(self): test_019(step=2) - -@TestCheck -def test_020(self, step=1): - manifest = f"manifests/chi/test-020-{step}-multi-volume.yaml" - chi = yaml_manifest.get_name(util.get_full_path(manifest)) - kubectl.create_and_check( - manifest=manifest, - check={ - "pod_count": 1, - "pod_volumes": { - "/var/lib/clickhouse", - "/var/lib/clickhouse2", - }, - "do_not_delete": 1, - }, - ) - kubectl.wait_chi_status(chi, "Completed") - - with Then("Test that ClickHouse recognizes two disks"): - cnt = clickhouse.query(chi, "select count() from system.disks") - assert cnt == "2" - - with When("Create a table and insert 1 row"): - clickhouse.query(chi, "create table test_disks(a Int8) Engine = MergeTree() order by a") - clickhouse.query(chi, "insert into test_disks values (1)") - - with Then("Data should be placed on default disk"): - disk = clickhouse.query(chi, "select disk_name from system.parts where table='test_disks'") - print(f"disk : {disk}") - print(f"want: default") - assert disk == "default" or True - - with When(f"alter table test_disks move partition tuple() to disk 'disk2'"): - clickhouse.query_with_error(chi, f"alter table test_disks move partition tuple() to disk 'disk2'") - - with Then(f"Data should be placed on disk2"): - disk = clickhouse.query(chi, "select disk_name from system.parts where table='test_disks'") - print(f"disk : {disk}") - print(f"want: disk2") - assert disk == "disk2" or True - - with Finally("I clean up"): - delete_test_namespace() - - -@TestScenario -@Name("test_010020_1. Test multi-volume configuration, step=1") -@Requirements(RQ_SRS_026_ClickHouseOperator_Deployments_MultipleStorageVolumes("1.0")) -def test_010020_1(self): - create_shell_namespace_clickhouse_template() - - test_020(step=1) - - -@TestScenario -@Name("test_010020_2. Test multi-volume configuration, step=2") -@Requirements(RQ_SRS_026_ClickHouseOperator_Deployments_MultipleStorageVolumes("1.0")) -def test_010020_2(self): - create_shell_namespace_clickhouse_template() - - test_020(step=2) - - def pause(): if settings.step_by_step: input("Press Enter to continue...") From 735e90d616c926f619da9c0509da12ba1d46bea4 Mon Sep 17 00:00:00 2001 From: alz Date: Fri, 5 Jun 2026 16:07:26 +0300 Subject: [PATCH 010/164] Remove test_018 that has functionality covered by test_016 --- .../manifests/chi/test-018-configmap-1.yaml | 14 ------- .../manifests/chi/test-018-configmap-2.yaml | 14 ------- tests/e2e/test_operator.py | 41 ------------------- 3 files changed, 69 deletions(-) delete mode 100644 tests/e2e/manifests/chi/test-018-configmap-1.yaml delete mode 100644 tests/e2e/manifests/chi/test-018-configmap-2.yaml diff --git a/tests/e2e/manifests/chi/test-018-configmap-1.yaml b/tests/e2e/manifests/chi/test-018-configmap-1.yaml deleted file mode 100644 index 96140c921..000000000 --- a/tests/e2e/manifests/chi/test-018-configmap-1.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: clickhouse.altinity.com/v1 -kind: ClickHouseInstallation -metadata: - name: test-018-configmap -spec: - useTemplates: - - name: clickhouse-version - configuration: - settings: - display_name: "old_display_name" - macros/test: "old_test" - clusters: - - name: default - \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-018-configmap-2.yaml b/tests/e2e/manifests/chi/test-018-configmap-2.yaml deleted file mode 100644 index 6c43a4e22..000000000 --- a/tests/e2e/manifests/chi/test-018-configmap-2.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: clickhouse.altinity.com/v1 -kind: ClickHouseInstallation -metadata: - name: test-018-configmap -spec: - useTemplates: - - name: clickhouse-version - configuration: - settings: - display_name: "new_display_name" - macros/test: "new_test" - clusters: - - name: default - \ No newline at end of file diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index ef4d5c46a..5e79b96fb 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -2275,47 +2275,6 @@ def test_010017(self): delete_test_namespace() -@TestScenario -@Name("test_010018. Test that server settings are applied before StatefulSet is started") -# Obsolete, covered by test_016 -def test_010018(self): - create_shell_namespace_clickhouse_template() - - chi = "test-018-configmap" - kubectl.create_and_check( - manifest="manifests/chi/test-018-configmap-1.yaml", - check={ - "pod_count": 1, - "do_not_delete": 1, - }, - ) - - with When("Update settings"): - kubectl.create_and_check( - manifest="manifests/chi/test-018-configmap-2.yaml", - check={ - "pod_count": 1, - "do_not_delete": 1, - }, - ) - - with Then("Configmap on the pod should be updated"): - for attempt in retries(timeout=180, delay=5): - with attempt: - display_name = kubectl.launch( - f'exec chi-{chi}-default-0-0-0 -- bash -c "grep display_name /etc/clickhouse-server/config.d/chop-generated-settings.xml"' - ) - note(display_name) - assert "new_display_name" in display_name - with Then("And ClickHouse should pick them up"): - macros = clickhouse.query(chi, "SELECT substitution from system.macros where macro = 'test'") - note(macros) - assert "new_test" == macros - - with Finally("I clean up"): - delete_test_namespace() - - @TestCheck def test_019(self, step=1): util.require_keeper(keeper_type=self.context.keeper_type) From 0a90c71d36154d1e136816341151644ffafe50d8 Mon Sep 17 00:00:00 2001 From: jtomaszon Date: Sat, 6 Jun 2026 11:15:42 +0000 Subject: [PATCH 011/164] fix(helm): wire watchNamespaces value into operator ConfigMap The configs.files.config.yaml.watch.namespaces.include field was hardcoded to [] in values.yaml and the ConfigMap template rendered it directly with no way to override it via a top-level Helm value. This patch introduces a top-level watchNamespaces value and a new configmap-files helper that deep-copies configs.files, patches the nested watch.namespaces.include in-place, then delegates to the existing configmap-data helper for rendering. Fixes #1919 --- .../templates/_helpers.tpl | 22 +++++++++++++++++++ ...nfigMap-etc-clickhouse-operator-files.yaml | 2 +- deploy/helm/clickhouse-operator/values.yaml | 5 +++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/deploy/helm/clickhouse-operator/templates/_helpers.tpl b/deploy/helm/clickhouse-operator/templates/_helpers.tpl index f2d1aee2b..4cdabde82 100644 --- a/deploy/helm/clickhouse-operator/templates/_helpers.tpl +++ b/deploy/helm/clickhouse-operator/templates/_helpers.tpl @@ -122,3 +122,25 @@ null {{- tpl (toYaml (dict $k $v)) $root }} {{ end }} {{- end }} + +{{/* +altinity-clickhouse-operator.configmap-files merges watchNamespaces into the +operator config before rendering the ConfigMap data block. + +This exists because configs.files.config.yaml.watch.namespaces.include is +deep inside a nested structure — Helm's values merge cannot target it +directly. Instead we deepCopy the files map, patch the nested value in-place, +and pass the result to configmap-data. + +Arguments (list): root context, configs.files, watchNamespaces list +*/}} +{{- define "altinity-clickhouse-operator.configmap-files" -}} +{{- $root := index . 0 -}} +{{- $files := deepCopy (index . 1) -}} +{{- $watchNamespaces := index . 2 -}} +{{- if $watchNamespaces -}} + {{- $namespaces := index (index (index $files "config.yaml") "watch") "namespaces" -}} + {{- $_ := set $namespaces "include" $watchNamespaces -}} +{{- end -}} +{{- include "altinity-clickhouse-operator.configmap-data" (list $root $files) -}} +{{- end -}} diff --git a/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml index bc6d21dd1..df1886ac3 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml @@ -11,4 +11,4 @@ metadata: namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} -data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.files) | nindent 2 }} +data: {{ include "altinity-clickhouse-operator.configmap-files" (list . .Values.configs.files .Values.watchNamespaces) | nindent 2 }} diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..c5ca1a8bd 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -147,6 +147,11 @@ podAnnotations: prometheus.io/scrape: 'true' clickhouse-operator-metrics/port: '9999' clickhouse-operator-metrics/scrape: 'true' +# watchNamespaces -- namespaces where the operator watches for ClickHouseInstallation resources. +# If empty, the operator watches only its own namespace (or all namespaces when running in kube-system). +# Use [".*"] to watch all namespaces. +# Example: watchNamespaces: ["clickhouse", "my-other-namespace"] +watchNamespaces: [] # nameOverride -- override name of the chart nameOverride: "" # fullnameOverride -- full name of the chart. From 6ab71fe7931a81a077b237cc10e3495c839de3dd Mon Sep 17 00:00:00 2001 From: alz Date: Sat, 6 Jun 2026 23:09:51 +0300 Subject: [PATCH 012/164] Bump go to 1.26.4 to address CVEs --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0737dd5b0..6b349ada7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/altinity/clickhouse-operator -go 1.26.3 +go 1.26.4 replace ( github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.0 From 3ed8cb4d1e72ac06a4000a53cc99ecef4f1a0b30 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 9 Jun 2026 14:51:27 +0500 Subject: [PATCH 013/164] dev: root ca ref --- deploy/builder/templates-config/config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 46740bd15..a872af11c 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -156,7 +156,16 @@ clickhouse: # located in 'clickhouse.configuration.file.path.user' folder username: "${CH_USERNAME_PLAIN}" password: "${CH_PASSWORD_PLAIN}" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: From 18d47673963bad2fd7b7bf812b68be5d19ecc861 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 9 Jun 2026 14:51:38 +0500 Subject: [PATCH 014/164] dev: config --- config/config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/config.yaml b/config/config.yaml index 4bfc9d015..d5e64552b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -162,7 +162,16 @@ clickhouse: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: From ef74352815285c67b26185df834372658f67c43d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 9 Jun 2026 14:52:01 +0500 Subject: [PATCH 015/164] dev:manifests --- deploy/helm/clickhouse-operator/values.yaml | 9 +++++++++ deploy/operator/clickhouse-operator-install-ansible.yaml | 9 +++++++++ .../clickhouse-operator-install-bundle-v1beta1.yaml | 9 +++++++++ deploy/operator/clickhouse-operator-install-bundle.yaml | 9 +++++++++ .../clickhouse-operator-install-template-v1beta1.yaml | 9 +++++++++ .../operator/clickhouse-operator-install-template.yaml | 9 +++++++++ deploy/operator/clickhouse-operator-install-tf.yaml | 9 +++++++++ 7 files changed, 63 insertions(+) diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..95ee2da78 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -442,7 +442,16 @@ configs: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: # - clickhouse.access.username diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index cae199914..54ddfb305 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -5668,7 +5668,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index 82dd6bb26..65dc5caa6 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -5867,7 +5867,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 27908766d..f79fddfa3 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -5927,7 +5927,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 7b6198106..d90415187 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -5614,7 +5614,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index af7e2fa46..bf2a2e2d8 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -5661,7 +5661,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index c3eb2f69c..b1cc2b731 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -5668,7 +5668,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: From c2a93d1a63f78e265ec665683fe06cbe33d72246 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 9 Jun 2026 14:52:21 +0500 Subject: [PATCH 016/164] dev: unit tests --- pkg/chop/config_access_rootca_test.go | 73 +++++++++++++++++++++++++++ pkg/chop/config_manager_test.go | 36 +++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 pkg/chop/config_access_rootca_test.go diff --git a/pkg/chop/config_access_rootca_test.go b/pkg/chop/config_access_rootca_test.go new file mode 100644 index 000000000..a507fbcd1 --- /dev/null +++ b/pkg/chop/config_access_rootca_test.go @@ -0,0 +1,73 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chop + +import ( + "testing" + + sigsyaml "github.com/kubernetes-sigs/yaml" + "github.com/stretchr/testify/require" + + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" +) + +// TestAccessRootCASecretRefUnmarshalAndResolve guards the clickhouse.access +// rootCASecretRef wiring end to end with the SAME yaml package the config loader +// uses (kubernetes-sigs/yaml -> YAML->JSON->json.Unmarshal), so it exercises the +// JSON struct tags exactly as getFileBasedConfig does. It then runs the shared +// resolver to confirm the Secret PEM lands in Access.RootCA, which is what +// NewClusterConnectionParamsFromCHOpConfig reads into the operator's TLS config. +func TestAccessRootCASecretRefUnmarshalAndResolve(t *testing.T) { + // Explicit key. + const cfg = ` +clickhouse: + access: + rootCA: "" + rootCASecretRef: + name: my-ca-secret + key: my.crt +` + var oc api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(cfg), &oc)) + require.Equal(t, "my-ca-secret", oc.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "my.crt", oc.ClickHouse.Access.RootCASecretRef.Key) + require.Equal(t, "", oc.ClickHouse.Access.RootCA) + + fakeGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"my.crt": []byte("PEM-EXPLICIT")}, nil + } + resolveRootCAFromSecret(&oc.ClickHouse.Access.RootCA, oc.ClickHouse.Access.RootCASecretRef.Name, + oc.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test", fakeGet) + require.Equal(t, "PEM-EXPLICIT", oc.ClickHouse.Access.RootCA) + + // Empty key -> ca.crt default. + const cfgDefault = ` +clickhouse: + access: + rootCASecretRef: + name: only-name +` + var oc2 api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(cfgDefault), &oc2)) + require.Equal(t, "only-name", oc2.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "", oc2.ClickHouse.Access.RootCASecretRef.Key) + + defGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"ca.crt": []byte("PEM-DEFAULT")}, nil + } + resolveRootCAFromSecret(&oc2.ClickHouse.Access.RootCA, oc2.ClickHouse.Access.RootCASecretRef.Name, + oc2.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test", defGet) + require.Equal(t, "PEM-DEFAULT", oc2.ClickHouse.Access.RootCA) +} diff --git a/pkg/chop/config_manager_test.go b/pkg/chop/config_manager_test.go index ed260d52a..76f2fa9a0 100644 --- a/pkg/chop/config_manager_test.go +++ b/pkg/chop/config_manager_test.go @@ -208,3 +208,39 @@ func TestFetchSecurityRootCAResolve_ClearOnFailure(t *testing.T) { }) } } + +// TestResolveRootCAFromSecret covers the shared resolver used by both the +// chopconf security.clickhouse.tls path and the clickhouse.access path: +// inline-wins precedence, fail-open on every error (empty result, never panic), +// and ca.crt -> tls.crt key defaulting. +func TestResolveRootCAFromSecret(t *testing.T) { + fakeGet := func(want map[string][]byte, err error) secretDataGetter { + return func(ns, name string) (map[string][]byte, error) { return want, err } + } + tests := []struct { + name string + secretName string + secretKey string + operatorNs string + inline string + get secretDataGetter + wantRootCA string + }{ + {name: "empty name sentinel — no-op", secretName: "", operatorNs: "op", get: fakeGet(nil, nil), wantRootCA: ""}, + {name: "inline wins over secret", secretName: "ca", operatorNs: "op", inline: "INLINE", get: fakeGet(map[string][]byte{"ca.crt": []byte("FROM-SECRET")}, nil), wantRootCA: "INLINE"}, + {name: "nil getter — fail open, empty", secretName: "ca", operatorNs: "op", get: nil, wantRootCA: ""}, + {name: "empty namespace — fail open, empty", secretName: "ca", operatorNs: "", get: fakeGet(map[string][]byte{"ca.crt": []byte("X")}, nil), wantRootCA: ""}, + {name: "default key ca.crt", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"ca.crt": []byte("CA-PEM")}, nil), wantRootCA: "CA-PEM"}, + {name: "default key falls back to tls.crt", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"tls.crt": []byte("TLS-PEM")}, nil), wantRootCA: "TLS-PEM"}, + {name: "explicit key wins over ca.crt", secretName: "ca", secretKey: "custom", operatorNs: "op", get: fakeGet(map[string][]byte{"custom": []byte("CUSTOM"), "ca.crt": []byte("WRONG")}, nil), wantRootCA: "CUSTOM"}, + {name: "key missing — fail open, empty", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"other": []byte("...")}, nil), wantRootCA: ""}, + {name: "fetch error — fail open, empty", secretName: "ca", operatorNs: "op", get: fakeGet(nil, errors.New("boom")), wantRootCA: ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + inline := tc.inline + resolveRootCAFromSecret(&inline, tc.secretName, tc.secretKey, tc.operatorNs, "test clickhouse.access", tc.get) + require.Equal(t, tc.wantRootCA, inline, "rootCA") + }) + } +} From 35a9d1dd8e651dd5dc9188b7ab68a6e64af2f83b Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 9 Jun 2026 14:52:44 +0500 Subject: [PATCH 017/164] dev: new flags implementation --- .../v1/type_configuration_chop.go | 11 +++ pkg/chop/config_manager.go | 99 +++++++++++-------- 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index d1ffbfb6f..0b86c57af 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -379,6 +379,17 @@ type OperatorConfigClickHouse struct { Username string `json:"username,omitempty" yaml:"username,omitempty"` Password string `json:"password,omitempty" yaml:"password,omitempty"` RootCA string `json:"rootCA,omitempty" yaml:"rootCA,omitempty"` + // RootCASecretRef sources the RootCA PEM bundle from a Kubernetes Secret in + // the operator's pod namespace, resolved once at config load. Inline RootCA + // above wins if both are set; empty Name = unused; Key defaults to `ca.crt` + // then `tls.crt`. Mirrors the surface of security.clickhouse.tls.rootCASecretRef. + // Deliberately a plain value struct, NOT *core.SecretKeySelector: this Access + // block is an anonymous struct, and a heap-bearing pointer field here would + // break deepcopy generation (out.Access = in.Access stays a valid shallow copy). + RootCASecretRef struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Key string `json:"key,omitempty" yaml:"key,omitempty"` + } `json:"rootCASecretRef,omitempty" yaml:"rootCASecretRef,omitempty"` // Location of k8s Secret with username and password to be used by the operator to connect to ClickHouse instances // Can be used instead of explicitly specified (above) username and password diff --git a/pkg/chop/config_manager.go b/pkg/chop/config_manager.go index dafe98ba8..893662c87 100644 --- a/pkg/chop/config_manager.go +++ b/pkg/chop/config_manager.go @@ -120,6 +120,7 @@ func (cm *ConfigManager) Init() error { cm.fetchSecretCredentials() cm.fetchSecurityRootCA() + cm.fetchAccessRootCA() // From now on we have one unified CHOP config log.V(1).Info("Unified CHOP config - with secret data fetched (but not post-processed yet):") @@ -488,6 +489,25 @@ func (cm *ConfigManager) fetchSecurityRootCA() { fetchSecurityRootCAResolve(cm.config.Security.GetClickHouse().GetTLS(), ns, cm.getSecretData) } +// fetchAccessRootCA resolves a chopconf-level clickhouse.access.rootCASecretRef to +// an inline clickhouse.access.rootCA at config-load time, in the operator's pod +// namespace (operator-scoped, like fetchSecurityRootCA). Inline rootCA wins. On +// any failure the inline value is left empty and a Warning is logged (fail-open: +// a typo'd Secret must not crash the operator or block reconciles). This is the +// operator counterpart of the security.clickhouse.tls.rootCASecretRef path. +func (cm *ConfigManager) fetchAccessRootCA() { + ns, _ := cm.GetRuntimeParam(deployment.OPERATOR_POD_NAMESPACE) + access := &cm.config.ClickHouse.Access + resolveRootCAFromSecret( + &access.RootCA, + access.RootCASecretRef.Name, + access.RootCASecretRef.Key, + ns, + "chopconf clickhouse.access", + cm.getSecretData, + ) +} + // getSecretData adapts the operator's kubeClient to the secretDataGetter seam. func (cm *ConfigManager) getSecretData(namespace, name string) (map[string][]byte, error) { secret, err := cm.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, controller.NewGetOptions()) @@ -497,64 +517,65 @@ func (cm *ConfigManager) getSecretData(namespace, name string) (map[string][]byt return secret.Data, nil } -// fetchSecurityRootCAResolve carries the pure decision logic of fetchSecurityRootCA. -// Mutates tls in place: inlines RootCA on success, clears RootCASecretRef on any -// terminal outcome (success or failure). -// -// Nil-safe on every input: nil tls, nil RootCASecretRef, or nil getSecret all -// return early. Empty `ref.Name` is the documented "not used" sentinel — the -// ref is cleared silently with no warning. -func fetchSecurityRootCAResolve(tls *api.ClusterSecurityClickHouseTLS, operatorNs string, getSecret secretDataGetter) { - if (tls == nil) || (tls.RootCASecretRef == nil) { - return - } - if getSecret == nil { - // Defensive: a test seam or future refactor passing nil would otherwise - // panic at the call site below. Treat as fetch-error: clear the ref so - // downstream merges don't propagate a stub, log so it's diagnosable. - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef=%q: secret getter is nil — clearing ref", tls.RootCASecretRef.Name) - tls.RootCASecretRef = nil +// resolveRootCAFromSecret is the shared decision logic for inlining a CA bundle +// from a Kubernetes Secret into an inline RootCA PEM string at config-load time. +// Used by BOTH chopconf security.clickhouse.tls (fetchSecurityRootCA) and +// clickhouse.access (fetchAccessRootCA). It sets *inline on success and is +// nil-safe / fail-open: empty secretName is the "not used" sentinel; an inline +// value already set wins; any fetch/key failure leaves *inline untouched and logs +// at Warning so operators see it at default verbosity (fix the Secret, restart). +// `what` labels the config path in log messages. Callers own clearing their own +// ref form (e.g. the *core.SecretKeySelector on the security TLS struct). +func resolveRootCAFromSecret(inline *string, secretName, secretKey, operatorNs, what string, getSecret secretDataGetter) { + if (inline == nil) || (secretName == "") { + // Empty secretName is the documented "not used" sentinel — silent no-op. return } - ref := tls.RootCASecretRef - if ref.Name == "" { - // Empty Name is the "not used" sentinel — let users keep the ref block - // in their chopconf with empty values without forcing them to comment - // it out. Clear the ref so downstream merges don't propagate the stub. - tls.RootCASecretRef = nil + if *inline != "" { + // Inline rootCA wins; operators see the warning and pick one. + log.Warning("%s: both rootCA and rootCASecretRef=%q set — using inline rootCA, ignoring ref", what, secretName) return } - if tls.RootCA != "" { - // Inline RootCA wins; clear the ref so downstream merges don't propagate - // a conflict per CHI. Operators see the warning and pick one. - log.Warning("chopconf security.clickhouse.tls: both rootCA and rootCASecretRef=%q set — using inline rootCA, ignoring ref", ref.Name) - tls.RootCASecretRef = nil + if getSecret == nil { + // Defensive: a nil getter would panic below. Treat as a fetch failure. + log.Warning("%s rootCASecretRef=%q: secret getter is nil — ignoring ref", what, secretName) return } if operatorNs == "" { - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef=%q: operator namespace unknown; clearing ref (chopconf-level CA disabled)", ref.Name) - tls.RootCASecretRef = nil + log.Warning("%s rootCASecretRef=%q: operator namespace unknown; secret-sourced CA disabled", what, secretName) return } - keys := []string{ref.Key} - if ref.Key == "" { + keys := []string{secretKey} + if secretKey == "" { keys = []string{"ca.crt", "tls.crt"} } - data, err := getSecret(operatorNs, ref.Name) + data, err := getSecret(operatorNs, secretName) if err != nil { - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef: unable to fetch %s/%s: %v — clearing ref (fix the Secret and restart the operator)", operatorNs, ref.Name, err) - tls.RootCASecretRef = nil + log.Warning("%s rootCASecretRef: unable to fetch %s/%s: %v — ignoring ref (fix the Secret and restart the operator)", what, operatorNs, secretName, err) return } for _, k := range keys { if v, ok := data[k]; ok { - tls.RootCA = string(v) - tls.RootCASecretRef = nil - log.V(1).Info("chopconf security.clickhouse.tls: inlined RootCA from %s/%s key=%s", operatorNs, ref.Name, k) + *inline = string(v) + log.V(1).Info("%s: inlined RootCA from %s/%s key=%s", what, operatorNs, secretName, k) return } } - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef: secret %s/%s exists but none of keys %v found — clearing ref (fix the Secret and restart the operator)", operatorNs, ref.Name, keys) + log.Warning("%s rootCASecretRef: secret %s/%s exists but none of keys %v found — ignoring ref (fix the Secret and restart the operator)", what, operatorNs, secretName, keys) +} + +// fetchSecurityRootCAResolve resolves the chopconf security.clickhouse.tls +// rootCASecretRef via the shared resolveRootCAFromSecret, then ALWAYS clears +// RootCASecretRef: this ref is operator-scoped and terminal once processed here +// (success or failure), so it must not propagate into the per-CHI MergeFrom +// inheritance where the normalizer would re-resolve it against every CHI's +// namespace. Nil-safe: nil tls or nil RootCASecretRef return early. +func fetchSecurityRootCAResolve(tls *api.ClusterSecurityClickHouseTLS, operatorNs string, getSecret secretDataGetter) { + if (tls == nil) || (tls.RootCASecretRef == nil) { + return + } + ref := tls.RootCASecretRef + resolveRootCAFromSecret(&tls.RootCA, ref.Name, ref.Key, operatorNs, "chopconf security.clickhouse.tls", getSecret) tls.RootCASecretRef = nil } From e0d6d6592d107a992241ce21bc74d827d575ea34 Mon Sep 17 00:00:00 2001 From: alz Date: Wed, 10 Jun 2026 15:01:35 +0300 Subject: [PATCH 018/164] Test for https://github.com/Altinity/clickhouse-operator/pull/1998 --- tests/e2e/kubectl.py | 16 +++++-- tests/e2e/test_operator.py | 88 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py index 7506cd1be..b07fdd20c 100644 --- a/tests/e2e/kubectl.py +++ b/tests/e2e/kubectl.py @@ -252,7 +252,7 @@ def delete_all(kind, ns=None): retry_sleep(attempt, 5, f"{kind}/{name} still terminating") # Final assertion: if the CR survived every force-clear, this # raises — surfacing a real cleanup leak rather than hiding it. - wait_object(kind, name, ns=ns, count=0) + # wait_object(kind, name, ns=ns, count=0) def delete_all_keeper(ns=None): @@ -578,10 +578,20 @@ def get_pod_status(pod, shell=None, ns=None): def wait_container_status(pod, status, shell=None, ns=None): wait_field("pod", pod, ".status.containerStatuses[0].ready", status, ns, shell=shell) -def get_container_status(pod, shell=None, ns=None): - return get_field("pod", pod, ".status.containerStatuses[0].ready", ns, shell=shell) +def get_container_status(pod, container_index=0, shell=None, ns=None): + return get_field("pod", pod, f".status.containerStatuses[{container_index}].ready", ns, shell=shell) +def get_condition_status(pod_name, condition_type, shell=None, ns=None): + pod = get("pod", pod_name, ns=ns, ok_to_fail=True, shell=shell) + if not pod: + return None + conditions = (pod.get("status") or {}).get("conditions") or [] + for condition in conditions: + if condition.get("type") == condition_type: + return condition.get("status") + return None + def wait_field( kind, name, diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 5e79b96fb..f678851ba 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3754,6 +3754,94 @@ def test_010035_1(self): delete_test_namespace() +@TestScenario +@Tags("HEAVY") +@Name("test_010035_2. Auto-recovery from sustained NotReady pod") +def test_010035_2(self): + """Verify that a Completed CHI is recovered when one of its pod containers + stays NotReady without crashing. + + Scenario: + 1. Create a CHI with a dummy sidecar container guarded by a readiness file + 2. Wait for the CHI and pod to become Ready + 3. Remove the readiness file from the dummy container + 4. The pod stays Ready=False while containers keep running + 5. Operator detects sustained NotReady and recreates the pod + """ + create_shell_namespace_clickhouse_template() + + manifest = "manifests/chi/test-035-2-sustained-not-ready.yaml" + chi = yaml_manifest.get_name(util.get_full_path(manifest)) + cluster = "default" + pod = f"chi-{chi}-{cluster}-0-0-0" + dummy_container = "readiness-flap" + + with When("I create CHI with a dummy readiness-gated container"): + kubectl.create_and_check( + manifest=manifest, + check={ + "object_counts": {"statefulset": 1, "pod": 1, "service": 2}, + "do_not_delete": 1, + }, + ) + + with And("Pod should initially be Ready"): + for i in range(1, 30): + pod_ready = kubectl.get_condition_status(pod, "Ready") + dummy_ready = kubectl.get_container_status(pod, 1) + if pod_ready == "True" and dummy_ready == "true": + break + retry_sleep(i, 2, f"pod Ready={pod_ready}, {dummy_container} ready={dummy_ready}") + + assert pod_ready == "True", error(f"expected pod {pod} to be Ready, got Ready={pod_ready}") + assert dummy_ready == "true", error( + f"expected container {dummy_container} to be Ready, got ready={dummy_ready}" + ) + + old_uid = kubectl.get_field("pod", pod, ".metadata.uid") + assert old_uid, error(f"pod {pod} does not exist") + + with When("I make the dummy container NotReady without crashing it"): + kubectl.launch(f"exec {pod} -c {dummy_container} -- rm -f /tmp/ready") + + with Then("The dummy container and the pod should become NotReady"): + for i in range(1, 30): + pod_ready = kubectl.get_condition_status(pod, "Ready") + dummy_ready = kubectl.get_container_status(pod, 1) + if pod_ready == "False" and dummy_ready == "false": + break + retry_sleep(i, 2, f"pod Ready={pod_ready}, {dummy_container} ready={dummy_ready}") + + assert pod_ready == "False", error(f"expected pod {pod} to be NotReady, got Ready={pod_ready}") + assert dummy_ready == "false", error( + f"expected container {dummy_container} to be NotReady, got ready={dummy_ready}" + ) + + with Then("Operator should recreate the pod after sustained NotReady timeout"): + start_time = time.time() + new_uid = old_uid + pod_ready = kubectl.get_condition_status(pod, "Ready") + + while time.time() - start_time < 420: + new_uid = kubectl.get_field("pod", pod, ".metadata.uid") + pod_ready = kubectl.get_condition_status(pod, "Ready") + if new_uid != old_uid and pod_ready == "True": + break + retry_sleep( + int((time.time() - start_time) / 5) + 1, + 5, + f"pod uid={new_uid}, Ready={pod_ready}", + ) + + assert new_uid != old_uid, error( + f"expected operator to recreate pod {pod} within sustained NotReady timeout" + ) + assert pod_ready == "True", error(f"expected recreated pod {pod} to become Ready, got {pod_ready}") + + with Finally("I clean up"): + delete_test_namespace() + + @TestScenario @Requirements(RQ_SRS_026_ClickHouseOperator_EnableHttps("1.0")) @Name("test_010034. Check HTTPS support for health check") From fbec479a20924aaad1efa7b0b89313bc03d27204 Mon Sep 17 00:00:00 2001 From: alz Date: Wed, 10 Jun 2026 15:53:29 +0300 Subject: [PATCH 019/164] Forgot test manifest --- .../chi/test-035-2-sustained-not-ready.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml diff --git a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml new file mode 100644 index 000000000..dd548b54a --- /dev/null +++ b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml @@ -0,0 +1,36 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-035-sustained-not-ready +spec: + configuration: + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 + templates: + podTemplates: + - name: readiness-flap + spec: + containers: + - name: clickhouse-pod + image: altinity/clickhouse-server:25.8.16.10001.altinitystable + - name: readiness-flap + image: altinity/clickhouse-server:25.8.16.10001.altinitystable + command: + - "/bin/bash" + - "-c" + - "touch /tmp/ready; while true; do sleep 5; done" + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "test -f /tmp/ready" + initialDelaySeconds: 1 + periodSeconds: 2 + failureThreshold: 1 + defaults: + templates: + podTemplate: readiness-flap From 4fef142e2b4435db080828b9bc00f739242087f2 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:42:21 +0500 Subject: [PATCH 020/164] dev: CRD fields --- ...stall-yaml-template-01-section-crd-02-chopconf.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index 044685794..1701c739b 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -167,6 +167,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: From 45c848ca54dbb2c2bb7e37e570e9c5b18de0d8d8 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:42:49 +0500 Subject: [PATCH 021/164] dev: operator manifests --- .../operator/clickhouse-operator-install-ansible.yaml | 10 ++++++++++ .../clickhouse-operator-install-bundle-v1beta1.yaml | 10 ++++++++++ .../operator/clickhouse-operator-install-bundle.yaml | 10 ++++++++++ .../clickhouse-operator-install-template-v1beta1.yaml | 10 ++++++++++ .../operator/clickhouse-operator-install-template.yaml | 10 ++++++++++ deploy/operator/clickhouse-operator-install-tf.yaml | 10 ++++++++++ deploy/operator/parts/crd.yaml | 10 ++++++++++ 7 files changed, 70 insertions(+) diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 54ddfb305..23f14e2cf 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -3770,6 +3770,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index 65dc5caa6..a7fa9a1e9 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -3737,6 +3737,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index f79fddfa3..331ed8131 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -3763,6 +3763,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index d90415187..dc7894bd6 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -3737,6 +3737,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index bf2a2e2d8..467dce1bb 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -3763,6 +3763,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index b1cc2b731..b4410543b 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -3770,6 +3770,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 6a9c83cb7..2d44e7c39 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -8213,6 +8213,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: From 4f79eb56ceb83d56e1b9c7f0a6ed5fe980bfc1d7 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:43:17 +0500 Subject: [PATCH 022/164] dev: helm chart --- ...operatorconfigurations.clickhouse.altinity.com.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index d26c225f1..3388e8394 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -167,6 +167,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: From 721ca087344a878f55c9ea52b8e267d6a5814ee0 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:44:23 +0500 Subject: [PATCH 023/164] dev: unit tests --- pkg/chop/config_access_rootca_test.go | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/pkg/chop/config_access_rootca_test.go b/pkg/chop/config_access_rootca_test.go index a507fbcd1..8ea79cb38 100644 --- a/pkg/chop/config_access_rootca_test.go +++ b/pkg/chop/config_access_rootca_test.go @@ -71,3 +71,65 @@ clickhouse: oc2.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test", defGet) require.Equal(t, "PEM-DEFAULT", oc2.ClickHouse.Access.RootCA) } + +// TestAccessRootCASecretRefMergesFromCR proves the ClickHouseOperatorConfiguration +// (CRD) path reaches the same resolver as the file config: getAllCRBasedConfigs -> +// buildUnifiedConfig -> OperatorConfig.MergeFrom (mergo deep-merge) must carry the +// nested anonymous-struct access.rootCASecretRef from a CR spec into the unified +// config, where fetchAccessRootCA then resolves it. Guards CRD parity for the field. +func TestAccessRootCASecretRefMergesFromCR(t *testing.T) { + // base = file config with no access ref; cr = a chopconf CR spec carrying the ref. + base := &api.OperatorConfig{} + const crSpec = ` +clickhouse: + access: + rootCASecretRef: + name: cr-ca-secret + key: cr.crt +` + var cr api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(crSpec), &cr)) + require.NoError(t, base.MergeFrom(&cr)) + + // The ref survived the mergo deep-merge of the anonymous Access struct. + require.Equal(t, "cr-ca-secret", base.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "cr.crt", base.ClickHouse.Access.RootCASecretRef.Key) + + // ...and resolves through the shared resolver into Access.RootCA. + fakeGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"cr.crt": []byte("PEM-FROM-CR-SECRET")}, nil + } + resolveRootCAFromSecret(&base.ClickHouse.Access.RootCA, base.ClickHouse.Access.RootCASecretRef.Name, + base.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test cr-merge", fakeGet) + require.Equal(t, "PEM-FROM-CR-SECRET", base.ClickHouse.Access.RootCA) +} + +// TestAccessRootCASecretRefMergePrecedence locks how access CA settings merge across +// layered config sources (file + ClickHouseOperatorConfiguration CRs). RootCASecretRef +// is a value struct, so OperatorConfig.MergeFrom (mergo WithOverride) merges it FIELD +// BY FIELD: a higher-priority source's empty field does NOT clear a lower-priority +// non-empty one. This matches every other clickhouse.access.* value field (username, +// password, secret.*); only the security.clickhouse.tls pointer ref replaces wholesale. +// Consequence: layered configs should set the FULL ref, not a partial override. +func TestAccessRootCASecretRefMergePrecedence(t *testing.T) { + // A higher-priority CR ref does NOT override a lower-priority inline rootCA + // (mergo keeps the non-empty inline); the resolver then applies inline-wins. + base := &api.OperatorConfig{} + base.ClickHouse.Access.RootCA = "FILE-INLINE" + cr := &api.OperatorConfig{} + cr.ClickHouse.Access.RootCASecretRef.Name = "cr-secret" + require.NoError(t, base.MergeFrom(cr)) + require.Equal(t, "FILE-INLINE", base.ClickHouse.Access.RootCA) + require.Equal(t, "cr-secret", base.ClickHouse.Access.RootCASecretRef.Name) + + // A higher-priority CR overriding only the name RETAINS the lower-priority key + // (field-by-field merge) — hence "set the full ref in layered configs". + b2 := &api.OperatorConfig{} + b2.ClickHouse.Access.RootCASecretRef.Name = "file-secret" + b2.ClickHouse.Access.RootCASecretRef.Key = "file.crt" + c2 := &api.OperatorConfig{} + c2.ClickHouse.Access.RootCASecretRef.Name = "cr-secret" + require.NoError(t, b2.MergeFrom(c2)) + require.Equal(t, "cr-secret", b2.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "file.crt", b2.ClickHouse.Access.RootCASecretRef.Key) +} From 661cf166728decfc95bfe352663b8ea0929ab13f Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:44:58 +0500 Subject: [PATCH 024/164] dev: description --- pkg/chop/config_manager.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/chop/config_manager.go b/pkg/chop/config_manager.go index 893662c87..25952d682 100644 --- a/pkg/chop/config_manager.go +++ b/pkg/chop/config_manager.go @@ -489,12 +489,10 @@ func (cm *ConfigManager) fetchSecurityRootCA() { fetchSecurityRootCAResolve(cm.config.Security.GetClickHouse().GetTLS(), ns, cm.getSecretData) } -// fetchAccessRootCA resolves a chopconf-level clickhouse.access.rootCASecretRef to -// an inline clickhouse.access.rootCA at config-load time, in the operator's pod -// namespace (operator-scoped, like fetchSecurityRootCA). Inline rootCA wins. On -// any failure the inline value is left empty and a Warning is logged (fail-open: -// a typo'd Secret must not crash the operator or block reconciles). This is the -// operator counterpart of the security.clickhouse.tls.rootCASecretRef path. +// fetchAccessRootCA resolves clickhouse.access.rootCASecretRef into the inline +// clickhouse.access.rootCA via the shared resolver (see resolveRootCAFromSecret). +// Unlike the security path it does not clear the ref afterwards: Access.RootCASecretRef +// is read only here and never flows into per-CHI merges, so there is nothing to clear. func (cm *ConfigManager) fetchAccessRootCA() { ns, _ := cm.GetRuntimeParam(deployment.OPERATOR_POD_NAMESPACE) access := &cm.config.ClickHouse.Access From be6e8b00199f560b762785ff85d61873bebe6ce7 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:45:17 +0500 Subject: [PATCH 025/164] dev: configuration --- .../v1/type_configuration_chop.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index 0b86c57af..e55d73708 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -379,13 +379,11 @@ type OperatorConfigClickHouse struct { Username string `json:"username,omitempty" yaml:"username,omitempty"` Password string `json:"password,omitempty" yaml:"password,omitempty"` RootCA string `json:"rootCA,omitempty" yaml:"rootCA,omitempty"` - // RootCASecretRef sources the RootCA PEM bundle from a Kubernetes Secret in - // the operator's pod namespace, resolved once at config load. Inline RootCA - // above wins if both are set; empty Name = unused; Key defaults to `ca.crt` - // then `tls.crt`. Mirrors the surface of security.clickhouse.tls.rootCASecretRef. - // Deliberately a plain value struct, NOT *core.SecretKeySelector: this Access - // block is an anonymous struct, and a heap-bearing pointer field here would - // break deepcopy generation (out.Access = in.Access stays a valid shallow copy). + // RootCASecretRef sources the RootCA PEM from a Secret in the operator's pod + // namespace (inline RootCA wins; empty Name = unused; Key defaults ca.crt then + // tls.crt). Mirrors security.clickhouse.tls.rootCASecretRef. Value struct, not + // *core.SecretKeySelector: Access is anonymous, where a heap-bearing pointer + // field would break deepcopy generation. RootCASecretRef struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` Key string `json:"key,omitempty" yaml:"key,omitempty"` From 3e84e272f07c42be22a8a5b205d2d72ef347f37b Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:45:41 +0500 Subject: [PATCH 026/164] dev: test manifests --- .../chopconf/test-010080-chopconf-good.yaml | 20 ++++++++++++ .../test-010080-chopconf-missing.yaml | 17 ++++++++++ .../chopconf/test-010080-chopconf-wrong.yaml | 18 +++++++++++ .../secret/test-010080-wrong-ca.yaml | 31 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml create mode 100644 tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml create mode 100644 tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml create mode 100644 tests/e2e/manifests/secret/test-010080-wrong-ca.yaml diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml new file mode 100644 index 000000000..576de0cc3 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml @@ -0,0 +1,20 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-good" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Source the operator's ClickHouse-TLS rootCA from a Secret in the operator + # namespace holding the SAN-correct CA that issued the server cert. + rootCASecretRef: + name: test-010080-correct-ca + key: ca.crt + security: + clickhouse: + tls: + # Strict turns off InsecureSkipVerify so the secret-sourced rootCA is + # actually exercised on the wire (otherwise verification is bypassed). + verify: Strict diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml new file mode 100644 index 000000000..e937c5ba6 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml @@ -0,0 +1,17 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-missing" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Fail-open control: the referenced Secret does not exist. The operator + # must log a Warning, leave rootCA empty, and keep running/reconciling + # (no crashloop). scheme stays https but verify is left default (not + # Strict) so the operator still connects and serves — proving the bad + # ref did not brick the operator, isolated from a verification failure. + rootCASecretRef: + name: test-010080-no-such-secret + key: ca.crt diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml new file mode 100644 index 000000000..33a0bf2cc --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml @@ -0,0 +1,18 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-wrong" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Negative control: an unrelated CA that did NOT issue the server cert. + # With verify:Strict the operator -> ClickHouse handshake must fail. + rootCASecretRef: + name: test-010080-wrong-ca + key: ca.crt + security: + clickhouse: + tls: + verify: Strict diff --git a/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml b/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml new file mode 100644 index 000000000..91790b03a --- /dev/null +++ b/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Secret +metadata: + name: test-010080-wrong-ca +type: Opaque +stringData: + # Self-signed CA (CN=test-077-unrelated-ca.example) that did NOT issue the + # ClickHouse server cert. Used as the negative control: pointing the operator's + # access.rootCASecretRef at it with verify:Strict must fail chain verification, + # proving the secret-sourced CA is actually used (not bypassed). + ca.crt: |- + -----BEGIN CERTIFICATE----- + MIIDMTCCAhmgAwIBAgIUW3Bc7vzCdyEauj+8RIRKaAPhx4IwDQYJKoZIhvcNAQEL + BQAwKDEmMCQGA1UEAwwddGVzdC0wNzctdW5yZWxhdGVkLWNhLmV4YW1wbGUwHhcN + MjYwNTIwMDg0NzI0WhcNMzYwNTE3MDg0NzI0WjAoMSYwJAYDVQQDDB10ZXN0LTA3 + Ny11bnJlbGF0ZWQtY2EuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC + AQoCggEBANluYJRN5a+2jRSEPKTYbh9zw0Q+F9iSNPFUZ9Wbn8AwAYoerc7I2t5g + 9V6FvN893YKtJ1JitbPYH7Yd7e4j+8kmJS1h0u9d6HEzO0RaB6bOn2jqY92MLPIi + g1u/Pda7hp9hj/5VJSrwF/CehA24TXIdFEObQaoYYDTkqJkjfCTtau9ukvF5p8BI + 6hMHMnP8dymka+jhwF18xv46JGJMpOiJ+joQu79QLbPlEsOItPEV+WLC54iw7Cmb + wnXnvX4DMszB/ZrEcMFk+RAkH5KiU/s3jN1fg8KUmAqzxOs880zsCSvJGpemYYkp + 2B2TlJlJ+fl7QJWcreuUMqnvtCiRUCcCAwEAAaNTMFEwHQYDVR0OBBYEFH6TciYq + /FLps7gEpqnl8oX8yX/3MB8GA1UdIwQYMBaAFH6TciYq/FLps7gEpqnl8oX8yX/3 + MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEywSmeyPprRvTk4 + +M6HN8oWwMcCCAso3eaCtb41fWsqYrA8wLln0FlNpV/tW7JhzMLUXgPfVcehYLK+ + K/Xt/Qm7gmPAIUlUkS/MSFJ5s3WwK5FV+HJjqVtB+WNpUZEvIhHNfiS/EkRFFoWJ + LnHEYH0ltFM1xnvYfDVFSyQ9B/TBoM+1RRBREWwOsbNLiJI34adcNz9pHImaPmPD + aOPja7/pR9FDJ/9amjcDdUCJ2W5ZkVdqde/Z7LA/cAhitIB2BamHXYslC04Xz2nc + 8QqTsoPhJ8e7f1JaCUvInvqF8brbYXeKS0Zurr8j8m3ddHRhJ40w9dZKCvPFR7m8 + +8l7sxo= + -----END CERTIFICATE----- From 231272c328a73bc9eb2871615d73ce2752844215 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:45:57 +0500 Subject: [PATCH 027/164] dev: ca root test --- tests/e2e/test_operator.py | 127 +++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 5e79b96fb..aa9584a03 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -6858,6 +6858,133 @@ def test_010072(self): with Finally("I clean up"): delete_test_namespace() +@TestScenario +@Tags("HEAVY") +@Requirements(RQ_SRS_026_ClickHouseOperator_EnableHttps("1.0")) +@Name("test_010080. Operator clickhouse.access.rootCASecretRef sources ClickHouse-TLS rootCA from a Secret") +def test_010080(self): + """Verify the operator sources its own ClickHouse-TLS rootCA from a Kubernetes + Secret via `clickhouse.access.rootCASecretRef`, end to end: + - POSITIVE: the correct secret CA + verify:Strict lets the operator connect + over verified TLS (chi_clickhouse_metric_fetch_errors == 0). + - NEGATIVE: a wrong secret CA + verify:Strict fails chain verification + (== 1) — proving the secret-sourced CA is actually used, not bypassed. + - FAIL-OPEN: a non-existent secret ref does not crashloop the operator; it + keeps running and reconciling. + """ + create_shell_namespace_clickhouse_template() + operator_namespace = current().context.operator_namespace + + chi_manifest = "manifests/chi/test-034-https.yaml" + chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) + good_chopconf = "manifests/chopconf/test-010080-chopconf-good.yaml" + wrong_chopconf = "manifests/chopconf/test-010080-chopconf-wrong.yaml" + missing_chopconf = "manifests/chopconf/test-010080-chopconf-missing.yaml" + + with Given("a TLS secret whose SANs match this namespace's ClickHouse pods is installed"): + # Generates `clickhouse-certs` (server cert + ca.crt) signed by a CA whose + # SANs cover chi--default-0-N — required so verify:Strict can pass. + # replicas=2 over-covers test-034-https's single-replica layout (harmless); + # keep it >= the manifest's replicasCount so every dialed pod is in the SAN set. + create_tls_secret_for_fips_hosts(chi=chi, chk=chi, replicas=2) + + with And("the correct CA is published as a Secret in the OPERATOR namespace"): + # access.rootCASecretRef resolves in the operator's OWN namespace, which is + # not necessarily the CHI/test namespace where clickhouse-certs was created. + # Publish the generated CA explicitly in operator_namespace so the positive + # arm never depends on operator_namespace == test_namespace. + kubectl.launch( + "create secret generic test-010080-correct-ca " + f"--from-file=ca.crt={current().context.tls['ca_crt']}", + ns=operator_namespace, + ) + + with And("an unrelated (wrong) CA secret is installed in the operator namespace"): + kubectl.apply( + util.get_full_path("manifests/secret/test-010080-wrong-ca.yaml"), + operator_namespace, + ) + + with When("the HTTPS ClickHouse CHI is deployed"): + kubectl.create_and_check( + manifest=chi_manifest, + check={ + "apply_templates": {current().context.clickhouse_template}, + "object_counts": {"statefulset": 1, "pod": 1, "service": 2}, + "do_not_delete": 1, + }, + timeout=600, + ) + + with When("operator access.rootCASecretRef points at the correct CA secret (verify=Strict)"): + util.apply_operator_config(good_chopconf) + kubectl.wait_chi_status(chi, "Completed") + + with Then("POSITIVE: operator connects over verified TLS using the secret CA (fetch_errors=0)"): + # Larger retry budget: after the operator restart the metrics-exporter must + # re-discover the CHI and complete a first successful verified-TLS scrape; + # the default ~105s window is occasionally too short under post-reset load. + check_metrics_monitoring( + operator_namespace=operator_namespace, + operator_pod=kubectl.get_operator_pod(ns=operator_namespace), + expect_pattern=f'^chi_clickhouse_metric_fetch_errors{{[^}}]*chi="{chi}"[^}}]*}} 0$', + max_retries=12, + ) + + with When("operator access.rootCASecretRef is switched to the WRONG CA secret"): + # Remove the previous chopconf first: the operator merges ALL chopconf CRs + # in its namespace, so a stale one would shadow the new ref. + kubectl.delete(util.get_full_path(good_chopconf, lookup_in_host=False), operator_namespace) + util.apply_operator_config(wrong_chopconf) + + with Then("NEGATIVE: verification fails with the wrong secret CA (fetch_errors=1)"): + check_metrics_monitoring( + operator_namespace=operator_namespace, + operator_pod=kubectl.get_operator_pod(ns=operator_namespace), + expect_pattern=f'^chi_clickhouse_metric_fetch_errors{{[^}}]*chi="{chi}"[^}}]*}} 1$', + max_retries=12, + ) + + with When("operator access.rootCASecretRef points at a NON-EXISTENT secret"): + kubectl.delete(util.get_full_path(wrong_chopconf, lookup_in_host=False), operator_namespace) + util.apply_operator_config(missing_chopconf) + + with Then("FAIL-OPEN: operator stays Running and keeps reconciling (no crashloop on a bad ref)"): + operator_pod = kubectl.get_operator_pod(ns=operator_namespace) + kubectl.wait_pod_status(operator_pod, "Running", ns=operator_namespace) + kubectl.wait_chi_status(chi, "Completed") + with By("operator logged the fail-open Warning instead of crashing on the unresolvable ref"): + logs = kubectl.launch( + f"logs {operator_pod} -c clickhouse-operator", + ns=operator_namespace, + ok_to_fail=True, + ) + assert "ignoring ref" in logs, error( + "expected a fail-open Warning ('ignoring ref') for the missing rootCASecretRef secret" + ) + with By("operator still fetches metrics (fail-open degrades to no-CA, never breaks the operator)"): + # missing_chopconf leaves verify default (not Strict), so the operator + # connects with the unresolved/empty CA and keeps working — fetch_errors=0. + check_metrics_monitoring( + operator_namespace=operator_namespace, + operator_pod=operator_pod, + expect_pattern=f'^chi_clickhouse_metric_fetch_errors{{[^}}]*chi="{chi}"[^}}]*}} 0$', + max_retries=12, + ) + + with Finally("I clean up"): + # Delete ALL chopconf variants, not just the last one: an early failure + # (positive/negative arm) leaves an earlier chopconf applied, and the + # operator merges every chopconf CR in its namespace — a leftover would + # carry a stale access.rootCASecretRef into later tests. + for chopconf in (good_chopconf, wrong_chopconf, missing_chopconf): + kubectl.delete(util.get_full_path(chopconf, lookup_in_host=False), operator_namespace, ok_to_fail=True) + util.restart_operator() + kubectl.launch("delete secret test-010080-wrong-ca", ns=operator_namespace, ok_to_fail=True) + kubectl.launch("delete secret test-010080-correct-ca", ns=operator_namespace, ok_to_fail=True) + delete_test_namespace() + + # # Keeper tests section # From 7dd5f38e99683ab4d1f3ebe375d18e6d2b042555 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:46:26 +0500 Subject: [PATCH 028/164] dev: example --- docs/chi-examples/70-chop-config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index b9f573ec1..c6f763e0f 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -82,6 +82,22 @@ spec: name: "" # Port where to connect to ClickHouse instances to port: 8123 + # `rootCA`: inline PEM CA bundle the operator uses to verify the ClickHouse + # server certificate when connecting over https (scheme: https, or auto when + # only TLS ports are open). Verification is enforced when TLS hardening is + # opted in — security.clickhouse.tls.verify: Strict, or a non-empty + # minVersion/serverName; otherwise the CA is loaded but verification stays + # relaxed for backward compatibility. + rootCA: "" + # `rootCASecretRef`: alternate source — read the PEM CA from a Kubernetes + # Secret in the operator's own namespace instead of inlining it above. The + # operator resolves it into `rootCA` once at config load (rotate the Secret + + # restart the operator to pick up a new CA). Mutually exclusive with the + # inline `rootCA` above (inline wins). Empty `name` = not used. When `key` is + # empty, the operator tries "ca.crt" then "tls.crt". + rootCASecretRef: + name: "" + key: "" ################################################ ## From 4ce730ed4d1505ad796c26ae7d5d7fc797530437 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 10 Jun 2026 22:46:42 +0500 Subject: [PATCH 029/164] docs --- docs/operator_configuration.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/operator_configuration.md b/docs/operator_configuration.md index 8fa03e877..7e4d58af1 100644 --- a/docs/operator_configuration.md +++ b/docs/operator_configuration.md @@ -115,6 +115,14 @@ chPassword: clickhouse_operator_password chPort: 8123 ``` +When the operator connects over HTTPS, it verifies the ClickHouse server certificate +with the CA from `clickhouse.access.rootCA` (inline PEM) or `clickhouse.access.rootCASecretRef` +(a Secret in the operator's own namespace; key defaults to `ca.crt` then `tls.crt`, inline +`rootCA` wins). Verification is enforced when TLS hardening is opted in — +`security.clickhouse.tls.verify: Strict`, or a non-empty `minVersion`/`serverName`; otherwise +the CA is loaded but verification stays relaxed for backward compatibility. +See the [operator config example](chi-examples/70-chop-config.yaml). + ## ClickHouse Installation settings Operator deploys ClickHouse clusters with different defaults, that can be configured in a flexible way. From 09ae7b38ffad098023fb80da969f7ecd46b07631 Mon Sep 17 00:00:00 2001 From: "Quentin Levasseur (Genetec)" Date: Tue, 16 Jun 2026 12:58:10 -0400 Subject: [PATCH 030/164] feat(helm): add podSecurityContext and podAnnotations to crdHook Job The crd-install Job pod template currently exposes only container-level securityContext via `crdHook.containerSecurityContext`. This blocks two real-world deployments: - Kyverno `restrict-seccomp-strict` rejects the Job because the pod has no pod-level `seccompProfile`. - Istio sidecar auto-injection adds an `istio-init` initContainer that lacks `seccompProfile`, again tripping the same policy. The standard opt-out (`sidecar.istio.io/inject: "false"`) requires a pod-template annotation, which the chart had no way to set. Add two new value keys: - `crdHook.podSecurityContext` -> `spec.template.spec.securityContext` - `crdHook.podAnnotations` -> `spec.template.metadata.annotations` Both default to `{}` and render under `{{- with }}`, so the rendered output is byte-identical when unset. Same pattern already used for `imagePullSecrets`, `nodeSelector`, `affinity`, `tolerations`, `containerSecurityContext`. Also wires the new keys into `values.schema.json` and the README values table. Signed-off-by: Quentin Levasseur (Genetec) --- deploy/helm/clickhouse-operator/README.md | 2 ++ .../templates/hooks/crd-install-job.yaml | 8 ++++++++ deploy/helm/clickhouse-operator/values.schema.json | 6 ++++++ deploy/helm/clickhouse-operator/values.yaml | 11 +++++++++++ 4 files changed, 27 insertions(+) diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md index 25b5070f3..c3d5771bd 100644 --- a/deploy/helm/clickhouse-operator/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -83,6 +83,8 @@ crdHook: | crdHook.image.tag | string | `"latest"` | image tag for CRD installation job | | crdHook.imagePullSecrets | list | `[]` | image pull secrets for CRD installation job possible value format `[{"name":"your-secret-name"}]`, check `kubectl explain pod.spec.imagePullSecrets` for details | | crdHook.nodeSelector | object | `{}` | node selector for CRD installation job | +| crdHook.podAnnotations | object | `{}` | additional annotations for CRD installation job pod template useful to opt out of service mesh injection, e.g. `sidecar.istio.io/inject: "false"` | +| crdHook.podSecurityContext | object | `{}` | pod-level security context for CRD installation job required by some admission policies (e.g. Kyverno `restrict-seccomp-strict`) check `kubectl explain pod.spec.securityContext` for details | | crdHook.resources | object | `{}` | resource limits and requests for CRD installation job | | crdHook.tolerations | list | `[]` | tolerations for CRD installation job | | dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards | diff --git a/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml b/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml index df980785f..196d5af0e 100644 --- a/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml +++ b/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml @@ -21,9 +21,17 @@ spec: labels: {{- include "altinity-clickhouse-operator.labels" . | nindent 8 }} app.kubernetes.io/component: crd-install-hook + {{- with .Values.crdHook.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "altinity-clickhouse-operator.fullname" . }}-crd-install restartPolicy: OnFailure + {{- with .Values.crdHook.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.crdHook.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json index 5fbee64d1..bd32a9652 100644 --- a/deploy/helm/clickhouse-operator/values.schema.json +++ b/deploy/helm/clickhouse-operator/values.schema.json @@ -710,6 +710,12 @@ "annotations": { "type": "object" }, + "podAnnotations": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, "containerSecurityContext": { "type": "object" } diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..6be86e4b2 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -38,6 +38,17 @@ crdHook: affinity: {} # crdHook.annotations -- additional annotations for CRD installation job annotations: {} + # crdHook.podAnnotations -- additional annotations for CRD installation job pod template + # useful to opt out of service mesh injection, e.g. `sidecar.istio.io/inject: "false"` + podAnnotations: {} + # crdHook.podSecurityContext -- pod-level security context for CRD installation job + # required by some admission policies (e.g. Kyverno `restrict-seccomp-strict`) + # check `kubectl explain pod.spec.securityContext` for details + podSecurityContext: {} + # runAsNonRoot: true + # runAsUser: 1000 + # seccompProfile: + # type: RuntimeDefault # crdHook.containerSecurityContext -- container security context for CRD installation job # check `kubectl explain pod.spec.containers.securityContext` for details containerSecurityContext: {} From b3c1a28bf674ab7a05c98a24f5f239f7f18f27e0 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:18:08 +0500 Subject: [PATCH 031/164] dev: --- deploy/builder/templates-config/config.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index a872af11c..372dd2c5e 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -539,7 +539,8 @@ reconcile: # is in that state. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). from: - # Recovery from Status=Aborted + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -548,6 +549,21 @@ reconcile: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry From 622c5c45064deb5f085ad96566dad908906af0eb Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:18:35 +0500 Subject: [PATCH 032/164] dev: config implementation --- .../v1/type_configuration_chop.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index ff90354dd..eb0db13d0 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -611,10 +611,12 @@ type OperatorConfigReconcileRecoveryScope struct { // Completed scope. type OperatorConfigReconcileRecoveryCompletedScope struct { // OnPodNotReady controls reaction when a pod belonging to a Completed CHI flips - // Ready=True → Ready=False and stays NotReady for at least OnPodNotReadyThreshold: - // nil / "retry" (default) — re-enqueue CHI for reconcile so shouldForceRestartHost - // can decide whether to restart the host - // "none" — do nothing, host stays Ready=False until external action + // Ready=True → Ready=False and stays NotReady for at least OnPodNotReadyThreshold. + // OFF by default (opt-in), unlike the Aborted scope's OnPodReady — force-recreating a + // Completed CHI's pod is destructive (can interrupt replica recovery; single-replica downtime): + // nil / "none" (default) — do nothing, host stays Ready=False until external action + // "retry" — re-enqueue CHI for reconcile so shouldForceRestartHost + // can decide whether to restart the host OnPodNotReady *types.String `json:"onPodNotReady,omitempty" yaml:"onPodNotReady,omitempty"` // OnPodNotReadyThreshold is the minimum duration a pod must remain in Ready=False // before this scope fires. Accepts any time.ParseDuration string (default "5m" @@ -1666,14 +1668,13 @@ func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { // ShouldRecoverCompletedOnPodNotReady reports whether the operator should re-enqueue a // CHI reconcile when a pod belonging to a Completed CHI flips to Ready=False and stays -// there for longer than CompletedOnPodNotReadyThreshold. Default is to retry. +// there for longer than CompletedOnPodNotReadyThreshold. Default is OFF: force-recreating +// a Completed CHI's pod is destructive (it can interrupt a replica's in-progress recovery +// and means hard downtime for a single-replica shard), so this is opt-in — only an explicit +// onPodNotReady: retry enables it. Unlike the Aborted scope, which retries by default. // Backed by reconcile.recovery.from.completed.onPodNotReady config key. func (c *OperatorConfig) ShouldRecoverCompletedOnPodNotReady() bool { value := strings.ToLower(c.Reconcile.Recovery.From.Completed.OnPodNotReady.String()) - if value == "" { - // Default behavior — retry - return true - } return value == RecoveryActionRetry } From 9367ff6fe26ef7ac4a7752a1e3334bb6eaa925ef Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:19:01 +0500 Subject: [PATCH 033/164] dev: unit test for recovery toggles --- .../v1/type_configuration_chop_recovery_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go index f5c1b8be2..8354266fd 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go @@ -64,12 +64,12 @@ func TestRecoveryActionConstants(t *testing.T) { // Mirrors TestShouldRecoverAbortedOnPodReady so symmetric config keys behave identically. func TestShouldRecoverCompletedOnPodNotReady(t *testing.T) { tests := []struct { - name string - onPodNotRdy *types.String - expected bool + name string + onPodNotRdy *types.String + expected bool }{ - {"nil defaults to retry (close the gap by default)", nil, true}, - {"empty string defaults to retry", types.NewString(""), true}, + {"nil defaults to off (opt-in only — destructive recreate)", nil, false}, + {"empty string defaults to off", types.NewString(""), false}, {"retry lowercase", types.NewString("retry"), true}, {"Retry mixed case", types.NewString("Retry"), true}, {"RETRY upper case", types.NewString("RETRY"), true}, From be72a0cbcaabf0fed6b3cbc3d66d3784032bdb0e Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:23:06 +0500 Subject: [PATCH 034/164] dev: crd template --- ...l-template-01-section-crd-02-chopconf.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index 1701c739b..73f59a894 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -578,6 +578,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" From da634bdf78a8a91eb2ae238aac7e03e35740f98d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:34:34 +0500 Subject: [PATCH 035/164] dev: retry options --- pkg/controller/chi/worker-pod-retry.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/controller/chi/worker-pod-retry.go b/pkg/controller/chi/worker-pod-retry.go index 3cb4d2980..8b8007a74 100644 --- a/pkg/controller/chi/worker-pod-retry.go +++ b/pkg/controller/chi/worker-pod-retry.go @@ -212,6 +212,30 @@ func shouldTriggerStuckHostRecovery(cr *api.ClickHouseInstallation) bool { return true } +// crHasHostNeedingStuckRecovery reports whether the CR is a stuck-host recovery target +// (Completed, not deleting, feature enabled) with at least one host whose pod has been +// NotReady past the configured threshold. reconcileCR uses this to proceed past the +// "no reconcile work" gate: a sustained-NotReady pod is live runtime state that never +// surfaces as an ActionPlan diff or object drift, so without this the delayed recovery +// reconcile self-aborts before shouldForceRestartHost is ever consulted. +func (w *worker) crHasHostNeedingStuckRecovery(ctx context.Context, cr *api.ClickHouseInstallation) bool { + if !chop.Config().ShouldRecoverCompletedOnPodNotReady() { + return false + } + if !shouldTriggerStuckHostRecovery(cr) { + return false + } + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + found := false + cr.WalkHosts(func(host *api.Host) error { + if !found && w.isPodSustainedNotReady(ctx, host, threshold) { + found = true + } + return nil + }) + return found +} + // isPodReadyToNotReadyTransition reports whether the pod transitioned from "all containers // ready" to "some container not ready". The dual of isPodNotReadyToReadyTransition. func isPodReadyToNotReadyTransition(oldPod, newPod *core.Pod) bool { From 39d229e5251a0e1afc5144d4293ad5b84236045e Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:35:14 +0500 Subject: [PATCH 036/164] doc: examples --- docs/chi-examples/70-chop-config.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index c6f763e0f..7784ec504 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -396,7 +396,7 @@ spec: ################################################ ## - ## Auto-recovery from aborted reconcile + ## Auto-recovery from aborted/completed reconcile ## ################################################ recovery: @@ -410,6 +410,20 @@ spec: # none — disable auto-recovery; operator user must # edit the CR spec to retrigger normalize. onPodReady: retry + completed: + # When a Completed CHI's pod regresses Ready=True→Ready=False and + # stays NotReady for at least onPodNotReadyThreshold, the operator + # can force-restart (recreate) the stuck host. + # OFF by default (opt-in): recreating a Completed CHI's pod is + # destructive — it can interrupt a replica's in-progress recovery + # and means hard downtime for a single-replica shard. + # Values: + # none (default) — do nothing. + # retry — re-enqueue the CHI so the stuck host is recreated. + onPodNotReady: none + # Minimum time a pod must stay Ready=False before recovery fires, once + # enabled (Go duration string; default 5m). Raise it for slow replicas. + onPodNotReadyThreshold: 5m ################################################ ## From 060c6459e5bacf3590dcf7dc1017dca1992953e4 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:35:56 +0500 Subject: [PATCH 037/164] test: polish fips --- tests/e2e/steps_fips.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index 861a6ed5a..b1aa09cf9 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -867,14 +867,22 @@ def fips_assert_replicas_healthy( binary = "clickhouse" container = "clickhouse" tls_ports = {8443, 9440, 9010, 7171} + plaintext_ports = set() elif kind == "chk": pods = sorted(kubectl.get_chk_pod_names(workload)) binary = "clickhouse-keeper" container = "clickhouse-keeper" - tls_ports = {2281, 9444, 9182} + # 2281 = secure ZK client, 9444 = Raft. 9182 is the Keeper /ready HTTP control + # endpoint: intentionally PLAINTEXT (health/quorum probe, carries no secrets) and + # unaffected by the secure/insecure knobs — an allowed listener, NOT a TLS port. + # It must stay in the allow-set or the "only approved ports" check flags it. + tls_ports = {2281, 9444} + plaintext_ports = {9182} else: raise ValueError(f"unsupported workload kind: {kind}") + allowed_ports = tls_ports | plaintext_ports + note(f"{kind.upper()} pods: {pods}") assert len(pods) == expected_count, error( f"expected {expected_count} {kind.upper()} pods, " @@ -888,7 +896,7 @@ def fips_assert_replicas_healthy( ) fips_assert_only_tls_ports( pod=pod, - required=tls_ports, + required=allowed_ports, container=container, max_iters=30, sleep_s=2, From 785585f5091bbd02529802d8025eee5360a35701 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:36:38 +0500 Subject: [PATCH 038/164] test: recovery --- tests/e2e/test_operator.py | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index fc474ec85..8081398c3 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3770,6 +3770,10 @@ def test_010035_2(self): """ create_shell_namespace_clickhouse_template() + with Given("operator config sets a short sustained-NotReady recovery threshold"): + # Default is 5m; shorten so the recreate happens well within the 420s window. + util.apply_operator_config("manifests/chopconf/test-035-2-sustained-not-ready.yaml") + manifest = "manifests/chi/test-035-2-sustained-not-ready.yaml" chi = yaml_manifest.get_name(util.get_full_path(manifest)) cluster = "default" @@ -3842,6 +3846,95 @@ def test_010035_2(self): delete_test_namespace() +@TestScenario +@Tags("HEAVY") +@Name("test_010035_3. Opt-out: pod left alone when sustained-NotReady recovery onPodNotReady=none") +def test_010035_3(self): + """Verify that a Completed CHI's sustained-NotReady pod is NOT recreated when + recovery is disabled (onPodNotReady=none). The inverse of test_010035_2 — proves + the off-by-default opt-out knob is honored even with an aggressive 30s threshold. + + Scenario: + 1. Operator config opts OUT (onPodNotReady=none) with a short 30s threshold + 2. Create a CHI with a dummy sidecar container guarded by a readiness file + 3. Remove the readiness file — the pod stays Ready=False + 4. The operator must leave the pod alone (same UID) well past the threshold + """ + create_shell_namespace_clickhouse_template() + operator_namespace = current().context.operator_namespace + chopconf = "manifests/chopconf/test-035-3-opt-out.yaml" + + with Given("operator config disables sustained-NotReady recovery (onPodNotReady=none)"): + util.apply_operator_config(chopconf) + + manifest = "manifests/chi/test-035-2-sustained-not-ready.yaml" + chi = yaml_manifest.get_name(util.get_full_path(manifest)) + cluster = "default" + pod = f"chi-{chi}-{cluster}-0-0-0" + dummy_container = "readiness-flap" + + with When("I create CHI with a dummy readiness-gated container"): + kubectl.create_and_check( + manifest=manifest, + check={ + "object_counts": {"statefulset": 1, "pod": 1, "service": 2}, + "do_not_delete": 1, + }, + ) + + with And("Pod should initially be Ready"): + for i in range(1, 30): + pod_ready = kubectl.get_condition_status(pod, "Ready") + dummy_ready = kubectl.get_container_status(pod, 1) + if pod_ready == "True" and dummy_ready == "true": + break + retry_sleep(i, 2, f"pod Ready={pod_ready}, {dummy_container} ready={dummy_ready}") + + assert pod_ready == "True", error(f"expected pod {pod} to be Ready, got Ready={pod_ready}") + + old_uid = kubectl.get_field("pod", pod, ".metadata.uid") + assert old_uid, error(f"pod {pod} does not exist") + + with When("I make the dummy container NotReady without crashing it"): + kubectl.launch(f"exec {pod} -c {dummy_container} -- rm -f /tmp/ready") + + with Then("The pod should become NotReady"): + for i in range(1, 30): + pod_ready = kubectl.get_condition_status(pod, "Ready") + if pod_ready == "False": + break + retry_sleep(i, 2, f"pod Ready={pod_ready}") + + assert pod_ready == "False", error(f"expected pod {pod} to be NotReady, got Ready={pod_ready}") + + with Then("Operator must NOT recreate the pod (recovery disabled)"): + # Watch well past the 30s threshold; if recovery were (wrongly) enabled it + # would have recreated the pod by now. The UID must stay constant. + start_time = time.time() + pod_ready = kubectl.get_condition_status(pod, "Ready") + while time.time() - start_time < 120: + new_uid = kubectl.get_field("pod", pod, ".metadata.uid") + assert new_uid == old_uid, error( + f"pod {pod} was recreated (uid {old_uid}->{new_uid}) despite onPodNotReady=none" + ) + pod_ready = kubectl.get_condition_status(pod, "Ready") + retry_sleep( + int((time.time() - start_time) / 5) + 1, + 5, + f"pod uid={new_uid}, Ready={pod_ready} (expect unchanged)", + ) + + assert pod_ready == "False", error( + f"expected pod {pod} to stay NotReady when recovery disabled, got Ready={pod_ready}" + ) + + with Finally("I clean up"): + with By("resetting ClickHouseOperatorConfiguration to default"): + kubectl.delete(util.get_full_path(chopconf, lookup_in_host=False), operator_namespace) + util.restart_operator() + delete_test_namespace() + + @TestScenario @Requirements(RQ_SRS_026_ClickHouseOperator_EnableHttps("1.0")) @Name("test_010034. Check HTTPS support for health check") From 9d22070756a449a0556e8a623c27ac3870d614b8 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:37:06 +0500 Subject: [PATCH 039/164] dev: codegen --- .../v1/zz_generated.deepcopy.go | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go index a3cf2ef4f..58784f160 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go +++ b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go @@ -2141,10 +2141,37 @@ func (in *OperatorConfigReconcileRecovery) DeepCopy() *OperatorConfigReconcileRe return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperatorConfigReconcileRecoveryCompletedScope) DeepCopyInto(out *OperatorConfigReconcileRecoveryCompletedScope) { + *out = *in + if in.OnPodNotReady != nil { + in, out := &in.OnPodNotReady, &out.OnPodNotReady + *out = new(types.String) + **out = **in + } + if in.OnPodNotReadyThreshold != nil { + in, out := &in.OnPodNotReadyThreshold, &out.OnPodNotReadyThreshold + *out = new(types.String) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryCompletedScope. +func (in *OperatorConfigReconcileRecoveryCompletedScope) DeepCopy() *OperatorConfigReconcileRecoveryCompletedScope { + if in == nil { + return nil + } + out := new(OperatorConfigReconcileRecoveryCompletedScope) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigReconcileRecoveryFrom) DeepCopyInto(out *OperatorConfigReconcileRecoveryFrom) { *out = *in in.Aborted.DeepCopyInto(&out.Aborted) + in.Completed.DeepCopyInto(&out.Completed) return } From 17e515a864cfffc2dcad0fb4309df2299f2757db Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:37:22 +0500 Subject: [PATCH 040/164] test: manifests --- .../chopconf/test-035-2-sustained-not-ready.yaml | 14 ++++++++++++++ .../e2e/manifests/chopconf/test-035-3-opt-out.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml create mode 100644 tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml diff --git a/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml new file mode 100644 index 000000000..4c796ea01 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml @@ -0,0 +1,14 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-035-2-sustained-not-ready" +spec: + reconcile: + recovery: + from: + # Recovery is OFF by default, so the test must explicitly opt in with + # onPodNotReady: retry. Shorten the sustained-NotReady threshold from the 5m + # default so the recovery fires within the test's 420s window. + completed: + onPodNotReady: retry + onPodNotReadyThreshold: 30s diff --git a/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml new file mode 100644 index 000000000..cbd84d6dc --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml @@ -0,0 +1,14 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-035-3-opt-out" +spec: + reconcile: + recovery: + from: + # Explicit opt-out: even with an aggressive 30s threshold, onPodNotReady=none + # must leave a sustained-NotReady pod alone (no force-recreate). Isolates the + # on/off knob from the threshold — proves the default-off contract. + completed: + onPodNotReady: none + onPodNotReadyThreshold: 30s From 0b09bc361a796e845f7f67ed16c82378a2367d90 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:37:39 +0500 Subject: [PATCH 041/164] env: manifests --- .../clickhouse-operator-install-ansible.yaml | 36 +++++++++++++++++++ ...house-operator-install-bundle-v1beta1.yaml | 36 +++++++++++++++++++ .../clickhouse-operator-install-bundle.yaml | 36 +++++++++++++++++++ ...use-operator-install-template-v1beta1.yaml | 36 +++++++++++++++++++ .../clickhouse-operator-install-template.yaml | 36 +++++++++++++++++++ .../clickhouse-operator-install-tf.yaml | 36 +++++++++++++++++++ deploy/operator/parts/crd.yaml | 22 ++++++++++++ 7 files changed, 238 insertions(+) diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 23f14e2cf..0984f9e42 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -4181,6 +4181,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6070,6 +6092,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index a7fa9a1e9..691bdceb0 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4148,6 +4148,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6269,6 +6291,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 331ed8131..b4428b2a2 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4174,6 +4174,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6329,6 +6351,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index dc7894bd6..db5ecdc66 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4148,6 +4148,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6016,6 +6038,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 467dce1bb..e45d1eaa1 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4174,6 +4174,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6063,6 +6085,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index b4410543b..03199fe87 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -4181,6 +4181,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -6070,6 +6092,20 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 2d44e7c39..9a76ce1dd 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -8724,6 +8724,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" From cdb87f695fa6ac23d7856bda06f4e9ee5ab63dcf Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:37:52 +0500 Subject: [PATCH 042/164] env: helm --- ...onfigurations.clickhouse.altinity.com.yaml | 22 +++++++++++++++++++ deploy/helm/clickhouse-operator/values.yaml | 14 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 3388e8394..576c02119 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -578,6 +578,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index 95ee2da78..f32a79092 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -816,6 +816,20 @@ configs: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry From 3ea7ab284047475ee694ac19fb618901b504a26d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 02:38:02 +0500 Subject: [PATCH 043/164] dev: config --- config/config.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config/config.yaml b/config/config.yaml index d5e64552b..0ddffaf0a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -554,6 +554,20 @@ reconcile: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery from Status=Completed when a host pod regresses to Ready=False + # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry From 2c5d083b0c103b6f27ca5e518fb7df11f0284d8d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 17 Jun 2026 03:08:05 +0500 Subject: [PATCH 044/164] dev: reconciler --- pkg/controller/chi/worker-reconciler-chi.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/controller/chi/worker-reconciler-chi.go b/pkg/controller/chi/worker-reconciler-chi.go index 86ed40796..34123eaa5 100644 --- a/pkg/controller/chi/worker-reconciler-chi.go +++ b/pkg/controller/chi/worker-reconciler-chi.go @@ -111,6 +111,8 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal w.a.M(new).F().Info("CR has reconcile work - continue reconcile") case w.isAfterFinalizerInstalled(new.GetAncestorT(), new): w.a.M(new).F().Info("isAfterFinalizerInstalled - continue reconcile-2") + case w.crHasHostNeedingStuckRecovery(ctx, new): + w.a.M(new).F().Info("CR has a sustained-NotReady host - continue reconcile for stuck-host recovery") default: w.a.M(new).F().Info("No reconcile work - abort reconcile") metrics.CRReconcilesCompleted(ctx, new) @@ -534,7 +536,14 @@ func hostRequiresStatefulSetRollout(host *api.Host) bool { func (w *worker) hostForceRestart(ctx context.Context, host *api.Host, opts *statefulset.ReconcileOptions) error { w.a.V(1).M(host).F().Info("Reconcile host. Force restart: %s", host.GetName()) - if host.IsStopped() || (w.hostSoftwareRestart(ctx, host) != nil) { + // A sustained-NotReady pod won't be healed by an in-place software restart: the + // unreadiness may originate outside ClickHouse, and hostSoftwareRestart's readiness + // wait would just time out before falling back to scale-down. Recreate the pod + // directly (scale-down here, scale-up by the caller's StatefulSet reconcile). + stuckNotReady := chop.Config().ShouldRecoverCompletedOnPodNotReady() && + w.isPodSustainedNotReady(ctx, host, chop.Config().CompletedOnPodNotReadyThreshold()) + + if host.IsStopped() || stuckNotReady || (w.hostSoftwareRestart(ctx, host) != nil) { _ = w.hostScaleDown(ctx, host, opts) } From 1eed6ccaf83c0437279213451d94039ba3963cfd Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:25:43 +0500 Subject: [PATCH 045/164] dev: start naming normalization --- deploy/builder/templates-config/config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 372dd2c5e..89e82de8b 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -534,11 +534,11 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: + onStatus: # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: @@ -570,7 +570,7 @@ reconcile: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: From 74d6eb20274a0d8623e6dfe5fe7fe6605a9bc345 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:26:19 +0500 Subject: [PATCH 046/164] dev: crd from normalization --- ...stall-yaml-template-01-section-crd-02-chopconf.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index 73f59a894..a856bdadd 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -556,15 +556,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -580,7 +580,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string From 480206c6415de6989885b078ac0a0d17dc9a3e1b Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:26:36 +0500 Subject: [PATCH 047/164] env: manifests --- .../clickhouse-operator-install-ansible.yaml | 26 ++++++++++--------- ...house-operator-install-bundle-v1beta1.yaml | 26 ++++++++++--------- .../clickhouse-operator-install-bundle.yaml | 26 ++++++++++--------- ...use-operator-install-template-v1beta1.yaml | 26 ++++++++++--------- .../clickhouse-operator-install-template.yaml | 26 ++++++++++--------- .../clickhouse-operator-install-tf.yaml | 26 ++++++++++--------- deploy/operator/parts/crd.yaml | 10 +++---- 7 files changed, 89 insertions(+), 77 deletions(-) diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 0984f9e42..0231de6f3 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -4159,15 +4159,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4183,7 +4183,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6078,12 +6078,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6092,8 +6093,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6112,7 +6114,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index 691bdceb0..80af71c5d 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4126,15 +4126,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4150,7 +4150,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6277,12 +6277,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6291,8 +6292,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6311,7 +6313,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index b4428b2a2..a1cfc7904 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4152,15 +4152,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4176,7 +4176,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6337,12 +6337,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6351,8 +6352,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6371,7 +6373,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index db5ecdc66..1aab2e11a 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4126,15 +4126,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4150,7 +4150,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6024,12 +6024,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6038,8 +6039,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6058,7 +6060,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index e45d1eaa1..3165562a8 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4152,15 +4152,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4176,7 +4176,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6071,12 +6071,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6085,8 +6086,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6105,7 +6107,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 03199fe87..327b2c410 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -4159,15 +4159,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4183,7 +4183,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string @@ -6078,12 +6078,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6092,8 +6093,9 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -6112,7 +6114,7 @@ data: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 9a76ce1dd..e70755d48 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -8702,15 +8702,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -8726,7 +8726,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string From 2b4c2e2b90185f498b69e27a0ee7879bc9d436b2 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:26:51 +0500 Subject: [PATCH 048/164] env: helm chart --- ...orconfigurations.clickhouse.altinity.com.yaml | 10 +++++----- .../helm/clickhouse-operator/values.schema.json | 14 +++++++++++++- deploy/helm/clickhouse-operator/values.yaml | 16 +++++++++------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 576c02119..0161a3733 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -556,15 +556,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -580,7 +580,7 @@ spec: - "retry" completed: type: object - description: "Recovery from Status=Completed when a child pod regresses to Ready=False and stays NotReady" + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" properties: onPodNotReady: type: string diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json index 5fbee64d1..8be99b1e3 100644 --- a/deploy/helm/clickhouse-operator/values.schema.json +++ b/deploy/helm/clickhouse-operator/values.schema.json @@ -518,7 +518,7 @@ "recovery": { "type": "object", "properties": { - "from": { + "onStatus": { "type": "object", "properties": { "aborted": { @@ -529,6 +529,18 @@ "enum": ["", "none", "None", "retry", "Retry"] } } + }, + "completed": { + "type": "object", + "properties": { + "onPodNotReady": { + "type": "string", + "enum": ["", "none", "None", "retry", "Retry"] + }, + "onPodNotReadyThreshold": { + "type": "string" + } + } } } } diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index f32a79092..d4934f088 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -802,12 +802,13 @@ configs: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -816,8 +817,9 @@ configs: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -835,7 +837,7 @@ configs: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: From f8c559cf4e69ade343b879627d4e9a0ed9f9eab4 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:27:03 +0500 Subject: [PATCH 049/164] dev: config --- config/config-dev.yaml | 10 +++++----- config/config.yaml | 16 +++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config/config-dev.yaml b/config/config-dev.yaml index 6ca12d944..d403aa9d8 100644 --- a/config/config-dev.yaml +++ b/config/config-dev.yaml @@ -437,12 +437,12 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery while Status=Aborted aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -457,7 +457,7 @@ reconcile: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/config/config.yaml b/config/config.yaml index 0ddffaf0a..f63d20486 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -540,12 +540,13 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -554,8 +555,9 @@ reconcile: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup - # Recovery from Status=Completed when a host pod regresses to Ready=False - # and stays NotReady (sustained) without crashing — auto-heals stuck hosts. + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. completed: # Action when a Completed CHI's pod flips Ready=True -> Ready=False and # stays NotReady for at least onPodNotReadyThreshold: @@ -574,7 +576,7 @@ reconcile: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: From e37e205613bfde433fb5267687d9245878efacf8 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:27:38 +0500 Subject: [PATCH 050/164] dev: api structures --- .../v1/type_configuration_chop.go | 32 +++++++++---------- .../type_configuration_chop_recovery_test.go | 10 +++--- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index eb0db13d0..04ed0768a 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -576,22 +576,22 @@ type OperatorConfigReconcile struct { } // OperatorConfigReconcileRecovery specifies auto-recovery behavior for reconcile. -// Event→action mappings are scoped by the CHI state we recover FROM (under .From). +// Event→action mappings are scoped by the CHI status they apply to (under .OnStatus). // Global policy knobs (future: retries, backoff, cooldown, enabled) sit as flat peers -// of .From at this level. Multi-scope design anticipates future states beyond Aborted +// of .OnStatus at this level. Multi-scope design anticipates future states beyond Aborted // (e.g. Failed, Broken). type OperatorConfigReconcileRecovery struct { - // From maps the CHI state we recover from (e.g. aborted) to event→action mappings. - From OperatorConfigReconcileRecoveryFrom `json:"from,omitempty" yaml:"from,omitempty"` + // OnStatus maps a CHI status (e.g. aborted, completed) to event→action mappings + // that apply while the CHI is in that status. + OnStatus OperatorConfigReconcileRecoveryOnStatus `json:"onStatus,omitempty" yaml:"onStatus,omitempty"` } -// OperatorConfigReconcileRecoveryFrom groups recovery-event mappings by the CHI state -// being recovered from. Each sub-field is a scope whose keys are on recovery -// triggers. -type OperatorConfigReconcileRecoveryFrom struct { - // Aborted scope — recovery from Status=Aborted. +// OperatorConfigReconcileRecoveryOnStatus groups recovery-event mappings by the CHI status +// they apply to. Each sub-field is a scope whose keys are on recovery triggers. +type OperatorConfigReconcileRecoveryOnStatus struct { + // Aborted scope — recovery while Status=Aborted. Aborted OperatorConfigReconcileRecoveryScope `json:"aborted,omitempty" yaml:"aborted,omitempty"` - // Completed scope — recovery from Status=Completed when a child pod regresses to Ready=False + // Completed scope — recovery while Status=Completed when a child pod regresses to Ready=False Completed OperatorConfigReconcileRecoveryCompletedScope `json:"completed,omitempty" yaml:"completed,omitempty"` // Future: Failed, Broken, etc. } @@ -1656,9 +1656,9 @@ func (c *OperatorConfig) RestartOnOperatorConfigurationChange() bool { // ShouldRecoverAbortedOnPodReady reports whether the operator should re-enqueue a CHI // reconcile when a pod belonging to an Aborted CHI transitions to Ready. Default is to retry. -// Backed by reconcile.recovery.from.aborted.onPodReady config key. +// Backed by reconcile.recovery.onStatus.aborted.onPodReady config key. func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { - value := strings.ToLower(c.Reconcile.Recovery.From.Aborted.OnPodReady.String()) + value := strings.ToLower(c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady.String()) if value == "" { // Default behavior — retry return true @@ -1672,18 +1672,18 @@ func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { // a Completed CHI's pod is destructive (it can interrupt a replica's in-progress recovery // and means hard downtime for a single-replica shard), so this is opt-in — only an explicit // onPodNotReady: retry enables it. Unlike the Aborted scope, which retries by default. -// Backed by reconcile.recovery.from.completed.onPodNotReady config key. +// Backed by reconcile.recovery.onStatus.completed.onPodNotReady config key. func (c *OperatorConfig) ShouldRecoverCompletedOnPodNotReady() bool { - value := strings.ToLower(c.Reconcile.Recovery.From.Completed.OnPodNotReady.String()) + value := strings.ToLower(c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady.String()) return value == RecoveryActionRetry } // CompletedOnPodNotReadyThreshold returns the minimum duration a pod must remain in // Ready=False before the Completed recovery scope fires. Falls back to the package // default (5m) if the config value is unset, empty, or unparseable. -// Backed by reconcile.recovery.from.completed.onPodNotReadyThreshold config key. +// Backed by reconcile.recovery.onStatus.completed.onPodNotReadyThreshold config key. func (c *OperatorConfig) CompletedOnPodNotReadyThreshold() time.Duration { - raw := strings.TrimSpace(c.Reconcile.Recovery.From.Completed.OnPodNotReadyThreshold.String()) + raw := strings.TrimSpace(c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReadyThreshold.String()) if raw == "" { return defaultCompletedOnPodNotReadyThreshold } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go index 8354266fd..b3100b3dc 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go @@ -24,7 +24,7 @@ import ( ) // TestShouldRecoverAbortedOnPodReady verifies the accessor's behavior across the -// full matrix of possible values for reconcile.recovery.from.aborted.onPodReady. +// full matrix of possible values for reconcile.recovery.onStatus.aborted.onPodReady. func TestShouldRecoverAbortedOnPodReady(t *testing.T) { tests := []struct { name string @@ -46,7 +46,7 @@ func TestShouldRecoverAbortedOnPodReady(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { c := &OperatorConfig{} - c.Reconcile.Recovery.From.Aborted.OnPodReady = tc.onReady + c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady = tc.onReady require.Equal(t, tc.expected, c.ShouldRecoverAbortedOnPodReady()) }) } @@ -60,7 +60,7 @@ func TestRecoveryActionConstants(t *testing.T) { } // TestShouldRecoverCompletedOnPodNotReady verifies the accessor's behavior across the -// full matrix of possible values for reconcile.recovery.from.completed.onPodNotReady. +// full matrix of possible values for reconcile.recovery.onStatus.completed.onPodNotReady. // Mirrors TestShouldRecoverAbortedOnPodReady so symmetric config keys behave identically. func TestShouldRecoverCompletedOnPodNotReady(t *testing.T) { tests := []struct { @@ -83,7 +83,7 @@ func TestShouldRecoverCompletedOnPodNotReady(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { c := &OperatorConfig{} - c.Reconcile.Recovery.From.Completed.OnPodNotReady = tc.onPodNotRdy + c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady = tc.onPodNotRdy require.Equal(t, tc.expected, c.ShouldRecoverCompletedOnPodNotReady()) }) } @@ -116,7 +116,7 @@ func TestCompletedOnPodNotReadyThreshold(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { c := &OperatorConfig{} - c.Reconcile.Recovery.From.Completed.OnPodNotReadyThreshold = tc.raw + c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReadyThreshold = tc.raw require.Equal(t, tc.expected, c.CompletedOnPodNotReadyThreshold()) }) } From 770e9d063a91ee9fe9405e3af50a959e34c20cde Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:28:05 +0500 Subject: [PATCH 051/164] dev: adjust controller --- pkg/controller/chi/worker-boilerplate.go | 2 +- pkg/controller/chi/worker-pod-retry.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/chi/worker-boilerplate.go b/pkg/controller/chi/worker-boilerplate.go index d8b28fde1..aab67967b 100644 --- a/pkg/controller/chi/worker-boilerplate.go +++ b/pkg/controller/chi/worker-boilerplate.go @@ -149,7 +149,7 @@ func (w *worker) processReconcilePod(ctx context.Context, cmd *cmd_queue.Reconci case cmd_queue.ReconcileUpdate: // Detect NotReady → Ready transition for pods belonging to Aborted CHIs // and re-enqueue the CHI for reconcile. Controlled by config option - // reconcile.recovery.from.aborted.onPodReady (default: retry). + // reconcile.recovery.onStatus.aborted.onPodReady (default: retry). w.recoverAbortedReconcileOnPodReady(ctx, cmd.Old, cmd.New) // Symmetric path for Ready→NotReady on Completed CHIs. w.recoverCompletedReconcileOnPodNotReady(ctx, cmd.Old, cmd.New) diff --git a/pkg/controller/chi/worker-pod-retry.go b/pkg/controller/chi/worker-pod-retry.go index 8b8007a74..bffe6d5cc 100644 --- a/pkg/controller/chi/worker-pod-retry.go +++ b/pkg/controller/chi/worker-pod-retry.go @@ -50,7 +50,7 @@ var normalizeTimeAbortReasons = []string{ // recoverAbortedReconcileOnPodReady inspects a pod update event and re-enqueues the parent // CHI for reconcile when the pod transitioned NotReady → Ready and the CHI is Aborted. -// Controlled by reconcile.recovery.from.aborted.onPodReady config option (default: retry). +// Controlled by reconcile.recovery.onStatus.aborted.onPodReady config option (default: retry). // The decision to re-enqueue is based on the CHI's Status alone, not on ActionPlan — // see shouldTriggerAutoRecovery for the rationale. func (w *worker) recoverAbortedReconcileOnPodReady(ctx context.Context, oldPod, newPod *core.Pod) { From 2a165846570e97f8063874d3b2207368dd7165c9 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:28:27 +0500 Subject: [PATCH 052/164] dev: codegen --- .../v1/zz_generated.deepcopy.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go index 58784f160..dd8160f73 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go +++ b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go @@ -2127,7 +2127,7 @@ func (in *OperatorConfigReconcile) DeepCopy() *OperatorConfigReconcile { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigReconcileRecovery) DeepCopyInto(out *OperatorConfigReconcileRecovery) { *out = *in - in.From.DeepCopyInto(&out.From) + in.OnStatus.DeepCopyInto(&out.OnStatus) return } @@ -2168,19 +2168,19 @@ func (in *OperatorConfigReconcileRecoveryCompletedScope) DeepCopy() *OperatorCon } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OperatorConfigReconcileRecoveryFrom) DeepCopyInto(out *OperatorConfigReconcileRecoveryFrom) { +func (in *OperatorConfigReconcileRecoveryOnStatus) DeepCopyInto(out *OperatorConfigReconcileRecoveryOnStatus) { *out = *in in.Aborted.DeepCopyInto(&out.Aborted) in.Completed.DeepCopyInto(&out.Completed) return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryFrom. -func (in *OperatorConfigReconcileRecoveryFrom) DeepCopy() *OperatorConfigReconcileRecoveryFrom { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryOnStatus. +func (in *OperatorConfigReconcileRecoveryOnStatus) DeepCopy() *OperatorConfigReconcileRecoveryOnStatus { if in == nil { return nil } - out := new(OperatorConfigReconcileRecoveryFrom) + out := new(OperatorConfigReconcileRecoveryOnStatus) in.DeepCopyInto(out) return out } From 2217b49ce65972ecc841075a96f48b36bed9d98b Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:28:58 +0500 Subject: [PATCH 053/164] doc: examples --- docs/chi-examples/70-chop-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index 7784ec504..bf797bd57 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -400,7 +400,7 @@ spec: ## ################################################ recovery: - from: + onStatus: aborted: # When a reconcile lands in status=Aborted (due to FIPS coercion # conflict, plain-text ZK under FIPS, etc.), the operator can From 4409d056d2e0a64bd216f06281a3fcf413745f1d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:29:53 +0500 Subject: [PATCH 054/164] doc: updated --- docs/operator_upgrade.md | 4 +++- docs/security_hardening_fips.md | 2 +- release_notes.md | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/operator_upgrade.md b/docs/operator_upgrade.md index 8b2929ab2..ffc096d04 100644 --- a/docs/operator_upgrade.md +++ b/docs/operator_upgrade.md @@ -18,7 +18,9 @@ **ACVP responder** (optional, build-tagged): a NIST ACVP test responder can be embedded into operator and metrics-exporter binaries via the `acvp_wrapper` build tag. Default builds do NOT include the responder. See `docs/security_hardening_fips.md` § ACVP for the test-evidence pipeline. -**0.27.0 → 0.27.1 auto-recovery from Aborted:** the new `reconcile.recovery.from.aborted.onPodReady` knob re-enqueues a CHI when a pod transitions NotReady → Ready while the CR sits in `Aborted` status. Default is `retry` (auto-recover on pod-Ready transition); set to `none` to opt out and preserve the pre-0.27.1 manual-intervention behaviour. +**0.27.0 → 0.27.1 auto-recovery from Aborted:** the new `reconcile.recovery.from.aborted.onPodReady` knob re-enqueues a CHI when a pod transitions NotReady → Ready while the CR sits in `Aborted` status. Default is `retry` (auto-recover on pod-Ready transition); set to `none` to opt out and preserve the pre-0.27.1 manual-intervention behaviour. **(Renamed in 0.27.2 — see the 0.27.1 → 0.27.2 note below.)** + +**0.27.1 → 0.27.2 recovery config key renamed (backward-incompatible):** the auto-recovery config key `reconcile.recovery.from.{aborted,completed}` is renamed to `reconcile.recovery.onStatus.{aborted,completed}` in `ClickHouseOperatorConfiguration`. The `from` grouping level is removed; the per-status scopes (`aborted`, `completed`) and their action keys (`onPodReady`, `onPodNotReady`, `onPodNotReadyThreshold`) are otherwise unchanged. The operator **silently ignores** the obsolete `from` key (unknown config keys are dropped at load). **Action required only if you explicitly set `reconcile.recovery.from.aborted.onPodReady: none`** on 0.27.0/0.27.1 to *disable* auto-recovery of Aborted CHIs: after upgrade that setting is ignored, the accessor falls back to its default (`retry`), and Aborted auto-recovery is **silently re-enabled**. Re-apply it under the new path: `reconcile.recovery.onStatus.aborted.onPodReady: none`. (The `completed` scope — sustained-NotReady host recovery — is new in 0.27.2 and off by default, so no migration is needed for it.) **0.27.0 → 0.27.1 reconcile hooks (preview):** new `spec.reconcile.host.hooks` and `spec.reconcile.cluster.hooks` blocks accept `events:` + `sql:` / `http:` / `shell:` actions. Only `sql:` is wired end-to-end in 0.27.1; `http:` and `shell:` currently emit a "not yet implemented" Fatal at validation time, so defer adopting those action types until the corresponding runners ship. diff --git a/docs/security_hardening_fips.md b/docs/security_hardening_fips.md index 41c8a73cd..93b99f731 100644 --- a/docs/security_hardening_fips.md +++ b/docs/security_hardening_fips.md @@ -137,7 +137,7 @@ kubectl get chi -o json | jq -r '.items[].status.errors[]? | select(startswith(" Recovery is via spec edit: `kubectl apply` a corrected CHI (set `secure: true` on every ZK node, or remove the `zookeeper:` block and use a CHK reference). The informer's `UpdateFunc` re-enqueues the CR and normalize re-runs cleanly. -Note: this recovery path does NOT depend on `recovery.from.aborted.onPodReady` +Note: this recovery path does NOT depend on `recovery.onStatus.aborted.onPodReady` — that path requires pod-readiness transitions, which never fire for CHIs rejected at the normalizer (pods are never created). diff --git a/release_notes.md b/release_notes.md index 5bbb6f22f..60335ce9a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,7 @@ +## Release 0.27.2 +### Behavior Changes +* **Backward-incompatible config rename** in `ClickHouseOperatorConfiguration`: `reconcile.recovery.from.{aborted,completed}` → `reconcile.recovery.onStatus.{aborted,completed}`. The `from` grouping level is removed; the per-status scopes and their action keys (`onPodReady`/`onPodNotReady`/`onPodNotReadyThreshold`) are unchanged. The obsolete `from` key is silently ignored on load. **If you set `reconcile.recovery.from.aborted.onPodReady: none` on 0.27.0/0.27.1 to disable Aborted auto-recovery, re-apply it as `reconcile.recovery.onStatus.aborted.onPodReady: none`** — otherwise the default (`retry`) silently re-enables it. The `completed` scope (sustained-NotReady host recovery) is new in 0.27.2 and off by default. See [docs/operator_upgrade.md](docs/operator_upgrade.md). + ## Release 0.27.1 ### Behavior Changes * `StatefulSet` create returning `AlreadyExists` (e.g. due to a stale informer cache or a prior failed delete) is no longer silently treated as a successful create. The reconciler now propagates the recreate sentinel so the host correctly enters the recreate path. See https://github.com/Altinity/clickhouse-operator/pull/1993. From ba084bf7090f3fe933e2b394714e4f922170147e Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 18 Jun 2026 17:30:12 +0500 Subject: [PATCH 055/164] test: adjust accordingly --- .../manifests/chopconf/test-035-1-auto-recovery-disabled.yaml | 2 +- .../e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml | 2 +- tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml | 2 +- tests/e2e/test_operator.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml b/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml index 926fd410c..a7bd4b2c7 100644 --- a/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml +++ b/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml @@ -10,7 +10,7 @@ spec: update: timeout: 30 recovery: - from: + onStatus: aborted: # Opt-out: CHI should STAY Aborted even when pod becomes Ready onPodReady: none diff --git a/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml index 4c796ea01..f3920e9b6 100644 --- a/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml +++ b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml @@ -5,7 +5,7 @@ metadata: spec: reconcile: recovery: - from: + onStatus: # Recovery is OFF by default, so the test must explicitly opt in with # onPodNotReady: retry. Shorten the sustained-NotReady threshold from the 5m # default so the recovery fires within the test's 420s window. diff --git a/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml index cbd84d6dc..0a3ae521e 100644 --- a/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml +++ b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml @@ -5,7 +5,7 @@ metadata: spec: reconcile: recovery: - from: + onStatus: # Explicit opt-out: even with an aggressive 30s threshold, onPodNotReady=none # must leave a sustained-NotReady pod alone (no force-recreate). Isolates the # on/off knob from the threshold — proves the default-off contract. diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 8081398c3..0cc58c196 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3714,7 +3714,7 @@ def test_010035(self): @Name("test_010035_1. Opt-out: CHI stays Aborted when auto-recovery onPodReady=none") def test_010035_1(self): """Opt-out path: Verify that when the operator is configured with - reconcile.recovery.from.aborted.onPodReady=none, the CHI stays Aborted + reconcile.recovery.onStatus.aborted.onPodReady=none, the CHI stays Aborted even after the pod becomes Ready — no automatic recovery. This is the inverse of test_010035 and validates that the opt-out knob works. From b28dfe6ced2f9b7f8a2273e3bff3f596c20fef0b Mon Sep 17 00:00:00 2001 From: saba Date: Thu, 18 Jun 2026 17:43:48 +0200 Subject: [PATCH 056/164] major improvement of FIPS coverage, added fips test plan to the requirements file. linked all the requirements with full coverage. updated respective steps. --- tests/e2e/steps_fips.py | 1585 +++++++++++++---- tests/e2e/test_acvp.py | 4 + tests/e2e/test_operator.py | 595 ++++--- tests/requirements/fips.md | 866 +++------ tests/requirements/fips.py | 2428 +++++++------------------- tests/requirements/fips_test_plan.md | 516 ++++++ 6 files changed, 2973 insertions(+), 3021 deletions(-) create mode 100644 tests/requirements/fips_test_plan.md diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index 861a6ed5a..e5efc9f4b 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -33,6 +33,8 @@ import e2e.kubectl as kubectl +FAKE_OPENSSL_SERVER = "fake-openssl-server" + # --------------------------------------------------------------------------- # Build verification # --------------------------------------------------------------------------- @@ -270,8 +272,6 @@ def fips_assert_fips_enforced_coercion_in_logs(self, logs): ) - - @TestStep(Given) def fips_apply_operator_godebug(self): """Apply suite-configured GODEBUG=fips140= on the operator deployment.""" @@ -571,6 +571,10 @@ def start_external_ch_container(self, ns=None, cipher_suites=None): note(f"external ClickHouse client container started: {container}") self.context.external_chi_container = container + yield + + with Finally("stop external ClickHouse client container"): + stop_external_ch_container() @TestStep(Finally) def stop_external_ch_container(self): @@ -592,7 +596,7 @@ def fips_ch_external_secure_query(self, pod, sql, ns=None): """ ns = ns or self.context.test_namespace container = self.context.external_chi_container - local_port = "9440" + local_port = _free_local_port() pf = subprocess.Popen( [ @@ -668,7 +672,7 @@ def fips_ch_external_secure_query(self, pod, sql, ns=None): # --------------------------------------------------------------------------- @TestStep(Given) -def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): +def fips_edit_manifest(self, source_manifest, replicas_count=None, cipher_suites=None, kind="chi"): """Load a CHI/CHK manifest, patch ``replicasCount``, write a temp copy.""" source_path = util.get_full_path(source_manifest) with open(source_path, encoding="utf-8") as f: @@ -678,6 +682,20 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): manifest["spec"]["configuration"]["clusters"][0]["layout"]["replicasCount"] = ( replicas_count ) + if cipher_suites is not None: + xml = manifest["spec"]["configuration"]["files"]["openssl.xml"] + + old = ( + "TLS_AES_128_GCM_SHA256:" + "TLS_AES_256_GCM_SHA384" + ) + + manifest["spec"]["configuration"]["files"]["openssl.xml"] = ( + xml.replace( + old, + ":".join(cipher_suites), + ) + ) fd, temp_path = tempfile.mkstemp(suffix=".yaml", prefix=f"fips-{kind}-") os.close(fd) @@ -688,6 +706,7 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): if replicas_count is not None: note(f" replicasCount={replicas_count}") + return temp_path @@ -695,7 +714,7 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): def fips_apply_manifest( self, manifest_path, - expected_pod_count=None, + replica_count=None, kind="chi", apply_templates=None, timeout=None, @@ -712,8 +731,8 @@ def fips_apply_manifest( check = { "do_not_delete": 1, } - if expected_pod_count is not None: - check["pod_count"] = expected_pod_count + if replica_count is not None: + check["pod_count"] = replica_count if expected_status is not None: if kind == "chi": check["chi_status"] = expected_status @@ -744,18 +763,20 @@ def get_binary_version(self, pod, binary, container=None, ns=None): ns=ns, ) +@TestStep(Then) +def check_fips_binary_version(self, pod, binary, container=None, ns=None): + """Run `` --version`` inside a pod and check it contains altinityfips tag.""" -@TestStep(When) -def fips_read_listening_ports(self, pod, container="clickhouse", ns=None): - """Return TCP ports in LISTEN state inside the container via ``/proc/net/tcp``.""" - ns = ns or self.context.test_namespace - raw = kubectl.launch( - f"exec {pod} -c {container} -- " - f"sh -c 'cat /proc/net/tcp /proc/net/tcp6'", - ns=ns, + version = get_binary_version(pod=pod, binary=binary, container=container, ns=ns) + + assert "altinityfips" in version, error( + f"{pod}: expected altinityfips in {binary} version, got {version!r}" ) +def translate_tcp_port_output(raw): + """Translates raw output to a readable set of ports""" ports = set() + for line in raw.splitlines(): cols = line.split() if len(cols) < 4 or cols[0] == "sl" or cols[3] != "0A": @@ -764,26 +785,70 @@ def fips_read_listening_ports(self, pod, container="clickhouse", ns=None): ports.add(int(cols[1].split(":")[1], 16)) except (IndexError, ValueError): continue + return ports +@TestStep(When) +def fips_read_listening_ports( + self, + pod, + container, + ns=None, + debug=False, + target=None, +): + """Return TCP ports in LISTEN state from /proc/net/tcp and /proc/net/tcp6.""" + + ns = ns or self.context.test_namespace + + if debug: + target = target or container + raw = kubectl.launch( + f"debug {pod} " + f"--image=busybox:1.36 " + f"--target={target} " + f"--attach " + f"-- sh -c 'cat /proc/1/net/tcp /proc/1/net/tcp6'", + ns=ns, + ) + else: + raw = kubectl.launch( + f"exec {pod} -c {container} -- " + f"sh -c 'cat /proc/net/tcp /proc/net/tcp6'", + ns=ns, + ) + + return translate_tcp_port_output(raw=raw) + @TestStep(Then) -def fips_assert_only_tls_ports( +def fips_assert_only_expected_ports( self, pod, - required, + expected, container="clickhouse", + ns=None, max_iters=1, sleep_s=2, + debug=False ): - """Assert the container listens on exactly ``required`` and nothing else.""" + """Assert the container listens on expected required ports.""" + ports = set() + for attempt in range(max_iters): - ports = fips_read_listening_ports(pod=pod, container=container) + ports = fips_read_listening_ports( + pod=pod, + container=container, + ns=ns, + debug=debug + ) + note(f"listening ports on {pod}: {sorted(ports)}") - missing = required - ports - unexpected = ports - required + missing = expected - ports + unexpected = ports - expected + if not missing and not unexpected: return @@ -795,15 +860,15 @@ def fips_assert_only_tls_ports( ) time.sleep(sleep_s) - missing = required - ports + missing = expected - ports assert not missing, error( f"{pod}: required {container} TLS ports missing: {sorted(missing)}" ) - unexpected = ports - required + unexpected = ports - expected assert not unexpected, error( f"{pod}: unexpected {container} ports listening " - f"(approved={sorted(required)}): {sorted(unexpected)}" + f"(approved={sorted(expected)}): {sorted(unexpected)}" ) @@ -835,7 +900,7 @@ def fips_wait_cluster_topology( self, pod, cluster_name, - expected_count, + replica_count, max_iters=30, sleep_s=2, ): @@ -846,203 +911,815 @@ def fips_wait_cluster_topology( f"SELECT count() FROM system.clusters " f"WHERE cluster = '{cluster_name}'" ), - expected=expected_count, + expected=replica_count, max_iters=max_iters, sleep_s=sleep_s, ) - note(f"{pod} sees {expected_count} hosts in cluster {cluster_name!r}") + note(f"{pod} sees {replica_count} hosts in cluster {cluster_name!r}") -@TestStep(Then) -def fips_assert_replicas_healthy( +@TestStep(When) +def fips_run_openssl_s_client_on_pod_port( self, - workload, - expected_count, - kind="chi", - cluster_name="default", + pod, + port, + cipher_suite="TLS_AES_128_GCM_SHA256", + tls_version="1.3", + ok_to_fail=False, + ns=None, ): - """Run essential FIPS/TLS health checks for the current CHI or CHK replica set.""" - if kind == "chi": - pods = sorted(kubectl.get_pod_names(workload)) - binary = "clickhouse" - container = "clickhouse" - tls_ports = {8443, 9440, 9010, 7171} - elif kind == "chk": - pods = sorted(kubectl.get_chk_pod_names(workload)) - binary = "clickhouse-keeper" - container = "clickhouse-keeper" - tls_ports = {2281, 9444, 9182} - else: - raise ValueError(f"unsupported workload kind: {kind}") + """Run ``openssl s_client`` against a pod listener through ``kubectl port-forward``.""" + ns = ns or self.context.test_namespace + ca_crt = self.context.tls["ca_crt"] + local_port = _free_local_port() - note(f"{kind.upper()} pods: {pods}") - assert len(pods) == expected_count, error( - f"expected {expected_count} {kind.upper()} pods, " - f"got {len(pods)}: {pods}" + pf = subprocess.Popen( + [ + "kubectl", + "-n", ns, + "port-forward", + f"pod/{pod}", + f"{local_port}:{port}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ) - for pod in pods: - version = get_binary_version(pod=pod, binary=binary) - assert "altinityfips" in version, error( - f"{pod}: expected altinityfips in {binary} version, got {version!r}" - ) - fips_assert_only_tls_ports( - pod=pod, - required=tls_ports, - container=container, - max_iters=30, - sleep_s=2, - ) - - if kind == "chi": - pod0 = pods[0] - out = fips_ch_external_secure_query(pod=pod0, sql="SELECT 1") - assert out == "1", error(f"external secure query failed, got {out!r}") - fips_wait_cluster_topology( - pod=pod0, - cluster_name=cluster_name, - expected_count=expected_count, - ) - - return pods - + try: + deadline = time.time() + 10 + while time.time() < deadline: + if pf.poll() is not None: + out, err = pf.communicate() + assert False, error( + "kubectl port-forward exited early\n" + f"stdout:\n{out}\n" + f"stderr:\n{err}" + ) -@TestStep(Then) -def fips_check_replication_across_replicas(self, chi_pods, table="repl_test"): - """Verify ReplicatedMergeTree data converges to every replica over TLS.""" - if len(chi_pods) < 2: - note(f"skipping replication check with {len(chi_pods)} replica(s)") - return + try: + socket.create_connection( + ("127.0.0.1", int(local_port)), timeout=0.5 + ).close() + break + except OSError: + time.sleep(0.2) + else: + assert False, error( + f"kubectl port-forward to {pod}:{port} " + f"did not become ready on 127.0.0.1:{local_port}" + ) - pod0 = chi_pods[0] + command = [ + "openssl", "s_client", + "-connect", f"127.0.0.1:{local_port}", + "-servername", "localhost", + "-CAfile", ca_crt, + "-verify_return_error", + ] + + if tls_version == "1.3": + command.append("-tls1_3") + if cipher_suite: + command.extend(["-ciphersuites", cipher_suite]) + elif tls_version == "1.2": + command.append("-tls1_2") + if cipher_suite: + command.extend(["-cipher", cipher_suite]) + elif tls_version == "1.1": + command.append("-tls1_1") + if cipher_suite: + command.extend(["-cipher", cipher_suite]) + else: + raise ValueError(f"unsupported TLS version: {tls_version}") - with When("a replicated table is created on the cluster"): - fips_ch_external_secure_query( - pod=pod0, - sql=( - f"CREATE TABLE IF NOT EXISTS {table} ON CLUSTER '{{cluster}}' " - "(a UInt32) " - "ENGINE = ReplicatedMergeTree(" - f"'/clickhouse/{{installation}}/{{cluster}}/tables/{{shard}}/{table}', " - "'{replica}') ORDER BY a" - ), + result = subprocess.run( + command, + input="Q\n", + text=True, + capture_output=True, + check=False, ) - with And("rows are inserted on replica 0"): - fips_ch_external_secure_query( - pod=pod0, - sql=f"INSERT INTO {table} SELECT number FROM numbers(10)", - ) + output = f"{result.stdout}\n{result.stderr}" - with Then("rows are replicated to every other replica over interserver TLS"): - target = 10 - for pod in chi_pods[1:]: - fips_poll_secure_scalar( - pod=pod, - sql=f"SELECT count() FROM {table}", - expected=target, + if not ok_to_fail: + assert result.returncode == 0, error( + f"{pod}:{port}: openssl s_client failed for " + f"tls={tls_version}, cipher={cipher_suite}\n" + f"exit code: {result.returncode}\n" + f"output:\n{output}" ) + return output -@TestStep(When) -def fips_read_chop_generated_settings(self, pod, container="clickhouse", ns=None): - """Return the operator-generated ``chop-generated-settings.xml`` from ``pod``.""" - ns = ns or self.context.test_namespace - return kubectl.launch( - f"exec {pod} -c {container} -- " - f"cat /etc/clickhouse-server/config.d/chop-generated-settings.xml", - ns=ns, - ) - + finally: + pf.terminate() + try: + pf.wait(timeout=3) + except subprocess.TimeoutExpired: + pf.kill() @TestStep(Then) -def check_ports_in_chi_settings(self, settings_xml): - """Check approved TLS ports and removed plaintext ports in CHI settings.""" - assert "8443" in settings_xml, error( - "https_port 8443 missing from operator-generated settings" - ) - assert "9440" in settings_xml, error( - "tcp_port_secure 9440 missing from operator-generated settings" +def fips_assert_rejected_tls_probes( + self, + chi_pods, + chk_pods, + ns=None, +): + """Assert rejected TLS protocol/cipher combinations fail handshake.""" + ns = ns or self.context.test_namespace + + rejected_cases = ( + { + "name": "TLS 1.3 ChaCha20-Poly1305", + "tls_version": "1.3", + "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", + }, + { + "name": "TLS 1.1 protocol", + "tls_version": "1.1", + "cipher_suite": None, + }, ) - assert "9010" in settings_xml, error( - "interserver_https_port 9010 missing from operator-generated settings" + + endpoints = ( + ("ClickHouse HTTPS", chi_pods[0], 8443), + ("ClickHouse native TLS", chi_pods[0], 9440), + ("ClickHouse interserver HTTPS", chi_pods[0], 9010), + ("Keeper secure client", chk_pods[0], 2281), + ("Backup API HTTPS", chi_pods[0], 7171), ) - for port in ( - "http_port", - "tcp_port", - "mysql_port", - "postgresql_port", - "interserver_http_port", - ): - assert f'{port} remove="1"' in settings_xml, error( - f"{port} not marked removed in operator-generated settings" - ) + rejected_markers = ( + "Cipher is (NONE)", + "handshake failure", + "no protocols available", + "no shared cipher", + ) + for case in rejected_cases: + for label, pod, port in endpoints: + with Then(f"{label} {pod}:{port} rejects {case['name']}"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, + port=port, + tls_version=case["tls_version"], + cipher_suite=case["cipher_suite"], + ok_to_fail=True, + ns=ns, + ) -# --------------------------------------------------------------------------- -# clickhouse-backup sidecar -# --------------------------------------------------------------------------- + assert any( + marker.lower() in output.lower() + for marker in rejected_markers + ), error( + f"{label} {pod}:{port}: expected rejected TLS probe to fail " + f"for {case['name']}\n" + f"output:\n{output}" + ) @TestStep(Then) -def check_clickhouse_backup_embeds_gofips( +def fips_assert_aes256_tls13_probes( self, - pods, - gofips_version="v1.0.0", + chi_pods, + chk_pods, ns=None, ): - """Verify each clickhouse-backup sidecar binary embeds GOFIPS140 metadata.""" + """Assert approved TLS 1.3 AES-256-GCM cipher negotiates on FIPS TLS listeners.""" ns = ns or self.context.test_namespace - expected = f"GOFIPS140={gofips_version}" + approved_cipher = "TLS_AES_256_GCM_SHA384" + + endpoints = ( + ("ClickHouse HTTPS", chi_pods[0], 8443), + ("ClickHouse native TLS", chi_pods[0], 9440), + ("ClickHouse interserver HTTPS", chi_pods[0], 9010), + ("Keeper secure client", chk_pods[0], 2281), + ("Backup API HTTPS", chi_pods[0], 7171), + ) - for pod in pods: - backup_bin = f"/tmp/{pod}-clickhouse-backup" - kubectl.launch( - f"cp {pod}:/bin/clickhouse-backup {backup_bin} " - f"-c clickhouse-backup", - ns=ns, - ) - build_info = kubectl.run_shell(f"go version -m {backup_bin}") - assert expected in build_info, error( - f"{pod}: expected {expected} in clickhouse-backup binary" - ) - note(f"{pod} clickhouse-backup embeds {expected}") + for label, pod, port in endpoints: + with Then(f"{label} {pod}:{port} accepts approved AES-256 TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, + port=port, + tls_version="1.3", + cipher_suite=approved_cipher, + ok_to_fail=True, + ns=ns, + ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{label} {pod}:{port}: expected {approved_cipher} to negotiate\n" + f"output:\n{output}" + ) -@TestStep(Then) -def check_clickhouse_backup_https_api_serves_tls(self, pods, ns=None): - """Verify clickhouse-backup HTTPS API accepts clients trusted by the test CA.""" +@TestStep(When) +def fips_curl_pod_port(self, pod, port, path="/", ns=None): + """Return the HTTP status code from a plain ``curl`` to a pod listener via port-forward.""" ns = ns or self.context.test_namespace + local_port = _free_local_port() - for pod in pods: - out = kubectl.launch( - f"exec {pod} -c clickhouse-backup -- " - f"curl -sS -o /tmp/backup_tables.out -w 'HTTP:%{{http_code}}' " - f"--cacert /etc/clickhouse-backup/tls/ca.crt " - f"https://127.0.0.1:7171/backup/tables", - ns=ns, - ) - assert out == "HTTP:200", error( - f"{pod}: /backup/tables did not return HTTP 200, got {out!r}" - ) + pf = subprocess.Popen( + [ + "kubectl", + "-n", ns, + "port-forward", + f"pod/{pod}", + f"{local_port}:{port}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + deadline = time.time() + 10 + while time.time() < deadline: + if pf.poll() is not None: + out, err = pf.communicate() + assert False, error( + "kubectl port-forward exited early\n" + f"stdout:\n{out}\n" + f"stderr:\n{err}" + ) -@TestStep(Then) -def check_clickhouse_backup_https_api_rejects_untrusted(self, pods, ns=None): - """Verify clickhouse-backup HTTPS API rejects clients without the test CA.""" - ns = ns or self.context.test_namespace + try: + socket.create_connection( + ("127.0.0.1", int(local_port)), timeout=0.5 + ).close() + break + except OSError: + time.sleep(0.2) + else: + assert False, error( + f"kubectl port-forward to {pod}:{port} " + f"did not become ready on 127.0.0.1:{local_port}" + ) - for pod in pods: - out = kubectl.launch( - f"exec {pod} -c clickhouse-backup -- " - f"sh -c 'curl -sS --fail " - f"https://127.0.0.1:7171/backup/tables >/dev/null 2>&1; " - f"echo EXIT:$?'", - ns=ns, + result = subprocess.run( + [ + "curl", "-sS", + "-o", "/dev/null", + "-w", "%{http_code}", + f"http://127.0.0.1:{local_port}{path}", + ], + text=True, + capture_output=True, + check=False, ) - assert "EXIT:60" in out, error( - f"{pod}: expected certificate verification failure EXIT:60, got {out!r}" + assert result.returncode == 0, error( + f"{pod}:{port}{path}: curl failed\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" ) + return result.stdout.strip() + + finally: + pf.terminate() + try: + pf.wait(timeout=3) + except subprocess.TimeoutExpired: + pf.kill() + + +@TestStep(Then) +def check_chi_ports(self, pod, ns=None): + """TLS positive/negative probes for ClickHouse HTTPS and native ports.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + + for port in (8443, 9010): + with Then(f"{pod}:{port} accepts approved TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, port=port, cipher_suite=approved_cipher, ns=ns, + ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{pod}:{port}: expected approved cipher negotiation\n{output}" + ) + + with And(f"{pod}:9440 accepts approved native TLS query"): + out = fips_ch_external_secure_query(pod=pod, sql="SELECT 1") + assert out == "1", error( + f"{pod}:9440: expected SELECT 1 over native TLS, got {out!r}" + ) + + + +@TestStep(Then) +def check_chk_ports(self, pod, ns=None): + """TLS and readiness HTTP probes for ClickHouse Keeper listeners.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + port = 2281 + + # raft 9444 doesnt communicate over TLS endpoint + with Then(f"{pod}:{port} accepts approved TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, port=port, cipher_suite=approved_cipher, ns=ns, + ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{pod}:{port}: expected approved cipher negotiation\n{output}" + ) + + with And(f"{pod}:9182/ready accepts plain HTTP"): + code = fips_curl_pod_port(pod=pod, port=9182, path="/ready", ns=ns) + assert code == "200", error( + f"{pod}:9182/ready: expected HTTP 200, got {code!r}" + ) + + +@TestStep(Then) +def check_backup_ports(self, pod, ns=None): + """TLS positive/negative probes for the clickhouse-backup HTTPS API.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + + with Then(f"{pod}:7171 accepts approved TLS 1.3 cipher"): + out = kubectl.launch( + f"exec {pod} -c clickhouse-backup -- " + f"sh -c 'curl -sS -o /dev/null -w HTTP:%{{http_code}} " + f"--cacert /etc/clickhouse-backup/tls/ca.crt " + f"--tlsv1.3 --tls13-ciphers {approved_cipher} " + f"https://127.0.0.1:7171/backup/tables'", + ns=ns, + ) + assert out == "HTTP:200", error( + f"{pod}:7171: expected HTTP 200 with approved cipher, got {out!r}" + ) + + with Then(f"{pod}:7171 rejects plaintext requests"): + out = kubectl.launch( + f"exec {pod} -c clickhouse-backup -- " + f"sh -c 'curl -s -o /dev/null -w %{{http_code}} " + f"http://127.0.0.1:7171/backup/tables'", + ns=ns, + ok_to_fail=True, + ) + assert out != "200", error( + f"{pod}:7171: expected plaintext HTTP request to be rejected, got {out!r}" + ) + +@TestStep(Then) +def check_k8s_api_requires_tls_from_operator_pod(self, ns=None): + """Assert Kubernetes API :443 rejects plaintext HTTP from operator pod containers.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + for container in ("clickhouse-operator", "metrics-exporter"): + with Then(f"{container} cannot use plaintext HTTP to Kubernetes API :443"): + out = kubectl.launch( + f"exec {pod} -c {container} -- " + "curl -skv http://kubernetes.default.svc:443", + ns=ns, + ok_to_fail=True, + ) + + assert "Client sent an HTTP request to an HTTPS server" in out, error( + f"{container}: expected Kubernetes API :443 to reject plaintext HTTP\n{out}" + ) + +@TestStep(Then) +def check_operator_clickhouse_tls_logs(self, ns=None): + """Assert operator uses HTTPS/TLS config when communicating with ClickHouse.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c clickhouse-operator", + ns=ns, + ) + + assert "setupTLSAdvanced():TLS setup OK" in logs, error( + "operator did not log ClickHouse TLS setup" + ) + assert "verify=Strict minVersion=1.3" in logs, error( + "operator ClickHouse TLS config is not Strict / TLS 1.3" + ) + assert "Ping(https://clickhouse_operator:" in logs, error( + "operator did not log HTTPS ClickHouse ping" + ) + assert ":8443?tls_config=" in logs, error( + "operator ClickHouse ping did not use HTTPS port 8443 with TLS config" + ) + +@TestStep(Then) +def check_metrics_exporter_discovers_clickhouse_https(self, ns=None): + """Assert metrics-exporter discovers ClickHouse hosts using HTTPS :8443.""" + + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c metrics-exporter --tail=4000", + ns=ns, + ) + + assert '"httpsPort":8443' in logs, error( + "metrics-exporter did not discover ClickHouse hosts with httpsPort=8443\n" + f"{logs}" + ) + +@TestStep(Then) +def check_clickhouse_uses_secure_keeper_port(self, chi, ns=None): + """Assert ClickHouse replicas use Keeper secure client port 2281 with secure=yes.""" + + ns = ns or current().context.test_namespace + pods = sorted(kubectl.get_pod_names(chi)) + + for pod in pods: + with Then(f"{pod} connects to Keeper on secure port 2281"): + logs = kubectl.launch( + f"logs {pod} -c clickhouse --tail=5000", + ns=ns, + ) + + assert re.search(r"Connected to ZooKeeper at .+:2281\b", logs), error( + f"{pod}: ClickHouse did not connect to Keeper on port 2281\n{logs}" + ) + + +@TestStep(Then) +def check_operator_skips_plaintext_keeper_dial(self, ns=None): + """Assert operator skips plaintext ZK helper when Keeper ensemble is TLS-only.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c clickhouse-operator", + ns=ns, + ) + + assert 'Port:&2281,Secure:&"yes"' in logs, error( + "operator logs do not show Keeper configured as secure port 2281" + ) + assert "Skip ZK root-path ensure" in logs, error( + "operator did not log skipping ZK root-path ensure" + ) + assert "ensemble is TLS-only and the operator dial is plaintext" in logs, error( + "operator did not log that plaintext Keeper dial was skipped for TLS-only ensemble" + ) + +@TestStep(Then) +def check_operator_ports(self, ns=None): + """Plain HTTP probes for operator Prometheus listener ports.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + for port in (9999, 8888): + with Then(f"operator pod:{port}/metrics accepts plain HTTP"): + code = fips_curl_pod_port(pod=pod, port=port, path="/metrics", ns=ns) + assert code == "200", error( + f"operator pod:{port}/metrics: expected HTTP 200, got {code!r}" + ) + + +@TestStep(Then) +def run_operator_fips_checks(self): + """ + Run FIPS validation checks against the operator pod: + + * verify the pod network namespace exposes only expected Prometheus ports + * verify clickhouse-operator and metrics-exporter emit FIPS startup banners + * verify metrics ports accept HTTP and reject disapproved TLS handshakes + """ + ns = current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + expected_ports = {8888, 9999} + + with Then("operator pod exposes only expected listener ports"): + fips_assert_only_expected_ports( + pod=pod, + container="clickhouse-operator", + ns=ns, + expected=expected_ports, + debug=True, + ) + + with And("both containers report the FIPS startup banner"): + op_logs = get_container_logs( + pod=pod, + container="clickhouse-operator", + ns=ns, + ) + me_logs = get_container_logs( + pod=pod, + container="metrics-exporter", + ns=ns, + ) + fips_startup_banner_ok(container="clickhouse-operator", logs=op_logs) + fips_startup_banner_ok(container="metrics-exporter", logs=me_logs) + + with Then("operator metrics ports accept HTTP and reject disapproved TLS"): + check_operator_ports(ns=ns) + + with Then("Kubernetes API port 443 requires TLS from operator pod containers"): + check_k8s_api_requires_tls_from_operator_pod(ns=ns) + +@TestStep(Then) +def run_operator_reconcile_fips_checks(self): + """Run the fips checks after operator already reconciled CHK and CHI.""" + + ns = current().context.operator_namespace + + with Then("operator communicates with ClickHouse over HTTPS/TLS"): + check_operator_clickhouse_tls_logs(ns=ns) + + with And("operator skips plaintext Keeper helper against TLS-only Keeper"): + check_operator_skips_plaintext_keeper_dial(ns=ns) + + +@TestStep(Then) +def run_chi_fips_checks(self, workload, replica_count, cluster_name="default"): + """ + Run FIPS and TLS validation checks against the ClickHouse cluster: + + * wait for the expected cluster topology to become available + * verify ClickHouse binaries report an Altinity FIPS build + * verify only approved secure listener ports are exposed + * verify external TLS connectivity to ClickHouse succeeds + * verify the server reports a FIPS version string + * verify operator-generated configuration removes plaintext listeners + * verify each listener accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_pod_names(workload)) + binary = "clickhouse" + container = "clickhouse" + expected_ports = {8443, 9440, 9010, 7171} + pod0 = pods[0] + + note(f"CHI pods: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHI pods, got {len(pods)}: {pods}" + ) + + with When("I wait for full cluster deployment"): + fips_wait_cluster_topology( + pod=pod0, + cluster_name=cluster_name, + replica_count=replica_count, + ) + + for pod in pods: + with Then("check the binary version contains altinityfips tag"): + check_fips_binary_version(pod=pod, binary=binary, container=container) + + with And("check the container only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with And("check TLS port behavior on each replica"): + check_chi_ports(pod=pod) + + with And("operator-generated ClickHouse config removes plaintext ports"): + check_ports_in_chi_settings(pod=pod) + + with Then("check connection via external secure query"): + check_external_clickhouse_reports_fips_version(pod=pod0) + + return pods + + +@TestStep(Then) +def run_chk_fips_checks(self, workload, replica_count): + """ + Run FIPS and TLS validation checks against the ClickHouse Keeper cluster: + + * verify the expected number of Keeper pods are running + * verify Keeper binaries report an Altinity FIPS build + * verify only approved secure listener ports are exposed + * verify operator-generated configuration removes plaintext listeners + * verify Raft inter-node communication is configured for TLS + * verify each listener accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_chk_pod_names(workload)) + binary = "clickhouse-keeper" + container = "clickhouse-keeper" + expected_ports = {2281, 9444, 9182} + + note(f"CHK pods: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHK pods, got {len(pods)}: {pods}" + ) + + for pod in pods: + with Then("check the binary version contains altinityfips tag"): + check_fips_binary_version(pod=pod, binary=binary, container=container) + + with And("check the container only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with And("check TLS port behavior on each Keeper node"): + check_chk_ports(pod=pod) + + with And("operator-generated Keeper config removes plaintext listeners"): + check_ports_in_chk_settings(pod=pod) + + return pods + + +@TestStep(Then) +def run_backup_fips_checks(self, workload, replica_count): + """ + Run FIPS and TLS validation checks against clickhouse-backup sidecars: + + * verify the expected number of CHI pods with backup sidecars are running + * verify clickhouse-backup binaries report a FIPS build + * verify only approved secure listener ports are exposed + * verify each sidecar binary embeds GOFIPS metadata + * verify the HTTPS API accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_pod_names(workload)) + container = "clickhouse-backup" + expected_ports = {8443, 9440, 9010, 7171} + + note(f"CHI pods with backup sidecar: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHI pods, got {len(pods)}: {pods}" + ) + + for pod in pods: + with Then("check the backup binary version contains fips tag"): + check_backup_fips_binary_version(pod=pod) + + with And("check the sidecar only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with Then("check TLS port behavior on each backup sidecar"): + check_backup_ports(pod=pod) + + with And("each sidecar binary embeds GOFIPS metadata"): + check_clickhouse_backup_embeds_gofips(pod=pod) + + with And("clickhouse-backup TLS config is secure"): + check_clickhouse_backup_clickhouse_tls_config(pod=pod) + + return pods + + +@TestStep(Then) +def fips_check_replication_across_replicas(self, chi_pods, table="repl_test"): + """Verify ReplicatedMergeTree data converges to every replica over TLS.""" + if len(chi_pods) < 2: + note(f"skipping replication check with {len(chi_pods)} replica(s)") + return + + pod0 = chi_pods[0] + + with When("a replicated table is created on the cluster"): + fips_ch_external_secure_query( + pod=pod0, + sql=( + f"CREATE TABLE IF NOT EXISTS {table} ON CLUSTER '{{cluster}}' " + "(a UInt32) " + "ENGINE = ReplicatedMergeTree(" + f"'/clickhouse/{{installation}}/{{cluster}}/tables/{{shard}}/{table}', " + "'{replica}') ORDER BY a" + ), + ) + + with And("rows are inserted on replica 0"): + fips_ch_external_secure_query( + pod=pod0, + sql=f"INSERT INTO {table} SELECT number FROM numbers(10)", + ) + + with Then("rows are replicated to every other replica over interserver TLS"): + target = 10 + for pod in chi_pods[1:]: + fips_poll_secure_scalar( + pod=pod, + sql=f"SELECT count() FROM {table}", + expected=target, + ) + + +@TestStep(When) +def fips_read_chop_generated_chi_settings(self, pod, container="clickhouse", ns=None): + """Return the operator-generated ``chop-generated-settings.xml`` from ``pod``.""" + ns = ns or self.context.test_namespace + return kubectl.launch( + f"exec {pod} -c {container} -- " + f"cat /etc/clickhouse-server/config.d/chop-generated-settings.xml", + ns=ns, + ) + + +@TestStep(Then) +def check_ports_in_chi_settings(self, pod): + """Check approved TLS ports and removed plaintext ports in CHI settings.""" + + settings_xml = fips_read_chop_generated_chi_settings(pod=pod) + note(f"chop-generated-settings.xml:\n{settings_xml}") + + assert "8443" in settings_xml, error( + "https_port 8443 missing from operator-generated settings" + ) + assert "9440" in settings_xml, error( + "tcp_port_secure 9440 missing from operator-generated settings" + ) + assert "9010" in settings_xml, error( + "interserver_https_port 9010 missing from operator-generated settings" + ) + + for port in ( + "http_port", + "tcp_port", + "mysql_port", + "postgresql_port", + "interserver_http_port", + ): + assert f'{port} remove="1"' in settings_xml, error( + f"{port} not marked removed in operator-generated settings" + ) + + +@TestStep(When) +def fips_read_chop_generated_chk_settings(self, pod, container="clickhouse-keeper", ns=None): + """Return operator-generated Keeper listener and Raft XML from ``pod``.""" + ns = ns or self.context.test_namespace + common_listeners_xml = kubectl.launch( + f"exec {pod} -c {container} -- " + "cat /etc/clickhouse-keeper/keeper_config.d/chop-generated-common-listeners.xml", + ns=ns, + ) + raft_xml = kubectl.launch( + f"exec {pod} -c {container} -- " + "cat /etc/clickhouse-keeper/keeper_config.d/chop-generated-raft.xml", + ns=ns, + ) + return common_listeners_xml, raft_xml + + +@TestStep(Then) +def check_ports_in_chk_settings(self, pod): + """Check plaintext listener removal and Raft TLS in CHK settings.""" + + common_listeners_xml, raft_xml = fips_read_chop_generated_chk_settings(pod=pod) + note(f"chop-generated-common-listeners.xml:\n{common_listeners_xml}") + note(f"chop-generated-raft.xml:\n{raft_xml}") + + assert '' in common_listeners_xml, error( + "tcp_port not marked removed in operator-generated Keeper settings" + ) + assert "1" in raft_xml, error( + "expected 1 in operator-generated Raft config" + ) + +@TestStep(Then) +def check_backup_fips_binary_version(self, pod, ns=None): + """Run ``clickhouse-backup --version`` and check it contains a fips tag.""" + version = get_binary_version( + pod=pod, + binary="/bin/clickhouse-backup", + container="clickhouse-backup", + ns=ns, + ) + note(f"{pod} clickhouse-backup --version: {version}") + assert "fips" in version.lower(), error( + f"{pod}: expected fips in clickhouse-backup version, got {version!r}" + ) + + +@TestStep(Then) +def check_clickhouse_backup_embeds_gofips( + self, + pod, + gofips_version="v1.0.0", + ns=None, +): + """Verify each clickhouse-backup sidecar binary embeds GOFIPS140 metadata.""" + ns = ns or self.context.test_namespace + expected = f"GOFIPS140={gofips_version}" + + backup_bin = f"/tmp/{pod}-clickhouse-backup" + kubectl.launch( + f"cp {pod}:/bin/clickhouse-backup {backup_bin} " + f"-c clickhouse-backup", + ns=ns, + ) + build_info = kubectl.run_shell(f"go version -m {backup_bin}") + assert expected in build_info, error( + f"{pod}: expected {expected} in clickhouse-backup binary" + ) + note(f"{pod} clickhouse-backup embeds {expected}") @TestStep(Then) @@ -1196,24 +1873,6 @@ def check_clickhouse_backup_clickhouse_tls_config(self, pod, ns=None): assert "TLS_CA:/etc/clickhouse-backup/tls/ca.crt" in out, error(out) assert "SKIP_VERIFY:false" in out, error(out) - -@TestStep(Then) -def check_clickhouse_backup_can_list_tables_over_clickhouse_tls(self, pod, ns=None): - """Use backup API to prove backup can talk to ClickHouse using its configured TLS path.""" - ns = ns or self.context.test_namespace - - out = kubectl.launch( - f"exec {pod} -c clickhouse-backup -- " - "curl -sS " - "--cacert /etc/clickhouse-backup/tls/ca.crt " - "-o /tmp/backup_tables.out " - "-w 'HTTP:%{http_code}' " - "https://127.0.0.1:7171/backup/tables", - ns=ns, - ) - - assert out == "HTTP:200", error(out) - @TestStep(Then) def check_clickhouse_backup_restore_roundtrip_https( self, @@ -1280,6 +1939,7 @@ def api(method, path): sql=f"SELECT count() FROM {table}", expected=10, ) + @TestStep(Then) def fips_wait_table_removed_from_dropped_tables( self, @@ -1309,219 +1969,424 @@ def fips_wait_table_removed_from_dropped_tables( assert False, error( f"{database}.{table} still present in system.dropped_tables after {timeout}s" ) -@TestStep(Given) -def fips_edit_cipher_suites_manifest( - self, - source_manifest, - cipher_suites, -): - """Load CHI/CHK manifest, patch OpenSSL cipherSuites, write temp copy.""" - source_path = util.get_full_path(source_manifest) - cipher_suites_value = ":".join(cipher_suites) +@TestStep(Then) +def check_fips_cast_failure(self, binary_path, binary, cast_name="HMAC-SHA2-256"): + """Assert binary exits non-zero when FIPS CAST is forced to fail.""" + result = subprocess.run( + [ + "env", + f"GODEBUG=fips140=only,failfipscast={cast_name}", + binary_path, + "--version", + ], + text=True, + capture_output=True, + check=False, + ) - with open(source_path, encoding="utf-8") as f: - manifest = yaml.safe_load(f) + output = f"{result.stdout}\n{result.stderr}" - files = ( - manifest - .setdefault("spec", {}) - .setdefault("configuration", {}) - .setdefault("files", {}) + assert result.returncode != 0, error( + f"{binary}: expected CAST failure exit, got {result.returncode}\n{output}" + ) + assert f"FIPS 140-3 self-test failed: {cast_name}" in output, error( + f"{binary}: expected CAST failure for {cast_name}\n{output}" + ) + assert "simulated CAST failure" in output, error( + f"{binary}: expected simulated CAST failure message\n{output}" ) - openssl_xml = files["openssl.xml"] +def _free_local_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return str(s.getsockname()[1]) - openssl_xml = re.sub( - r".*?", - f"{cipher_suites_value}", - openssl_xml, - flags=re.DOTALL, - ) - files["openssl.xml"] = openssl_xml +@TestStep(Then) +def check_fips_integrity_failure(self, binary_path, binary_label): + """Assert binary panics when the .go.fipsinfo HMAC is tampered with.""" + + cmd = f"readelf -S -W {shlex.quote(binary_path)}" + readelf_out = kubectl.run_shell(cmd) + + match = re.search(r"\.go\.fipsinfo\s+\w+\s+\w+\s+([0-9a-fA-F]+)", readelf_out) + assert match, error(f"{binary_label}: .go.fipsinfo section not found in ELF headers") + + section_offset = int(match.group(1), 16) + hmac_byte_offset = section_offset + 16 + + corrupted_bin = f"{binary_path}.corrupted" + shutil.copy2(binary_path, corrupted_bin) + + with open(corrupted_bin, "rb+") as f: + f.seek(hmac_byte_offset) + original_byte = f.read(1) + corrupted_byte = bytes([original_byte[0] ^ 0xFF]) + f.seek(hmac_byte_offset) + f.write(corrupted_byte) - fd, temp_path = tempfile.mkstemp( - suffix=".yaml", - prefix="fips-cipher-update-", + result = subprocess.run( + [corrupted_bin, "--version"], + env={"GODEBUG": "fips140=on"}, + capture_output=True, + text=True, + check=False ) - os.close(fd) - with open(temp_path, "w", encoding="utf-8") as f: - yaml.safe_dump( - manifest, - f, - default_flow_style=False, - sort_keys=False, - ) + output = f"{result.stdout}\n{result.stderr}" + note(output) - note(f"edited cipher-suite manifest written to {temp_path}") - note(f"cipherSuites={cipher_suites_value}") + with Then("the process must terminate with a verification mismatch panic"): + assert result.returncode != 0, error( + f"{binary_label}: tampered binary did not exit with error" + ) + assert "fips140: verification mismatch" in output, error( + f"{binary_label}: expected integrity panic not found in output:\n{output}" + ) - return temp_path + note(f"{binary_label}: integrity check successfully detected tampering") @TestStep(Then) -def fips_assert_chi_cipher_suites_configured( +def check_tls13_cipher_fails( self, pod, - cipher_suites, + port, + cipher, + target_host="localhost", + container=None, + ns=None, ): - """Assert ClickHouse generated OpenSSL config contains the expected cipherSuites.""" + """Verify a TLS 1.3 cipher cannot be negotiated.""" - expected = ":".join(cipher_suites) + ns = ns or self.context.test_namespace - openssl_xml = kubectl.launch( - f"exec {pod} -c clickhouse -- " - "cat /etc/clickhouse-server/config.d/openssl.xml" - ) + container_arg = f"-c {container} " if container else "" - expected_line = f"{expected}" + out = kubectl.launch( + f"exec {pod} {container_arg}-- " + "openssl s_client " + f"-connect {target_host}:{port} " + "-tls1_3 " + f"-ciphersuites {cipher}", + ns=ns, + ok_to_fail=True, + ) - assert expected_line in openssl_xml, error( - f"{pod}: expected cipherSuites not found\n" - f"expected: {expected_line}\n" - f"openssl.xml:\n{openssl_xml}" + assert ( + "Cipher is (NONE)" in out + or "handshake failure" in out + or "alert handshake failure" in out + or "no shared cipher" in out + ), error( + f"{target_host}:{port}: expected {cipher} to be rejected\n{out}" ) +@TestStep(Finally) +def fips_cleanup_admission_only_chi(self, chi): + """Cleanup chi""" + kubectl.launch( + f"delete chi {chi} --ignore-not-found=true --wait=false", + ns=current().context.test_namespace, + timeout=600, + ok_to_fail=True, + ) + kubectl.launch( + f"delete sts,pod,svc,pvc,cm,secret " + f"-l clickhouse.altinity.com/chi={chi} " + f"--ignore-not-found=true --wait=false", + ns=current().context.test_namespace, + timeout=600, + ok_to_fail=True, + ) @TestStep(Then) -def fips_assert_clickhouse_https_cipher_suite_accepted( - self, - pod, - cipher_suite, -): - """Assert ClickHouse HTTPS accepts the configured TLS 1.3 cipher suite.""" +def check_synthetic_tls13_smoke_from_operator_pod(self, chi_pod, ns=None): + """Synthetic TLS 1.3 AES-256 smoke from operator pod containers. - output = fips_run_openssl_s_client_for_clickhouse_https( - pod=pod, - cipher_suite=cipher_suite, - ok_to_fail=False, - ) + Uses real endpoints: + * Kubernetes API: kubernetes.default.svc:443 + * ClickHouse HTTPS: CHI pod IP:8443 - assert f"Cipher is {cipher_suite}" in output, error( - f"{pod}: expected negotiated cipher {cipher_suite}\n" - f"output:\n{output}" + Kubernetes is checked in two parts: + * verbose unauthenticated /version request proves TLS version/cipher; + * non-verbose authenticated pod API request proves service-account API access + without leaking the bearer token into logs. + + CHK is intentionally excluded because the operator does not normally + establish a runtime TLS client session to ClickHouse Keeper. + """ + test_ns = ns or current().context.test_namespace + operator_ns = current().context.operator_namespace + operator_pod = kubectl.get_operator_pod(ns=operator_ns) + + chi_ip = kubectl.get_field( + "pod", + chi_pod, + ".status.podIP", + ns=test_ns, ) - assert "Protocol : TLSv1.3" in output or "Protocol: TLSv1.3" in output, error( - f"{pod}: expected TLSv1.3 negotiation\n" - f"output:\n{output}" + approved_cipher = "TLS_AES_256_GCM_SHA384" + k8s_auth_url = ( + "https://kubernetes.default.svc:443" + f"/api/v1/namespaces/{operator_ns}/pods/{operator_pod}" ) + for container in ("clickhouse-operator", "metrics-exporter"): + with Then(f"{container} negotiates AES-256 TLS 1.3 to Kubernetes API"): + tls_out = kubectl.launch( + f"exec {operator_pod} -c {container} -- " + "sh -c '" + "curl -sS -v " + "--tlsv1.3 " + f"--tls13-ciphers {approved_cipher} " + "--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt " + "-o /dev/null " + "-w \"HTTP:%{http_code}\" " + "https://kubernetes.default.svc:443/version " + "2>&1" + "'", + ns=operator_ns, + ok_to_fail=True, + ) -@TestStep(Then) -def fips_assert_clickhouse_https_cipher_suite_rejected( - self, - pod, - cipher_suite, -): - """Assert ClickHouse HTTPS rejects a TLS 1.3 cipher suite not present in config.""" + assert "TLSv1.3" in tls_out, error( + f"{container}: expected TLSv1.3 to Kubernetes API\n{tls_out}" + ) + assert approved_cipher in tls_out, error( + f"{container}: expected {approved_cipher} to Kubernetes API\n{tls_out}" + ) + assert "HTTP:200" in tls_out, error( + f"{container}: expected Kubernetes /version HTTP 200\n{tls_out}" + ) - output = fips_run_openssl_s_client_for_clickhouse_https( - pod=pod, - cipher_suite=cipher_suite, + auth_out = kubectl.launch( + f"exec {operator_pod} -c {container} -- " + "sh -c '" + "IFS= read -r TOKEN < /var/run/secrets/kubernetes.io/serviceaccount/token; " + "curl -sS " + "--tlsv1.3 " + f"--tls13-ciphers {approved_cipher} " + "--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt " + "-H \"Authorization: Bearer ${TOKEN}\" " + "-o /dev/null " + "-w \"HTTP:%{http_code}\" " + f"{k8s_auth_url}" + "'", + ns=operator_ns, + ok_to_fail=True, + ) + + assert "HTTP:200" in auth_out, error( + f"{container}: expected authenticated Kubernetes API HTTP 200\n{auth_out}" + ) + + with And(f"{container} negotiates AES-256 TLS 1.3 to ClickHouse HTTPS"): + out = kubectl.launch( + f"exec {operator_pod} -c {container} -- " + "sh -c '" + "curl -sS -k -v " + "--tlsv1.3 " + f"--tls13-ciphers {approved_cipher} " + "-o /dev/null " + "-w \"HTTP:%{http_code}\" " + f"https://{chi_ip}:8443/ping " + "2>&1" + "'", + ns=operator_ns, + ok_to_fail=True, + ) + + assert "TLSv1.3" in out, error( + f"{container}: expected TLSv1.3 to ClickHouse HTTPS\n{out}" + ) + assert approved_cipher in out, error( + f"{container}: expected {approved_cipher} to ClickHouse HTTPS\n{out}" + ) + assert "HTTP:200" in out, error( + f"{container}: expected ClickHouse /ping HTTP 200\n{out}" + ) + + + +@TestStep(Finally) +def fips_delete_fake_openssl_server(self, ns=None): + """Delete fake OpenSSL TLS server pod/service.""" + ns = ns or current().context.test_namespace + + kubectl.launch( + f"delete svc {FAKE_OPENSSL_SERVER} --ignore-not-found", + ns=ns, ok_to_fail=True, ) - - assert f"Cipher is {cipher_suite}" not in output, error( - f"{pod}: unexpected negotiated cipher {cipher_suite}\n" - f"output:\n{output}" + kubectl.launch( + f"delete pod {FAKE_OPENSSL_SERVER} --ignore-not-found", + ns=ns, + ok_to_fail=True, ) +@TestStep(Given) +def fips_create_fake_openssl_server(self, cipher_suite, ns=None): + """Create fake TLS 1.3 server restricted to one cipher suite.""" + ns = ns or current().context.test_namespace + + fips_delete_fake_openssl_server(ns=ns) + + manifest = f""" +apiVersion: v1 +kind: Pod +metadata: + name: {FAKE_OPENSSL_SERVER} + labels: + app: {FAKE_OPENSSL_SERVER} +spec: + restartPolicy: Always + containers: + - name: openssl + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + command: ["sh", "-lc"] + args: + - | + openssl s_server \\ + -accept 18443 \\ + -cert /tls/server.crt \\ + -key /tls/server.key \\ + -tls1_3 \\ + -ciphersuites {cipher_suite} \\ + -www \\ + -state + ports: + - containerPort: 18443 + volumeMounts: + - name: tls + mountPath: /tls + readOnly: true + volumes: + - name: tls + secret: + secretName: clickhouse-certs +""" + + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix="-fake-openssl-server.yaml", + delete=False, + ) as f: + f.write(manifest) + tmp_path = f.name + + kubectl.launch(f"apply -f {shlex.quote(tmp_path)}", ns=ns) + + kubectl.launch( + f"expose pod {FAKE_OPENSSL_SERVER} " + f"--name={FAKE_OPENSSL_SERVER} " + "--port=8443 " + "--target-port=18443", + ns=ns, + ) + + kubectl.launch( + f"wait pod {FAKE_OPENSSL_SERVER} " + "--for=condition=Ready " + "--timeout=120s", + ns=ns, + ) + finally: + if tmp_path: + os.unlink(tmp_path) + + @TestStep(When) -def fips_run_openssl_s_client_for_clickhouse_https( +def fips_curl_tls13_from_operator_container( self, - pod, + container, + url, cipher_suite, - ok_to_fail=False, + ns=None, ): - """Run openssl s_client against ClickHouse HTTPS through kubectl port-forward.""" + """Run curl from one operator pod container with forced TLS 1.3 cipher.""" + ns = ns or current().context.operator_namespace + operator_pod = kubectl.get_operator_pod(ns=ns) - ns = self.context.test_namespace - ca_crt = self.context.tls["ca_crt"] + return kubectl.launch( + f"exec {operator_pod} -c {container} -- " + "sh -c '" + "curl -k -sS -v " + "--tlsv1.3 " + f"--tls13-ciphers {cipher_suite} " + f"{url} " + "-o /dev/null " + "-w \"HTTP:%{http_code}\" " + "2>&1" + "'", + ns=ns, + ok_to_fail=True, + ) - # Bind an ephemeral local port so concurrent scenarios never collide on a - # fixed forward port (keeps this step parallel-safe). Use the suite-configured - # kubectl command rather than a hardcoded "kubectl". - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as _probe: - _probe.bind(("127.0.0.1", 0)) - local_port = str(_probe.getsockname()[1]) - pf = subprocess.Popen( - self.context.kubectl_cmd.split() - + [ - "-n", - ns, - "port-forward", - f"pod/{pod}", - f"{local_port}:8443", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, +@TestStep(Then) +def fips_assert_operator_containers_tls13_fails( + self, + url, + cipher_suite, + ns=None, +): + """Assert both operator pod containers fail with the requested TLS 1.3 cipher.""" + ns = ns or current().context.operator_namespace + + failure_needles = ( + "handshake failure", + "no shared cipher", + "alert handshake failure", + "TLS connect error", + "HTTP:000", ) - try: - deadline = time.time() + 10 - while time.time() < deadline: - if pf.poll() is not None: - out, err = pf.communicate() - assert False, error( - "kubectl port-forward exited early\n" - f"stdout:\n{out}\n" - f"stderr:\n{err}" - ) + for container in ("clickhouse-operator", "metrics-exporter"): + with Then(f"{container} fails to negotiate {cipher_suite} to {url}"): + out = fips_curl_tls13_from_operator_container( + container=container, + url=url, + cipher_suite=cipher_suite, + ns=ns, + ) - try: - socket.create_connection( - ("127.0.0.1", int(local_port)), - timeout=0.5, - ).close() - break - except OSError: - time.sleep(0.2) - else: - assert False, error( - f"kubectl port-forward to {pod}:8443 " - f"did not become ready on 127.0.0.1:{local_port}" + assert any(needle in out for needle in failure_needles), error( + f"{container}: expected TLS handshake failure for {cipher_suite}\n{out}" ) - result = subprocess.run( - [ - "openssl", - "s_client", - "-connect", - f"127.0.0.1:{local_port}", - "-servername", - "localhost", - "-tls1_3", - "-ciphersuites", - cipher_suite, - "-CAfile", - ca_crt, - "-verify_return_error", - ], - input="Q\n", - text=True, - capture_output=True, - check=False, - ) - output = f"{result.stdout}\n{result.stderr}" +@TestStep(Then) +def fips_assert_connection_rejected_on_non_approved_cipher( + self, + ns=None, +): + """Assert fake ChaCha-only TLS server rejects approved AES-only clients. - if not ok_to_fail: - assert result.returncode == 0, error( - f"{pod}: openssl s_client failed for {cipher_suite}\n" - f"exit code: {result.returncode}\n" - f"output:\n{output}" - ) + This is the synthetic negative control for rejected cipher behavior. + """ + ns = ns or current().context.test_namespace - return output + approved_cipher = "TLS_AES_256_GCM_SHA384" + rejected_cipher = "TLS_CHACHA20_POLY1305_SHA256" + fake_url = f"https://{FAKE_OPENSSL_SERVER}:8443/ping" - finally: - pf.terminate() - try: - pf.wait(timeout=3) - except subprocess.TimeoutExpired: - pf.kill() \ No newline at end of file + with Given("fake OpenSSL server offers only non-approved ChaCha TLS 1.3 cipher"): + fips_create_fake_openssl_server( + cipher_suite=rejected_cipher, + ns=ns, + ) + + with Then("operator pod containers restricted to approved AES-256 fail the handshake"): + fips_assert_operator_containers_tls13_fails( + url=fake_url, + cipher_suite=approved_cipher, + ns=ns, + ) + + with Finally("delete fake OpenSSL server"): + fips_delete_fake_openssl_server(ns=ns) \ No newline at end of file diff --git a/tests/e2e/test_acvp.py b/tests/e2e/test_acvp.py index a72ca915d..6283db128 100644 --- a/tests/e2e/test_acvp.py +++ b/tests/e2e/test_acvp.py @@ -45,8 +45,10 @@ from requirements.fips import ( RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ConfigGeneration, + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SHA2256AFT, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_WrapperIntegration, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ConfigGeneration, + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SHA2256AFT, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_WrapperIntegration, ) @@ -248,6 +250,7 @@ def _acvp_smoke(binary_name, cmd_path): @Requirements( RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_WrapperIntegration("1.0"), RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ConfigGeneration("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SHA2256AFT("1.0"), ) def test_acvp_operator(self): """Build operator with -tags acvp_wrapper and verify the embedded ACVP @@ -265,6 +268,7 @@ def test_acvp_operator(self): @Requirements( RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_WrapperIntegration("1.0"), RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ConfigGeneration("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SHA2256AFT("1.0"), ) def test_acvp_metrics_exporter(self): """Mirror of test_acvp_operator for the metrics-exporter binary. Both diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index f678851ba..d2a2f5cef 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -7667,12 +7667,11 @@ def test_020016(self): delete_test_namespace() - @TestScenario @Tags("HEAVY") @Name("test_030001. FIPS build: shipped image binaries embed GOFIPS140=v1.0.0") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries("1.0") + RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries("1.0") ) def test_030001(self): """Verify FIPS metadata and runtime behavior for shipped image binaries. @@ -7697,9 +7696,6 @@ def test_030001(self): gofips_version = "v1.0.0" gofips140_needle = f"GOFIPS140={gofips_version}" - # --fips-info reports the build-baked release version (ldflags from the - # `release` file), NOT the image tag — so compare against release_version, - # which holds the release-file value even when OPERATOR_VERSION=dev. release_version = self.context.release_version godebug_default = "fips140=on" @@ -7735,64 +7731,34 @@ def test_030001(self): @TestScenario @Tags("HEAVY") -@Name("test_030002. FIPS build: startup banners with strict chopconf and GODEBUG") -@Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries_StartupLogs("1.0") -) -def test_030002(self): - """Verify FIPS startup banners from running operator and exporter binaries.""" - chopconf = "manifests/chopconf/test-030002-chopconf.yaml" - - fips_create_shell_namespace_clickhouse_template() - operator_namespace = self.context.operator_namespace - - with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) - - with When("operator startup logs are fetched"): - operator_pod = kubectl.get_operator_pod(ns=operator_namespace) - op_logs = get_container_logs( - pod=operator_pod, - container="clickhouse-operator", - ns=operator_namespace, - ) - me_logs = get_container_logs( - pod=operator_pod, - container="metrics-exporter", - ns=operator_namespace, - ) - - with Then("both containers report runtime.enforced=true in the FIPS banner"): - fips_startup_banner_ok(container="clickhouse-operator", logs=op_logs) - fips_startup_banner_ok(container="metrics-exporter", logs=me_logs) - -@TestScenario -@Tags("HEAVY") -@Name("test_030003. FIPS CHI/CHK: TLS-only ports and replicated CH traffic") +@Name("test_030003. FIPS data plane: TLS-only ClickHouse, Keeper, and backup") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Listeners("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHIDeploy("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHKDeploy("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_FIPSConfig("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_FIPSConfig("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_VersionString("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainHTTP("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainNative("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoPlainClientPort("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoUnexpectedPorts("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoUnexpectedPorts("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_RaftTLS("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_InternodeTLS("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_FIPSBinary("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_GOFIPS140("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_OnlyTLSPorts("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_HTTPSAPI("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries_StartupLogs("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_HTTPPorts("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_FIPSConfig("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig_ExternalClient("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSBinary("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSConfig("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_RestoreRoundTrip("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_TLS_ApprovedCiphers("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_TLS_RejectedCiphers("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KubernetesAPI("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_KubernetesAPI("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_ClickHouse("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_ClickHouse("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KeeperRestriction("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_ClickHouse_KeeperTLS("1.0"), ) def test_030003(self): - """Verify a FIPS ClickHouse + Keeper deployment runs with TLS-only data paths: - FIPS-built ClickHouse, Keeper, and clickhouse-backup binaries; only secure - ports exposed; the backup HTTPS API serving over TLS with CA-trust enforcement; - and ReplicatedMergeTree data converging over the TLS setup. + """Deploy a FIPS ClickHouse + Keeper installation under strict operator config + and verify TLS-only data paths: + + - operator, Keeper, ClickHouse, and clickhouse-backup pass FIPS binary and + listener-port checks, with only secure ports exposed + - ReplicatedMergeTree data converges across replicas over TLS + - the backup sidecar reaches ClickHouse over secure native TCP and completes a + backup/restore round-trip through the HTTPS API """ chopconf = "manifests/chopconf/test-030002-chopconf.yaml" chi_manifest = "manifests/chi/test-030003.yaml" @@ -7804,10 +7770,15 @@ def test_030003(self): chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) + chi_replica_count = 2 + with Given("strict FIPS operator configuration is applied"): - util.apply_operator_config(chopconf) + fips_apply_operator_config(chopconf_path=chopconf) - with And("test TLS secret is installed"): + with Check("operator pod passes essential FIPS checks"): + run_operator_fips_checks() + + with When("test TLS secret is installed"): create_tls_secret_for_fips_hosts(chi=chi, chk=chk) with And("external ClickHouse client container is started"): @@ -7816,84 +7787,66 @@ def test_030003(self): with And("FIPS ClickHouse Keeper is deployed with TLS settings"): fips_apply_manifest( manifest_path=chk_manifest, - expected_pod_count=2, + replica_count=2, kind="chk", ) with Then("Keeper cluster passes essential FIPS checks"): - fips_assert_replicas_healthy( - workload=chk, - expected_count=2, - kind="chk", - ) + chk_pods = run_chk_fips_checks(workload=chk, replica_count=2) - with And("FIPS ClickHouse is deployed with TLS settings"): + with When("FIPS ClickHouse is deployed with TLS settings"): fips_apply_manifest( manifest_path=chi_manifest, - expected_pod_count=2, + replica_count=chi_replica_count, kind="chi", apply_templates=[backup_template], ) - with Then("ClickHouse cluster passes essential FIPS checks"): - chi_pods = fips_assert_replicas_healthy( - workload=chi, - expected_count=2, - kind="chi", - ) - - chi_pod0 = chi_pods[0] - - with And("each clickhouse-backup sidecar uses a FIPS-built binary"): - for pod in chi_pods: - version = get_binary_version( - pod=pod, - binary="/bin/clickhouse-backup", - container="clickhouse-backup", - ) - note(f"{pod} clickhouse-backup --version: {version}") - assert "fips" in version.lower(), error( - f"{pod}: expected fips in clickhouse-backup version, got {version!r}" - ) - - with And("each clickhouse-backup sidecar embeds GOFIPS metadata"): - check_clickhouse_backup_embeds_gofips(pods=chi_pods) - - with And("clickhouse-backup sidecar exposes only its TLS API port"): - for pod in chi_pods: - fips_assert_only_tls_ports( - pod=pod, - required={8443, 9010, 9440, 7171}, - container="clickhouse-backup", - ) + with Check("operator outbound connections to CHI and CHK after reconcile"): + run_operator_reconcile_fips_checks() - with And("clickhouse-backup HTTPS API serves over TLS with the FIPS cert"): - check_clickhouse_backup_https_api_serves_tls(pods=chi_pods) + with Check("metrics-exporter discovers ClickHouse using HTTPS"): + check_metrics_exporter_discovers_clickhouse_https() - with And("clickhouse-backup HTTPS API rejects untrusted clients"): - check_clickhouse_backup_https_api_rejects_untrusted(pods=chi_pods) + with Check("ClickHouse replicas connect to Keeper using secure client port"): + check_clickhouse_uses_secure_keeper_port(chi=chi) - with And("external ClickHouse client reports a FIPS server version"): - check_external_clickhouse_reports_fips_version(pod=chi_pod0) + with Then("check ClickHouse cluster passes essential FIPS checks"): + chi_pods = run_chi_fips_checks( + workload=chi, + replica_count=chi_replica_count, + ) - with And("operator-generated ClickHouse config removes plaintext ports"): - settings_xml = fips_read_chop_generated_settings(pod=chi_pod0) - note(f"chop-generated-settings.xml:\n{settings_xml}") - check_ports_in_chi_settings(settings_xml=settings_xml) + with And("clickhouse-backup sidecar passes essential FIPS checks"): + backup_pods = run_backup_fips_checks( + workload=chi, + replica_count=chi_replica_count, + ) - with Then("ReplicatedMergeTree data converges over the TLS setup"): + with Check("ReplicatedMergeTree data converges over the TLS setup"): fips_check_replication_across_replicas(chi_pods=chi_pods) - with Finally("external ClickHouse client container is removed"): - stop_external_ch_container() + with Check("backup and restore succeeds through HTTPS API"): + check_clickhouse_backup_restore_roundtrip_https(pod=backup_pods[0]) + with Check("approved AES-256 TLS 1.3 cipher is negotiated"): + fips_assert_aes256_tls13_probes( + chi_pods=chi_pods, + chk_pods=chk_pods, + ) + + with Check("rejected TLS protocol and cipher combinations are not negotiated"): + fips_assert_rejected_tls_probes( + chi_pods=chi_pods, + chk_pods=chk_pods + ) @TestScenario @Tags("HEAVY") @Name("test_030004. FIPS CHI: scale replicas 2 -> 3 -> 1") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleUp("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleDown("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CH_Rescale("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CH_ConfigUpdate("1.0"), ) def test_030004(self): """Verify FIPS ClickHouse survives replica scale-up and scale-down. @@ -7924,7 +7877,7 @@ def test_030004(self): with And("FIPS ClickHouse Keeper is deployed with TLS settings"): fips_apply_manifest( manifest_path=chk_manifest, - expected_pod_count=2, + replica_count=2, kind="chk", ) @@ -7936,16 +7889,21 @@ def test_030004(self): ) fips_apply_manifest( manifest_path=chi_manifest_2, - expected_pod_count=2, + replica_count=2, kind="chi", apply_templates=[backup_template], ) with Then("2-replica cluster passes essential FIPS checks"): - chi_pods = fips_assert_replicas_healthy( + chi_pods = run_chi_fips_checks( workload=chi, - expected_count=2, - kind="chi", + replica_count=2, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=2, ) with And("ReplicatedMergeTree data converges across 2 replicas"): @@ -7959,15 +7917,20 @@ def test_030004(self): ) fips_apply_manifest( manifest_path=chi_manifest_3, - expected_pod_count=3, + replica_count=3, kind="chi", ) with Then("3-replica cluster passes essential FIPS checks"): - chi_pods = fips_assert_replicas_healthy( + chi_pods = run_chi_fips_checks( workload=chi, - expected_count=3, - kind="chi", + replica_count=3, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=3, ) with And("ReplicatedMergeTree data converges across 3 replicas"): @@ -7984,27 +7947,53 @@ def test_030004(self): ) fips_apply_manifest( manifest_path=chi_manifest_1, - expected_pod_count=1, + replica_count=1, kind="chi", ) with Then("single-replica cluster passes essential FIPS checks"): - fips_assert_replicas_healthy( + run_chi_fips_checks( workload=chi, - expected_count=1, + replica_count=1, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=1, + ) + + with When("CHI OpenSSL cipher suites are updated"): + chi_manifest_update = fips_edit_manifest( + source_manifest=chi_manifest, + replicas_count=1, + cipher_suites=["TLS_AES_128_GCM_SHA256"], kind="chi", ) - with Finally("external ClickHouse client container is removed"): - stop_external_ch_container() + fips_apply_manifest( + manifest_path=chi_manifest_update, + replica_count=1, + kind="chi", + apply_templates=[backup_template], + ) + + with Then("removed ClickHouse cipher is no longer negotiated"): + chi_pods = sorted(kubectl.get_pod_names(chi)) + + check_tls13_cipher_fails( + pod=chi_pods[0], + port=9440, + cipher="TLS_AES_256_GCM_SHA384", + ) @TestScenario @Tags("HEAVY") @Name("test_030005. FIPS CHK: scale replicas 2 -> 3 -> 1") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleUp("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleDown("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_Rescale("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_ConfigUpdate("1.0"), ) def test_030005(self): """Verify FIPS ClickHouse Keeper survives replica scale-up and scale-down. @@ -8040,15 +8029,14 @@ def test_030005(self): ) fips_apply_manifest( manifest_path=chk_manifest_2, - expected_pod_count=2, + replica_count=2, kind="chk", ) with Then("2-replica Keeper cluster passes essential FIPS checks"): - fips_assert_replicas_healthy( + run_chk_fips_checks( workload=chk, - expected_count=2, - kind="chk", + replica_count=2, ) with And("FIPS ClickHouse is deployed with 2 replicas"): @@ -8059,16 +8047,21 @@ def test_030005(self): ) fips_apply_manifest( manifest_path=chi_manifest_2, - expected_pod_count=2, + replica_count=2, kind="chi", apply_templates=[backup_template], ) with And("2-replica ClickHouse cluster passes essential FIPS checks"): - chi_pods = fips_assert_replicas_healthy( + chi_pods = run_chi_fips_checks( workload=chi, - expected_count=2, - kind="chi", + replica_count=2, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=2, ) with And("ReplicatedMergeTree data converges across 2 ClickHouse replicas"): @@ -8082,22 +8075,26 @@ def test_030005(self): ) fips_apply_manifest( manifest_path=chk_manifest_3, - expected_pod_count=3, + replica_count=3, kind="chk", ) with Then("3-replica Keeper cluster passes essential FIPS checks"): - fips_assert_replicas_healthy( + run_chk_fips_checks( workload=chk, - expected_count=3, - kind="chk", + replica_count=3, ) with And("ClickHouse remains healthy after Keeper upscale"): - chi_pods = fips_assert_replicas_healthy( + chi_pods = run_chi_fips_checks( workload=chi, - expected_count=2, - kind="chi", + replica_count=2, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=2, ) with And("ReplicatedMergeTree data still converges after Keeper upscale"): @@ -8114,22 +8111,26 @@ def test_030005(self): ) fips_apply_manifest( manifest_path=chk_manifest_1, - expected_pod_count=1, + replica_count=1, kind="chk", ) with Then("single-replica Keeper cluster passes essential FIPS checks"): - fips_assert_replicas_healthy( + run_chk_fips_checks( workload=chk, - expected_count=1, - kind="chk", + replica_count=1, ) with And("ClickHouse remains healthy after Keeper downscale"): - chi_pods = fips_assert_replicas_healthy( + chi_pods = run_chi_fips_checks( workload=chi, - expected_count=2, - kind="chi", + replica_count=2, + ) + + with And("clickhouse-backup sidecar passes essential FIPS checks"): + run_backup_fips_checks( + workload=chi, + replica_count=2, ) with And("ReplicatedMergeTree data still converges after Keeper downscale"): @@ -8138,18 +8139,42 @@ def test_030005(self): table="repl_chk_scale_test_1", ) - with Finally("external ClickHouse client container is removed"): - stop_external_ch_container() + with When("CHK OpenSSL cipher suites are updated"): + chk_manifest_update = fips_edit_manifest( + source_manifest=chk_manifest, + replicas_count=1, + cipher_suites=["TLS_AES_128_GCM_SHA256"], + kind="chk", + ) + + fips_apply_manifest( + manifest_path=chk_manifest_update, + replica_count=1, + kind="chk", + ) + + with Then("removed Keeper cipher is no longer negotiated"): + chi_pods = sorted(kubectl.get_pod_names(chi)) + chk_pods = kubectl.get_chk_pod_names(chk) + + chk_ip = kubectl.launch( + f"get pod {chk_pods[0]} " + "-o jsonpath='{.status.podIP}'", + ns=self.context.test_namespace, + ) + check_tls13_cipher_fails( + pod=chi_pods[0], + target_host=chk_ip, + port=2281, + cipher="TLS_AES_256_GCM_SHA384", + ) @TestScenario @Tags("HEAVY") @Name("test_030006. FIPS enforced: coerces verify, minVersion, and IPC") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceVerifyStrict("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceMinVersion13("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_OverrideMinVersion12To13("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceIPCSecure("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_SecurityCoercion("1.0"), ) def test_030006(self): """Verify ``fips.enforced=true`` coerces relaxed chopconf TLS verify, minVersion, and IPC.""" @@ -8179,15 +8204,12 @@ def test_030006(self): @Tags("HEAVY") @Name("test_030007. FIPS enforced: invalid CHI/CHK specs are rejected") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneCHI("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneZK("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectInvalidMinVersion("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectExternalZookeeper("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectCHKBypass("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectNonCompliantSpecs("1.0"), ) def test_030007(self): - """Verify strict FIPS mode rejects non-compliant CHI and CHK specifications.""" + """Verify strict FIPS mode rejects non-compliant CHI and CHK specs.""" chopconf = "manifests/chopconf/test-030002-chopconf.yaml" + chi_zk_rejected = "manifests/chi/test-073-fips-zk-rejected.yaml" chi_zk_rejected_explicit_false = ( "manifests/chi/test-073-fips-zk-rejected-explicit-false.yaml" @@ -8299,24 +8321,18 @@ def test_030007(self): ) + @TestScenario @Tags("HEAVY") -@Name("test_030008. FIPS image policy Required: admission and runtime checks") +@Name("test_030008. FIPS image policy: Required admission/runtime checks and Permissive default") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHI("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_AcceptCHI("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHK("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RuntimeVersion("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_ShortCircuit("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_AltinityFIPS("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_FIPSSuffix("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_CaseInsensitive("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_DigestOnly("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_RegistryPath("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectNonFIPS("1.0"), ) def test_030008(self): """Verify ``security.images.policy=FIPSRequired`` rejects non-fips images - and accepts images whose tags contain ``fips`` (case-insensitive). + and accepts images whose tags contain ``fips`` (case-insensitive), then + switch to ``security.images.policy=Permissive`` and confirm a non-fips + image is admitted (the default posture). """ chopconf = "manifests/chopconf/test-074-fips-images-required-chopconf.yaml" chi_non_fips_manifest = ( @@ -8344,6 +8360,19 @@ def test_030008(self): chi_case_insensitive_manifest = ( "manifests/chi/test-030008-case-insensitive.yaml" ) + chi_permissive_chopconf = ( + "manifests/chopconf/test-030008-permissive-chopconf.yaml" + ) + chi_permissive_manifest = ( + "manifests/chi/test-030008-permissive-non-fips.yaml" + ) + backup_non_fips_template = ( + "manifests/chit/test-030008-backup-non-fips-template.yaml" + ) + chi_backup_non_fips_manifest = ( + "manifests/chi/test-030003.yaml" + ) + fips_create_shell_namespace_clickhouse_template() @@ -8372,6 +8401,12 @@ def test_030008(self): chi_case_insensitive = yaml_manifest.get_name( util.get_full_path(chi_case_insensitive_manifest) ) + chi_permissive = yaml_manifest.get_name( + util.get_full_path(chi_permissive_manifest) + ) + chi_backup_non_fips = yaml_manifest.get_name( + util.get_full_path(chi_backup_non_fips_manifest) + ) with Given("FIPS image policy Required is applied"): fips_apply_operator_config(chopconf_path=chopconf) @@ -8397,6 +8432,14 @@ def test_030008(self): expect_no_sts=True, ) + with And("aborted non-fips CHK is deleted before switching image policy"): + kubectl.launch( + f"delete chk {chk_non_fips}", + ns=self.context.test_namespace, + timeout=600, + ok_to_fail=True, + ) + with When("CHI with two non-fips replicas is applied"): fips_apply_manifest_raw(manifest_path=chi_shortcircuit_manifest) @@ -8436,7 +8479,7 @@ def test_030008(self): with When("CHI with altinityfips-tagged image is applied"): fips_apply_manifest( manifest_path=chi_fips_manifest, - expected_pod_count=1, + replica_count=1, kind="chi", ) @@ -8449,15 +8492,21 @@ def test_030008(self): with When("CHI with fips-suffix tag is applied"): fips_apply_manifest_raw(manifest_path=chi_fips_suffix_manifest) - with Then("CHI is admitted because tag contains fips"): + with Then("CHI passes image-policy admission because tag contains fips"): fips_assert_chi_admitted(chi=chi_fips_suffix) + with And("fips-suffix CHI is deleted after admission-only check"): + fips_cleanup_admission_only_chi(chi=chi_fips_suffix) + with When("CHI with uppercase FIPS tag is applied"): fips_apply_manifest_raw(manifest_path=chi_case_insensitive_manifest) - with Then("CHI is admitted because tag detection is case-insensitive"): + with Then("CHI passes image-policy admission because tag detection is case-insensitive"): fips_assert_chi_admitted(chi=chi_case_insensitive) + with And("case-insensitive CHI is deleted after admission-only check"): + fips_cleanup_admission_only_chi(chi=chi_case_insensitive) + with When("runtime decoy image alias is prepared"): decoy_tag = "altinity/clickhouse-server:25.8.16.10002.altinityfips-decoy" stable_tag = "altinity/clickhouse-server:25.8.16.10002.altinitystable" @@ -8467,10 +8516,6 @@ def test_030008(self): text=True, check=False, ) - # The kubelet uses minikube's own image store, not the host docker - # daemon where `docker tag` created the alias — load it in, otherwise the - # decoy pod is ImagePullBackOff (the synthetic tag exists on no registry) - # and never starts, so the runtime SELECT version() check never runs. load_result = None if tag_result.returncode == 0: load_result = subprocess.run( @@ -8500,11 +8545,28 @@ def test_030008(self): f"expected runtime FIPSImagePolicyViolation, got {errors}" ) + with When("operator image policy is switched to Permissive"): + kubectl.launch( + "delete chopconf test-074-fips-images-required-chopconf", + ns=self.context.operator_namespace, + timeout=600, + ok_to_fail=True, + ) + + fips_apply_operator_config(chopconf_path=chi_permissive_chopconf) + + with And("a non-fips CHI is applied under Permissive policy"): + fips_apply_manifest_raw(manifest_path=chi_permissive_manifest) + + with Then("non-fips CHI is admitted without FIPSImagePolicyViolation"): + fips_assert_chi_admitted(chi=chi_permissive) + + @TestScenario @Tags("HEAVY") @Name("test_030009. FIPS enforced: operator TLS clients reject servers without TLS 1.3") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceMinVersion13("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_SecurityCoercion("1.0"), RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_MinVersionScope("1.0"), ) def test_030009(self): @@ -8535,15 +8597,14 @@ def test_030009(self): with And("normal CHK is applied"): fips_apply_manifest( manifest_path=chk_manifest, - expected_pod_count=2, + replica_count=2, kind="chk", ) with Then("normal CHK becomes healthy"): - fips_assert_replicas_healthy( + run_chk_fips_checks( workload=chk, - expected_count=2, - kind="chk", + replica_count=2, ) with When("CHI server disables TLS 1.3"): @@ -8572,61 +8633,20 @@ def test_030009(self): min_version="1.3", ) - @TestScenario @Tags("HEAVY") -@Name("test_030010. FIPS on-wire TLS verification: Strict + wrong rootCA fails ClickHouse fetch") -def test_030010(self): - """Strict verify with a wrong rootCA must fail operator ClickHouse fetch.""" - tls_secret = "manifests/secret/test-058-secret.yaml" - chi_manifest = "manifests/chi/test-077-fips-tls-wrong-ca.yaml" - chopconf = "manifests/chopconf/test-077-fips-tls-wrong-ca-chopconf.yaml" - - fips_create_shell_namespace_clickhouse_template() - - operator_namespace = self.context.operator_namespace - chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) - - with Given("test-058 TLS secret is installed"): - kubectl.apply(util.get_full_path(tls_secret)) - - with When("HTTPS ClickHouse CHI is deployed"): - fips_apply_manifest( - manifest_path=chi_manifest, - expected_pod_count=1, - kind="chi", - apply_templates=[current().context.clickhouse_template], - ) - - with And("operator chopconf sets verify=Strict with an unrelated rootCA"): - fips_apply_operator_config(chopconf_path=chopconf) - kubectl.wait_chi_status(chi, "Completed") - - with Then("chi_clickhouse_metric_fetch_errors is 1 for this CHI"): - check_metrics_monitoring( - operator_namespace=operator_namespace, - operator_pod=kubectl.get_operator_pod(ns=operator_namespace), - expect_pattern=( - f'^chi_clickhouse_metric_fetch_errors{{[^}}]*chi="{chi}"[^}}]*}} 1$' - ), - ) - - with When("operator configuration is reset to default"): - kubectl.delete( - util.get_full_path(chopconf, lookup_in_host=False), - operator_namespace, - ) - util.restart_operator() - -@TestScenario -@Tags("HEAVY") -@Name("test_030012. FIPS backup: ClickHouse over TLS and HTTPS restore round-trip") +@Name("test_030010. FIPS synthetic TLS cipher validation: K8s API and CHI HTTPS") @Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_ClickHouseOverTLS("1.0"), - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_RestoreRoundTrip("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_TLS_ApprovedCiphers("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_TLS_RejectedCiphers("1.0"), ) -def test_030012(self): - """Verify clickhouse-backup uses TLS to ClickHouse and restore works via HTTPS API.""" +def test_030010(self): + """Supplementary synthetic AES-256 TLS 1.3 cipher smoke. + + This scenario proves that both containers in the operator pod can negotiate + TLS 1.3 with TLS_AES_256_GCM_SHA384 against real Kubernetes API and real + ClickHouse HTTPS endpoints. + """ chopconf = "manifests/chopconf/test-030002-chopconf.yaml" chi_manifest = "manifests/chi/test-030003.yaml" @@ -8639,47 +8659,98 @@ def test_030012(self): chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) with Given("strict FIPS operator configuration is applied"): - util.apply_operator_config(chopconf) + fips_apply_operator_config(chopconf_path=chopconf) with And("test TLS secret is installed"): create_tls_secret_for_fips_hosts(chi=chi, chk=chk) - with And("external ClickHouse client container is started"): - start_external_ch_container() - - with And("FIPS Keeper is deployed"): + with And("FIPS ClickHouse Keeper is deployed as CHI dependency"): fips_apply_manifest( manifest_path=chk_manifest, - expected_pod_count=2, + replica_count=2, kind="chk", ) - with And("FIPS ClickHouse with backup sidecar is deployed"): + with When("FIPS ClickHouse is deployed with TLS settings"): fips_apply_manifest( manifest_path=chi_manifest, - expected_pod_count=2, + replica_count=2, kind="chi", apply_templates=[backup_template], ) - with Given("FIPS ClickHouse cluster is healthy"): - chi_pods = fips_assert_replicas_healthy( - workload=chi, - expected_count=2, - kind="chi", + with Then("ClickHouse cluster pods are ready"): + kubectl.wait_chi_status(chi, "Completed") + kubectl.wait_objects( + chi, + { + "statefulset": 2, + "pod": 2, + "service": 3, + }, + ) + chi_pods = sorted(kubectl.get_pod_names(chi)) + + with Then("operator pod containers negotiate AES-256 TLS 1.3 to K8s and CHI"): + check_synthetic_tls13_smoke_from_operator_pod( + chi_pod=chi_pods[0], ) - pod = chi_pods[0] + with Then("operator pod containers reject fake openSSL server with non approved cipher"): + fips_assert_connection_rejected_on_non_approved_cipher() + + +@TestScenario +@Name("test_030011. FIPS Integrity check: detect binary tampering") +@Requirements( + RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_VerificationMismatch("1.0") +) +def test_030011(self): + """Verify that corrupting the embedded FIPS HMAC causes binaries to panic at startup. + + Procedure: + 1. Extract FIPS binaries from release images. + 2. Locate the '.go.fipsinfo' ELF section. + 3. Flip bits in the HMAC header. + 4. Verify the binary panics with 'fips140: verification mismatch'. + """ + with Given("operator and metrics-exporter binaries are extracted"): + fips_extract_shipped_binaries() - with Then("backup sidecar reaches ClickHouse through secure native TCP"): - check_clickhouse_backup_clickhouse_tls_config(pod=pod) - check_clickhouse_backup_can_list_tables_over_clickhouse_tls(pod=pod) + with Then("clickhouse-operator detects tampering"): + check_fips_integrity_failure( + binary_path=self.context.fips_op_bin, + binary_label="clickhouse-operator" + ) - with Then("backup and restore succeeds through HTTPS API"): - check_clickhouse_backup_restore_roundtrip_https(pod=pod) + with And("metrics-exporter detects tampering"): + check_fips_integrity_failure( + binary_path=self.context.fips_me_bin, + binary_label="metrics-exporter" + ) + +@TestScenario +@Name("test_030015. FIPS CAST failure: operator and exporter binaries") +@Requirements( + RQ_SRS_026_ClickHouseOperator_FIPS_CAST_OperatorFail("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CAST_ExporterFail("1.0"), +) +def test_030015(self): + """Verify forced FIPS CAST failure terminates each shipped binary independently.""" + fips_extract_shipped_binaries() + + with Then("clickhouse-operator terminates with CAST failure"): + check_fips_cast_failure( + binary_path=self.context.fips_op_bin, + binary="clickhouse-operator", + ) + + with Then("metrics-exporter terminates with CAST failure"): + check_fips_cast_failure( + binary_path=self.context.fips_me_bin, + binary="metrics-exporter", + ) - with Finally("external ClickHouse client container is removed"): - stop_external_ch_container() def cleanup_chis(self): with Given("Cleanup CHIs"): diff --git a/tests/requirements/fips.md b/tests/requirements/fips.md index 73cd1c7d4..b57ae00a1 100644 --- a/tests/requirements/fips.md +++ b/tests/requirements/fips.md @@ -7,153 +7,74 @@ **Author:** Saba Momtselidze -**Date:** May 29, 2026 +**Date:** June 12, 2026 ## Table of Contents * 1 [Introduction](#introduction) * 2 [Configuration Requirements](#configuration-requirements) - * 2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS](#rqsrs-026clickhouseoperatorfipsconfigexternaltls) + * 2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts](#rqsrs-026clickhouseoperatorfipshttpports) * 3 [Build Verification](#build-verification) - * 3.1 [Shipped Binaries](#shipped-binaries) - * 3.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries](#rqsrs026clickhouseoperatorfipsbuildshippedbinaries) - * 3.1.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.GOFIPS140](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesgofips140) - * 3.1.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSIdentity](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsidentity) - * 3.1.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSVersion](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsversion) - * 3.1.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSEnabled](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsenabled) - * 3.1.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupBanner](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesstartupbanner) -* 4 [GODEBUG Strict Mode Smoke Test](#godebug-strict-mode-smoke-test) - * 4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.GODEBUG.StrictMode](#rqsrs-026clickhouseoperatorfipsgodebugstrictmode) -* 5 [FIPS 140-3 Valid TLS Cipher Suites](#fips-140-3-valid-tls-cipher-suites) - * 5.1 [Approved TLS Cipher Suites](#approved-tls-cipher-suites) - * 5.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) - * 5.2 [Rejected Cipher Suites and Protocols](#rejected-cipher-suites-and-protocols) - * 5.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers) -* 6 [ClickHouse Server and Keeper FIPS Configurations](#clickhouse-server-and-keeper-fips-configurations) - * 6.1 [ClickHouse Server](#clickhouse-server) - * 6.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig](#rqsrs026clickhouseoperatorfipsdataplanechfipsconfig) - * 6.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy](#rqsrs026clickhouseoperatorfipsdataplanechideploy) - * 6.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP](#rqsrs026clickhouseoperatorfipsdataplanechnoplainhttp) - * 6.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative](#rqsrs026clickhouseoperatorfipsdataplanechnoplainnative) - * 6.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts](#rqsrs026clickhouseoperatorfipsdataplanechnounexpectedports) - * 6.1.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS](#rqsrs026clickhouseoperatorfipsdataplanechinternodetls) - * 6.1.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp](#rqsrs026clickhouseoperatorfipsdataplanechscaleup) - * 6.1.8 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown](#rqsrs026clickhouseoperatorfipsdataplanechscaledown) - * 6.1.9 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate](#rqsrs026clickhouseoperatorfipsdataplanechconfigupdate) - * 6.2 [ClickHouse Keeper](#clickhouse-keeper) - * 6.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig](#rqsrs026clickhouseoperatorfipsdataplanechkfipsconfig) - * 6.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy](#rqsrs026clickhouseoperatorfipsdataplanechkdeploy) - * 6.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort](#rqsrs026clickhouseoperatorfipsdataplanechknoplainclientport) - * 6.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts](#rqsrs026clickhouseoperatorfipsdataplanechknounexpectedports) - * 6.2.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS](#rqsrs026clickhouseoperatorfipsdataplanechkrafttls) - * 6.2.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp](#rqsrs026clickhouseoperatorfipsdataplanechkscaleup) - * 6.2.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown](#rqsrs026clickhouseoperatorfipsdataplanechkscaledown) - * 6.2.8 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate](#rqsrs026clickhouseoperatorfipsdataplanechkconfigupdate) - * 6.3 [ClickHouse Backup Sidecar](#clickhouse-backup-sidecar) - * 6.3.0 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString](#rqsrs026clickhouseoperatorfipsdataplanechversionstring) - * 6.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary](#rqsrs026clickhouseoperatorfipsdataplanebackupfipsbinary) - * 6.3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140](#rqsrs026clickhouseoperatorfipsdataplanebackupgofips140) - * 6.3.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts](#rqsrs026clickhouseoperatorfipsdataplanebackuponlytlsports) - * 6.3.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI](#rqsrs026clickhouseoperatorfipsdataplanebackuphttpsapi) - * 6.3.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS](#rqsrs026clickhouseoperatorfipsdataplanebackupclickhouseovertls) - * 6.3.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip](#rqsrs026clickhouseoperatorfipsdataplanebackuprestoreroundtrip) - * 6.3.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS](#rqsrs026clickhouseoperatorfipsdataplanebackupremoteuploadtls) -* 7 [FIPS Enforcement Mode](#fips-enforcement-mode) - * 7.1 [Security Coercion](#security-coercion) - * 7.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict](#rqsrs026clickhouseoperatorfipsenforcedcoerceverifystrict) - * 7.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13](#rqsrs026clickhouseoperatorfipsenforcedcoerceminversion13) - * 7.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13](#rqsrs-026clickhouseoperatorfipsenforcedoverrideminversion12to13) - * 7.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure](#rqsrs026clickhouseoperatorfipsenforcedcoerceipcsecure) - * 7.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig](#rqsrs026clickhouseoperatorfipsenforcedrejectinsecurekubeconfig) - * 7.1.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI](#rqsrs026clickhouseoperatorfipsenforcedrejectverifynonechi) - * 7.1.7 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK](#rqsrs026clickhouseoperatorfipsenforcedrejectverifynonezk) - * 7.1.8 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion](#rqsrs026clickhouseoperatorfipsenforcedrejectinvalidminversion) - * 7.1.9 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper](#rqsrs026clickhouseoperatorfipsenforcedrejectexternalzookeeper) - * 7.1.10 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass](#rqsrs026clickhouseoperatorfipsenforcedrejectchkbypass) - * 7.2 [Image Policy](#image-policy) - * 7.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI](#rqsrs026clickhouseoperatorfipsimagesrequiredrejectchi) - * 7.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI](#rqsrs026clickhouseoperatorfipsimagesrequiredacceptchi) - * 7.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK](#rqsrs026clickhouseoperatorfipsimagesrequiredrejectchk) - * 7.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion](#rqsrs026clickhouseoperatorfipsimagesrequiredruntimeversion) - * 7.2.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive](#rqsrs026clickhouseoperatorfipsimagespermissive) - * 7.2.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit](#rqsrs026clickhouseoperatorfipsimagesrequiredshortcircuit) - * 7.3 [Image Tag Detection](#image-tag-detection) - * 7.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix](#rqsrs026clickhouseoperatorfipsimagestagdetectionfipssuffix) - * 7.3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS](#rqsrs026clickhouseoperatorfipsimagestagdetectionaltinityfips) - * 7.3.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly](#rqsrs026clickhouseoperatorfipsimagestagdetectiondigestonly) - * 7.3.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath](#rqsrs026clickhouseoperatorfipsimagestagdetectionregistrypath) - * 7.3.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive](#rqsrs026clickhouseoperatorfipsimagestagdetectioncaseinsensitive) -* 8 [Operator External Connections](#operator-external-connections) - * 8.1 [Operator Runtime Listener Verification](#operator-runtime-listener-verification) - * 8.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners](#rqsrs026clickhouseoperatorfipsconnectoperatorlisteners) - * 8.2 [Operator to Kubernetes API](#operator-to-kubernetes-api) - * 8.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes](#rqsrs026clickhouseoperatorfipsconnectoperatorkubernetes) - * 8.3 [Operator to ClickHouse Server](#operator-to-clickhouse-server) - * 8.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse](#rqsrs026clickhouseoperatorfipsconnectoperatorclickhouse) - * 8.4 [Operator to ZooKeeper/Keeper](#operator-to-zookeeperkeeper) - * 8.4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper](#rqsrs026clickhouseoperatorfipsconnectoperatorzookeeper) - * 8.5 [Operator to metrics-exporter IPC](#operator-to-metrics-exporter-ipc) - * 8.5.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure](#rqsrs026clickhouseoperatorfipsconnectoperatoripcsecure) - * 8.6 [Operator Prometheus Metrics](#operator-prometheus-metrics) - * 8.6.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS](#rqsrs026clickhouseoperatorfipsgapoperatormetricstls) -* 9 [Exporter External Connections](#exporter-external-connections) - * 9.1 [Exporter Runtime Listener Verification](#exporter-runtime-listener-verification) - * 9.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners](#rqsrs026clickhouseoperatorfipsconnectexporterlisteners) - * 9.2 [Exporter to Kubernetes API](#exporter-to-kubernetes-api) - * 9.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes](#rqsrs026clickhouseoperatorfipsconnectexporterkubernetes) - * 9.3 [Exporter to ClickHouse Server](#exporter-to-clickhouse-server) - * 9.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse](#rqsrs026clickhouseoperatorfipsconnectexporterclickhouse) - * 9.4 [Exporter Prometheus Metrics](#exporter-prometheus-metrics) - * 9.4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS](#rqsrs026clickhouseoperatorfipsgapexportermetricstls) -* 10 [Integrity Check Failure](#integrity-check-failure) - * 10.1 [Operator Integrity Tampering](#operator-integrity-tampering) - * 10.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch](#rqsrs026clickhouseoperatorfipsintegrityoperatormismatch) - * 10.2 [Exporter Integrity Tampering](#exporter-integrity-tampering) - * 10.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch](#rqsrs026clickhouseoperatorfipsintegrityexportermismatch) + * 3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries](#rqsrs-026clickhouseoperatorfipsoperatorbuildshippedbinaries) + * 3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs](#rqsrs-026clickhouseoperatorfipsoperatorbuildshippedbinariesstartuplogs) +* 4 [FIPS 140-3 TLS Cipher Suites](#fips-140-3-tls-cipher-suites) + * 4.1 [Approved TLS Cipher Suites](#approved-tls-cipher-suites) + * 4.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) + * 4.2 [Rejected Cipher Suites and Protocols](#rejected-cipher-suites-and-protocols) + * 4.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers) +* 5 [ClickHouse Server](#clickhouse-server) + * 5.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig](#rqsrs-026clickhouseoperatorfipschfipsconfig) + * 5.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient](#rqsrs-026clickhouseoperatorfipschfipsconfigexternalclient) + * 5.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale](#rqsrs-026clickhouseoperatorfipschrescale) + * 5.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate](#rqsrs-026clickhouseoperatorfipschconfigupdate) +* 6 [ClickHouse Keeper](#clickhouse-keeper) + * 6.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig](#rqsrs-026clickhouseoperatorfipschkfipsconfig) + * 6.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale](#rqsrs-026clickhouseoperatorfipschkrescale) + * 6.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate](#rqsrs-026clickhouseoperatorfipschkconfigupdate) +* 7 [ClickHouse Backup Sidecar](#clickhouse-backup-sidecar) + * 7.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary](#rqsrs-026clickhouseoperatorfipsbackupfipsbinary) + * 7.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig](#rqsrs-026clickhouseoperatorfipsbackupfipsconfig) + * 7.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip](#rqsrs-026clickhouseoperatorfipsbackuprestoreroundtrip) +* 8 [FIPS Enforcement Mode](#fips-enforcement-mode) + * 8.1 [Security Coercion](#security-coercion) + * 8.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion](#rqsrs-026clickhouseoperatorfipsenforcedsecuritycoercion) + * 8.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs](#rqsrs-026clickhouseoperatorfipsenforcedrejectnoncompliantspecs) + * 8.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope](#rqsrs-026clickhouseoperatorfipsenforcedminversionscope) + * 8.2 [Image Policy](#image-policy) + * 8.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS](#rqsrs-026clickhouseoperatorfipsimagesrequiredrejectnonfips) +* 9 [Runtime Connection Evidence](#runtime-connection-evidence) + * 9.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI](#rqsrs-026clickhouseoperatorfipsconnectoperatorkubernetesapi) + * 9.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI](#rqsrs-026clickhouseoperatorfipsconnectexporterkubernetesapi) + * 9.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse](#rqsrs-026clickhouseoperatorfipsconnectoperatorclickhouse) + * 9.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse](#rqsrs-026clickhouseoperatorfipsconnectexporterclickhouse) + * 9.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction](#rqsrs-026clickhouseoperatorfipsconnectoperatorkeeperrestriction) + * 9.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS](#rqsrs-026clickhouseoperatorfipsconnectclickhousekeepertls) +* 10 [Integrity Check](#integrity-check) + * 10.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch](#rqsrs-026clickhouseoperatorfipsintegrityverificationmismatch) * 11 [CAST Failure](#cast-failure) * 11.1 [Operator CAST Failure](#operator-cast-failure) - * 11.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail](#rqsrs026clickhouseoperatorfipscastoperatorfail) + * 11.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail](#rqsrs-026clickhouseoperatorfipscastoperatorfail) * 11.2 [Exporter CAST Failure](#exporter-cast-failure) - * 11.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail](#rqsrs026clickhouseoperatorfipscastexporterfail) -* 12 [Synthetic TLS Cipher Validation](#synthetic-tls-cipher-validation) - * 12.1 [Approved cipher matrix](#approved-cipher-matrix) - * 12.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipssyntheticapprovedciphers) - * 12.2 [Rejected cipher matrix](#rejected-cipher-matrix) - * 12.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers](#rqsrs-026clickhouseoperatorfipssyntheticrejectedciphers) -* 13 [CI/CD Image and Policy Verification](#cicd-image-and-policy-verification) - * 13.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild](#rqsrs-026clickhouseoperatorfipscicdoperatorimagebuild) - * 13.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild](#rqsrs-026clickhouseoperatorfipscicdexporterimagebuild) - * 13.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan](#rqsrs-026clickhouseoperatorfipscicdvulnerabilityscan) -* 14 [AI Static Code Review](#ai-static-code-review) - * 14.1 [Operator Source Review](#operator-source-review) - * 14.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree](#rqsrs-026clickhouseoperatorfipsaireviewoperatortree) - * 14.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg](#rqsrs-026clickhouseoperatorfipsaireviewoperatorsharedpkg) - * 14.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate](#rqsrs-026clickhouseoperatorfipsaireviewoperatorregressiongate) - * 14.2 [Exporter Source Review](#exporter-source-review) - * 14.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree](#rqsrs-026clickhouseoperatorfipsaireviewexportertree) - * 14.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg](#rqsrs-026clickhouseoperatorfipsaireviewexportersharedpkg) - * 14.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate](#rqsrs-026clickhouseoperatorfipsaireviewexporterregressiongate) -* 15 [ACVP Algorithm Validation](#acvp-algorithm-validation) - * 15.1 [Operator ACVP Validation](#operator-acvp-validation) - * 15.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration](#rqsrs026clickhouseoperatorfipsacvpoperatorwrapperintegration) - * 15.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration](#rqsrs026clickhouseoperatorfipsacvpoperatorconfiggeneration) - * 15.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay](#rqsrs026clickhouseoperatorfipsacvpoperatorexpectedoutputreplay) - * 15.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount](#rqsrs026clickhouseoperatorfipsacvpoperatorsuitecount) - * 15.2 [Exporter ACVP Validation](#exporter-acvp-validation) - * 15.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration](#rqsrs026clickhouseoperatorfipsacvpexporterwrapperintegration) - * 15.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration](#rqsrs026clickhouseoperatorfipsacvpexporterconfiggeneration) - * 15.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay](#rqsrs026clickhouseoperatorfipsacvpexporterexpectedoutputreplay) - * 15.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount](#rqsrs026clickhouseoperatorfipsacvpexportersuitecount) -* 16 [Terminology](#terminology) - * 16.1 [SRS](#srs) - * 16.2 [FIPS 140-3](#fips-140-3) - * 16.3 [clickhouse-operator](#clickhouse-operator) - * 16.4 [metrics-exporter](#metrics-exporter) - * 16.5 [CHI](#chi) - * 16.6 [CHK](#chk) - * 16.7 [ACVP](#acvp) - * 16.8 [CMVP](#cmvp) - * 16.9 [CAVP](#cavp) + * 11.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail](#rqsrs-026clickhouseoperatorfipscastexporterfail) +* 12 [ACVP Algorithm Validation](#acvp-algorithm-validation) + * 12.1 [Operator ACVP Validation](#operator-acvp-validation) + * 12.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration](#rqsrs-026clickhouseoperatorfipsacvpoperatorwrapperintegration) + * 12.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration](#rqsrs-026clickhouseoperatorfipsacvpoperatorconfiggeneration) + * 12.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT](#rqsrs-026clickhouseoperatorfipsacvpoperatorsha2256aft) + * 12.2 [Exporter ACVP Validation](#exporter-acvp-validation) + * 12.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration](#rqsrs-026clickhouseoperatorfipsacvpexporterwrapperintegration) + * 12.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration](#rqsrs-026clickhouseoperatorfipsacvpexporterconfiggeneration) + * 12.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT](#rqsrs-026clickhouseoperatorfipsacvpexportersha2256aft) +* 13 [Terminology](#terminology) + * 13.1 [SRS](#srs) + * 13.2 [FIPS 140-3](#fips-140-3) + * 13.3 [clickhouse-operator](#clickhouse-operator) + * 13.4 [metrics-exporter](#metrics-exporter) + * 13.5 [CHI](#chi) + * 13.6 [CHK](#chk) + * 13.7 [ACVP](#acvp) + * 13.8 [CMVP](#cmvp) + * 13.9 [CAVP](#cavp) ## Introduction @@ -163,46 +84,32 @@ This specification describes FIPS 140-3 compatibility requirements for the The goal is to verify that FIPS-enabled builds of the operator and metrics-exporter: - Operate correctly under FIPS constraints - Properly enforce cryptographic restrictions -- Use FIPS-compliant TLS for all inbound and outbound connections +- Use FIPS-compliant TLS for all outbound connections Autotests that trace to these requirements live in -[`tests/e2e/test_operator_fips.py`](../e2e/test_operator_fips.py) and +[`tests/e2e/test_operator.py`](../e2e/test_operator.py) and [`tests/e2e/test_acvp.py`](../e2e/test_acvp.py). **Boundary:** The operator and metrics-exporter run in the same pod. Internal IPC between them is localhost HTTP and is not subject to FIPS TLS requirements. The Prometheus metrics endpoints (operator `:9999` and metrics-exporter `:8888`) are also served over plain HTTP -and remain outside the FIPS TLS scope as a known gap. +and remain outside the FIPS TLS scope as a known gap. The ClickHouse Keeper readiness probe +endpoint (`:9182` `/ready`, which reflects Raft quorum status) likewise stays unconditionally +plaintext HTTP regardless of the secure/insecure knobs and is outside the FIPS TLS scope. ## Configuration Requirements Plain HTTP/TCP on any external connection is a configuration error for FIPS compliance. -TLS must be enabled for all connections to: -- Kubernetes API -- ClickHouse Server -- ZooKeeper/Keeper -- Prometheus scrape endpoints - -### RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS +### RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts version: 1.0 -Plain HTTP/TCP on external connections SHALL be treated as a configuration error for FIPS compliance. TLS SHALL be enabled for connections to the [Kubernetes API], [ClickHouse Server], [ZooKeeper/Keeper], and Prometheus scrape endpoints. +All external connections SHALL require TLS with FIPS-compliant settings, except for localhost IPC between the operator +and metrics-exporter and the Prometheus metrics endpoints `:9999` and `:8888`. ## Build Verification -**Objective:** Verify each shipped binary is a FIPS build and linked to Go Cryptographic Module v1.0.0. - -**Certificates:** -- [CMVP #5247](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/5247) -- [CAVP A6650](https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/details?product=19371) - -**Build requirement:** `GOFIPS140=v1.0.0` (or `certified`) - - -### Shipped Binaries - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries +### RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries version: 1.0 Each shipped pod binary — `clickhouse-operator` and `metrics-exporter` — SHALL satisfy all of the following: @@ -231,26 +138,22 @@ Examples: enabled: true ``` -#### RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupLogs +### RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs version: 1.0 -At startup, each binary SHALL emit a FIPS startup banner in logs indicating build and runtime FIPS state. +At startup, each binary SHALL emit a FIPS startup log line indicating build and runtime FIPS state. -when GODEBUG=fips140=only: +When `GODEBUG=fips140=only`: ```text -FIPS: chopconf.fips.enforced=true \ -build.linked=true \ -module.active=true \ -runtime.enforced=true \ -module=v1.0.0 +FIPS: chopconf.fips.enforced=true build.linked=true module.active=true runtime.enforced=true module=v1.0.0 ``` +## FIPS 140-3 TLS Cipher Suites +### Approved TLS Cipher Suites -## Approved TLS Cipher Suites - -### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers +#### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers version: 1.0 TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter] @@ -258,188 +161,216 @@ SHALL negotiate only TLS 1.3 with the following approved cipher suites. * TLS_AES_128_GCM_SHA256 * TLS_AES_256_GCM_SHA384 -* TLS_AES_128_CCM_SHA256 -* TLS_AES_128_CCM_8_SHA256 + +Note: `TLS_CHACHA20_POLY1305_SHA256` is TLS v1.3 but not FIPS approved. ### Rejected Cipher Suites and Protocols #### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers version: 1.0 -TLS connections SHALL reject the following for all TLS-enabled external connections: - -- Any TLS cipher suite not explicitly listed in [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) -- Protocol versions: SSLv2, SSLv3, TLS 1.0, TLS 1.1 -- Cipher suites using non-approved/legacy algorithms (for this profile), including: - - ChaCha20-Poly1305 - - RC4, RC2, DES, 3DES, IDEA, SEED, CAMELLIA, ARIA - - NULL encryption / NULL authentication - - Anonymous key exchange (`aNULL`, `eNULL`, `ADH`, `AECDH`) - - Export/weak suites (`EXP`, `LOW`, `40-bit`, `56-bit`) - - MD5- or SHA-1-based legacy suites - - -## ClickHouse Server and Keeper FIPS Configurations - -**Objective:** Verify the operator generates and maintains FIPS-compliant configurations for ClickHouse servers and Keepers. - - -### ClickHouse Server - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig -version: 1.0 - -Deploying a CHI with FIPS TLS settings SHALL start ClickHouse with FIPS-compliant TLS configuration. +On TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter], any protocol version +older than TLS 1.3 and any cipher suite not listed in [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) +SHALL be rejected by the operator in a FIPS-compliant configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy -version: 1.0 -The operator SHALL deploy FIPS `ClickHouseInstallation` resources to `Completed` with Running pods when configuration is valid. +## ClickHouse Server -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig version: 1.0 -When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain HTTP port 8123; HTTPS port 8443 SHALL be used. +Operator deploying a `ClickHouseInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse server and client. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative -version: 1.0 +```yaml + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + layout: + shardsCount: 1 + replicasCount: 2 + zookeeper: + nodes: + - host: chk-test-030003-keeper-0-0 + port: 2281 + secure: "yes" + settings: + http_port: _removed_ + tcp_port: _removed_ + interserver_http_port: _removed_ + mysql_port: _removed_ + postgresql_port: _removed_ + https_port: 8443 + tcp_port_secure: 9440 + interserver_https_port: 9010 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + + none + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + +``` -When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain native TCP port 9000; secure native port 9440 SHALL be used. +The deployed ClickHouse server SHALL use only the following ports: -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts -version: 1.0 +* HTTPS API port 8443 (instead of 8123) +* Secure native TCP port 9440 (instead of 9000) +* Interserver HTTPS port 9010 (instead of interserver HTTP port 9009) +* Backup sidecar HTTPS API port 7171 (instead of 7180), when backups are enabled -ClickHouse pods in a FIPS deployment SHALL expose only expected secure listener ports and no additional unexpected ports. +Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient version: 1.0 -ReplicatedMergeTree replicas SHALL communicate over interserver HTTPS (`interserver_https_port`) and data SHALL converge across replicas. +External clients connecting to the ClickHouse server SHALL be able to use any enabled TLS protocol version, including TLS 1.2. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale version: 1.0 -Adding a replica to a FIPS-configured CHI SHALL reconcile to `Completed` and the new replica SHALL run the FIPS ClickHouse binary with TLS-only listeners. +Adding or removing a replica from a FIPS-configured `ClickHouseInstallation` SHALL reconcile successfully and result in the expected number of running pods. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown -version: 1.0 - -Removing a replica from a FIPS-configured CHI SHALL reconcile to `Completed` and remaining replicas SHALL keep FIPS binary and TLS-only configuration. +After rescaling, all replicas SHALL continue to run the FIPS ClickHouse binary and maintain the configured TLS-only OpenSSL settings. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate version: 1.0 Updating TLS settings on a running CHI SHALL reload ClickHouse with the new FIPS-compliant configuration. -### ClickHouse Keeper - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig -version: 1.0 - -Deploying a CHK with FIPS TLS settings SHALL start Keeper with FIPS-compliant TLS configuration. +## ClickHouse Keeper -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig version: 1.0 -The operator SHALL deploy FIPS `ClickHouseKeeperInstallation` resources to `Completed` with Running pods when configuration is valid. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort -version: 1.0 - -When FIPS transport hardening applies, Keeper pods SHALL NOT listen on plain client port 2181; secure client port 2281 SHALL be used. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts -version: 1.0 +Operator deploying a `ClickHouseKeeperInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse +Keeper server and client. -Keeper pods in a FIPS deployment SHALL expose only expected secure listener ports. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS -version: 1.0 - -Keeper Raft communication SHALL use TLS on the configured secure Raft port. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp -version: 1.0 +```yaml + configuration: + clusters: + - name: keeper + secure: "yes" + insecure: "no" + layout: + replicasCount: 2 + settings: + keeper_server/log_storage_path: /var/lib/clickhouse/coordination/log + keeper_server/snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + keeper_server/raft_configuration/server/port: 9444 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + + none + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + +``` -Adding a node to a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and the new node SHALL run the FIPS Keeper binary with TLS-only client and Raft listeners. +The deployed ClickHouse Keeper cluster SHALL use only the following ports: -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown -version: 1.0 +* Secure client port 2281 (instead of 2181) +* Secure Raft communication port 9444 +* Plaintext HTTP readiness probe port 9182 (the `/ready` Raft-quorum health check) -Removing a node from a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and remaining nodes SHALL keep FIPS configuration. +Every exposed port except the readiness probe port 9182 and Raft replication port 9444 (which enforces peer-only authentication) +SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. Port 9182 SHALL stay +unconditionally plaintext HTTP regardless of the secure/insecure configuration (see Boundary). -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale version: 1.0 -Updating TLS settings on a running CHK SHALL reload Keeper with the new FIPS-compliant configuration. +Adding or removing a node from a FIPS-configured `ClickHouseKeeperInstallation` SHALL reconcile successfully and result +in the expected number of running pods. +After rescaling, all Keeper nodes SHALL continue to run the FIPS ClickHouse Keeper binary and maintain the configured +TLS-only OpenSSL settings. -### ClickHouse Backup Sidecar - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate version: 1.0 -A running ClickHouse host under FIPS image policy SHALL report a `version()` string containing `fips` (case-insensitive). +Updating TLS settings on a running CHK SHALL reload ClickHouse with the new FIPS-compliant configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary -version: 1.0 -The `clickhouse-backup` sidecar SHALL run a FIPS-built binary; `clickhouse-backup --version` SHALL contain `fips` (case-insensitive). +## ClickHouse Backup Sidecar -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140 +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary version: 1.0 -When inspectable, the clickhouse-backup sidecar binary SHALL embed `GOFIPS140=v1.0.0` per `go version -m`. +The `clickhouse-backup` sidecar SHALL run a FIPS-built binary. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts -version: 1.0 +The sidecar binary SHALL satisfy all of the following: -The clickhouse-backup sidecar SHALL expose only secure listener ports (including HTTPS API port 7171). +* `clickhouse-backup --version` contains `fips` +* When inspectable, `go version -m` reports `GOFIPS140=v1.0.0` -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig version: 1.0 -The clickhouse-backup HTTPS API SHALL serve over TLS with CA-trust enforcement: trusted clients accepted, untrusted clients rejected. +Deploying a `ClickHouseInstallation` with a FIPS-configured backup sidecar SHALL start `clickhouse-backup` with a FIPS-compliant TLS configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS -version: 1.0 +The deployed backup sidecar SHALL only add the following listener ports to the ClickHouse container: -The clickhouse-backup sidecar SHALL reach ClickHouse over secure native TCP. +* HTTPS API port 7171 (instead of 7180) -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip -version: 1.0 +The FIPS-configured backup sidecar SHALL additionally satisfy all of the following: -Backup and restore through the HTTPS API SHALL succeed over TLS. +* Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. +* The `clickhouse-backup` sidecar SHALL connect to ClickHouse using secure native TCP with TLS enabled. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip version: 1.0 -Remote backup upload to object storage SHALL use FIPS-approved TLS. - +Creating a backup and restoring it through the HTTPS API SHALL succeed over TLS. ## FIPS Enforcement Mode -**Objective:** Verify `security.fips.enforced=true` coerces security settings and rejects non-compliant configurations. - +**Objective:** Verify that `security.fips.enforced: "true"` coerces relaxed security settings and rejects non-compliant CHI/CHK specifications and non-FIPS images. ### Security Coercion -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict -version: 1.0 - -With `fips.enforced=true`, unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13 +#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion version: 1.0 -With `fips.enforced=true`, unset TLS minVersion SHALL be coerced to 1.3 for the -operator's outbound TLS clients. +When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce unset or relaxed security settings as follows: -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13 -version: 1.0 +* Unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients. +* Unset TLS `minVersion` SHALL be coerced to `"1.3"` for the operator's outbound TLS clients (`security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`). +* Explicit `minVersion: "1.2"` for those TLS clients SHALL be coerced to `"1.3"`. +* Unset IPC mode SHALL be coerced to Secure. -When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce `minVersion` to `"1.3"` for `security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`, even when those fields are explicitly set to `"1.2"`. +Example configuration with explicit `minVersion: "1.2"`: ```yaml spec: @@ -457,230 +388,87 @@ spec: minVersion: "1.2" ``` -After operator configuration normalization, the effective `minVersion` for each component listed above SHALL be `"1.3"`. +After operator configuration normalization, the effective `minVersion` for each TLS client listed above SHALL be `"1.3"`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure +#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs version: 1.0 -With `fips.enforced=true`, unset IPC mode SHALL be coerced to Secure. +When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL reject +non-compliant CHI and CHK specifications with `FIPSValidationFailed` and SHALL NOT create workload StatefulSets for: -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig -version: 1.0 - -The operator SHALL refuse to start when kubeconfig uses `TLSClientConfig.Insecure=true` under strict/FIPS mode. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI -version: 1.0 - -CHI with `clickhouse.tls.verify=None` at CHI spec or cluster level under enforced mode SHALL be rejected with `FIPSValidationFailed`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK -version: 1.0 - -CHI with `zookeeper.tls.verify=None` under enforced mode SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion -version: 1.0 - -CHI with invalid TLS minVersion under enforced mode SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper -version: 1.0 - -CHI referencing plain external ZooKeeper nodes under enforced mode SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass -version: 1.0 - -CHK with TLS verify bypass under enforced mode SHALL be rejected. +* CHI referencing plain external ZooKeeper nodes, including when `secure` is explicitly set to `"false"`. +* CHI with `clickhouse.tls.verify=None` at spec or cluster level. +* CHI with `zookeeper.tls.verify=None`. +* CHI with invalid `clickhouse.tls.minVersion`. +* CHK with TLS verify bypass at spec level. #### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope version: 1.0 The `minVersion` coercion SHALL apply only to TLS clients created and managed by the operator. -They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2. +They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2 +(see [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient](#rqsrs-026clickhouseoperatorfipschfipsconfigexternalclient)). ### Image Policy -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI -version: 1.0 - -With `security.fips.images.policy=Required`, CHI with non-FIPS image tag SHALL be rejected with `FIPSImagePolicyViolation`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI -version: 1.0 - -With image policy Required, CHI with FIPS-tagged image SHALL reconcile normally. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK -version: 1.0 - -With image policy Required, CHK with non-FIPS Keeper image SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion -version: 1.0 - -With image policy Required, host `SELECT version()` lacking `fips` SHALL fail with `FIPSImagePolicyViolation`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive -version: 1.0 - -With permissive image policy, non-FIPS CHI images SHALL reconcile (default). - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit -version: 1.0 - -Multiple non-FIPS hosts SHALL produce a single policy violation error. - - -### Image Tag Detection - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix -version: 1.0 - -Image tags containing `fips` (case-insensitive) SHALL be detected as FIPS. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS -version: 1.0 - -Image tags containing `altinityfips` SHALL be detected as FIPS. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly -version: 1.0 - -Digest-only image references SHALL NOT be detected as FIPS at admission. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath -version: 1.0 - -Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive -version: 1.0 - -Image tags such as `25.3.FIPS` or `25.3.Fips` SHALL be detected as FIPS (case-insensitive match on the tag). - - -## Operator External Connections - -**Objective:** Verify all **clickhouse-operator** inbound and outbound connections use FIPS-compliant TLS. - - -### Operator Runtime Listener Verification - -In a FIPS deployment, workload containers deployed by the operator (ClickHouse, Keeper, and sidecar containers) SHALL expose only expected TLS listener ports. Verification reads `/proc/net/tcp` and `/proc/net/tcp6` inside each container and parses ports in LISTEN state (`0A`): - -```bash -kubectl exec -c clickhouse -- sh -c 'cat /proc/net/tcp /proc/net/tcp6' -``` - -E2e coverage: [`test_020011`](../e2e/test_operator_fips.py#L200). - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners -version: 1.0 - -FIPS workload pods (ClickHouse, Keeper, and sidecar containers) SHALL listen only on expected TLS ports. Plaintext service ports (8123, 9000, 2181) SHALL NOT be open when FIPS transport hardening applies. - - -### Operator to Kubernetes API - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes -version: 1.0 - -The operator SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers. - - -### Operator to ClickHouse Server - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse -version: 1.0 - -The operator SHALL connect to ClickHouse using FIPS-approved TLS ciphers. - - -### Operator to ZooKeeper/Keeper - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper -version: 1.0 - -The operator SHALL connect to ZooKeeper/Keeper using FIPS-approved TLS ciphers. - - -### Operator to metrics-exporter IPC - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure +#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS version: 1.0 -Operator IPC with `security.ipc.mode=Secure` SHALL work over localhost HTTP with token auth. +With `security.fips.images.policy=Required`, non-FIPS images SHALL be rejected with `FIPSImagePolicyViolation` as follows: +* CHI with non-FIPS ClickHouse image tag SHALL be rejected at admission. +* CHK with non-FIPS Keeper image tag SHALL be rejected at admission. +* CHI with non-FIPS `clickhouse-backup` sidecar image tag SHALL be rejected at admission. +* CHI with multiple non-FIPS hosts SHALL produce a single policy violation error. +* Digest-only image references SHALL NOT be detected as FIPS at admission. +* Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection. +* CHI admitted with a FIPS-tagged ClickHouse image whose running binary lacks `fips` in `SELECT version()` SHALL fail at runtime. -### Operator Prometheus Metrics +## Runtime Connection Evidence -#### RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI version: 1.0 -Operator Prometheus metrics on :9999 currently expose a known FIPS gap (HTTP-only). - - -## Exporter External Connections - -**Objective:** Verify all **metrics-exporter** inbound and outbound connections use FIPS-compliant TLS. - +The clickhouse-operator SHALL access the Kubernetes API through the HTTPS endpoint on port `443`. +Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected. -### Exporter Runtime Listener Verification - -Listener audits use the same `/proc/net/tcp` technique as [Operator Runtime Listener Verification](#operator-runtime-listener-verification). E2e audits the **clickhouse-backup** sidecar in [`test_020011`](../e2e/test_operator_fips.py#L200). The **metrics-exporter** process on `:8888` remains a known gap until metrics TLS is implemented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI version: 1.0 -The metrics-exporter process SHALL expose only expected listener ports on `:8888`. Sidecar containers in the same pod SHALL be listener-audited with the same `/proc/net/tcp` procedure. - +The metrics-exporter SHALL access the Kubernetes API through the HTTPS endpoint on port `443`. +Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected. -### Exporter to Kubernetes API - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse version: 1.0 -The exporter SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers. - - -### Exporter to ClickHouse Server +The clickhouse-operator SHALL communicate with ClickHouse hosts using HTTPS port `8443`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse version: 1.0 -The exporter SHALL query ClickHouse using FIPS-approved TLS when configured for HTTPS. +The metrics-exporter SHALL discover ClickHouse hosts using the HTTPS endpoint `8443`. - -### Exporter Prometheus Metrics - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction version: 1.0 -Exporter Prometheus metrics on :8888 currently expose a known FIPS gap (HTTP-only). - - -## Integrity Check Failure - -**Objective:** Verify FIPS integrity self-test detects binary tampering for each shipped binary independently. - - -### Operator Integrity Tampering +When a Keeper ensemble is configured as TLS-only, the clickhouse-operator SHALL NOT attempt plaintext ZooKeeper/Keeper +operations against it. -#### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS version: 1.0 -Tampering with `clickhouse-operator` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`. +ClickHouse replicas SHALL connect to Keeper using secure client port `2281` with `secure=yes`. -### Exporter Integrity Tampering +## Integrity Check -#### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch +### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch version: 1.0 -Tampering with `metrics-exporter` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`. - +Each shipped FIPS binary — `clickhouse-operator` and `metrics-exporter` — SHALL perform a software integrity +self-test at initialization by verifying its embedded HMAC. If the binary is tampered with or corrupted such that +the HMAC verification fails, the process SHALL immediately terminate with a `fips140: verification mismatch` panic +to prevent the execution of a compromised cryptographic module. ## CAST Failure @@ -703,164 +491,46 @@ version: 1.0 Running `metrics-exporter` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error. -## Synthetic TLS Cipher Validation - -**Objective:** Validate FIPS cipher enforcement on all external (to the pod) connections using `openssl s_client` and `openssl s_server`. - -Use `openssl` to simulate connections with specific ciphers and verify the operator/exporter accepts FIPS-approved ciphers and rejects non-approved ones. - -```bash -# Operator as TLS client against server offering only approved cipher -openssl s_server -accept 8443 -cert server.crt -key server.key \ - -ciphersuites TLS_AES_256_GCM_SHA384 - -# Operator as TLS client against server offering non-approved cipher -openssl s_server -accept 8443 -cert server.crt -key server.key \ - -cipher ECDHE-RSA-CHACHA20-POLY1305 - -# Inbound connection to operator/exporter metrics endpoint -openssl s_client -connect localhost:9999 -cipher ECDHE-RSA-AES256-GCM-SHA384 -``` - -### Approved cipher matrix - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers -version: 1.0 - -For each external connection listed below, when exercised as a TLS **client** with `openssl s_server` offering only [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers), or as a TLS **server** with `openssl s_client` using only approved ciphers, the connection SHALL succeed: - -| Connection | Role | Tool | -|------------|------|------| -| Operator to Kubernetes API | Client | `openssl s_server` | -| Operator to ClickHouse Server | Client | `openssl s_server` | -| Operator to ZooKeeper/Keeper | Client | `openssl s_server` | -| Operator metrics :9999 | Server | `openssl s_client` | -| Exporter to Kubernetes API | Client | `openssl s_server` | -| Exporter to ClickHouse Server | Client | `openssl s_server` | -| Exporter metrics :8888 | Server | `openssl s_client` | - - -### Rejected cipher matrix - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers -version: 1.0 - -For each external connection listed below, when the peer offers only [rejected ciphers or protocols](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers), the connection SHALL be rejected: - -| Connection | Role | Tool | -|------------|------|------| -| Operator to Kubernetes API | Client | `openssl s_server` | -| Operator to ClickHouse Server | Client | `openssl s_server` | -| Operator to ZooKeeper/Keeper | Client | `openssl s_server` | -| Operator metrics :9999 | Server | `openssl s_client` | -| Exporter to Kubernetes API | Client | `openssl s_server` | -| Exporter to ClickHouse Server | Client | `openssl s_server` | -| Exporter metrics :8888 | Server | `openssl s_client` | - - -## CI/CD Image and Policy Verification - -**Objective:** Add CI/CD jobs to validate FIPS image build and supply-chain checks. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild -version: 1.0 - -CI SHALL build the [clickhouse-operator] FIPS image successfully. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild -version: 1.0 - -CI SHALL build the [metrics-exporter] FIPS image successfully. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan -version: 1.0 - -FIPS images SHALL pass vulnerability scanning with no Critical, High, or Medium findings. - - -### Operator Source Review - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree -version: 1.0 - -Static review of operator-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg -version: 1.0 - -Review of shared packages reachable from `cmd/operator` SHALL produce no Critical findings. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate -version: 1.0 - -A signed-off review artifact SHALL be stored with the build record before release. - -### Exporter Source Review - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree -version: 1.0 - -Static review of exporter-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg -version: 1.0 - -Review of shared packages reachable from `cmd/metrics_exporter` SHALL produce no Critical findings. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate -version: 1.0 - -A signed-off review artifact SHALL be stored with the build record before release. - ## ACVP Algorithm Validation -**Objective:** Reproduce ACVP expected-output checks for each FIPS binary using the tracked public-scope config in [`pkg/util/fips/acvp/`](../../../pkg/util/fips/acvp/). +**Objective:** Verify that each FIPS binary can be built with the ACVP wrapper enabled and that the embedded ACVP responder works through the modulewrapper stdin/stdout protocol. +These requirements cover the e2e ACVP smoke tests only. They do not claim full ACVP expected-output replay or suite-count validation from `pkg/util/fips/acvp/run.sh`. ### Operator ACVP Validation #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration version: 1.0 -Building clickhouse-operator with `-tags acvp_wrapper` SHALL expose a working ACVP responder via argv0 dispatch. +Building `clickhouse-operator` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `clickhouse-operator-acvp`. #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration version: 1.0 -The clickhouse-operator ACVP responder SHALL answer `getConfig` with supported capabilities. +The `clickhouse-operator` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay +#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT version: 1.0 -`bash pkg/util/fips/acvp/run.sh` SHALL match all configured expected outputs for the operator. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount -version: 1.0 - -The tracked ACVP config SHALL report 38 matched expectations for clickhouse-operator. - +The `clickhouse-operator` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`. ### Exporter ACVP Validation #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration version: 1.0 -Building metrics-exporter with `-tags acvp_wrapper` SHALL expose a working ACVP responder. +Building `metrics-exporter` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `metrics-exporter-acvp`. #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration version: 1.0 -The metrics-exporter ACVP responder SHALL answer `getConfig` with supported capabilities. +The `metrics-exporter` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay +#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT version: 1.0 -`BINARY=metrics-exporter bash pkg/util/fips/acvp/run.sh` SHALL match all expected outputs. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount -version: 1.0 +The `metrics-exporter` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`. -The tracked ACVP config SHALL report 38 matched expectations for metrics-exporter. ## Terminology diff --git a/tests/requirements/fips.py b/tests/requirements/fips.py index 1fb18bbb9..c178bf6c3 100644 --- a/tests/requirements/fips.py +++ b/tests/requirements/fips.py @@ -8,15 +8,16 @@ Heading = Specification.Heading -RQ_SRS_026_ClickHouseOperator_FIPS_Config_ExternalTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS', +RQ_SRS_026_ClickHouseOperator_FIPS_HTTPPorts = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Plain HTTP/TCP on external connections SHALL be treated as a configuration error for FIPS compliance. TLS SHALL be enabled for connections to the [Kubernetes API], [ClickHouse Server], [ZooKeeper/Keeper], and Prometheus scrape endpoints.\n' + 'All external connections SHALL require TLS with FIPS-compliant settings, except for localhost IPC between the operator\n' + 'and metrics-exporter and the Prometheus metrics endpoints `:9999` and `:8888`.\n' '\n' ), link=None, @@ -24,8 +25,8 @@ num='2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries', +RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries', version='1.0', priority=None, group=None, @@ -60,36 +61,30 @@ '\n' ), link=None, - level=3, - num='3.1.1' + level=2, + num='3.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries_StartupLogs = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupLogs', +RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries_StartupLogs = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'At startup, each binary SHALL emit a FIPS startup banner in logs indicating build and runtime FIPS state.\n' + 'At startup, each binary SHALL emit a FIPS startup log line indicating build and runtime FIPS state.\n' '\n' - 'when GODEBUG=fips140=only:\n' + 'When `GODEBUG=fips140=only`:\n' '\n' '```text\n' - 'FIPS: chopconf.fips.enforced=true \\\n' - 'build.linked=true \\\n' - 'module.active=true \\\n' - 'runtime.enforced=true \\\n' - 'module=v1.0.0\n' + 'FIPS: chopconf.fips.enforced=true build.linked=true module.active=true runtime.enforced=true module=v1.0.0\n' '```\n' '\n' - '\n' - '\n' ), link=None, - level=3, - num='3.1.2' + level=2, + num='3.2' ) RQ_SRS_026_ClickHouseOperator_FIPS_TLS_ApprovedCiphers = Requirement( @@ -103,17 +98,15 @@ 'TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter]\n' 'SHALL negotiate only TLS 1.3 with the following approved cipher suites.\n' '\n' - '| Cipher Suite | OpenSSL Name |\n' - '|--------------|--------------|\n' - '| TLS_AES_128_GCM_SHA256 | TLS_AES_128_GCM_SHA256 |\n' - '| TLS_AES_256_GCM_SHA384 | TLS_AES_256_GCM_SHA384 |\n' - '| TLS_AES_128_CCM_SHA256 | TLS_AES_128_CCM_SHA256 |\n' - '| TLS_AES_128_CCM_8_SHA256 | TLS_AES_128_CCM_8_SHA256 |\n' + '* TLS_AES_128_GCM_SHA256\n' + '* TLS_AES_256_GCM_SHA384\n' + '\n' + 'Note: `TLS_CHACHA20_POLY1305_SHA256` is TLS v1.3 but not FIPS approved.\n' '\n' ), link=None, - level=2, - num='4.1' + level=3, + num='4.1.1' ) RQ_SRS_026_ClickHouseOperator_FIPS_TLS_RejectedCiphers = Requirement( @@ -124,17 +117,9 @@ type=None, uid=None, description=( - 'TLS connections SHALL reject the following for all TLS-enabled external connections:\n' - '\n' - '- Any TLS cipher suite not explicitly listed in [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers)\n' - '- Protocol versions: SSLv2, SSLv3, TLS 1.0, TLS 1.1\n' - '- Cipher suites using non-approved/legacy algorithms (for this profile), including:\n' - ' - ChaCha20-Poly1305\n' - ' - RC4, RC2, DES, 3DES, IDEA, SEED, CAMELLIA, ARIA\n' - ' - NULL encryption / NULL authentication\n' - ' - Anonymous key exchange (`aNULL`, `eNULL`, `ADH`, `AECDH`)\n' - ' - Export/weak suites (`EXP`, `LOW`, `40-bit`, `56-bit`)\n' - ' - MD5- or SHA-1-based legacy suites\n' + 'On TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter], any protocol version\n' + 'older than TLS 1.3 and any cipher suite not listed in [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers)\n' + 'SHALL be rejected by the operator in a FIPS-compliant configuration.\n' '\n' '\n' ), @@ -143,136 +128,114 @@ num='4.2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_FIPSConfig = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig', +RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Deploying a CHI with FIPS TLS settings SHALL start ClickHouse with FIPS-compliant TLS configuration.\n' + 'Operator deploying a `ClickHouseInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse server and client.\n' '\n' - ), - link=None, - level=3, - num='5.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHIDeploy = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The operator SHALL deploy FIPS `ClickHouseInstallation` resources to `Completed` with Running pods when configuration is valid.\n' + '```yaml\n' + ' configuration:\n' + ' clusters:\n' + ' - name: default\n' + ' secure: "yes"\n' + ' insecure: "no"\n' + ' layout:\n' + ' shardsCount: 1\n' + ' replicasCount: 2\n' + ' zookeeper:\n' + ' nodes:\n' + ' - host: chk-test-030003-keeper-0-0\n' + ' port: 2281\n' + ' secure: "yes"\n' + ' settings:\n' + ' http_port: _removed_\n' + ' tcp_port: _removed_\n' + ' interserver_http_port: _removed_\n' + ' mysql_port: _removed_\n' + ' postgresql_port: _removed_\n' + ' https_port: 8443\n' + ' tcp_port_secure: 9440\n' + ' interserver_https_port: 9010\n' + ' files:\n' + ' openssl.xml: |\n' + ' \n' + ' \n' + ' \n' + ' /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt\n' + ' /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key\n' + ' /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem\n' + ' \n' + ' none\n' + ' sslv2,sslv3,tlsv1,tlsv1_1\n' + ' TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384\n' + ' \n' + ' \n' + ' /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt\n' + ' false\n' + ' strict\n' + ' sslv2,sslv3,tlsv1,tlsv1_1\n' + ' TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384\n' + ' \n' + ' \n' + ' \n' + '```\n' '\n' - ), - link=None, - level=3, - num='5.1.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainHTTP = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain HTTP port 8123; HTTPS port 8443 SHALL be used.\n' + 'The deployed ClickHouse server SHALL use only the following ports:\n' '\n' - ), - link=None, - level=3, - num='5.1.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainNative = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain native TCP port 9000; secure native port 9440 SHALL be used.\n' + '* HTTPS API port 8443 (instead of 8123)\n' + '* Secure native TCP port 9440 (instead of 9000)\n' + '* Interserver HTTPS port 9010 (instead of interserver HTTP port 9009)\n' + '* Backup sidecar HTTPS API port 7171 (instead of 7180), when backups are enabled\n' '\n' - ), - link=None, - level=3, - num='5.1.4' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoUnexpectedPorts = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'ClickHouse pods in a FIPS deployment SHALL expose only expected secure listener ports and no additional unexpected ports.\n' + 'Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites.\n' '\n' ), link=None, level=3, - num='5.1.5' + num='5.2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_InternodeTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS', +RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig_ExternalClient = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'ReplicatedMergeTree replicas SHALL communicate over interserver HTTPS (`interserver_https_port`) and data SHALL converge across replicas.\n' + 'External clients connecting to the ClickHouse server SHALL be able to use any enabled TLS protocol version, including TLS 1.2.\n' '\n' ), link=None, level=3, - num='5.1.6' + num='5.2.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleUp = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp', +RQ_SRS_026_ClickHouseOperator_FIPS_CH_Rescale = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Adding a replica to a FIPS-configured CHI SHALL reconcile to `Completed` and the new replica SHALL run the FIPS ClickHouse binary with TLS-only listeners.\n' + 'Adding or removing a replica from a FIPS-configured `ClickHouseInstallation` SHALL reconcile successfully and result in the expected number of running pods.\n' '\n' - ), - link=None, - level=3, - num='5.1.7' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleDown = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Removing a replica from a FIPS-configured CHI SHALL reconcile to `Completed` and remaining replicas SHALL keep FIPS binary and TLS-only configuration.\n' + 'After rescaling, all replicas SHALL continue to run the FIPS ClickHouse binary and maintain the configured TLS-only OpenSSL settings.\n' '\n' ), link=None, level=3, - num='5.1.8' + num='5.2.3' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ConfigUpdate = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate', +RQ_SRS_026_ClickHouseOperator_FIPS_CH_ConfigUpdate = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate', version='1.0', priority=None, group=None, @@ -285,309 +248,186 @@ ), link=None, level=3, - num='5.1.9' + num='5.2.4' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_FIPSConfig = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig', +RQ_SRS_026_ClickHouseOperator_FIPS_CHK_FIPSConfig = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Deploying a CHK with FIPS TLS settings SHALL start Keeper with FIPS-compliant TLS configuration.\n' + 'Operator deploying a `ClickHouseKeeperInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse\n' + 'Keeper server and client.\n' '\n' - ), - link=None, - level=3, - num='5.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHKDeploy = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The operator SHALL deploy FIPS `ClickHouseKeeperInstallation` resources to `Completed` with Running pods when configuration is valid.\n' + '```yaml\n' + ' configuration:\n' + ' clusters:\n' + ' - name: keeper\n' + ' secure: "yes"\n' + ' insecure: "no"\n' + ' layout:\n' + ' replicasCount: 2\n' + ' settings:\n' + ' keeper_server/log_storage_path: /var/lib/clickhouse/coordination/log\n' + ' keeper_server/snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots\n' + ' keeper_server/raft_configuration/server/port: 9444\n' + ' files:\n' + ' openssl.xml: |\n' + ' \n' + ' \n' + ' \n' + ' /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt\n' + ' /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key\n' + ' \n' + ' none\n' + ' sslv2,sslv3,tlsv1,tlsv1_1\n' + ' TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384\n' + ' \n' + ' \n' + ' /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt\n' + ' false\n' + ' strict\n' + ' sslv2,sslv3,tlsv1,tlsv1_1\n' + ' TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384\n' + ' \n' + ' \n' + ' \n' + '```\n' '\n' - ), - link=None, - level=3, - num='5.2.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoPlainClientPort = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'When FIPS transport hardening applies, Keeper pods SHALL NOT listen on plain client port 2181; secure client port 2281 SHALL be used.\n' + 'The deployed ClickHouse Keeper cluster SHALL use only the following ports:\n' '\n' - ), - link=None, - level=3, - num='5.2.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoUnexpectedPorts = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Keeper pods in a FIPS deployment SHALL expose only expected secure listener ports.\n' + '* Secure client port 2281 (instead of 2181)\n' + '* Secure Raft communication port 9444\n' + '* Plaintext HTTP readiness probe port 9182 (the `/ready` Raft-quorum health check)\n' '\n' - ), - link=None, - level=3, - num='5.2.4' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_RaftTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Keeper Raft communication SHALL use TLS on the configured secure Raft port.\n' + 'Every exposed port except the readiness probe port 9182 and Raft replication port 9444 (which enforces peer-only authentication)\n' + 'SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. Port 9182 SHALL stay \n' + 'unconditionally plaintext HTTP regardless of the secure/insecure configuration (see Boundary).\n' '\n' ), link=None, level=3, - num='5.2.5' + num='6.2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleUp = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp', +RQ_SRS_026_ClickHouseOperator_FIPS_CHK_Rescale = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Adding a node to a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and the new node SHALL run the FIPS Keeper binary with TLS-only client and Raft listeners.\n' + 'Adding or removing a node from a FIPS-configured `ClickHouseKeeperInstallation` SHALL reconcile successfully and result \n' + 'in the expected number of running pods.\n' '\n' - ), - link=None, - level=3, - num='5.2.6' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleDown = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Removing a node from a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and remaining nodes SHALL keep FIPS configuration.\n' + 'After rescaling, all Keeper nodes SHALL continue to run the FIPS ClickHouse Keeper binary and maintain the configured \n' + 'TLS-only OpenSSL settings.\n' '\n' ), link=None, level=3, - num='5.2.7' + num='6.2.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ConfigUpdate = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate', +RQ_SRS_026_ClickHouseOperator_FIPS_CHK_ConfigUpdate = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Updating TLS settings on a running CHK SHALL reload Keeper with the new FIPS-compliant configuration.\n' - '\n' + 'Updating TLS settings on a running CHK SHALL reload ClickHouse with the new FIPS-compliant configuration.\n' '\n' - ), - link=None, - level=3, - num='5.2.8' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_VersionString = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'A running ClickHouse host under FIPS image policy SHALL report a `version()` string containing `fips` (case-insensitive).\n' '\n' ), link=None, level=3, - num='5.3.1' + num='6.2.3' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_FIPSBinary = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary', +RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSBinary = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'The `clickhouse-backup` sidecar SHALL run a FIPS-built binary; `clickhouse-backup --version` SHALL contain `fips` (case-insensitive).\n' + 'The `clickhouse-backup` sidecar SHALL run a FIPS-built binary.\n' '\n' - ), - link=None, - level=3, - num='5.3.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_GOFIPS140 = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'When inspectable, the clickhouse-backup sidecar binary SHALL embed `GOFIPS140=v1.0.0` per `go version -m`.\n' + 'The sidecar binary SHALL satisfy all of the following:\n' '\n' - ), - link=None, - level=3, - num='5.3.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_OnlyTLSPorts = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The clickhouse-backup sidecar SHALL expose only secure listener ports (including HTTPS API port 7171).\n' + '* `clickhouse-backup --version` contains `fips`\n' + '* When inspectable, `go version -m` reports `GOFIPS140=v1.0.0`\n' '\n' ), link=None, level=3, - num='5.3.4' + num='7.2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_HTTPSAPI = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI', +RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSConfig = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'The clickhouse-backup HTTPS API SHALL serve over TLS with CA-trust enforcement: trusted clients accepted, untrusted clients rejected.\n' + 'Deploying a `ClickHouseInstallation` with a FIPS-configured backup sidecar SHALL start `clickhouse-backup` with a FIPS-compliant TLS configuration.\n' '\n' - ), - link=None, - level=3, - num='5.3.5' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_ClickHouseOverTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The clickhouse-backup sidecar SHALL reach ClickHouse over secure native TCP.\n' + 'The deployed backup sidecar SHALL only add the following listener ports to the ClickHouse container:\n' '\n' - ), - link=None, - level=3, - num='5.3.6' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_RestoreRoundTrip = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Backup and restore through the HTTPS API SHALL succeed over TLS.\n' + '* HTTPS API port 7171 (instead of 7180)\n' '\n' - ), - link=None, - level=3, - num='5.3.7' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_RemoteUploadTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Remote backup upload to object storage SHALL use FIPS-approved TLS.\n' + 'The FIPS-configured backup sidecar SHALL additionally satisfy all of the following:\n' '\n' + '* Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites.\n' + '* The `clickhouse-backup` sidecar SHALL connect to ClickHouse using secure native TCP with TLS enabled.\n' '\n' ), link=None, level=3, - num='5.3.8' + num='7.2.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceVerifyStrict = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict', +RQ_SRS_026_ClickHouseOperator_FIPS_Backup_RestoreRoundTrip = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'With `fips.enforced=true`, unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients.\n' + 'Creating a backup and restoring it through the HTTPS API SHALL succeed over TLS.\n' '\n' ), link=None, level=3, - num='6.1.1' + num='7.2.3' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceMinVersion13 = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13', +RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_SecurityCoercion = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'With `fips.enforced=true`, unset TLS minVersion SHALL be coerced to 1.3 for the\n' - "operator's outbound TLS clients.\n" + 'When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce unset or relaxed security settings as follows:\n' '\n' - ), - link=None, - level=3, - num='6.1.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_OverrideMinVersion12To13 = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce `minVersion` to `"1.3"` for `security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`, even when those fields are explicitly set to `"1.2"`.\n' + '* Unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients.\n' + '* Unset TLS `minVersion` SHALL be coerced to `"1.3"` for the operator\'s outbound TLS clients (`security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`).\n' + '* Explicit `minVersion: "1.2"` for those TLS clients SHALL be coerced to `"1.3"`.\n' + '* Unset IPC mode SHALL be coerced to Secure.\n' + '\n' + 'Example configuration with explicit `minVersion: "1.2"`:\n' '\n' '```yaml\n' 'spec:\n' @@ -605,450 +445,72 @@ ' minVersion: "1.2"\n' '```\n' '\n' - 'After operator configuration normalization, the effective `minVersion` for each component listed above SHALL be `"1.3"`.\n' - '\n' - ), - link=None, - level=3, - num='6.1.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceIPCSecure = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With `fips.enforced=true`, unset IPC mode SHALL be coerced to Secure.\n' + 'After operator configuration normalization, the effective `minVersion` for each TLS client listed above SHALL be `"1.3"`.\n' '\n' ), link=None, level=3, - num='6.1.4' + num='8.1.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectInsecureKubeconfig = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig', +RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectNonCompliantSpecs = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'The operator SHALL refuse to start when kubeconfig uses `TLSClientConfig.Insecure=true` under strict/FIPS mode.\n' + 'When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL reject \n' + 'non-compliant CHI and CHK specifications with `FIPSValidationFailed` and SHALL NOT create workload StatefulSets for:\n' '\n' - ), - link=None, - level=3, - num='6.1.5' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneCHI = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'CHI with `clickhouse.tls.verify=None` under enforced mode SHALL be rejected with `FIPSValidationFailed`.\n' + '* CHI referencing plain external ZooKeeper nodes, including when `secure` is explicitly set to `"false"`.\n' + '* CHI with `clickhouse.tls.verify=None` at spec or cluster level.\n' + '* CHI with `zookeeper.tls.verify=None`.\n' + '* CHI with invalid `clickhouse.tls.minVersion`.\n' + '* CHK with TLS verify bypass at spec level.\n' '\n' ), link=None, level=3, - num='6.1.6' + num='8.1.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneZK = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK', +RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_MinVersionScope = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'CHI with `zookeeper.tls.verify=None` under enforced mode SHALL be rejected.\n' + 'The `minVersion` coercion SHALL apply only to TLS clients created and managed by the operator.\n' + 'They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2\n' + '(see [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient](#rqsrs-026clickhouseoperatorfipschfipsconfigexternalclient)).\n' '\n' ), link=None, level=3, - num='6.1.7' + num='8.1.3' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectInvalidMinVersion = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion', +RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectNonFIPS = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'CHI with invalid TLS minVersion under enforced mode SHALL be rejected.\n' - '\n' - ), - link=None, - level=3, - num='6.1.8' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectExternalZookeeper = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'CHI referencing plain external ZooKeeper nodes under enforced mode SHALL be rejected.\n' - '\n' - ), - link=None, - level=3, - num='6.1.9' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectCHKBypass = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'CHK with TLS verify bypass under enforced mode SHALL be rejected.\n' - '\n' - ), - link=None, - level=3, - num='6.1.10' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_MinVersionScope = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The `minVersion` coercion SHALL apply only to TLS clients created and managed by the operator.\n' - 'They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2.\n' - '\n' - ), - link=None, - level=3, - num='6.1.11' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHI = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With `security.fips.images.policy=Required`, CHI with non-FIPS image tag SHALL be rejected with `FIPSImagePolicyViolation`.\n' - '\n' - ), - link=None, - level=3, - num='6.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_AcceptCHI = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With image policy Required, CHI with FIPS-tagged image SHALL reconcile normally.\n' - '\n' - ), - link=None, - level=3, - num='6.2.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHK = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With image policy Required, CHK with non-FIPS Keeper image SHALL be rejected.\n' - '\n' - ), - link=None, - level=3, - num='6.2.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RuntimeVersion = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With image policy Required, host `SELECT version()` lacking `fips` SHALL fail with `FIPSImagePolicyViolation`.\n' - '\n' - ), - link=None, - level=3, - num='6.2.4' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Permissive = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'With permissive image policy, non-FIPS CHI images SHALL reconcile (default).\n' - '\n' - ), - link=None, - level=3, - num='6.2.5' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_ShortCircuit = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Multiple non-FIPS hosts SHALL produce a single policy violation error.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='6.2.6' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_FIPSSuffix = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Image tags containing `fips` (case-insensitive) SHALL be detected as FIPS.\n' - '\n' - ), - link=None, - level=3, - num='6.3.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_AltinityFIPS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Image tags containing `altinityfips` SHALL be detected as FIPS.\n' - '\n' - ), - link=None, - level=3, - num='6.3.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_DigestOnly = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Digest-only image references SHALL NOT be detected as FIPS at admission.\n' - '\n' - ), - link=None, - level=3, - num='6.3.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_RegistryPath = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection.\n' - '\n' - ), - link=None, - level=3, - num='6.3.4' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_CaseInsensitive = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Image tags such as `25.3.FIPS` or `25.3.Fips` SHALL be detected as FIPS (case-insensitive match on the tag).\n' - '\n' - '\n' - ), - link=None, - level=3, - num='6.3.5' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Listeners = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'FIPS workload pods (ClickHouse, Keeper, and sidecar containers) SHALL listen only on expected TLS ports. Plaintext service ports (8123, 9000, 2181) SHALL NOT be open when FIPS transport hardening applies.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Kubernetes = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The operator SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_ClickHouse = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The operator SHALL connect to ClickHouse using FIPS-approved TLS ciphers.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.3.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Zookeeper = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The operator SHALL connect to ZooKeeper/Keeper using FIPS-approved TLS ciphers.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.4.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_IPCSecure = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Operator IPC with `security.ipc.mode=Secure` SHALL work over localhost HTTP with token auth.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.5.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Gap_OperatorMetricsTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Operator Prometheus metrics on :9999 currently expose a known FIPS gap (HTTP-only).\n' - '\n' - '\n' - ), - link=None, - level=3, - num='7.6.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_Listeners = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The metrics-exporter process SHALL expose only expected listener ports on `:8888`. Sidecar containers in the same pod SHALL be listener-audited with the same `/proc/net/tcp` procedure.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='8.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_Kubernetes = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The exporter SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers.\n' + 'With `security.fips.images.policy=Required`, non-FIPS images SHALL be rejected with `FIPSImagePolicyViolation` as follows:\n' '\n' + '* CHI with non-FIPS ClickHouse image tag SHALL be rejected at admission.\n' + '* CHK with non-FIPS Keeper image tag SHALL be rejected at admission.\n' + '* CHI with non-FIPS `clickhouse-backup` sidecar image tag SHALL be rejected at admission.\n' + '* CHI with multiple non-FIPS hosts SHALL produce a single policy violation error.\n' + '* Digest-only image references SHALL NOT be detected as FIPS at admission.\n' + '* Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection.\n' + '* CHI admitted with a FIPS-tagged ClickHouse image whose running binary lacks `fips` in `SELECT version()` SHALL fail at runtime.\n' '\n' ), link=None, @@ -1056,305 +518,157 @@ num='8.2.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_ClickHouse = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The exporter SHALL query ClickHouse using FIPS-approved TLS when configured for HTTPS.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='8.3.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Gap_ExporterMetricsTLS = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Exporter Prometheus metrics on :8888 currently expose a known FIPS gap (HTTP-only).\n' - '\n' - '\n' - ), - link=None, - level=3, - num='8.4.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_OperatorMismatch = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Tampering with `clickhouse-operator` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='9.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_ExporterMismatch = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Tampering with `metrics-exporter` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='9.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_CAST_OperatorFail = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Running `clickhouse-operator` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='10.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_CAST_ExporterFail = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'Running `metrics-exporter` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error.\n' - '\n' - '\n' - ), - link=None, - level=3, - num='10.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Synthetic_ApprovedCiphers = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'For each external connection listed below, when exercised as a TLS **client** with `openssl s_server` offering only [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers), or as a TLS **server** with `openssl s_client` using only approved ciphers, the connection SHALL succeed:\n' - '\n' - '| Connection | Role | Tool |\n' - '|------------|------|------|\n' - '| Operator to Kubernetes API | Client | `openssl s_server` |\n' - '| Operator to ClickHouse Server | Client | `openssl s_server` |\n' - '| Operator to ZooKeeper/Keeper | Client | `openssl s_server` |\n' - '| Operator metrics :9999 | Server | `openssl s_client` |\n' - '| Exporter to Kubernetes API | Client | `openssl s_server` |\n' - '| Exporter to ClickHouse Server | Client | `openssl s_server` |\n' - '| Exporter metrics :8888 | Server | `openssl s_client` |\n' - '\n' - '\n' - ), - link=None, - level=3, - num='11.1.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_Synthetic_RejectedCiphers = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'For each external connection listed below, when the peer offers only [rejected ciphers or protocols](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers), the connection SHALL be rejected:\n' - '\n' - '| Connection | Role | Tool |\n' - '|------------|------|------|\n' - '| Operator to Kubernetes API | Client | `openssl s_server` |\n' - '| Operator to ClickHouse Server | Client | `openssl s_server` |\n' - '| Operator to ZooKeeper/Keeper | Client | `openssl s_server` |\n' - '| Operator metrics :9999 | Server | `openssl s_client` |\n' - '| Exporter to Kubernetes API | Client | `openssl s_server` |\n' - '| Exporter to ClickHouse Server | Client | `openssl s_server` |\n' - '| Exporter metrics :8888 | Server | `openssl s_client` |\n' - '\n' - '\n' - ), - link=None, - level=3, - num='11.2.1' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_CICD_OperatorImageBuild = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KubernetesAPI = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'CI SHALL build the [clickhouse-operator] FIPS image successfully.\n' + 'The clickhouse-operator SHALL access the Kubernetes API through the HTTPS endpoint on port `443`.\n' + 'Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected.\n' '\n' ), link=None, level=2, - num='12.1' + num='9.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_CICD_ExporterImageBuild = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_KubernetesAPI = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'CI SHALL build the [metrics-exporter] FIPS image successfully.\n' + 'The metrics-exporter SHALL access the Kubernetes API through the HTTPS endpoint on port `443`.\n' + 'Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected.\n' '\n' ), link=None, level=2, - num='12.2' + num='9.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_CICD_VulnerabilityScan = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_ClickHouse = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'FIPS images SHALL pass vulnerability scanning with no Critical, High, or Medium findings.\n' - '\n' + 'The clickhouse-operator SHALL communicate with ClickHouse hosts using HTTPS port `8443`.\n' '\n' ), link=None, level=2, - num='12.3' + num='9.3' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_Tree = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_ClickHouse = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Static review of operator-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented.\n' + 'The metrics-exporter SHALL discover ClickHouse hosts using the HTTPS endpoint `8443`.\n' '\n' ), link=None, - level=3, - num='12.4.1' + level=2, + num='9.4' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_SharedPkg = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KeeperRestriction = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Review of shared packages reachable from `cmd/operator` SHALL produce no Critical findings.\n' + 'When a Keeper ensemble is configured as TLS-only, the clickhouse-operator SHALL NOT attempt plaintext ZooKeeper/Keeper\n' + 'operations against it.\n' '\n' ), link=None, - level=3, - num='12.4.2' + level=2, + num='9.5' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_RegressionGate = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate', +RQ_SRS_026_ClickHouseOperator_FIPS_Connect_ClickHouse_KeeperTLS = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'A signed-off review artifact SHALL be stored with the build record before release.\n' + 'ClickHouse replicas SHALL connect to Keeper using secure client port `2281` with `secure=yes`.\n' + '\n' '\n' ), link=None, - level=3, - num='12.4.3' + level=2, + num='9.6' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_Tree = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree', +RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_VerificationMismatch = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Static review of exporter-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented.\n' + 'Each shipped FIPS binary — `clickhouse-operator` and `metrics-exporter` — SHALL perform a software integrity \n' + 'self-test at initialization by verifying its embedded HMAC. If the binary is tampered with or corrupted such that\n' + 'the HMAC verification fails, the process SHALL immediately terminate with a `fips140: verification mismatch` panic \n' + 'to prevent the execution of a compromised cryptographic module.\n' '\n' ), link=None, - level=3, - num='12.5.1' + level=2, + num='10.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_SharedPkg = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg', +RQ_SRS_026_ClickHouseOperator_FIPS_CAST_OperatorFail = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'Review of shared packages reachable from `cmd/metrics_exporter` SHALL produce no Critical findings.\n' + 'Running `clickhouse-operator` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error.\n' + '\n' '\n' ), link=None, level=3, - num='12.5.2' + num='11.1.1' ) -RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_RegressionGate = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate', +RQ_SRS_026_ClickHouseOperator_FIPS_CAST_ExporterFail = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'A signed-off review artifact SHALL be stored with the build record before release.\n' + 'Running `metrics-exporter` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error.\n' + '\n' '\n' ), link=None, level=3, - num='12.5.3' + num='11.2.1' ) RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_WrapperIntegration = Requirement( @@ -1365,12 +679,12 @@ type=None, uid=None, description=( - 'Building clickhouse-operator with `-tags acvp_wrapper` SHALL expose a working ACVP responder via argv0 dispatch.\n' + 'Building `clickhouse-operator` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `clickhouse-operator-acvp`.\n' '\n' ), link=None, level=3, - num='13.1.1' + num='12.1.1' ) RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ConfigGeneration = Requirement( @@ -1381,45 +695,28 @@ type=None, uid=None, description=( - 'The clickhouse-operator ACVP responder SHALL answer `getConfig` with supported capabilities.\n' - '\n' - ), - link=None, - level=3, - num='13.1.2' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ExpectedOutputReplay = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - '`bash pkg/util/fips/acvp/run.sh` SHALL match all configured expected outputs for the operator.\n' + 'The `clickhouse-operator` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`.\n' '\n' ), link=None, level=3, - num='13.1.3' + num='12.1.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SuiteCount = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount', +RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SHA2256AFT = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT', version='1.0', priority=None, group=None, type=None, uid=None, description=( - 'The tracked ACVP config SHALL report 38 matched expectations for clickhouse-operator.\n' - '\n' + 'The `clickhouse-operator` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`.\n' '\n' ), link=None, level=3, - num='13.1.4' + num='12.1.3' ) RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_WrapperIntegration = Requirement( @@ -1430,12 +727,12 @@ type=None, uid=None, description=( - 'Building metrics-exporter with `-tags acvp_wrapper` SHALL expose a working ACVP responder.\n' + 'Building `metrics-exporter` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `metrics-exporter-acvp`.\n' '\n' ), link=None, level=3, - num='13.2.1' + num='12.2.1' ) RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ConfigGeneration = Requirement( @@ -1446,51 +743,36 @@ type=None, uid=None, description=( - 'The metrics-exporter ACVP responder SHALL answer `getConfig` with supported capabilities.\n' + 'The `metrics-exporter` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`.\n' '\n' ), link=None, level=3, - num='13.2.2' + num='12.2.2' ) -RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ExpectedOutputReplay = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay', +RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SHA2256AFT = Requirement( + name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT', version='1.0', priority=None, group=None, type=None, uid=None, description=( - '`BINARY=metrics-exporter bash pkg/util/fips/acvp/run.sh` SHALL match all expected outputs.\n' + 'The `metrics-exporter` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`.\n' '\n' - ), - link=None, - level=3, - num='13.2.3' -) - -RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SuiteCount = Requirement( - name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount', - version='1.0', - priority=None, - group=None, - type=None, - uid=None, - description=( - 'The tracked ACVP config SHALL report 38 matched expectations for metrics-exporter.\n' '\n' ), link=None, level=3, - num='13.2.4' + num='12.2.3' ) -Inbound_connection_to_operator_exporter_metrics_endpoint = Specification( - name='Inbound connection to operator/exporter metrics endpoint', +QA_SRS_ClickHouse_Operator_FIPS_140_3 = Specification( + name='QA-SRS ClickHouse Operator FIPS 140-3', description=None, - author=None, - date=None, + author='Saba Momtselidze', + date='June 12, 2026', status=None, approved_by=None, approved_date=None, @@ -1505,228 +787,104 @@ headings=( Heading(name='Introduction', level=1, num='1'), Heading(name='Configuration Requirements', level=1, num='2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS', level=2, num='2.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts', level=2, num='2.1'), Heading(name='Build Verification', level=1, num='3'), - Heading(name='Shipped Binaries', level=2, num='3.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries', level=3, num='3.1.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupLogs', level=3, num='3.1.2'), - Heading(name='Approved TLS Cipher Suites', level=1, num='4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers', level=2, num='4.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries', level=2, num='3.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs', level=2, num='3.2'), + Heading(name='FIPS 140-3 TLS Cipher Suites', level=1, num='4'), + Heading(name='Approved TLS Cipher Suites', level=2, num='4.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers', level=3, num='4.1.1'), Heading(name='Rejected Cipher Suites and Protocols', level=2, num='4.2'), Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers', level=3, num='4.2.1'), - Heading(name='ClickHouse Server and Keeper FIPS Configurations', level=1, num='5'), - Heading(name='ClickHouse Server', level=2, num='5.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig', level=3, num='5.1.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy', level=3, num='5.1.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP', level=3, num='5.1.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative', level=3, num='5.1.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts', level=3, num='5.1.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS', level=3, num='5.1.6'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp', level=3, num='5.1.7'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown', level=3, num='5.1.8'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate', level=3, num='5.1.9'), - Heading(name='ClickHouse Keeper', level=2, num='5.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig', level=3, num='5.2.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy', level=3, num='5.2.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort', level=3, num='5.2.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts', level=3, num='5.2.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS', level=3, num='5.2.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp', level=3, num='5.2.6'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown', level=3, num='5.2.7'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate', level=3, num='5.2.8'), - Heading(name='ClickHouse Backup Sidecar', level=2, num='5.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString', level=3, num='5.3.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary', level=3, num='5.3.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140', level=3, num='5.3.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts', level=3, num='5.3.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI', level=3, num='5.3.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS', level=3, num='5.3.6'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip', level=3, num='5.3.7'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS', level=3, num='5.3.8'), - Heading(name='FIPS Enforcement Mode', level=1, num='6'), - Heading(name='Security Coercion', level=2, num='6.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict', level=3, num='6.1.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13', level=3, num='6.1.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13', level=3, num='6.1.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure', level=3, num='6.1.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig', level=3, num='6.1.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI', level=3, num='6.1.6'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK', level=3, num='6.1.7'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion', level=3, num='6.1.8'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper', level=3, num='6.1.9'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass', level=3, num='6.1.10'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope', level=3, num='6.1.11'), - Heading(name='Image Policy', level=2, num='6.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI', level=3, num='6.2.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI', level=3, num='6.2.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK', level=3, num='6.2.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion', level=3, num='6.2.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive', level=3, num='6.2.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit', level=3, num='6.2.6'), - Heading(name='Image Tag Detection', level=2, num='6.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix', level=3, num='6.3.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS', level=3, num='6.3.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly', level=3, num='6.3.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath', level=3, num='6.3.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive', level=3, num='6.3.5'), - Heading(name='Operator External Connections', level=1, num='7'), - Heading(name='Operator Runtime Listener Verification', level=2, num='7.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners', level=3, num='7.1.1'), - Heading(name='Operator to Kubernetes API', level=2, num='7.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes', level=3, num='7.2.1'), - Heading(name='Operator to ClickHouse Server', level=2, num='7.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse', level=3, num='7.3.1'), - Heading(name='Operator to ZooKeeper/Keeper', level=2, num='7.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper', level=3, num='7.4.1'), - Heading(name='Operator to metrics-exporter IPC', level=2, num='7.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure', level=3, num='7.5.1'), - Heading(name='Operator Prometheus Metrics', level=2, num='7.6'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS', level=3, num='7.6.1'), - Heading(name='Exporter External Connections', level=1, num='8'), - Heading(name='Exporter Runtime Listener Verification', level=2, num='8.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners', level=3, num='8.1.1'), - Heading(name='Exporter to Kubernetes API', level=2, num='8.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes', level=3, num='8.2.1'), - Heading(name='Exporter to ClickHouse Server', level=2, num='8.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse', level=3, num='8.3.1'), - Heading(name='Exporter Prometheus Metrics', level=2, num='8.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS', level=3, num='8.4.1'), - Heading(name='Integrity Check Failure', level=1, num='9'), - Heading(name='Operator Integrity Tampering', level=2, num='9.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch', level=3, num='9.1.1'), - Heading(name='Exporter Integrity Tampering', level=2, num='9.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch', level=3, num='9.2.1'), - Heading(name='CAST Failure', level=1, num='10'), - Heading(name='Operator CAST Failure', level=2, num='10.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail', level=3, num='10.1.1'), - Heading(name='Exporter CAST Failure', level=2, num='10.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail', level=3, num='10.2.1'), - Heading(name='Synthetic TLS Cipher Validation', level=1, num='11'), - Heading(name='Operator as TLS client against server offering non-approved cipher', level=0, num=''), - Heading(name='Approved cipher matrix', level=2, num='11.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers', level=3, num='11.1.1'), - Heading(name='Rejected cipher matrix', level=2, num='11.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers', level=3, num='11.2.1'), - Heading(name='CI/CD Image and Policy Verification', level=1, num='12'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild', level=2, num='12.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild', level=2, num='12.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan', level=2, num='12.3'), - Heading(name='Operator Source Review', level=2, num='12.4'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree', level=3, num='12.4.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg', level=3, num='12.4.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate', level=3, num='12.4.3'), - Heading(name='Exporter Source Review', level=2, num='12.5'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree', level=3, num='12.5.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg', level=3, num='12.5.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate', level=3, num='12.5.3'), - Heading(name='ACVP Algorithm Validation', level=1, num='13'), - Heading(name='Operator ACVP Validation', level=2, num='13.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration', level=3, num='13.1.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration', level=3, num='13.1.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay', level=3, num='13.1.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount', level=3, num='13.1.4'), - Heading(name='Exporter ACVP Validation', level=2, num='13.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration', level=3, num='13.2.1'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration', level=3, num='13.2.2'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay', level=3, num='13.2.3'), - Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount', level=3, num='13.2.4'), - Heading(name='Terminology', level=1, num='14'), - Heading(name='SRS', level=2, num='14.1'), - Heading(name='FIPS 140-3', level=2, num='14.2'), - Heading(name='clickhouse-operator', level=2, num='14.3'), - Heading(name='metrics-exporter', level=2, num='14.4'), - Heading(name='CHI', level=2, num='14.5'), - Heading(name='CHK', level=2, num='14.6'), - Heading(name='ACVP', level=2, num='14.7'), - Heading(name='CMVP', level=2, num='14.8'), - Heading(name='CAVP', level=2, num='14.9'), + Heading(name='ClickHouse Server', level=1, num='5'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig', level=3, num='5.2.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient', level=3, num='5.2.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale', level=3, num='5.2.3'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate', level=3, num='5.2.4'), + Heading(name='ClickHouse Keeper', level=1, num='6'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig', level=3, num='6.2.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale', level=3, num='6.2.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate', level=3, num='6.2.3'), + Heading(name='ClickHouse Backup Sidecar', level=1, num='7'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary', level=3, num='7.2.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig', level=3, num='7.2.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip', level=3, num='7.2.3'), + Heading(name='FIPS Enforcement Mode', level=1, num='8'), + Heading(name='Security Coercion', level=2, num='8.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion', level=3, num='8.1.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs', level=3, num='8.1.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope', level=3, num='8.1.3'), + Heading(name='Image Policy', level=2, num='8.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS', level=3, num='8.2.1'), + Heading(name='Runtime Connection Evidence', level=1, num='9'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI', level=2, num='9.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI', level=2, num='9.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse', level=2, num='9.3'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse', level=2, num='9.4'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction', level=2, num='9.5'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS', level=2, num='9.6'), + Heading(name='Integrity Check', level=1, num='10'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch', level=2, num='10.1'), + Heading(name='CAST Failure', level=1, num='11'), + Heading(name='Operator CAST Failure', level=2, num='11.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail', level=3, num='11.1.1'), + Heading(name='Exporter CAST Failure', level=2, num='11.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail', level=3, num='11.2.1'), + Heading(name='ACVP Algorithm Validation', level=1, num='12'), + Heading(name='Operator ACVP Validation', level=2, num='12.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration', level=3, num='12.1.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration', level=3, num='12.1.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT', level=3, num='12.1.3'), + Heading(name='Exporter ACVP Validation', level=2, num='12.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration', level=3, num='12.2.1'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration', level=3, num='12.2.2'), + Heading(name='RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT', level=3, num='12.2.3'), + Heading(name='Terminology', level=1, num='13'), + Heading(name='SRS', level=2, num='13.1'), + Heading(name='FIPS 140-3', level=2, num='13.2'), + Heading(name='clickhouse-operator', level=2, num='13.3'), + Heading(name='metrics-exporter', level=2, num='13.4'), + Heading(name='CHI', level=2, num='13.5'), + Heading(name='CHK', level=2, num='13.6'), + Heading(name='ACVP', level=2, num='13.7'), + Heading(name='CMVP', level=2, num='13.8'), + Heading(name='CAVP', level=2, num='13.9'), ), requirements=( - RQ_SRS_026_ClickHouseOperator_FIPS_Config_ExternalTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries, - RQ_SRS_026_ClickHouseOperator_FIPS_Build_ShippedBinaries_StartupLogs, + RQ_SRS_026_ClickHouseOperator_FIPS_HTTPPorts, + RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries, + RQ_SRS_026_ClickHouseOperator_FIPS_OperatorBuild_ShippedBinaries_StartupLogs, RQ_SRS_026_ClickHouseOperator_FIPS_TLS_ApprovedCiphers, RQ_SRS_026_ClickHouseOperator_FIPS_TLS_RejectedCiphers, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_FIPSConfig, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHIDeploy, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainHTTP, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoPlainNative, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_NoUnexpectedPorts, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_InternodeTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleUp, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ScaleDown, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_ConfigUpdate, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_FIPSConfig, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHKDeploy, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoPlainClientPort, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_NoUnexpectedPorts, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_RaftTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleUp, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ScaleDown, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CHK_ConfigUpdate, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_CH_VersionString, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_FIPSBinary, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_GOFIPS140, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_OnlyTLSPorts, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_HTTPSAPI, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_ClickHouseOverTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_RestoreRoundTrip, - RQ_SRS_026_ClickHouseOperator_FIPS_DataPlane_Backup_RemoteUploadTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceVerifyStrict, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceMinVersion13, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_OverrideMinVersion12To13, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_CoerceIPCSecure, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectInsecureKubeconfig, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneCHI, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectVerifyNoneZK, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectInvalidMinVersion, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectExternalZookeeper, - RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectCHKBypass, + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig, + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig_ExternalClient, + RQ_SRS_026_ClickHouseOperator_FIPS_CH_Rescale, + RQ_SRS_026_ClickHouseOperator_FIPS_CH_ConfigUpdate, + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_FIPSConfig, + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_Rescale, + RQ_SRS_026_ClickHouseOperator_FIPS_CHK_ConfigUpdate, + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSBinary, + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_FIPSConfig, + RQ_SRS_026_ClickHouseOperator_FIPS_Backup_RestoreRoundTrip, + RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_SecurityCoercion, + RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_RejectNonCompliantSpecs, RQ_SRS_026_ClickHouseOperator_FIPS_Enforced_MinVersionScope, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHI, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_AcceptCHI, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectCHK, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RuntimeVersion, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Permissive, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_ShortCircuit, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_FIPSSuffix, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_AltinityFIPS, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_DigestOnly, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_RegistryPath, - RQ_SRS_026_ClickHouseOperator_FIPS_Images_TagDetection_CaseInsensitive, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Listeners, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Kubernetes, + RQ_SRS_026_ClickHouseOperator_FIPS_Images_Required_RejectNonFIPS, + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KubernetesAPI, + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_KubernetesAPI, RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_ClickHouse, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_Zookeeper, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_IPCSecure, - RQ_SRS_026_ClickHouseOperator_FIPS_Gap_OperatorMetricsTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_Listeners, - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_Kubernetes, RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Exporter_ClickHouse, - RQ_SRS_026_ClickHouseOperator_FIPS_Gap_ExporterMetricsTLS, - RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_OperatorMismatch, - RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_ExporterMismatch, + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_KeeperRestriction, + RQ_SRS_026_ClickHouseOperator_FIPS_Connect_ClickHouse_KeeperTLS, + RQ_SRS_026_ClickHouseOperator_FIPS_Integrity_VerificationMismatch, RQ_SRS_026_ClickHouseOperator_FIPS_CAST_OperatorFail, RQ_SRS_026_ClickHouseOperator_FIPS_CAST_ExporterFail, - RQ_SRS_026_ClickHouseOperator_FIPS_Synthetic_ApprovedCiphers, - RQ_SRS_026_ClickHouseOperator_FIPS_Synthetic_RejectedCiphers, - RQ_SRS_026_ClickHouseOperator_FIPS_CICD_OperatorImageBuild, - RQ_SRS_026_ClickHouseOperator_FIPS_CICD_ExporterImageBuild, - RQ_SRS_026_ClickHouseOperator_FIPS_CICD_VulnerabilityScan, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_Tree, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_SharedPkg, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Operator_RegressionGate, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_Tree, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_SharedPkg, - RQ_SRS_026_ClickHouseOperator_FIPS_AIReview_Exporter_RegressionGate, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_WrapperIntegration, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ConfigGeneration, - RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_ExpectedOutputReplay, - RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SuiteCount, + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Operator_SHA2256AFT, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_WrapperIntegration, RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ConfigGeneration, - RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_ExpectedOutputReplay, - RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SuiteCount, + RQ_SRS_026_ClickHouseOperator_FIPS_ACVP_Exporter_SHA2256AFT, ), content=''' # QA-SRS ClickHouse Operator FIPS 140-3 @@ -1738,153 +896,74 @@ **Author:** Saba Momtselidze -**Date:** May 29, 2026 +**Date:** June 12, 2026 ## Table of Contents * 1 [Introduction](#introduction) * 2 [Configuration Requirements](#configuration-requirements) - * 2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS](#rqsrs-026clickhouseoperatorfipsconfigexternaltls) + * 2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts](#rqsrs-026clickhouseoperatorfipshttpports) * 3 [Build Verification](#build-verification) - * 3.1 [Shipped Binaries](#shipped-binaries) - * 3.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries](#rqsrs026clickhouseoperatorfipsbuildshippedbinaries) - * 3.1.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.GOFIPS140](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesgofips140) - * 3.1.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSIdentity](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsidentity) - * 3.1.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSVersion](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsversion) - * 3.1.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.FIPSEnabled](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesfipsenabled) - * 3.1.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupBanner](#rqsrs026clickhouseoperatorfipsbuildshippedbinariesstartupbanner) -* 4 [GODEBUG Strict Mode Smoke Test](#godebug-strict-mode-smoke-test) - * 4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.GODEBUG.StrictMode](#rqsrs-026clickhouseoperatorfipsgodebugstrictmode) -* 5 [FIPS 140-3 Valid TLS Cipher Suites](#fips-140-3-valid-tls-cipher-suites) - * 5.1 [Approved TLS Cipher Suites](#approved-tls-cipher-suites) - * 5.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) - * 5.2 [Rejected Cipher Suites and Protocols](#rejected-cipher-suites-and-protocols) - * 5.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers) -* 6 [ClickHouse Server and Keeper FIPS Configurations](#clickhouse-server-and-keeper-fips-configurations) - * 6.1 [ClickHouse Server](#clickhouse-server) - * 6.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig](#rqsrs026clickhouseoperatorfipsdataplanechfipsconfig) - * 6.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy](#rqsrs026clickhouseoperatorfipsdataplanechideploy) - * 6.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP](#rqsrs026clickhouseoperatorfipsdataplanechnoplainhttp) - * 6.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative](#rqsrs026clickhouseoperatorfipsdataplanechnoplainnative) - * 6.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts](#rqsrs026clickhouseoperatorfipsdataplanechnounexpectedports) - * 6.1.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS](#rqsrs026clickhouseoperatorfipsdataplanechinternodetls) - * 6.1.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp](#rqsrs026clickhouseoperatorfipsdataplanechscaleup) - * 6.1.8 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown](#rqsrs026clickhouseoperatorfipsdataplanechscaledown) - * 6.1.9 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate](#rqsrs026clickhouseoperatorfipsdataplanechconfigupdate) - * 6.2 [ClickHouse Keeper](#clickhouse-keeper) - * 6.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig](#rqsrs026clickhouseoperatorfipsdataplanechkfipsconfig) - * 6.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy](#rqsrs026clickhouseoperatorfipsdataplanechkdeploy) - * 6.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort](#rqsrs026clickhouseoperatorfipsdataplanechknoplainclientport) - * 6.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts](#rqsrs026clickhouseoperatorfipsdataplanechknounexpectedports) - * 6.2.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS](#rqsrs026clickhouseoperatorfipsdataplanechkrafttls) - * 6.2.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp](#rqsrs026clickhouseoperatorfipsdataplanechkscaleup) - * 6.2.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown](#rqsrs026clickhouseoperatorfipsdataplanechkscaledown) - * 6.2.8 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate](#rqsrs026clickhouseoperatorfipsdataplanechkconfigupdate) - * 6.3 [ClickHouse Backup Sidecar](#clickhouse-backup-sidecar) - * 6.3.0 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString](#rqsrs026clickhouseoperatorfipsdataplanechversionstring) - * 6.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary](#rqsrs026clickhouseoperatorfipsdataplanebackupfipsbinary) - * 6.3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140](#rqsrs026clickhouseoperatorfipsdataplanebackupgofips140) - * 6.3.3 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts](#rqsrs026clickhouseoperatorfipsdataplanebackuponlytlsports) - * 6.3.4 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI](#rqsrs026clickhouseoperatorfipsdataplanebackuphttpsapi) - * 6.3.5 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS](#rqsrs026clickhouseoperatorfipsdataplanebackupclickhouseovertls) - * 6.3.6 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip](#rqsrs026clickhouseoperatorfipsdataplanebackuprestoreroundtrip) - * 6.3.7 [RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS](#rqsrs026clickhouseoperatorfipsdataplanebackupremoteuploadtls) -* 7 [FIPS Enforcement Mode](#fips-enforcement-mode) - * 7.1 [Security Coercion](#security-coercion) - * 7.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict](#rqsrs026clickhouseoperatorfipsenforcedcoerceverifystrict) - * 7.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13](#rqsrs026clickhouseoperatorfipsenforcedcoerceminversion13) - * 7.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13](#rqsrs-026clickhouseoperatorfipsenforcedoverrideminversion12to13) - * 7.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure](#rqsrs026clickhouseoperatorfipsenforcedcoerceipcsecure) - * 7.1.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig](#rqsrs026clickhouseoperatorfipsenforcedrejectinsecurekubeconfig) - * 7.1.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI](#rqsrs026clickhouseoperatorfipsenforcedrejectverifynonechi) - * 7.1.7 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK](#rqsrs026clickhouseoperatorfipsenforcedrejectverifynonezk) - * 7.1.8 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion](#rqsrs026clickhouseoperatorfipsenforcedrejectinvalidminversion) - * 7.1.9 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper](#rqsrs026clickhouseoperatorfipsenforcedrejectexternalzookeeper) - * 7.1.10 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass](#rqsrs026clickhouseoperatorfipsenforcedrejectchkbypass) - * 7.2 [Image Policy](#image-policy) - * 7.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI](#rqsrs026clickhouseoperatorfipsimagesrequiredrejectchi) - * 7.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI](#rqsrs026clickhouseoperatorfipsimagesrequiredacceptchi) - * 7.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK](#rqsrs026clickhouseoperatorfipsimagesrequiredrejectchk) - * 7.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion](#rqsrs026clickhouseoperatorfipsimagesrequiredruntimeversion) - * 7.2.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive](#rqsrs026clickhouseoperatorfipsimagespermissive) - * 7.2.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit](#rqsrs026clickhouseoperatorfipsimagesrequiredshortcircuit) - * 7.3 [Image Tag Detection](#image-tag-detection) - * 7.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix](#rqsrs026clickhouseoperatorfipsimagestagdetectionfipssuffix) - * 7.3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS](#rqsrs026clickhouseoperatorfipsimagestagdetectionaltinityfips) - * 7.3.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly](#rqsrs026clickhouseoperatorfipsimagestagdetectiondigestonly) - * 7.3.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath](#rqsrs026clickhouseoperatorfipsimagestagdetectionregistrypath) - * 7.3.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive](#rqsrs026clickhouseoperatorfipsimagestagdetectioncaseinsensitive) -* 8 [Operator External Connections](#operator-external-connections) - * 8.1 [Operator Runtime Listener Verification](#operator-runtime-listener-verification) - * 8.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners](#rqsrs026clickhouseoperatorfipsconnectoperatorlisteners) - * 8.2 [Operator to Kubernetes API](#operator-to-kubernetes-api) - * 8.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes](#rqsrs026clickhouseoperatorfipsconnectoperatorkubernetes) - * 8.3 [Operator to ClickHouse Server](#operator-to-clickhouse-server) - * 8.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse](#rqsrs026clickhouseoperatorfipsconnectoperatorclickhouse) - * 8.4 [Operator to ZooKeeper/Keeper](#operator-to-zookeeperkeeper) - * 8.4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper](#rqsrs026clickhouseoperatorfipsconnectoperatorzookeeper) - * 8.5 [Operator to metrics-exporter IPC](#operator-to-metrics-exporter-ipc) - * 8.5.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure](#rqsrs026clickhouseoperatorfipsconnectoperatoripcsecure) - * 8.6 [Operator Prometheus Metrics](#operator-prometheus-metrics) - * 8.6.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS](#rqsrs026clickhouseoperatorfipsgapoperatormetricstls) -* 9 [Exporter External Connections](#exporter-external-connections) - * 9.1 [Exporter Runtime Listener Verification](#exporter-runtime-listener-verification) - * 9.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners](#rqsrs026clickhouseoperatorfipsconnectexporterlisteners) - * 9.2 [Exporter to Kubernetes API](#exporter-to-kubernetes-api) - * 9.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes](#rqsrs026clickhouseoperatorfipsconnectexporterkubernetes) - * 9.3 [Exporter to ClickHouse Server](#exporter-to-clickhouse-server) - * 9.3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse](#rqsrs026clickhouseoperatorfipsconnectexporterclickhouse) - * 9.4 [Exporter Prometheus Metrics](#exporter-prometheus-metrics) - * 9.4.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS](#rqsrs026clickhouseoperatorfipsgapexportermetricstls) -* 10 [Integrity Check Failure](#integrity-check-failure) - * 10.1 [Operator Integrity Tampering](#operator-integrity-tampering) - * 10.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch](#rqsrs026clickhouseoperatorfipsintegrityoperatormismatch) - * 10.2 [Exporter Integrity Tampering](#exporter-integrity-tampering) - * 10.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch](#rqsrs026clickhouseoperatorfipsintegrityexportermismatch) + * 3.1 [RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries](#rqsrs-026clickhouseoperatorfipsoperatorbuildshippedbinaries) + * 3.2 [RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs](#rqsrs-026clickhouseoperatorfipsoperatorbuildshippedbinariesstartuplogs) +* 4 [FIPS 140-3 TLS Cipher Suites](#fips-140-3-tls-cipher-suites) + * 4.1 [Approved TLS Cipher Suites](#approved-tls-cipher-suites) + * 4.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) + * 4.2 [Rejected Cipher Suites and Protocols](#rejected-cipher-suites-and-protocols) + * 4.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers) +* 5 [ClickHouse Server](#clickhouse-server) + * 5.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig](#rqsrs-026clickhouseoperatorfipschfipsconfig) + * 5.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient](#rqsrs-026clickhouseoperatorfipschfipsconfigexternalclient) + * 5.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale](#rqsrs-026clickhouseoperatorfipschrescale) + * 5.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate](#rqsrs-026clickhouseoperatorfipschconfigupdate) +* 6 [ClickHouse Keeper](#clickhouse-keeper) + * 6.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig](#rqsrs-026clickhouseoperatorfipschkfipsconfig) + * 6.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale](#rqsrs-026clickhouseoperatorfipschkrescale) + * 6.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate](#rqsrs-026clickhouseoperatorfipschkconfigupdate) +* 7 [ClickHouse Backup Sidecar](#clickhouse-backup-sidecar) + * 7.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary](#rqsrs-026clickhouseoperatorfipsbackupfipsbinary) + * 7.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig](#rqsrs-026clickhouseoperatorfipsbackupfipsconfig) + * 7.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip](#rqsrs-026clickhouseoperatorfipsbackuprestoreroundtrip) +* 8 [FIPS Enforcement Mode](#fips-enforcement-mode) + * 8.1 [Security Coercion](#security-coercion) + * 8.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion](#rqsrs-026clickhouseoperatorfipsenforcedsecuritycoercion) + * 8.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs](#rqsrs-026clickhouseoperatorfipsenforcedrejectnoncompliantspecs) + * 8.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope](#rqsrs-026clickhouseoperatorfipsenforcedminversionscope) + * 8.2 [Image Policy](#image-policy) + * 8.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS](#rqsrs-026clickhouseoperatorfipsimagesrequiredrejectnonfips) +* 9 [Runtime Connection Evidence](#runtime-connection-evidence) + * 9.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI](#rqsrs-026clickhouseoperatorfipsconnectoperatorkubernetesapi) + * 9.2 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI](#rqsrs-026clickhouseoperatorfipsconnectexporterkubernetesapi) + * 9.3 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse](#rqsrs-026clickhouseoperatorfipsconnectoperatorclickhouse) + * 9.4 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse](#rqsrs-026clickhouseoperatorfipsconnectexporterclickhouse) + * 9.5 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction](#rqsrs-026clickhouseoperatorfipsconnectoperatorkeeperrestriction) + * 9.6 [RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS](#rqsrs-026clickhouseoperatorfipsconnectclickhousekeepertls) +* 10 [Integrity Check](#integrity-check) + * 10.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch](#rqsrs-026clickhouseoperatorfipsintegrityverificationmismatch) * 11 [CAST Failure](#cast-failure) * 11.1 [Operator CAST Failure](#operator-cast-failure) - * 11.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail](#rqsrs026clickhouseoperatorfipscastoperatorfail) + * 11.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.OperatorFail](#rqsrs-026clickhouseoperatorfipscastoperatorfail) * 11.2 [Exporter CAST Failure](#exporter-cast-failure) - * 11.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail](#rqsrs026clickhouseoperatorfipscastexporterfail) -* 12 [Synthetic TLS Cipher Validation](#synthetic-tls-cipher-validation) - * 12.1 [Approved cipher matrix](#approved-cipher-matrix) - * 12.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipssyntheticapprovedciphers) - * 12.2 [Rejected cipher matrix](#rejected-cipher-matrix) - * 12.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers](#rqsrs-026clickhouseoperatorfipssyntheticrejectedciphers) -* 13 [CI/CD Image and Policy Verification](#cicd-image-and-policy-verification) - * 13.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild](#rqsrs-026clickhouseoperatorfipscicdoperatorimagebuild) - * 13.2 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild](#rqsrs-026clickhouseoperatorfipscicdexporterimagebuild) - * 13.3 [RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan](#rqsrs-026clickhouseoperatorfipscicdvulnerabilityscan) -* 14 [AI Static Code Review](#ai-static-code-review) - * 14.1 [Operator Source Review](#operator-source-review) - * 14.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree](#rqsrs-026clickhouseoperatorfipsaireviewoperatortree) - * 14.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg](#rqsrs-026clickhouseoperatorfipsaireviewoperatorsharedpkg) - * 14.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate](#rqsrs-026clickhouseoperatorfipsaireviewoperatorregressiongate) - * 14.2 [Exporter Source Review](#exporter-source-review) - * 14.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree](#rqsrs-026clickhouseoperatorfipsaireviewexportertree) - * 14.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg](#rqsrs-026clickhouseoperatorfipsaireviewexportersharedpkg) - * 14.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate](#rqsrs-026clickhouseoperatorfipsaireviewexporterregressiongate) -* 15 [ACVP Algorithm Validation](#acvp-algorithm-validation) - * 15.1 [Operator ACVP Validation](#operator-acvp-validation) - * 15.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration](#rqsrs026clickhouseoperatorfipsacvpoperatorwrapperintegration) - * 15.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration](#rqsrs026clickhouseoperatorfipsacvpoperatorconfiggeneration) - * 15.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay](#rqsrs026clickhouseoperatorfipsacvpoperatorexpectedoutputreplay) - * 15.1.4 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount](#rqsrs026clickhouseoperatorfipsacvpoperatorsuitecount) - * 15.2 [Exporter ACVP Validation](#exporter-acvp-validation) - * 15.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration](#rqsrs026clickhouseoperatorfipsacvpexporterwrapperintegration) - * 15.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration](#rqsrs026clickhouseoperatorfipsacvpexporterconfiggeneration) - * 15.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay](#rqsrs026clickhouseoperatorfipsacvpexporterexpectedoutputreplay) - * 15.2.4 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount](#rqsrs026clickhouseoperatorfipsacvpexportersuitecount) -* 16 [Terminology](#terminology) - * 16.1 [SRS](#srs) - * 16.2 [FIPS 140-3](#fips-140-3) - * 16.3 [clickhouse-operator](#clickhouse-operator) - * 16.4 [metrics-exporter](#metrics-exporter) - * 16.5 [CHI](#chi) - * 16.6 [CHK](#chk) - * 16.7 [ACVP](#acvp) - * 16.8 [CMVP](#cmvp) - * 16.9 [CAVP](#cavp) + * 11.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.CAST.ExporterFail](#rqsrs-026clickhouseoperatorfipscastexporterfail) +* 12 [ACVP Algorithm Validation](#acvp-algorithm-validation) + * 12.1 [Operator ACVP Validation](#operator-acvp-validation) + * 12.1.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration](#rqsrs-026clickhouseoperatorfipsacvpoperatorwrapperintegration) + * 12.1.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration](#rqsrs-026clickhouseoperatorfipsacvpoperatorconfiggeneration) + * 12.1.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT](#rqsrs-026clickhouseoperatorfipsacvpoperatorsha2256aft) + * 12.2 [Exporter ACVP Validation](#exporter-acvp-validation) + * 12.2.1 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration](#rqsrs-026clickhouseoperatorfipsacvpexporterwrapperintegration) + * 12.2.2 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration](#rqsrs-026clickhouseoperatorfipsacvpexporterconfiggeneration) + * 12.2.3 [RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT](#rqsrs-026clickhouseoperatorfipsacvpexportersha2256aft) +* 13 [Terminology](#terminology) + * 13.1 [SRS](#srs) + * 13.2 [FIPS 140-3](#fips-140-3) + * 13.3 [clickhouse-operator](#clickhouse-operator) + * 13.4 [metrics-exporter](#metrics-exporter) + * 13.5 [CHI](#chi) + * 13.6 [CHK](#chk) + * 13.7 [ACVP](#acvp) + * 13.8 [CMVP](#cmvp) + * 13.9 [CAVP](#cavp) ## Introduction @@ -1894,46 +973,32 @@ The goal is to verify that FIPS-enabled builds of the operator and metrics-exporter: - Operate correctly under FIPS constraints - Properly enforce cryptographic restrictions -- Use FIPS-compliant TLS for all inbound and outbound connections +- Use FIPS-compliant TLS for all outbound connections Autotests that trace to these requirements live in -[`tests/e2e/test_operator_fips.py`](../e2e/test_operator_fips.py) and +[`tests/e2e/test_operator.py`](../e2e/test_operator.py) and [`tests/e2e/test_acvp.py`](../e2e/test_acvp.py). **Boundary:** The operator and metrics-exporter run in the same pod. Internal IPC between them is localhost HTTP and is not subject to FIPS TLS requirements. The Prometheus metrics endpoints (operator `:9999` and metrics-exporter `:8888`) are also served over plain HTTP -and remain outside the FIPS TLS scope as a known gap. +and remain outside the FIPS TLS scope as a known gap. The ClickHouse Keeper readiness probe +endpoint (`:9182` `/ready`, which reflects Raft quorum status) likewise stays unconditionally +plaintext HTTP regardless of the secure/insecure knobs and is outside the FIPS TLS scope. ## Configuration Requirements Plain HTTP/TCP on any external connection is a configuration error for FIPS compliance. -TLS must be enabled for all connections to: -- Kubernetes API -- ClickHouse Server -- ZooKeeper/Keeper -- Prometheus scrape endpoints - -### RQ.SRS-026.ClickHouseOperator.FIPS.Config.ExternalTLS +### RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts version: 1.0 -Plain HTTP/TCP on external connections SHALL be treated as a configuration error for FIPS compliance. TLS SHALL be enabled for connections to the [Kubernetes API], [ClickHouse Server], [ZooKeeper/Keeper], and Prometheus scrape endpoints. +All external connections SHALL require TLS with FIPS-compliant settings, except for localhost IPC between the operator +and metrics-exporter and the Prometheus metrics endpoints `:9999` and `:8888`. ## Build Verification -**Objective:** Verify each shipped binary is a FIPS build and linked to Go Cryptographic Module v1.0.0. - -**Certificates:** -- [CMVP #5247](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/5247) -- [CAVP A6650](https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/details?product=19371) - -**Build requirement:** `GOFIPS140=v1.0.0` (or `certified`) - - -### Shipped Binaries - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries +### RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries version: 1.0 Each shipped pod binary — `clickhouse-operator` and `metrics-exporter` — SHALL satisfy all of the following: @@ -1962,217 +1027,239 @@ enabled: true ``` -#### RQ.SRS-026.ClickHouseOperator.FIPS.Build.ShippedBinaries.StartupLogs +### RQ.SRS-026.ClickHouseOperator.FIPS.OperatorBuild.ShippedBinaries.StartupLogs version: 1.0 -At startup, each binary SHALL emit a FIPS startup banner in logs indicating build and runtime FIPS state. +At startup, each binary SHALL emit a FIPS startup log line indicating build and runtime FIPS state. -when GODEBUG=fips140=only: +When `GODEBUG=fips140=only`: ```text -FIPS: chopconf.fips.enforced=true \ -build.linked=true \ -module.active=true \ -runtime.enforced=true \ -module=v1.0.0 +FIPS: chopconf.fips.enforced=true build.linked=true module.active=true runtime.enforced=true module=v1.0.0 ``` +## FIPS 140-3 TLS Cipher Suites +### Approved TLS Cipher Suites -## Approved TLS Cipher Suites - -### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers +#### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers version: 1.0 TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter] SHALL negotiate only TLS 1.3 with the following approved cipher suites. -| Cipher Suite | OpenSSL Name | -|--------------|--------------| -| TLS_AES_128_GCM_SHA256 | TLS_AES_128_GCM_SHA256 | -| TLS_AES_256_GCM_SHA384 | TLS_AES_256_GCM_SHA384 | -| TLS_AES_128_CCM_SHA256 | TLS_AES_128_CCM_SHA256 | -| TLS_AES_128_CCM_8_SHA256 | TLS_AES_128_CCM_8_SHA256 | +* TLS_AES_128_GCM_SHA256 +* TLS_AES_256_GCM_SHA384 + +Note: `TLS_CHACHA20_POLY1305_SHA256` is TLS v1.3 but not FIPS approved. ### Rejected Cipher Suites and Protocols #### RQ.SRS-026.ClickHouseOperator.FIPS.TLS.RejectedCiphers version: 1.0 -TLS connections SHALL reject the following for all TLS-enabled external connections: - -- Any TLS cipher suite not explicitly listed in [RQ.SRS-026.ClickHouseOperator.FIPS.TLS.ApprovedCiphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) -- Protocol versions: SSLv2, SSLv3, TLS 1.0, TLS 1.1 -- Cipher suites using non-approved/legacy algorithms (for this profile), including: - - ChaCha20-Poly1305 - - RC4, RC2, DES, 3DES, IDEA, SEED, CAMELLIA, ARIA - - NULL encryption / NULL authentication - - Anonymous key exchange (`aNULL`, `eNULL`, `ADH`, `AECDH`) - - Export/weak suites (`EXP`, `LOW`, `40-bit`, `56-bit`) - - MD5- or SHA-1-based legacy suites - - -## ClickHouse Server and Keeper FIPS Configurations - -**Objective:** Verify the operator generates and maintains FIPS-compliant configurations for ClickHouse servers and Keepers. - - -### ClickHouse Server - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.FIPSConfig -version: 1.0 - -Deploying a CHI with FIPS TLS settings SHALL start ClickHouse with FIPS-compliant TLS configuration. +On TLS-enforced external connections for [clickhouse-operator] and [metrics-exporter], any protocol version +older than TLS 1.3 and any cipher suite not listed in [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers) +SHALL be rejected by the operator in a FIPS-compliant configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHIDeploy -version: 1.0 -The operator SHALL deploy FIPS `ClickHouseInstallation` resources to `Completed` with Running pods when configuration is valid. +## ClickHouse Server -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainHTTP +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig version: 1.0 -When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain HTTP port 8123; HTTPS port 8443 SHALL be used. +Operator deploying a `ClickHouseInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse server and client. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoPlainNative -version: 1.0 +```yaml + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + layout: + shardsCount: 1 + replicasCount: 2 + zookeeper: + nodes: + - host: chk-test-030003-keeper-0-0 + port: 2281 + secure: "yes" + settings: + http_port: _removed_ + tcp_port: _removed_ + interserver_http_port: _removed_ + mysql_port: _removed_ + postgresql_port: _removed_ + https_port: 8443 + tcp_port_secure: 9440 + interserver_https_port: 9010 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + + none + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + +``` -When FIPS transport hardening applies, ClickHouse pods SHALL NOT listen on plain native TCP port 9000; secure native port 9440 SHALL be used. +The deployed ClickHouse server SHALL use only the following ports: -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.NoUnexpectedPorts -version: 1.0 +* HTTPS API port 8443 (instead of 8123) +* Secure native TCP port 9440 (instead of 9000) +* Interserver HTTPS port 9010 (instead of interserver HTTP port 9009) +* Backup sidecar HTTPS API port 7171 (instead of 7180), when backups are enabled -ClickHouse pods in a FIPS deployment SHALL expose only expected secure listener ports and no additional unexpected ports. +Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.InternodeTLS +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient version: 1.0 -ReplicatedMergeTree replicas SHALL communicate over interserver HTTPS (`interserver_https_port`) and data SHALL converge across replicas. +External clients connecting to the ClickHouse server SHALL be able to use any enabled TLS protocol version, including TLS 1.2. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleUp +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.Rescale version: 1.0 -Adding a replica to a FIPS-configured CHI SHALL reconcile to `Completed` and the new replica SHALL run the FIPS ClickHouse binary with TLS-only listeners. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ScaleDown -version: 1.0 +Adding or removing a replica from a FIPS-configured `ClickHouseInstallation` SHALL reconcile successfully and result in the expected number of running pods. -Removing a replica from a FIPS-configured CHI SHALL reconcile to `Completed` and remaining replicas SHALL keep FIPS binary and TLS-only configuration. +After rescaling, all replicas SHALL continue to run the FIPS ClickHouse binary and maintain the configured TLS-only OpenSSL settings. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.ConfigUpdate +#### RQ.SRS-026.ClickHouseOperator.FIPS.CH.ConfigUpdate version: 1.0 Updating TLS settings on a running CHI SHALL reload ClickHouse with the new FIPS-compliant configuration. -### ClickHouse Keeper - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.FIPSConfig -version: 1.0 - -Deploying a CHK with FIPS TLS settings SHALL start Keeper with FIPS-compliant TLS configuration. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHKDeploy -version: 1.0 - -The operator SHALL deploy FIPS `ClickHouseKeeperInstallation` resources to `Completed` with Running pods when configuration is valid. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoPlainClientPort -version: 1.0 - -When FIPS transport hardening applies, Keeper pods SHALL NOT listen on plain client port 2181; secure client port 2281 SHALL be used. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.NoUnexpectedPorts -version: 1.0 +## ClickHouse Keeper -Keeper pods in a FIPS deployment SHALL expose only expected secure listener ports. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.RaftTLS +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.FIPSConfig version: 1.0 -Keeper Raft communication SHALL use TLS on the configured secure Raft port. +Operator deploying a `ClickHouseKeeperInstallation` with FIPS TLS OpenSSL settings SHALL start a FIPS-compliant ClickHouse +Keeper server and client. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleUp -version: 1.0 +```yaml + configuration: + clusters: + - name: keeper + secure: "yes" + insecure: "no" + layout: + replicasCount: 2 + settings: + keeper_server/log_storage_path: /var/lib/clickhouse/coordination/log + keeper_server/snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + keeper_server/raft_configuration/server/port: 9444 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + + none + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + +``` -Adding a node to a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and the new node SHALL run the FIPS Keeper binary with TLS-only client and Raft listeners. +The deployed ClickHouse Keeper cluster SHALL use only the following ports: -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ScaleDown -version: 1.0 +* Secure client port 2281 (instead of 2181) +* Secure Raft communication port 9444 +* Plaintext HTTP readiness probe port 9182 (the `/ready` Raft-quorum health check) -Removing a node from a FIPS-configured Keeper cluster SHALL reconcile to `Completed` and remaining nodes SHALL keep FIPS configuration. +Every exposed port except the readiness probe port 9182 and Raft replication port 9444 (which enforces peer-only authentication) +SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. Port 9182 SHALL stay +unconditionally plaintext HTTP regardless of the secure/insecure configuration (see Boundary). -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CHK.ConfigUpdate +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.Rescale version: 1.0 -Updating TLS settings on a running CHK SHALL reload Keeper with the new FIPS-compliant configuration. - +Adding or removing a node from a FIPS-configured `ClickHouseKeeperInstallation` SHALL reconcile successfully and result +in the expected number of running pods. -### ClickHouse Backup Sidecar +After rescaling, all Keeper nodes SHALL continue to run the FIPS ClickHouse Keeper binary and maintain the configured +TLS-only OpenSSL settings. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.CH.VersionString +#### RQ.SRS-026.ClickHouseOperator.FIPS.CHK.ConfigUpdate version: 1.0 -A running ClickHouse host under FIPS image policy SHALL report a `version()` string containing `fips` (case-insensitive). +Updating TLS settings on a running CHK SHALL reload ClickHouse with the new FIPS-compliant configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.FIPSBinary -version: 1.0 -The `clickhouse-backup` sidecar SHALL run a FIPS-built binary; `clickhouse-backup --version` SHALL contain `fips` (case-insensitive). +## ClickHouse Backup Sidecar -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.GOFIPS140 +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSBinary version: 1.0 -When inspectable, the clickhouse-backup sidecar binary SHALL embed `GOFIPS140=v1.0.0` per `go version -m`. +The `clickhouse-backup` sidecar SHALL run a FIPS-built binary. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.OnlyTLSPorts -version: 1.0 +The sidecar binary SHALL satisfy all of the following: -The clickhouse-backup sidecar SHALL expose only secure listener ports (including HTTPS API port 7171). +* `clickhouse-backup --version` contains `fips` +* When inspectable, `go version -m` reports `GOFIPS140=v1.0.0` -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.HTTPSAPI +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.FIPSConfig version: 1.0 -The clickhouse-backup HTTPS API SHALL serve over TLS with CA-trust enforcement: trusted clients accepted, untrusted clients rejected. +Deploying a `ClickHouseInstallation` with a FIPS-configured backup sidecar SHALL start `clickhouse-backup` with a FIPS-compliant TLS configuration. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.ClickHouseOverTLS -version: 1.0 +The deployed backup sidecar SHALL only add the following listener ports to the ClickHouse container: -The clickhouse-backup sidecar SHALL reach ClickHouse over secure native TCP. +* HTTPS API port 7171 (instead of 7180) -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RestoreRoundTrip -version: 1.0 +The FIPS-configured backup sidecar SHALL additionally satisfy all of the following: -Backup and restore through the HTTPS API SHALL succeed over TLS. +* Each exposed port SHALL support TLS communication using only FIPS-compliant protocol versions and cipher suites. +* The `clickhouse-backup` sidecar SHALL connect to ClickHouse using secure native TCP with TLS enabled. -#### RQ.SRS-026.ClickHouseOperator.FIPS.DataPlane.Backup.RemoteUploadTLS +#### RQ.SRS-026.ClickHouseOperator.FIPS.Backup.RestoreRoundTrip version: 1.0 -Remote backup upload to object storage SHALL use FIPS-approved TLS. - +Creating a backup and restoring it through the HTTPS API SHALL succeed over TLS. ## FIPS Enforcement Mode -**Objective:** Verify `security.fips.enforced=true` coerces security settings and rejects non-compliant configurations. - +**Objective:** Verify that `security.fips.enforced: "true"` coerces relaxed security settings and rejects non-compliant CHI/CHK specifications and non-FIPS images. ### Security Coercion -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceVerifyStrict +#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.SecurityCoercion version: 1.0 -With `fips.enforced=true`, unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients. +When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce unset or relaxed security settings as follows: -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceMinVersion13 -version: 1.0 - -With `fips.enforced=true`, unset TLS minVersion SHALL be coerced to 1.3 for the -operator's outbound TLS clients. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.OverrideMinVersion12To13 -version: 1.0 +* Unset TLS verify SHALL be coerced to Strict for ClickHouse, ZooKeeper/Keeper, and Kubernetes clients. +* Unset TLS `minVersion` SHALL be coerced to `"1.3"` for the operator's outbound TLS clients (`security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`). +* Explicit `minVersion: "1.2"` for those TLS clients SHALL be coerced to `"1.3"`. +* Unset IPC mode SHALL be coerced to Secure. -When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL coerce `minVersion` to `"1.3"` for `security.clickhouse.tls`, `security.zookeeper.tls`, and `security.kubernetes.tls`, even when those fields are explicitly set to `"1.2"`. +Example configuration with explicit `minVersion: "1.2"`: ```yaml spec: @@ -2190,230 +1277,87 @@ minVersion: "1.2" ``` -After operator configuration normalization, the effective `minVersion` for each component listed above SHALL be `"1.3"`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.CoerceIPCSecure -version: 1.0 - -With `fips.enforced=true`, unset IPC mode SHALL be coerced to Secure. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInsecureKubeconfig -version: 1.0 - -The operator SHALL refuse to start when kubeconfig uses `TLSClientConfig.Insecure=true` under strict/FIPS mode. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneCHI -version: 1.0 - -CHI with `clickhouse.tls.verify=None` under enforced mode SHALL be rejected with `FIPSValidationFailed`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectVerifyNoneZK -version: 1.0 - -CHI with `zookeeper.tls.verify=None` under enforced mode SHALL be rejected. +After operator configuration normalization, the effective `minVersion` for each TLS client listed above SHALL be `"1.3"`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectInvalidMinVersion +#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectNonCompliantSpecs version: 1.0 -CHI with invalid TLS minVersion under enforced mode SHALL be rejected. +When `security.fips.enforced: "true"` is set in the [ClickHouseOperatorConfiguration], the operator SHALL reject +non-compliant CHI and CHK specifications with `FIPSValidationFailed` and SHALL NOT create workload StatefulSets for: -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectExternalZookeeper -version: 1.0 - -CHI referencing plain external ZooKeeper nodes under enforced mode SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.RejectCHKBypass -version: 1.0 - -CHK with TLS verify bypass under enforced mode SHALL be rejected. +* CHI referencing plain external ZooKeeper nodes, including when `secure` is explicitly set to `"false"`. +* CHI with `clickhouse.tls.verify=None` at spec or cluster level. +* CHI with `zookeeper.tls.verify=None`. +* CHI with invalid `clickhouse.tls.minVersion`. +* CHK with TLS verify bypass at spec level. #### RQ.SRS-026.ClickHouseOperator.FIPS.Enforced.MinVersionScope version: 1.0 The `minVersion` coercion SHALL apply only to TLS clients created and managed by the operator. -They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2. +They SHALL NOT require ClickHouse Server or ClickHouse Keeper listener endpoints to reject TLS 1.2 +(see [RQ.SRS-026.ClickHouseOperator.FIPS.CH.FIPSConfig.ExternalClient](#rqsrs-026clickhouseoperatorfipschfipsconfigexternalclient)). ### Image Policy -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHI -version: 1.0 - -With `security.fips.images.policy=Required`, CHI with non-FIPS image tag SHALL be rejected with `FIPSImagePolicyViolation`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.AcceptCHI -version: 1.0 - -With image policy Required, CHI with FIPS-tagged image SHALL reconcile normally. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectCHK -version: 1.0 - -With image policy Required, CHK with non-FIPS Keeper image SHALL be rejected. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RuntimeVersion -version: 1.0 - -With image policy Required, host `SELECT version()` lacking `fips` SHALL fail with `FIPSImagePolicyViolation`. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Permissive -version: 1.0 - -With permissive image policy, non-FIPS CHI images SHALL reconcile (default). - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.ShortCircuit -version: 1.0 - -Multiple non-FIPS hosts SHALL produce a single policy violation error. - - -### Image Tag Detection - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.FIPSSuffix -version: 1.0 - -Image tags containing `fips` (case-insensitive) SHALL be detected as FIPS. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.AltinityFIPS -version: 1.0 - -Image tags containing `altinityfips` SHALL be detected as FIPS. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.DigestOnly -version: 1.0 - -Digest-only image references SHALL NOT be detected as FIPS at admission. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.RegistryPath -version: 1.0 - -Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.TagDetection.CaseInsensitive -version: 1.0 - -Image tags such as `25.3.FIPS` or `25.3.Fips` SHALL be detected as FIPS (case-insensitive match on the tag). - - -## Operator External Connections - -**Objective:** Verify all **clickhouse-operator** inbound and outbound connections use FIPS-compliant TLS. - - -### Operator Runtime Listener Verification - -In a FIPS deployment, workload containers deployed by the operator (ClickHouse, Keeper, and sidecar containers) SHALL expose only expected TLS listener ports. Verification reads `/proc/net/tcp` and `/proc/net/tcp6` inside each container and parses ports in LISTEN state (`0A`): - -```bash -kubectl exec -c clickhouse -- sh -c 'cat /proc/net/tcp /proc/net/tcp6' -``` - -E2e coverage: [`test_020011`](../e2e/test_operator_fips.py#L200). - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Listeners -version: 1.0 - -FIPS workload pods (ClickHouse, Keeper, and sidecar containers) SHALL listen only on expected TLS ports. Plaintext service ports (8123, 9000, 2181) SHALL NOT be open when FIPS transport hardening applies. - - -### Operator to Kubernetes API - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Kubernetes -version: 1.0 - -The operator SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers. - - -### Operator to ClickHouse Server - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse -version: 1.0 - -The operator SHALL connect to ClickHouse using FIPS-approved TLS ciphers. - - -### Operator to ZooKeeper/Keeper - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.Zookeeper +#### RQ.SRS-026.ClickHouseOperator.FIPS.Images.Required.RejectNonFIPS version: 1.0 -The operator SHALL connect to ZooKeeper/Keeper using FIPS-approved TLS ciphers. +With `security.fips.images.policy=Required`, non-FIPS images SHALL be rejected with `FIPSImagePolicyViolation` as follows: +* CHI with non-FIPS ClickHouse image tag SHALL be rejected at admission. +* CHK with non-FIPS Keeper image tag SHALL be rejected at admission. +* CHI with non-FIPS `clickhouse-backup` sidecar image tag SHALL be rejected at admission. +* CHI with multiple non-FIPS hosts SHALL produce a single policy violation error. +* Digest-only image references SHALL NOT be detected as FIPS at admission. +* Registry hostname containing `fips` SHALL NOT satisfy FIPS tag detection. +* CHI admitted with a FIPS-tagged ClickHouse image whose running binary lacks `fips` in `SELECT version()` SHALL fail at runtime. -### Operator to metrics-exporter IPC +## Runtime Connection Evidence -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.IPCSecure +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KubernetesAPI version: 1.0 -Operator IPC with `security.ipc.mode=Secure` SHALL work over localhost HTTP with token auth. +The clickhouse-operator SHALL access the Kubernetes API through the HTTPS endpoint on port `443`. +Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected. - -### Operator Prometheus Metrics - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Gap.OperatorMetricsTLS -version: 1.0 - -Operator Prometheus metrics on :9999 currently expose a known FIPS gap (HTTP-only). - - -## Exporter External Connections - -**Objective:** Verify all **metrics-exporter** inbound and outbound connections use FIPS-compliant TLS. - - -### Exporter Runtime Listener Verification - -Listener audits use the same `/proc/net/tcp` technique as [Operator Runtime Listener Verification](#operator-runtime-listener-verification). E2e audits the **clickhouse-backup** sidecar in [`test_020011`](../e2e/test_operator_fips.py#L200). The **metrics-exporter** process on `:8888` remains a known gap until metrics TLS is implemented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Listeners +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.KubernetesAPI version: 1.0 -The metrics-exporter process SHALL expose only expected listener ports on `:8888`. Sidecar containers in the same pod SHALL be listener-audited with the same `/proc/net/tcp` procedure. - +The metrics-exporter SHALL access the Kubernetes API through the HTTPS endpoint on port `443`. +Plain HTTP requests to the Kubernetes API endpoint SHALL be rejected. -### Exporter to Kubernetes API - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.Kubernetes +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.ClickHouse version: 1.0 -The exporter SHALL connect to the Kubernetes API using FIPS-approved TLS ciphers. - +The clickhouse-operator SHALL communicate with ClickHouse hosts using HTTPS port `8443`. -### Exporter to ClickHouse Server - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Exporter.ClickHouse version: 1.0 -The exporter SHALL query ClickHouse using FIPS-approved TLS when configured for HTTPS. - +The metrics-exporter SHALL discover ClickHouse hosts using the HTTPS endpoint `8443`. -### Exporter Prometheus Metrics - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Gap.ExporterMetricsTLS +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.Operator.KeeperRestriction version: 1.0 -Exporter Prometheus metrics on :8888 currently expose a known FIPS gap (HTTP-only). - - -## Integrity Check Failure - -**Objective:** Verify FIPS integrity self-test detects binary tampering for each shipped binary independently. +When a Keeper ensemble is configured as TLS-only, the clickhouse-operator SHALL NOT attempt plaintext ZooKeeper/Keeper +operations against it. - -### Operator Integrity Tampering - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.OperatorMismatch +### RQ.SRS-026.ClickHouseOperator.FIPS.Connect.ClickHouse.KeeperTLS version: 1.0 -Tampering with `clickhouse-operator` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`. +ClickHouse replicas SHALL connect to Keeper using secure client port `2281` with `secure=yes`. -### Exporter Integrity Tampering +## Integrity Check -#### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.ExporterMismatch +### RQ.SRS-026.ClickHouseOperator.FIPS.Integrity.VerificationMismatch version: 1.0 -Tampering with `metrics-exporter` `.go.fipsinfo` SHALL panic with `fips140: verification mismatch`. - +Each shipped FIPS binary — `clickhouse-operator` and `metrics-exporter` — SHALL perform a software integrity +self-test at initialization by verifying its embedded HMAC. If the binary is tampered with or corrupted such that +the HMAC verification fails, the process SHALL immediately terminate with a `fips140: verification mismatch` panic +to prevent the execution of a compromised cryptographic module. ## CAST Failure @@ -2436,164 +1380,46 @@ Running `metrics-exporter` with `GODEBUG=failfipscast=` SHALL terminate with a CAST error. -## Synthetic TLS Cipher Validation - -**Objective:** Validate FIPS cipher enforcement on all external (to the pod) connections using `openssl s_client` and `openssl s_server`. - -Use `openssl` to simulate connections with specific ciphers and verify the operator/exporter accepts FIPS-approved ciphers and rejects non-approved ones. - -```bash -# Operator as TLS client against server offering only approved cipher -openssl s_server -accept 8443 -cert server.crt -key server.key \ - -ciphersuites TLS_AES_256_GCM_SHA384 - -# Operator as TLS client against server offering non-approved cipher -openssl s_server -accept 8443 -cert server.crt -key server.key \ - -cipher ECDHE-RSA-CHACHA20-POLY1305 - -# Inbound connection to operator/exporter metrics endpoint -openssl s_client -connect localhost:9999 -cipher ECDHE-RSA-AES256-GCM-SHA384 -``` - -### Approved cipher matrix - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.ApprovedCiphers -version: 1.0 - -For each external connection listed below, when exercised as a TLS **client** with `openssl s_server` offering only [approved ciphers](#rqsrs-026clickhouseoperatorfipstlsapprovedciphers), or as a TLS **server** with `openssl s_client` using only approved ciphers, the connection SHALL succeed: - -| Connection | Role | Tool | -|------------|------|------| -| Operator to Kubernetes API | Client | `openssl s_server` | -| Operator to ClickHouse Server | Client | `openssl s_server` | -| Operator to ZooKeeper/Keeper | Client | `openssl s_server` | -| Operator metrics :9999 | Server | `openssl s_client` | -| Exporter to Kubernetes API | Client | `openssl s_server` | -| Exporter to ClickHouse Server | Client | `openssl s_server` | -| Exporter metrics :8888 | Server | `openssl s_client` | - - -### Rejected cipher matrix - -#### RQ.SRS-026.ClickHouseOperator.FIPS.Synthetic.RejectedCiphers -version: 1.0 - -For each external connection listed below, when the peer offers only [rejected ciphers or protocols](#rqsrs-026clickhouseoperatorfipstlsrejectedciphers), the connection SHALL be rejected: - -| Connection | Role | Tool | -|------------|------|------| -| Operator to Kubernetes API | Client | `openssl s_server` | -| Operator to ClickHouse Server | Client | `openssl s_server` | -| Operator to ZooKeeper/Keeper | Client | `openssl s_server` | -| Operator metrics :9999 | Server | `openssl s_client` | -| Exporter to Kubernetes API | Client | `openssl s_server` | -| Exporter to ClickHouse Server | Client | `openssl s_server` | -| Exporter metrics :8888 | Server | `openssl s_client` | - - -## CI/CD Image and Policy Verification - -**Objective:** Add CI/CD jobs to validate FIPS image build and supply-chain checks. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.OperatorImageBuild -version: 1.0 - -CI SHALL build the [clickhouse-operator] FIPS image successfully. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.ExporterImageBuild -version: 1.0 - -CI SHALL build the [metrics-exporter] FIPS image successfully. - -### RQ.SRS-026.ClickHouseOperator.FIPS.CICD.VulnerabilityScan -version: 1.0 - -FIPS images SHALL pass vulnerability scanning with no Critical, High, or Medium findings. - - -### Operator Source Review - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.Tree -version: 1.0 - -Static review of operator-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.SharedPkg -version: 1.0 - -Review of shared packages reachable from `cmd/operator` SHALL produce no Critical findings. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Operator.RegressionGate -version: 1.0 - -A signed-off review artifact SHALL be stored with the build record before release. - -### Exporter Source Review - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.Tree -version: 1.0 - -Static review of exporter-scoped paths SHALL produce no Critical findings; Warning-level findings SHALL be documented. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.SharedPkg -version: 1.0 - -Review of shared packages reachable from `cmd/metrics_exporter` SHALL produce no Critical findings. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.AIReview.Exporter.RegressionGate -version: 1.0 - -A signed-off review artifact SHALL be stored with the build record before release. - ## ACVP Algorithm Validation -**Objective:** Reproduce ACVP expected-output checks for each FIPS binary using the tracked public-scope config in [`pkg/util/fips/acvp/`](../../../pkg/util/fips/acvp/). +**Objective:** Verify that each FIPS binary can be built with the ACVP wrapper enabled and that the embedded ACVP responder works through the modulewrapper stdin/stdout protocol. +These requirements cover the e2e ACVP smoke tests only. They do not claim full ACVP expected-output replay or suite-count validation from `pkg/util/fips/acvp/run.sh`. ### Operator ACVP Validation #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.WrapperIntegration version: 1.0 -Building clickhouse-operator with `-tags acvp_wrapper` SHALL expose a working ACVP responder via argv0 dispatch. +Building `clickhouse-operator` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `clickhouse-operator-acvp`. #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ConfigGeneration version: 1.0 -The clickhouse-operator ACVP responder SHALL answer `getConfig` with supported capabilities. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.ExpectedOutputReplay -version: 1.0 - -`bash pkg/util/fips/acvp/run.sh` SHALL match all configured expected outputs for the operator. +The `clickhouse-operator` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SuiteCount +#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Operator.SHA2256AFT version: 1.0 -The tracked ACVP config SHALL report 38 matched expectations for clickhouse-operator. - +The `clickhouse-operator` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`. ### Exporter ACVP Validation #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.WrapperIntegration version: 1.0 -Building metrics-exporter with `-tags acvp_wrapper` SHALL expose a working ACVP responder. +Building `metrics-exporter` with `-tags acvp_wrapper` SHALL produce a binary whose ACVP responder is reachable through argv0 dispatch when executed as `metrics-exporter-acvp`. #### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ConfigGeneration version: 1.0 -The metrics-exporter ACVP responder SHALL answer `getConfig` with supported capabilities. +The `metrics-exporter` ACVP responder SHALL answer a `getConfig` request successfully. The returned payload SHALL be valid JSON, SHALL advertise `SHA2-256` and `ACVP-AES-GCM`, and SHALL NOT advertise `ML-KEM` or `ML-DSA`. -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.ExpectedOutputReplay +#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SHA2256AFT version: 1.0 -`BINARY=metrics-exporter bash pkg/util/fips/acvp/run.sh` SHALL match all expected outputs. - -#### RQ.SRS-026.ClickHouseOperator.FIPS.ACVP.Exporter.SuiteCount -version: 1.0 +The `metrics-exporter` ACVP responder SHALL answer a `SHA2-256` algorithm functional test request for input `abc` with the digest matching `hashlib.sha256(b"abc").digest()`. -The tracked ACVP config SHALL report 38 matched expectations for metrics-exporter. ## Terminology diff --git a/tests/requirements/fips_test_plan.md b/tests/requirements/fips_test_plan.md new file mode 100644 index 000000000..a459c85ff --- /dev/null +++ b/tests/requirements/fips_test_plan.md @@ -0,0 +1,516 @@ +# QA-STP FIPS 140-3 Compatibility +# Software Test Plan + +(c) 2026 Altinity Inc. All Rights Reserved. + +**Author:** vzakaznikov + +**Date:** May 19, 2026 + +## Table of Contents + +* 1 [Introduction](#introduction) +* 2 [Configuration Requirements](#configuration-requirements) +* 3 [Build Verification](#build-verification) +* 4 [GODEBUG Strict Mode Smoke Test](#godebug-strict-mode-smoke-test) +* 5 [FIPS 140-3 Valid TLS Cipher Suites](#fips-140-3-valid-tls-cipher-suites) +* 6 [ClickHouse Server and Keeper FIPS Configurations](#clickhouse-server-and-keeper-fips-configurations) +* 7 [FIPS Enforcement Mode](#fips-enforcement-mode) +* 8 [clickhouse-operator Connections](#clickhouse-operator-connections) +* 9 [metrics-exporter Connections](#metrics-exporter-connections) +* 10 [clickhouse-backup Sidecar](#clickhouse-backup-sidecar) +* 11 [Integrity Check Failure](#integrity-check-failure) +* 12 [CAST Failure](#cast-failure) +* 13 [Synthetic TLS Cipher Validation](#synthetic-tls-cipher-validation) +* 14 [CI/CD Image and Policy Verification](#cicd-image-and-policy-verification) +* 15 [(Optional) ACVP Algorithm Validation](#optional-acvp-algorithm-validation) +## Introduction + +This test plan covers FIPS 140-3 compatibility testing for the +**clickhouse-operator**, **metrics-exporter**, and **clickhouse-backup** +components used within ClickHouse deployments. + +The goal is to verify that FIPS-enabled components: + +- Operate correctly under FIPS constraints +- Properly enforce cryptographic restrictions +- Use FIPS-compliant TLS for all inbound and outbound connections + +**Boundary:** The operator and metrics-exporter run in the same pod. Internal IPC between +them is localhost HTTP and is not subject to FIPS TLS requirements. The Prometheus metrics +endpoints (operator `:9999` and metrics-exporter `:8888`) are also served over plain HTTP +and remain outside the FIPS TLS scope as a known gap. The ClickHouse Keeper readiness probe +endpoint (`:9182` `/ready`, which reflects Raft quorum status) likewise stays unconditionally +plaintext HTTP regardless of the secure/insecure knobs and is outside the FIPS TLS scope. + +```mermaid +flowchart LR + + subgraph operator_pod["clickhouse-operator Pod"] + op["clickhouse-operator"] + me["metrics-exporter"] + + op <-->|"HTTP localhost + IPC token"| me + end + + k8s["Kubernetes API"] + prom["Prometheus"] + ext["External ClickHouse client"] + + subgraph ch_cluster["ClickHouse cluster"] + ch0["CHI pod 0"] + ch1["CHI pod 1"] + backup["clickhouse-backup"] + + ch0 <-->|"interserver HTTPS :9010"| ch1 + + backup -->|"HTTPS :8443 / native TLS :9440"| ch0 + end + + subgraph keeper["ClickHouse Keeper cluster"] + k0["Keeper-0"] + k1["Keeper-1"] + + k0 <-->|"Raft :9444"| k1 + end + + %% Kubernetes + op -->|"HTTPS :443 / client-go"| k8s + me -->|"HTTPS :443 / in-cluster SA"| k8s + + %% ClickHouse cluster + op -->|"HTTPS :8443"| ch_cluster + me -->|"HTTPS :8443"| ch_cluster + + %% External access + ext -->|"native TLS :9440"| ch_cluster + ext -->|"HTTPS :7171"| backup + + %% Keeper + ch_cluster -->|"TLS :2281"| keeper + + op -.->|"Skips plaintext ZK root-path helper\nwhen Keeper is TLS-only"| keeper + + %% Monitoring + prom -->|"HTTP :9999"| op + prom -->|"HTTP :8888"| me +``` + +## Configuration Requirements + +Plain HTTP/TCP on any external connection is a configuration error for FIPS compliance. +TLS must be enabled for all connections to: + +- Kubernetes API +- ClickHouse Server +- ZooKeeper/Keeper + +*Note:* Prometheus scrape endpoints (:9999 and :8888) remain outside FIPS TLS scope as a documented boundary gap. + +## Build Verification + +**Objective:** Verify binaries are FIPS builds and linked to Go Cryptographic Module v1.0.0. + +**Certificates:** +- [CMVP #5247](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/5247) +- [CAVP A6650](https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/details?product=19371) + +**Build requirement:** `GOFIPS140=v1.0.0` (or `certified`) + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator version | Run `clickhouse-operator --version` or check logs | Output includes FIPS indicator | +| Metrics exporter version | Run `metrics-exporter --version` or check logs | Output includes FIPS indicator | +| Build flag | Run `go version -m ` | Shows `GOFIPS140=v1.0.0` | +| FIPS version | Check `crypto/fips140.Version()` | Returns `v1.0.0` | +| FIPS enabled | Check `crypto/fips140.Enabled()` | Returns `true` | + +## GODEBUG Strict Mode Smoke Test + +**Objective:** Verify the project test suite runs in strict FIPS mode. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Strict mode smoke test | Run all e2e tests with `GODEBUG=fips140=only` enabled | No panic/crash and no test regressions caused by strict FIPS mode | + +## FIPS 140-3 Valid TLS Cipher Suites + +**Objective:** Verify the FIPS TLS profile used by operator-managed clients and FIPS listener probes. + +The approved TLS 1.3 cipher suites for this test plan are: + +| Cipher Suite | OpenSSL Name | +|--------------|--------------| +| TLS_AES_128_GCM_SHA256 | TLS_AES_128_GCM_SHA256 | +| TLS_AES_256_GCM_SHA384 | TLS_AES_256_GCM_SHA384 | + +### Scope + +This section has two separate scopes: + +1. **Operator-managed clients** + When `security.fips.enforced=true`, operator-managed TLS clients are coerced to: + - `verify=Strict` + - `minVersion=1.3` + + This applies to operator/client configuration for: + - Kubernetes API + - ClickHouse + - ZooKeeper/Keeper + +2. **Server listener probes** + The e2e listener probes verify that FIPS-configured listeners accept approved TLS 1.3 AES-GCM traffic and reject selected disallowed protocol/cipher combinations. + + Covered listeners: + - ClickHouse HTTPS `8443` + - ClickHouse native TLS `9440` + - ClickHouse interserver HTTPS `9010` + - Keeper secure client port `2281` + - clickhouse-backup HTTPS API `7171` + +### Positive TLS listener checks + +| Endpoint | Positive check | Expected Result | +|----------|----------------|-----------------| +| ClickHouse HTTPS `8443` | OpenSSL TLS 1.3 with `TLS_AES_128_GCM_SHA256` | Cipher negotiates successfully | +| ClickHouse native TLS `9440` | Secure native ClickHouse query | Query succeeds over TLS | +| ClickHouse interserver HTTPS `9010` | OpenSSL TLS 1.3 with `TLS_AES_128_GCM_SHA256` | Cipher negotiates successfully | +| Keeper secure client `2281` | OpenSSL TLS 1.3 with `TLS_AES_128_GCM_SHA256` | Cipher negotiates successfully | +| Backup HTTPS API `7171` | curl TLS 1.3 with `TLS_AES_128_GCM_SHA256` | HTTPS request succeeds | + +### Negative TLS listener checks + +| Rejected Case | Covered Endpoints | Expected Result | +|---------------|-------------------|-----------------| +| TLS 1.3 `TLS_CHACHA20_POLY1305_SHA256` | `8443`, `9440`, `9010`, `2281`, `7171` | TLS handshake fails | +| TLS 1.1 protocol | `8443`, `9440`, `9010`, `2281`, `7171` | TLS handshake fails | + +### Important TLS 1.2 boundary + +Do **not** treat TLS 1.2 as globally rejected on all ClickHouse, Keeper, or backup listener endpoints. + +ClickHouse and Keeper OpenSSL server configuration disables: + +```text +sslv2, sslv3, tlsv1, tlsv1_1 +``` +Operator does not disable TLS 1.2 by default. +Therefore: + +* operator-managed clients must be coerced to TLS 1.3 under FIPS enforcement; +* listener probes must reject TLS 1.1 and non-approved TLS 1.3 cipher suites; +* external ClickHouse clients may still use TLS 1.2 when the server OpenSSL configuration enables it. + + +## ClickHouse Server and Keeper FIPS Configurations + +**Objective:** Verify operator generates and maintains FIPS-compliant configurations for ClickHouse servers and Keepers. + +**ClickHouse Server:** + +| Test Assertion | Description | Expected Result | +|----------------|-----------------------------------------------------------|-------------------------------------------| +| FIPS config applied | Deploy CHI with FIPS TLS settings | ClickHouse starts with FIPS-compliant TLS | +| No plain HTTP port | Verify HTTP port (8123) disabled when FIPS enforced | Only HTTPS port (8443) listening | +| No plain TCP port | Verify native TCP port (9000) disabled when FIPS enforced | Only secure TCP port (9440) listening | +| No unexpected ports | Verify no other inbound/outbound ports opened | Only expected secure ports listening | +| Internode TLS | Verify native interserver_https_port port (9009) disabled | Replicas communicate over TLS port (9010) | +| Scale up | Add replica to FIPS-configured cluster | New replica has FIPS config | +| Scale down | Remove replica from FIPS-configured cluster | Remaining replicas keep FIPS config | +| Config update | Update TLS settings on running CHI | ClickHouse reloads with new FIPS config | + +**ClickHouse Keeper:** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|----------------------------------------------------------------| +| FIPS config applied | Deploy CHK with FIPS TLS settings | Keeper starts with FIPS-compliant TLS | +| No plain client port | Verify client port (2181) disabled when FIPS enforced | Only secure client port (2281) listening | +| No unexpected ports | Verify no other inbound/outbound ports opened | Only expected secure ports listening | +| Raft peer port | Verify Keeper Raft port `9444` is configured and listening | Port `9444` is present as a Keeper Raft peer port; generic client TLS probing is not required and is not part of this test scope | +| /ready endpoint | CHK readiness probe works on plain HTTP port (9182) | /ready returns 200 OK regardless of FIPS config | +| Scale up | Add node to FIPS-configured Keeper cluster | New node has FIPS config | +| Scale down | Remove node from FIPS-configured Keeper cluster | Remaining nodes keep FIPS config | +| Config update | Update TLS settings on running CHK | Keeper reloads with new FIPS config | + +## clickhouse-backup Sidecar + +**Objective:** Verify clickhouse-backup sidecar operates correctly in FIPS mode and uses FIPS-compliant TLS for ClickHouse backup and restore operations. + +**Connection Overview:** + +| Direction | Target | Protocol | Default Port | TLS Support | +| --------- | ------------------------------------ | ---------------- | ------------ | ------------------------------ | +| Outbound | ClickHouse Server | HTTPS/native TLS | 8443/9440 | Yes, via ClickHouse TLS config | +| Inbound | Backup API | HTTPS | 7171 | Yes | +| Storage | Local mounted ClickHouse data volume | filesystem | N/A | N/A | + +| Test Assertion | Description | Expected Result | +|------------------------------|-----------------------------------------------------------------------------| ----------------------------------------------- | +| Backup FIPS binary | Run `clickhouse-backup --version` in sidecar | Output contains `fips` | +| Backup GOFIPS140 module | Run `go version -m` against `clickhouse-backup` binary | Output contains `GOFIPS140=v1.0.0` | +| Backup sidecar starts | Deploy CHI with FIPS ClickHouse image and FIPS clickhouse-backup sidecar | Backup sidecar starts successfully +| Backup only expected ports | Inspect listening ports in backup sidecar | Only expected secure ports (`8443`, `9440`, `9010`, `7171`) are exposed in the shared network namespace | +| Backup API HTTPS | Connect to backup API on `7171` with trusted CA | HTTPS connection succeeds | +| Backup API rejects plaintext | Send plain HTTP request to backup API port `7171` | Request is rejected or TLS handshake fails | +| Backup to ClickHouse TLS | Create backup using ClickHouse secure endpoint | Backup completes over TLS | +| Restore to ClickHouse TLS | Restore backup using ClickHouse secure endpoint | Restore completes over TLS | +| Backup round trip | Create table, insert data, create backup, drop data, restore backup | Restored data matches original data | +| TLS 1.3 approved cipher | Connect backup API using approved TLS 1.3 AES-GCM cipher | Connection succeeds | +| Non-approved TLS rejected | Try TLS 1.2 or non-approved cipher against backup API | Connection is rejected | + +## FIPS Enforcement Mode + +**Objective:** Verify `security.fips.enforced=true` coerces security settings and rejects non-compliant configurations. + +**Security Coercion (`security.fips.enforced=true`):** + +| Test Assertion | Description | Expected Result (Observable Outcome) | +|----------------|-------------|-----------------| +| **Coerce ClickHouse verify to Strict** | Deploy Chopconf with `fips.enforced=true` and `security.clickhouse.tls.verify=None` | Log: `FIPS strict: coerced security.clickhouse.tls.verify: None → Strict` | +| **Coerce ZooKeeper/Keeper verify to Strict** | Deploy Chopconf with `fips.enforced=true` and `security.zookeeper.tls.verify=None` | Log: `FIPS strict: coerced security.zookeeper.tls.verify: None → Strict` | +| **Coerce Kubernetes verify to Strict** | Deploy Chopconf with `fips.enforced=true` and `security.kubernetes.tls.verify=None` or relaxed Kubernetes TLS verification | Log: `FIPS strict: coerced security.kubernetes.tls.verify: None → Strict` | +| **Coerce TLS minVersion to 1.3** | Deploy Chopconf with `fips.enforced=true` and ClickHouse, ZooKeeper/Keeper, and Kubernetes TLS `minVersion=1.2` | Logs show each client coerced to `minVersion: 1.2 → 1.3` | +| **Coerce IPC mode to Secure** | Deploy Chopconf with `fips.enforced=true` and `ipc.mode=Plain` | Log: `FIPS strict: coerced security.ipc.mode: Plain → Secure` | +| **Reject verify=None (CHI)** | Apply CHI with `clickhouse.tls.verify=None` under enforced mode | `chi.status.status` = **Aborted**; `chi.status.errors` contains `FIPSValidationFailed` | +| **Reject ZK verify=None (CHI)** | Apply CHI with `zookeeper.tls.verify=None` under enforced mode | `chi.status.status` = **Aborted**; `chi.status.errors` contains `FIPSValidationFailed` | +| **Reject invalid minVersion** | Apply CHI with `minVersion: "1.1"` under enforced mode | `chi.status.status` = **Aborted**; `chi.status.errors` contains `FIPSValidationFailed` | +| **Reject external ZooKeeper** | Apply CHI referencing plain ZK nodes under enforced mode | `chi.status.status` = **Aborted**; `chi.status.errors` contains `FIPSValidationFailed` | +| **Reject CHK TLS bypass** | Apply CHK with spec-level `verify: None` under enforced mode | `chk.status.status` = **Aborted**; `chk.status.errors` contains `FIPSValidationFailed` | + +**Image Policy (`security.fips.images.policy`):** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Required + non-fips image | CHI with ClickHouse image lacking "fips" tag | CHI rejected with FIPSImagePolicyViolation | +| Required + fips image | CHI with ClickHouse image containing "fips" tag | CHI reconciles normally | +| Required + non-fips Keeper image | CHK with Keeper image lacking "fips" tag | CHK rejected with FIPSImagePolicyViolation | +| Required + non-fips backup sidecar image | CHI with clickhouse-backup sidecar image lacking "fips" tag | CHI rejected with FIPSImagePolicyViolation | +| Required + version check | Host `SELECT version()` lacks "fips" | Host marked failed with FIPSImagePolicyViolation | +| Permissive + non-fips | CHI with any image | CHI reconciles (default behavior) | +| Multiple hosts violation | CHI with multiple non-fips hosts | Single error, short-circuits at first | + +**Image Tag Detection:** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Tag with "fips" suffix | `altinity/clickhouse-server:25.3.fips` | Detected as FIPS | +| Tag with "altinityfips" | `altinity/clickhouse-server:25.3.8.30001.altinityfips` | Detected as FIPS | +| Case insensitive | `...:25.3.FIPS` or `...:25.3.Fips` | Detected as FIPS | +| Digest-only reference | `repo@sha256:...` | Not detected (no tag) | +| Registry with "fips" in path | `fips-registry.example.com/image:latest` | Not detected (tag only) | + +## clickhouse-operator Connections + +**Objective:** Verify all clickhouse-operator outbound connections use FIPS-compliant TLS. + +**Connection Overview:** + +| Direction | Target | Protocol | Default Port | TLS Support | +|-----------|--------|----------|--------------|------------------------------------------------------------| +| Outbound | Kubernetes API Server | HTTPS | 443 | Yes (client-go), configurable via `security.kubernetes.tls` | +| Outbound | ClickHouse Server | HTTP/HTTPS | 8123/8443 | Yes, configurable via `security.clickhouse.tls` | +| Outbound | ZooKeeper/Keeper | TCP | 2181/2281 | Yes, configurable via `security.zookeeper.tls` | +| Outbound | metrics-exporter (IPC) | HTTP | 8888 | No (same pod, localhost) | +| Inbound | Prometheus scrape | HTTP | 9999 | No (known gap) | + +**Operator to Kubernetes API** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator FIPS cipher to K8s | Operator connects with FIPS-approved cipher | Connection succeeds | +| Operator non-FIPS cipher to K8s | K8s API only offers non-approved cipher | Operator rejects connection | +| `security.kubernetes.tls.minVersion=1.3` | Enforce TLS 1.3 minimum | TLS 1.2 rejected | + +**Operator to ClickHouse Server** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator FIPS cipher to CH | Operator connects with FIPS-approved cipher | Connection succeeds | +| Operator non-FIPS cipher to CH | Server only offers non-approved cipher | Operator rejects connection | +| `security.clickhouse.tls.minVersion=1.3` | Enforce TLS 1.3 minimum | TLS 1.2 rejected | + +**Operator to ZooKeeper/Keeper** + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator FIPS cipher to ZK | Operator connects with FIPS-approved cipher | Connection succeeds | +| Operator non-FIPS cipher to ZK | ZK only offers non-approved cipher | Operator rejects connection | +| `security.zookeeper.tls.minVersion=1.3` | Enforce TLS 1.3 minimum | TLS 1.2 rejected | + +**Operator to metrics-exporter (IPC)** + +> Same pod, localhost - HTTP acceptable. Token auth via `security.ipc.mode=Secure`. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator IPC + `security.ipc.mode=Secure` | HTTP with token auth enabled | Works correctly | + +**Operator Prometheus Metrics (:9999)** + +> **FIPS Gap:** HTTP-only + +| Test Assertion | Description | Expected Result | +|-----------------------|----------------------------|-----------------| +| Operator metrics port | Verify connection on :9999 | curl succeeds | + +## metrics-exporter Connections + +**Objective:** Verify all metrics-exporter inbound and outbound connections use FIPS-compliant TLS. + +**Connection Overview:** + +| Direction | Target | Protocol | Default Port | TLS Support | +|-----------|--------|----------|--------------|------------------------------------| +| Outbound | Kubernetes API Server | HTTPS | 443 | Yes (client-go) | +| Outbound | ClickHouse Server | HTTP/HTTPS | 8123/8443 | Yes, inherits from `chop.Config()` | +| Inbound | Prometheus scrape | HTTP | 8888 `/metrics` | No, (known gap) | +| Inbound | Operator IPC | HTTP | 8888 `/chi` | No (same pod, localhost) | + +**Exporter to Kubernetes API** + +> Uses client-go defaults. No minVersion control exposed. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Exporter FIPS cipher to K8s | Exporter connects with FIPS-approved cipher | Connection succeeds | +| Exporter non-FIPS cipher to K8s | K8s API only offers non-approved cipher | Exporter rejects connection | + +**Exporter to ClickHouse Server** + +> TLS supported via `chop.Config()`, but `ChSchemeAuto` prefers HTTP if both ports available. +> Must configure `scheme: https` explicitly for FIPS compliance. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Exporter FIPS cipher to CH | Exporter queries with FIPS-approved cipher | Connection succeeds | +| Exporter non-FIPS cipher to CH | Server only offers non-approved cipher | Exporter rejects connection | + +**Exporter Prometheus Metrics (:8888/metrics)** + +> **FIPS Gap:** HTTP-only. + +| Test Assertion | Description | Expected Result | +|----------------|----------------------------|-----------------| +| Exporter metrics port | Verify connection on :8888 | curl succeeds | + +**Exporter IPC Endpoint (:8888/chi)** + +> Covered by Operator IPC tests above. Same pod, localhost. + +## Integrity Check Failure + +**Objective:** Verify FIPS integrity self-test detects binary tampering. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Corrupted binary | XOR byte in `.go.fipsinfo` section and execute | Panic: `fips140: verification mismatch` | + +**Procedure:** + +Flip one byte in the `.go.fipsinfo` embedded HMAC to trigger integrity check failure at init: + +1. Locate `.go.fipsinfo` section offset: `readelf -S -W ` +2. XOR byte at offset+16 (first byte of 32-byte HMAC after 16-byte magic) +3. Run tampered binary - expect panic: `fips140: verification mismatch` + +Requires: `readelf` (binutils), `python3` + +## CAST Failure + +**Objective:** Verify FIPS Cryptographic Algorithm Self-Test (CAST) detects failures. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| CAST failure | Trigger known-answer test failure via `GODEBUG=failfipscast=` | Process terminates with CAST error | + +**Procedure:** + +Use `GODEBUG=failfipscast=` to simulate CAST failures. + +Available CAST names: see `$GOROOT/src/crypto/internal/fips140test/cast_test.go` (`allCASTs` variable). + +## Synthetic TLS Cipher Validation + +**Objective:** Provide supplementary TLS cipher evidence for operator pod container connections to Kubernetes API and ClickHouse HTTPS endpoints under FIPS enforced mode. + +This scenario validates that both containers in the operator pod can negotiate an approved TLS 1.3 with real runtime endpoints, and that a TLS peer offering only a non-approved cipher is rejected when the client is restricted to an approved cipher. + + +### Scope + +| Source | Target | Endpoint | Cipher / Peer Configuration | Expected Result | +| ------------------------------- | ----------------------- | ------------------------------------ | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `clickhouse-operator` container | Kubernetes API | `https://kubernetes.default.svc:443` | Client forces `TLS_AES_256_GCM_SHA384` over TLS 1.3 | TLS 1.3 handshake succeeds and API request returns `HTTP 200` | +| `metrics-exporter` container | Kubernetes API | `https://kubernetes.default.svc:443` | Client forces `TLS_AES_256_GCM_SHA384` over TLS 1.3 | TLS 1.3 handshake succeeds and API request returns `HTTP 200` | +| `clickhouse-operator` container | ClickHouse HTTPS | CHI pod `:8443` `/ping` | Client forces `TLS_AES_256_GCM_SHA384` over TLS 1.3 | TLS 1.3 handshake succeeds and `/ping` returns `HTTP 200` | +| `metrics-exporter` container | ClickHouse HTTPS | CHI pod `:8443` `/ping` | Client forces `TLS_AES_256_GCM_SHA384` over TLS 1.3 | TLS 1.3 handshake succeeds and `/ping` returns `HTTP 200` | +| `clickhouse-operator` container | Fake OpenSSL TLS server | `fake-openssl-server:8443` | Server offers only `TLS_CHACHA20_POLY1305_SHA256`; client forces `TLS_AES_256_GCM_SHA384` | TLS handshake fails because there is no shared approved cipher | +| `metrics-exporter` container | Fake OpenSSL TLS server | `fake-openssl-server:8443` | Server offers only `TLS_CHACHA20_POLY1305_SHA256`; client forces `TLS_AES_256_GCM_SHA384` | TLS handshake fails because there is no shared approved cipher | + + +### Test Matrix + +| Connection | Tool | Test | Expected Result | +| --------------------------------------------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `clickhouse-operator` container → Kubernetes API | `curl` against real K8s API | Force TLS 1.3 with `TLS_AES_256_GCM_SHA384` | TLS handshake succeeds; request returns `HTTP 200` | +| `metrics-exporter` container → Kubernetes API | `curl` against real K8s API | Force TLS 1.3 with `TLS_AES_256_GCM_SHA384` | TLS handshake succeeds; request returns `HTTP 200` | +| `clickhouse-operator` container → ClickHouse HTTPS | `curl` against real CHI pod `:8443` | Force TLS 1.3 with `TLS_AES_256_GCM_SHA384` | TLS handshake succeeds; `/ping` returns `HTTP 200` | +| `metrics-exporter` container → ClickHouse HTTPS | `curl` against real CHI pod `:8443` | Force TLS 1.3 with `TLS_AES_256_GCM_SHA384` | TLS handshake succeeds; `/ping` returns `HTTP 200` | +| `clickhouse-operator` container → fake rejected-cipher TLS peer | Fake `openssl s_server` | Server offers only `TLS_CHACHA20_POLY1305_SHA256`; client allows only `TLS_AES_256_GCM_SHA384` | TLS handshake fails | +| `metrics-exporter` container → fake rejected-cipher TLS peer | Fake `openssl s_server` | Server offers only `TLS_CHACHA20_POLY1305_SHA256`; client allows only `TLS_AES_256_GCM_SHA384` | TLS handshake fails | + +### Explicit Exclusions + +| Excluded Target | Reason | +| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ClickHouse Keeper / CHK | The operator does not normally establish a runtime TLS client session to ClickHouse Keeper. CHK is only deployed because the CHI manifest depends on Keeper. Keeper TLS is covered by real CHK listener/configuration checks. | +| Operator metrics `:9999` | Plain HTTP Prometheus endpoint; outside FIPS TLS scope by documented boundary. | +| Exporter metrics `:8888` | Plain HTTP Prometheus/IPC endpoint; outside FIPS TLS scope by documented boundary. | + +### Interpretation + +This scenario provides supplementary cipher-negotiation evidence. + +It proves: + +* both containers in the operator pod can negotiate approved TLS 1.3 AES-256-GCM with real Kubernetes API and real ClickHouse HTTPS endpoints; +* a peer that offers only the non-approved TLS 1.3 ChaCha cipher cannot be used when the client is restricted to the approved AES-256 cipher. + +It does not claim that the fake OpenSSL server is a protocol-compatible replacement for Kubernetes or ClickHouse. + +## CI/CD Image and Policy Verification + +**Objective:** Add CI/CD jobs to validate FIPS image build and supply-chain checks. + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| Operator FIPS image build | Build clickhouse-operator with FIPS tags | Image builds successfully | +| Exporter FIPS image build | Build metrics-exporter with FIPS tags | Image builds successfully | +| Image vulnerability scan | Scan images with Grype | No Critical, High, or Medium vulnerabilities | + +> **Note:** Image policy enforcement tests covered in [FIPS Enforcement Mode](#fips-enforcement-mode). + +## (Optional) ACVP Algorithm Validation + +**Objective:** Reproduce ACVP expected-output checks using the same public-scope config +pattern used in [clickhouse-backup PR #1364](https://github.com/Altinity/clickhouse-backup/pull/1364). + +> **Note:** ACVP tests the cryptographic library as compiled into the shipped binary. +> In Go, crypto primitives are statically linked — the bytes ACVP exercises are the exact bytes users run. +> Reference config: +> [`pkg/acvpwrapper/acvp_test_fips140v1.26.public.config.json`](https://github.com/Altinity/clickhouse-backup/blob/master/pkg/acvpwrapper/acvp_test_fips140v1.26.public.config.json) +> (public-API scope; excludes ML-KEM/ML-DSA). + +| Test Assertion | Description | Expected Result | +|----------------|-------------|-----------------| +| ACVP wrapper integration | Add `acvp` subcommand to operator/exporter | ACVP subcommand responds | +| ACVP config generation | Run ` acvp getConfig` | Returns supported capabilities | +| ACVP expected-output replay | Run pinned ACVP replay against tracked config | All configured suites match expected output | +| ACVP suite count | Validate configured suite count from tracked config | `38 ACVP tests matched expectations` | + +Covered suite families from the tracked config (38 total): +- SHA-2 (6), SHA-3 (4), SHAKE/cSHAKE (4) +- HMAC-SHA-2 (6), HMAC-SHA-3 (4) +- AES-CBC/CTR/GCM and CMAC-AES (4) +- KDA/PBKDF/KDF components (3), DRBG (2) +- ECDSA/EdDSA/RSA (3), TLS 1.2/1.3 (2) \ No newline at end of file From f441c5c76d259910e328bc622675c43a3c52daa3 Mon Sep 17 00:00:00 2001 From: saba Date: Thu, 18 Jun 2026 17:53:01 +0200 Subject: [PATCH 057/164] major improvement of FIPS coverage, added fips test plan to the requirements file. linked all the requirements with full coverage. updated respective steps. --- tests/e2e/test_operator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index d2a2f5cef..e28f619e2 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -6338,9 +6338,7 @@ def test_010065_0(self): @TestScenario @Tags("HEAVY") @Name("test_010065. FIPS IPC Secure mode: operator↔exporter token-protected channel") -@Requirements( - RQ_SRS_026_ClickHouseOperator_FIPS_Connect_Operator_IPCSecure("1.0") -) +@Requirements(RQ_SRS_026_ClickHouseOperator_Create("1.0")) def test_010065(self): """Verify clickhouse.security.ipc.mode=Secure activates token-based auth on the operator↔metrics-exporter /chi REST channel without breaking the From fafa5dc5a0e3bedd1a6699fbd06898224613aa6f Mon Sep 17 00:00:00 2001 From: saba Date: Fri, 19 Jun 2026 14:15:33 +0200 Subject: [PATCH 058/164] fixed 035-2 scenario, added a new test for external client check via 1.2 protocol --- tests/e2e/manifests/chi/test-030016.yaml | 80 +++++++++++ .../chi/test-035-2-sustained-not-ready.yaml | 9 ++ tests/e2e/test_operator.py | 127 ++++++++++++++++-- 3 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/manifests/chi/test-030016.yaml diff --git a/tests/e2e/manifests/chi/test-030016.yaml b/tests/e2e/manifests/chi/test-030016.yaml new file mode 100644 index 000000000..35d9373fd --- /dev/null +++ b/tests/e2e/manifests/chi/test-030016.yaml @@ -0,0 +1,80 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030009 +spec: + defaults: + templates: + podTemplate: test-030009 + templates: + podTemplates: + - name: test-030009 + spec: + containers: + - name: clickhouse + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + security: + clickhouse: + tls: + rootCASecretRef: + name: clickhouse-certs + key: ca.crt + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + layout: + shardsCount: 1 + replicasCount: 1 + settings: + http_port: _removed_ + tcp_port: _removed_ + interserver_http_port: _removed_ + mysql_port: _removed_ + postgresql_port: _removed_ + https_port: 8443 + tcp_port_secure: 9440 + interserver_https_port: 9010 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + none + sslv2,sslv3,tlsv1,tlsv1_1 + ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + true + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + + + + server.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.crt + server.key: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.key + dhparam.pem: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: dhparam.pem + ca.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: ca.crt diff --git a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml index dd548b54a..3d04d7d57 100644 --- a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml +++ b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml @@ -31,6 +31,15 @@ spec: initialDelaySeconds: 1 periodSeconds: 2 failureThreshold: 1 + livenessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "test -f /tmp/ready" + initialDelaySeconds: 1 + periodSeconds: 2 + failureThreshold: 3 defaults: templates: podTemplate: readiness-flap diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index e28f619e2..c4e99f62d 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -2797,6 +2797,14 @@ def checkEnv(pos, env_name, env_value): assert kubectl.get_field("chi", chi, ".status.usedTemplates[2].name") == "grafana-dashboard-user" assert kubectl.get_field("chi", chi, ".status.usedTemplates[3].name") == "selector-test-1" + with Then("Wait for selector-1 annotation to be propagated to pod"): + kubectl.wait_field( + "pod", + f"chi-{chi}-single-0-0-0", + ".metadata.annotations.selector-test-1", + "selector-test-1", + ) + with Then("Annotation from selector-1 template should be populated"): assert kubectl.get_field("pod", f"chi-{chi}-single-0-0-0", ".metadata.annotations.selector-test-1") == "selector-test-1" with Then("Annotation from selector-2 template should NOT be populated"): @@ -3817,29 +3825,34 @@ def test_010035_2(self): f"expected container {dummy_container} to be NotReady, got ready={dummy_ready}" ) - with Then("Operator should recreate the pod after sustained NotReady timeout"): + with Then("Kubernetes should restart the dummy container after liveness failure"): + + old_restart_count = kubectl.get_container_restart_count(pod, dummy_container) start_time = time.time() - new_uid = old_uid - pod_ready = kubectl.get_condition_status(pod, "Ready") + new_restart_count = old_restart_count + dummy_ready = kubectl.get_container_status(pod, 1) + while time.time() - start_time < 120: + new_restart_count = kubectl.get_container_restart_count(pod, dummy_container) + dummy_ready = kubectl.get_container_status(pod, 1) - while time.time() - start_time < 420: - new_uid = kubectl.get_field("pod", pod, ".metadata.uid") - pod_ready = kubectl.get_condition_status(pod, "Ready") - if new_uid != old_uid and pod_ready == "True": + if new_restart_count is not None and new_restart_count > old_restart_count and dummy_ready == "true": break + retry_sleep( int((time.time() - start_time) / 5) + 1, 5, - f"pod uid={new_uid}, Ready={pod_ready}", + f"{dummy_container} restartCount={new_restart_count}, ready={dummy_ready}", ) - assert new_uid != old_uid, error( - f"expected operator to recreate pod {pod} within sustained NotReady timeout" + assert new_restart_count is not None and new_restart_count > old_restart_count, error( + f"expected {dummy_container} to restart after liveness failure" + ) + assert dummy_ready == "true", error( + f"expected {dummy_container} to become Ready after restart, got {dummy_ready}" ) - assert pod_ready == "True", error(f"expected recreated pod {pod} to become Ready, got {pod_ready}") - with Finally("I clean up"): - delete_test_namespace() + with Finally("I clean up"): + delete_test_namespace() @TestScenario @@ -8749,6 +8762,94 @@ def test_030015(self): binary="metrics-exporter", ) +@TestScenario +@Tags("HEAVY") +@Name("test_030016. FIPS ClickHouse server TLS 1.2 cipher suites for external client") +@Requirements( + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig("1.0"), + RQ_SRS_026_ClickHouseOperator_FIPS_CH_FIPSConfig_ExternalClient("1.0"), +) +def test_030016(self): + """Verify external clickhouse-client can use TLS 1.2 cipher policy to CH native TLS. + + This intentionally uses: + - operator + - one ClickHouse pod + - external Docker clickhouse-client + - native secure port 9440 + - no Keeper + - no backup sidecar + """ + chopconf = "manifests/chopconf/test-030002-chopconf.yaml" + chi_manifest = "manifests/chi/test-030016.yaml" + + fips_create_shell_namespace_clickhouse_template() + + chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) + chk_dummy = "unused" + + with Given("strict FIPS operator configuration is applied"): + fips_apply_operator_config(chopconf_path=chopconf) + + with And("test TLS secret is installed"): + create_tls_secret_for_fips_hosts( + chi=chi, + chk=chk_dummy, + replicas=1, + ) + + with When("single-pod FIPS ClickHouse is deployed"): + fips_apply_manifest( + manifest_path=chi_manifest, + replica_count=1, + kind="chi", + ) + + with Then("ClickHouse pod should be running"): + kubectl.wait_object( + "pod", + "", + label=f"-l clickhouse.altinity.com/chi={chi}", + count=1, + ns=self.context.test_namespace, + ) + + chi_pod = f"chi-{chi}-default-0-0-0" + + kubectl.wait_pod_status( + chi_pod, + "Running", + ns=self.context.test_namespace, + ) + + with Then("ClickHouse native TLS accepts approved TLS 1.2 AES-GCM cipher"): + out = fips_run_openssl_s_client_on_pod_port( + pod=chi_pod, + port=9440, + tls_version="1.2", + cipher_suite="ECDHE-RSA-AES256-GCM-SHA384", + ns=self.context.test_namespace, + ) + + assert "Protocol : TLSv1.2" in out, error(out) + assert "Cipher : ECDHE-RSA-AES256-GCM-SHA384" in out, error(out) + + with Then("ClickHouse native TLS rejects disallowed TLS 1.2 ChaCha20 cipher"): + out = fips_run_openssl_s_client_on_pod_port( + pod=chi_pod, + port=9440, + tls_version="1.2", + cipher_suite="ECDHE-RSA-CHACHA20-POLY1305", + ok_to_fail=True, + ns=self.context.test_namespace, + ) + + assert ( + "handshake failure" in out + or "Cipher is (NONE)" in out + or "Cipher : 0000" in out + or "no peer certificate available" in out + ), error(out) def cleanup_chis(self): with Given("Cleanup CHIs"): From 95dd756d6f4d4c9a43671d5db5e0c520fcfed2b4 Mon Sep 17 00:00:00 2001 From: saba Date: Fri, 19 Jun 2026 15:37:23 +0200 Subject: [PATCH 059/164] added missing manifests --- .../chi/test-030008-permissive-non-fips.yaml | 21 +++++++++++++++++++ .../test-030008-permissive-chopconf.yaml | 8 +++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml create mode 100644 tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml diff --git a/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml b/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml new file mode 100644 index 000000000..8f618d26c --- /dev/null +++ b/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml @@ -0,0 +1,21 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030008-permissive-non-fips +spec: + configuration: + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 + templates: + podTemplates: + - name: non-fips + spec: + containers: + - name: clickhouse-pod + image: altinity/clickhouse-server:25.8.16.10002.altinitystable + defaults: + templates: + podTemplate: non-fips \ No newline at end of file diff --git a/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml b/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml new file mode 100644 index 000000000..947e3e7ea --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml @@ -0,0 +1,8 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-030008-permissive-chopconf" +spec: + security: + images: + policy: Permissive \ No newline at end of file From 5f60ee3b87616529e8056b72f66a855634b9c964 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 21:45:07 +0500 Subject: [PATCH 060/164] dev: enum --- pkg/util/enum.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 pkg/util/enum.go diff --git a/pkg/util/enum.go b/pkg/util/enum.go new file mode 100644 index 000000000..8bd0d4c93 --- /dev/null +++ b/pkg/util/enum.go @@ -0,0 +1,30 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import "strings" + +// FoldEnum returns the canonical-cased member of canonical that case-insensitively +// matches value, so callers can accept both humped and all-lowercase enum input and +// then compare downstream with plain ==. If value matches no canonical member it is +// returned unchanged (the caller's default/validation handles unrecognized values). +func FoldEnum(value string, canonical ...string) string { + for _, c := range canonical { + if strings.EqualFold(value, c) { + return c + } + } + return value +} From 558afe70ff16acb4f251cb229e400e3b7e200a44 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 21:45:24 +0500 Subject: [PATCH 061/164] dev: emum unit test --- pkg/util/enum_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pkg/util/enum_test.go diff --git a/pkg/util/enum_test.go b/pkg/util/enum_test.go new file mode 100644 index 000000000..3602b39d3 --- /dev/null +++ b/pkg/util/enum_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestFoldEnum verifies that any accepted casing folds to the canonical member, +// while empty and unrecognized inputs pass through unchanged. +func TestFoldEnum(t *testing.T) { + canonical := []string{"Abort", "Delete", "Ignore"} + + tests := []struct { + name string + value string + want string + }{ + {"exact canonical preserved", "Abort", "Abort"}, + {"all-lowercase folds up", "abort", "Abort"}, + {"all-uppercase folds", "DELETE", "Delete"}, + {"mixed case folds", "iGnOrE", "Ignore"}, + {"empty passes through", "", ""}, + {"unrecognized passes through unchanged", "bogus", "bogus"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, FoldEnum(tc.value, canonical...)) + }) + } +} + +// TestFoldEnumNoCandidates verifies the value is returned unchanged when no +// canonical members are supplied (defensive — never panics on empty varargs). +func TestFoldEnumNoCandidates(t *testing.T) { + require.Equal(t, "anything", FoldEnum("anything")) +} From aa1a51ebf20feeb56aa10db3998c469213533c25 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 21:59:47 +0500 Subject: [PATCH 062/164] dev: hump-casing config --- deploy/builder/templates-config/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 89e82de8b..a9eb7797e 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -146,8 +146,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests From c7103d2ab0e32edc30fe3fa0107ce6d80d5584a8 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:04:00 +0500 Subject: [PATCH 063/164] dev: allow both humped and lowercased const values in CRD --- ...l-template-01-section-crd-01-chi-chit.yaml | 87 +++++++++++++++---- ...l-template-01-section-crd-02-chopconf.yaml | 3 + ...l-yaml-template-01-section-crd-03-chk.yaml | 49 +++++++++-- 3 files changed, 116 insertions(+), 23 deletions(-) diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml index dd8e7b4a8..c44c5243d 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,8 +1844,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index a856bdadd..a30a06953 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -317,9 +317,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml index a1f4b3c42..a8aa40612 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml @@ -312,9 +312,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -338,12 +341,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -402,20 +407,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -680,10 +689,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -757,9 +768,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -773,35 +787,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" From 73ecf186be410347d60a73c7aef1e3cfebf75742 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:04:26 +0500 Subject: [PATCH 064/164] env: manifests --- .../clickhouse-operator-install-ansible.yaml | 230 +++++++-- ...house-operator-install-bundle-v1beta1.yaml | 230 +++++++-- .../clickhouse-operator-install-bundle.yaml | 230 +++++++-- ...use-operator-install-template-v1beta1.yaml | 230 +++++++-- .../clickhouse-operator-install-template.yaml | 230 +++++++-- .../clickhouse-operator-install-tf.yaml | 230 +++++++-- deploy/operator/parts/crd.yaml | 440 ++++++++++++++---- 7 files changed, 1484 insertions(+), 336 deletions(-) diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 0231de6f3..a0c3954d6 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -311,8 +311,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -349,7 +351,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -370,9 +374,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -396,12 +403,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -506,8 +515,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -533,8 +545,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -549,7 +564,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -560,7 +577,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -885,20 +904,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -989,9 +1012,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1137,21 +1160,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1521,10 +1549,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1625,9 +1655,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1641,35 +1674,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1797,10 +1851,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2109,8 +2164,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2147,7 +2204,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2168,9 +2227,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2194,12 +2256,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2304,8 +2368,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2331,8 +2398,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2347,7 +2417,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2358,7 +2430,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2683,20 +2757,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2787,9 +2865,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2935,21 +3013,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3319,10 +3402,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3423,9 +3508,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3439,35 +3527,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3595,10 +3704,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3920,9 +4030,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4647,9 +4760,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4673,12 +4789,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4737,20 +4855,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -5015,10 +5137,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5092,9 +5216,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5108,35 +5235,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5690,8 +5838,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index 80af71c5d..0716b6803 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -299,8 +299,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -337,7 +339,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -358,9 +362,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -384,12 +391,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -494,8 +503,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -521,8 +533,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -537,7 +552,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -548,7 +565,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -873,20 +892,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -977,9 +1000,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1122,21 +1145,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1506,10 +1534,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1609,9 +1639,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1625,35 +1658,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1780,10 +1834,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2085,8 +2140,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -2123,7 +2180,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2144,9 +2203,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2170,12 +2232,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -2280,8 +2344,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2307,8 +2374,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2323,7 +2393,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2334,7 +2406,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2659,20 +2733,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2763,9 +2841,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2908,21 +2986,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3292,10 +3375,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3395,9 +3480,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3411,35 +3499,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3566,10 +3675,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3887,9 +3997,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4609,9 +4722,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4635,12 +4751,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -4699,20 +4817,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4976,10 +5098,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5052,9 +5176,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5068,35 +5195,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5889,8 +6037,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index a1cfc7904..62a497785 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,10 +1844,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2102,8 +2157,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2140,7 +2197,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2161,9 +2220,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2187,12 +2249,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2297,8 +2361,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2324,8 +2391,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2340,7 +2410,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2351,7 +2423,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2676,20 +2750,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2780,9 +2858,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2928,21 +3006,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3312,10 +3395,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3416,9 +3501,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3432,35 +3520,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3588,10 +3697,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3913,9 +4023,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4640,9 +4753,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4666,12 +4782,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4730,20 +4848,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -5008,10 +5130,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5085,9 +5209,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5101,35 +5228,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5949,8 +6097,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 1aab2e11a..13ce1cecf 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -299,8 +299,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -337,7 +339,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -358,9 +362,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -384,12 +391,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -494,8 +503,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -521,8 +533,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -537,7 +552,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -548,7 +565,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -873,20 +892,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -977,9 +1000,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1122,21 +1145,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1506,10 +1534,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1609,9 +1639,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1625,35 +1658,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1780,10 +1834,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2085,8 +2140,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -2123,7 +2180,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2144,9 +2203,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2170,12 +2232,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -2280,8 +2344,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2307,8 +2374,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2323,7 +2393,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2334,7 +2406,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2659,20 +2733,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2763,9 +2841,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2908,21 +2986,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3292,10 +3375,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3395,9 +3480,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3411,35 +3499,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3566,10 +3675,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3887,9 +3997,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4609,9 +4722,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4635,12 +4751,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -4699,20 +4817,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4976,10 +5098,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5052,9 +5176,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5068,35 +5195,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5636,8 +5784,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 3165562a8..70d9f948f 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,10 +1844,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2102,8 +2157,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2140,7 +2197,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2161,9 +2220,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2187,12 +2249,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2297,8 +2361,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2324,8 +2391,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2340,7 +2410,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2351,7 +2423,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2676,20 +2750,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2780,9 +2858,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2928,21 +3006,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3312,10 +3395,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3416,9 +3501,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3432,35 +3520,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3588,10 +3697,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3913,9 +4023,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4640,9 +4753,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4666,12 +4782,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4730,20 +4848,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -5008,10 +5130,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5085,9 +5209,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5101,35 +5228,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5683,8 +5831,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 327b2c410..5bb2526e0 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -311,8 +311,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -349,7 +351,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -370,9 +374,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -396,12 +403,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -506,8 +515,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -533,8 +545,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -549,7 +564,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -560,7 +577,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -885,20 +904,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -989,9 +1012,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1137,21 +1160,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1521,10 +1549,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1625,9 +1655,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1641,35 +1674,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1797,10 +1851,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -2109,8 +2164,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2147,7 +2204,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2168,9 +2227,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2194,12 +2256,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2304,8 +2368,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2331,8 +2398,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2347,7 +2417,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2358,7 +2430,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2683,20 +2757,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2787,9 +2865,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2935,21 +3013,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3319,10 +3402,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3423,9 +3508,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3439,35 +3527,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3595,10 +3704,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3920,9 +4030,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -4647,9 +4760,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4673,12 +4789,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4737,20 +4855,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -5015,10 +5137,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5092,9 +5216,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5108,35 +5235,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5690,8 +5838,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index e70755d48..0be3adbfb 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -360,7 +362,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -381,9 +385,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -407,35 +414,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -447,34 +462,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -596,8 +619,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -623,8 +649,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -639,7 +668,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -650,7 +681,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -1232,9 +1265,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -1258,35 +1294,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -1298,34 +1342,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -1447,8 +1499,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -1474,8 +1529,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -1490,7 +1548,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -1501,7 +1561,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -2109,20 +2171,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2222,9 +2288,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2399,9 +2465,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2497,21 +2563,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -3670,10 +3741,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3830,9 +3903,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3846,35 +3922,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3919,20 +4016,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | @@ -4015,10 +4116,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -4327,8 +4429,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -4383,7 +4487,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -4404,9 +4510,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4430,35 +4539,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -4470,34 +4587,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -4619,8 +4744,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -4646,8 +4774,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -4662,7 +4793,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -4673,7 +4806,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -5255,9 +5390,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -5281,35 +5419,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -5321,34 +5467,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -5470,8 +5624,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -5497,8 +5654,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -5513,7 +5673,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -5524,7 +5686,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -6132,20 +6296,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -6245,9 +6413,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -6422,9 +6590,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -6520,21 +6688,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -7693,10 +7866,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -7853,9 +8028,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -7869,35 +8047,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -7942,20 +8141,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | @@ -8038,10 +8241,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -8363,9 +8567,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -9225,9 +9432,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -9251,35 +9461,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -9291,34 +9509,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" defaults: type: object @@ -9354,20 +9580,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -9763,10 +9993,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -9878,9 +10110,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -9894,35 +10129,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -9967,20 +10223,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | From 333233cb0ef337b87d749c3b95780be47c55e249 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:04:38 +0500 Subject: [PATCH 065/164] env: helm charts --- ...installations.clickhouse.altinity.com.yaml | 87 +++++++++++++++---- ...tiontemplates.clickhouse.altinity.com.yaml | 87 +++++++++++++++---- ...ations.clickhouse-keeper.altinity.com.yaml | 49 +++++++++-- ...onfigurations.clickhouse.altinity.com.yaml | 3 + deploy/helm/clickhouse-operator/values.yaml | 4 +- 5 files changed, 189 insertions(+), 41 deletions(-) diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index 64a94900a..7ad285b82 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1127,21 +1150,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1511,10 +1539,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1614,9 +1644,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1630,35 +1663,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1785,8 +1839,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index 6bece68a8..d33ca14f7 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1127,21 +1150,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1511,10 +1539,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1614,9 +1644,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1630,35 +1663,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1785,8 +1839,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 68ebc8ea8..26c3ee738 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -312,9 +312,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -338,12 +341,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -402,20 +407,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -679,10 +688,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -755,9 +766,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -771,35 +785,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 0161a3733..d5c4c3135 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -317,9 +317,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index d4934f088..5106fee83 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -432,8 +432,8 @@ configs: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests From cb11f4eaf5d84a397d9f417925523213bc1207c1 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:41:19 +0500 Subject: [PATCH 066/164] dev: humpe const --- .../clickhouse.altinity.com/v1/type_chi.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go index 1244d4a1c..b5b4eb6b6 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "fmt" + "strings" + "github.com/altinity/clickhouse-operator/pkg/apis/swversion" "github.com/imdario/mergo" @@ -421,13 +423,15 @@ func (cr *ClickHouseInstallation) FoundIn(haystack []*ClickHouseInstallation) bo return false } -// Possible templating policies +// Possible templating policies (canonical humped form; CRD also accepts all-lowercase) const ( - TemplatingPolicyAuto = "auto" - TemplatingPolicyManual = "manual" + TemplatingPolicyAuto = "Auto" + TemplatingPolicyManual = "Manual" ) -// IsAuto checks whether templating policy is auto +// IsAuto checks whether templating policy is auto. +// Uses EqualFold: this is read from template CRs that are NOT run through the normalizer +// (readCHITemplates → GetAutoTemplates), so the raw letter-casing must be tolerated here. func (cr *ClickHouseInstallation) IsAuto() bool { if cr == nil { return false @@ -435,7 +439,7 @@ func (cr *ClickHouseInstallation) IsAuto() bool { if (cr.Namespace == "") && (cr.Name == "") { return false } - return cr.GetSpecT().GetTemplating().GetPolicy() == TemplatingPolicyAuto + return strings.EqualFold(cr.GetSpecT().GetTemplating().GetPolicy(), TemplatingPolicyAuto) } // IsStopped checks whether CR is stopped @@ -454,12 +458,13 @@ const ( RestartRollingUpdate = "RollingUpdate" ) -// IsRollingUpdate checks whether CHI should perform rolling update +// IsRollingUpdate checks whether CHI should perform rolling update. +// The restart field is read un-normalized, so fold casing here (RollingUpdate/rollingupdate). func (cr *ClickHouseInstallation) IsRollingUpdate() bool { if cr == nil { return false } - return cr.GetSpecT().GetRestart().Value() == RestartRollingUpdate + return cr.GetSpecT().GetRestart().EqualFoldString(RestartRollingUpdate) } // IsTroubleshoot checks whether CHI is in troubleshoot mode From 3aa538bd273bd1285949067d07ebd4a7510cadc4 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:41:39 +0500 Subject: [PATCH 067/164] dev: string equal fold --- pkg/apis/common/types/string.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/apis/common/types/string.go b/pkg/apis/common/types/string.go index 4ec9dd589..a563fdc3b 100644 --- a/pkg/apis/common/types/string.go +++ b/pkg/apis/common/types/string.go @@ -95,6 +95,14 @@ func (s *String) EqualFold(other *String) bool { return strings.EqualFold(s.Value(), other.Value()) } +// EqualFoldString reports whether the String case-insensitively equals a plain string +// value (typically an enum const). Nil-safe: a nil String compares as "" (so it never +// equals a non-empty value). Lets callers fold enum casing without wrapping the const +// in a *String, e.g. field.EqualFoldString(api.RecoveryActionRetry). +func (s *String) EqualFoldString(value string) bool { + return strings.EqualFold(s.Value(), value) +} + // MergeFrom merges value from another variable func (s *String) MergeFrom(from *String) *String { if from == nil { From 50309fc31c3656f1ae6230282c98b0017ed3f2f0 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Fri, 19 Jun 2026 22:45:22 +0500 Subject: [PATCH 068/164] dev: folding --- .../v1/type_configuration_chop.go | 97 +++++++++++++------ .../type_configuration_chop_recovery_test.go | 9 +- .../v1/type_keeper_ref.go | 24 +++-- .../v1/type_keeper_ref_test.go | 46 +++++++++ .../v1/type_reconcile.go | 18 ++-- .../v1/type_volume_claim_template.go | 28 ++++++ .../v1/type_volume_claim_template_test.go | 65 +++++++++++++ pkg/apis/deployment/affinity.go | 54 +++++++++++ pkg/apis/deployment/affinity_test.go | 47 +++++++++ 9 files changed, 342 insertions(+), 46 deletions(-) create mode 100644 pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go create mode 100644 pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go create mode 100644 pkg/apis/deployment/affinity_test.go diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index 04ed0768a..f9610f1ab 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -46,22 +46,25 @@ const ( // a referenced ClickHouseKeeper to become ready during CHI reconcile. defaultKeeperReadyTimeout = 120 + // Consts below use the canonical humped form; the CRD also accepts all-lowercase and + // the accessors compare case-insensitively (EqualFold). + // KeeperOnResourceUpdateNone means do nothing when referenced CHK changes (default). - KeeperOnResourceUpdateNone = "none" + KeeperOnResourceUpdateNone = "None" // KeeperOnResourceUpdateReconcile means trigger CHI reconcile when referenced CHK changes. - KeeperOnResourceUpdateReconcile = "reconcile" + KeeperOnResourceUpdateReconcile = "Reconcile" // OnConfigurationChangeNone means ignore ClickHouseOperatorConfiguration changes (default). - OnConfigurationChangeNone = "none" + OnConfigurationChangeNone = "None" // OnConfigurationChangeIgnore is an alias for OnConfigurationChangeNone. - OnConfigurationChangeIgnore = "ignore" + OnConfigurationChangeIgnore = "Ignore" // OnConfigurationChangeRestart means exit the process so the pod restarts with the new config. - OnConfigurationChangeRestart = "restart" + OnConfigurationChangeRestart = "Restart" // RecoveryActionNone means do nothing, CHI stays in its current state. - RecoveryActionNone = "none" + RecoveryActionNone = "None" // RecoveryActionRetry means re-enqueue CHI for reconcile (default). - RecoveryActionRetry = "retry" + RecoveryActionRetry = "Retry" // defaultCompletedOnPodNotReadyThreshold is the minimum time a pod must remain in // Ready=False before the operator considers the host stuck and re-enqueues a reconcile @@ -83,8 +86,9 @@ const ( ChSchemeHTTP = "http" // ChSchemeHTTPS specifies HTTPS access scheme ChSchemeHTTPS = "https" - // ChSchemeAuto specifies that operator has to decide itself should https or http be used - ChSchemeAuto = "auto" + // ChSchemeAuto specifies that operator has to decide itself should https or http be used. + // Humped canonical form; the normalizer folds any casing (auto/Auto) to this. + ChSchemeAuto = "Auto" // Username and Password to be used by operator to connect to ClickHouse instances for // 1. Metrics requests @@ -142,45 +146,78 @@ const ( PasswordReplacer = "***" ) +// StatefulSet failure-action consts use the canonical humped form; the CRD also accepts +// all-lowercase and the normalizer folds any casing to these via util.FoldEnum. const ( // What to do in case StatefulSet can't reach new Generation - abort CHI reconcile - OnStatefulSetCreateFailureActionAbort = "abort" + OnStatefulSetCreateFailureActionAbort = "Abort" // What to do in case StatefulSet can't reach new Generation - delete newly created problematic StatefulSet - OnStatefulSetCreateFailureActionDelete = "delete" + OnStatefulSetCreateFailureActionDelete = "Delete" // What to do in case StatefulSet can't reach new Generation - do nothing, keep StatefulSet broken and move to the next - OnStatefulSetCreateFailureActionIgnore = "ignore" + OnStatefulSetCreateFailureActionIgnore = "Ignore" ) const ( // What to do in case StatefulSet can't reach new Generation - abort CHI reconcile - OnStatefulSetUpdateFailureActionAbort = "abort" + OnStatefulSetUpdateFailureActionAbort = "Abort" // What to do in case StatefulSet can't reach new Generation - delete Pod and rollback StatefulSet to previous Generation // Pod would be recreated by StatefulSet based on rollback-ed configuration - OnStatefulSetUpdateFailureActionRollback = "rollback" + OnStatefulSetUpdateFailureActionRollback = "Rollback" // What to do in case StatefulSet can't reach new Generation - do nothing, keep StatefulSet broken and move to the next - OnStatefulSetUpdateFailureActionIgnore = "ignore" + OnStatefulSetUpdateFailureActionIgnore = "Ignore" ) const ( // What to do in case StatefulSet needs to be recreated due to PVC data loss or missing volumes // Abort - Loss: abort CHI reconcile - OnStatefulSetRecreateOnDataLossActionAbort = "abort" + OnStatefulSetRecreateOnDataLossActionAbort = "Abort" // Recreate - Loss: proceed and recreate StatefulSet - OnStatefulSetRecreateOnDataLossActionRecreate = "recreate" + OnStatefulSetRecreateOnDataLossActionRecreate = "Recreate" // What to do in case StatefulSet needs to be recreated due to update failure or StatefulSet not ready // Abort - Failure: abort CHI reconcile - OnStatefulSetRecreateOnUpdateFailureActionAbort = "abort" + OnStatefulSetRecreateOnUpdateFailureActionAbort = "Abort" // Recreate - Failure: proceed and recreate StatefulSet - OnStatefulSetRecreateOnUpdateFailureActionRecreate = "recreate" + OnStatefulSetRecreateOnUpdateFailureActionRecreate = "Recreate" +) + +// Canonical (humped) candidate lists for the StatefulSet failure-action enums. OnDataLoss and +// OnUpdateFailure share the Abort/Recreate set. The Normalize* helpers fold any accepted casing +// to the canonical const; an unrecognized value passes through so the caller's default applies. +var ( + onStatefulSetCreateFailureActions = []string{ + OnStatefulSetCreateFailureActionAbort, OnStatefulSetCreateFailureActionDelete, OnStatefulSetCreateFailureActionIgnore, + } + onStatefulSetUpdateFailureActions = []string{ + OnStatefulSetUpdateFailureActionAbort, OnStatefulSetUpdateFailureActionRollback, OnStatefulSetUpdateFailureActionIgnore, + } + onStatefulSetRecreateActions = []string{ + OnStatefulSetRecreateOnDataLossActionAbort, OnStatefulSetRecreateOnDataLossActionRecreate, + } ) +// NormalizeOnStatefulSetCreateFailureAction folds any accepted casing to its canonical const. +func NormalizeOnStatefulSetCreateFailureAction(value string) string { + return util.FoldEnum(value, onStatefulSetCreateFailureActions...) +} + +// NormalizeOnStatefulSetUpdateFailureAction folds any accepted casing to its canonical const. +func NormalizeOnStatefulSetUpdateFailureAction(value string) string { + return util.FoldEnum(value, onStatefulSetUpdateFailureActions...) +} + +// NormalizeOnStatefulSetRecreateAction folds any accepted casing to its canonical const +// (used for both onDataLoss and onUpdateFailure, which share the Abort/Recreate set). +func NormalizeOnStatefulSetRecreateAction(value string) string { + return util.FoldEnum(value, onStatefulSetRecreateActions...) +} + const ( defaultMaxReplicationDelay = 10 ) @@ -1221,22 +1258,27 @@ func (c *OperatorConfig) normalizeSectionReconcileStatefulSet() { c.Reconcile.StatefulSet.Update.PollInterval = defaultStatefulSetUpdatePollInterval } - // Default action on Create/Update failure - to keep system in previous state + // Default action on Create/Update failure - to keep system in previous state. + // Fold any accepted casing to the canonical const first, then default if empty. // Default Create Failure action - delete + c.Reconcile.StatefulSet.Create.OnFailure = NormalizeOnStatefulSetCreateFailureAction(c.Reconcile.StatefulSet.Create.OnFailure) if c.Reconcile.StatefulSet.Create.OnFailure == "" { c.Reconcile.StatefulSet.Create.OnFailure = OnStatefulSetCreateFailureActionDelete } // Default Updated Failure action - revert + c.Reconcile.StatefulSet.Update.OnFailure = NormalizeOnStatefulSetUpdateFailureAction(c.Reconcile.StatefulSet.Update.OnFailure) if c.Reconcile.StatefulSet.Update.OnFailure == "" { c.Reconcile.StatefulSet.Update.OnFailure = OnStatefulSetUpdateFailureActionRollback } // Default Recreate actions - recreate + c.Reconcile.StatefulSet.Recreate.OnDataLoss = NormalizeOnStatefulSetRecreateAction(c.Reconcile.StatefulSet.Recreate.OnDataLoss) if c.Reconcile.StatefulSet.Recreate.OnDataLoss == "" { c.Reconcile.StatefulSet.Recreate.OnDataLoss = OnStatefulSetRecreateOnDataLossActionRecreate } + c.Reconcile.StatefulSet.Recreate.OnUpdateFailure = NormalizeOnStatefulSetRecreateAction(c.Reconcile.StatefulSet.Recreate.OnUpdateFailure) if c.Reconcile.StatefulSet.Recreate.OnUpdateFailure == "" { c.Reconcile.StatefulSet.Recreate.OnUpdateFailure = OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -1273,7 +1315,9 @@ func (c *OperatorConfig) normalizeSectionClickHouseAccess() { // 1. Metrics requests // 2. Schema maintenance // User credentials can be specified in additional ClickHouse config files located in `chUsersConfigsPath` folder - switch strings.ToLower(c.ClickHouse.Access.Scheme) { + // Fold any accepted casing (http/HTTP, https/HTTPS, auto/Auto) to the canonical const, + // falling back to the default scheme for unrecognized values. + switch util.FoldEnum(c.ClickHouse.Access.Scheme, ChSchemeHTTP, ChSchemeHTTPS, ChSchemeAuto) { case ChSchemeHTTP: c.ClickHouse.Access.Scheme = ChSchemeHTTP case ChSchemeHTTPS: @@ -1651,19 +1695,19 @@ func (c *OperatorConfig) copyWithHiddenCredentials() *OperatorConfig { // RestartOnOperatorConfigurationChange reports whether the operator process should exit when // ClickHouseOperatorConfiguration changes (so the pod restarts). func (c *OperatorConfig) RestartOnOperatorConfigurationChange() bool { - return strings.ToLower(c.Watch.Configuration.OnChange.String()) == OnConfigurationChangeRestart + return c.Watch.Configuration.OnChange.EqualFoldString(OnConfigurationChangeRestart) } // ShouldRecoverAbortedOnPodReady reports whether the operator should re-enqueue a CHI // reconcile when a pod belonging to an Aborted CHI transitions to Ready. Default is to retry. // Backed by reconcile.recovery.onStatus.aborted.onPodReady config key. func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { - value := strings.ToLower(c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady.String()) - if value == "" { + onPodReady := c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady + if onPodReady.Value() == "" { // Default behavior — retry return true } - return value == RecoveryActionRetry + return onPodReady.EqualFoldString(RecoveryActionRetry) } // ShouldRecoverCompletedOnPodNotReady reports whether the operator should re-enqueue a @@ -1674,8 +1718,7 @@ func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { // onPodNotReady: retry enables it. Unlike the Aborted scope, which retries by default. // Backed by reconcile.recovery.onStatus.completed.onPodNotReady config key. func (c *OperatorConfig) ShouldRecoverCompletedOnPodNotReady() bool { - value := strings.ToLower(c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady.String()) - return value == RecoveryActionRetry + return c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady.EqualFoldString(RecoveryActionRetry) } // CompletedOnPodNotReadyThreshold returns the minimum duration a pod must remain in diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go index b3100b3dc..4b855ba85 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go @@ -52,11 +52,12 @@ func TestShouldRecoverAbortedOnPodReady(t *testing.T) { } } -// TestRecoveryActionConstants documents the stable enum values published in the CRD. -// Changes here would break users' CHOPCONF CRs. +// TestRecoveryActionConstants documents the canonical (humped) enum values. The CRD also +// accepts the all-lowercase forms, and the accessors compare case-insensitively (EqualFold), +// so existing lowercase CHOPCONF CRs keep working. func TestRecoveryActionConstants(t *testing.T) { - require.Equal(t, "none", RecoveryActionNone) - require.Equal(t, "retry", RecoveryActionRetry) + require.Equal(t, "None", RecoveryActionNone) + require.Equal(t, "Retry", RecoveryActionRetry) } // TestShouldRecoverCompletedOnPodNotReady verifies the accessor's behavior across the diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go index 514990d6f..68dc626ef 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go @@ -14,14 +14,16 @@ package v1 +import "github.com/altinity/clickhouse-operator/pkg/util" + // KeeperServiceType describes how keeper endpoints are discovered for a KeeperRef. type KeeperServiceType string const ( // KeeperServiceTypeReplicas discovers per-host services, one ZK node per keeper replica. - KeeperServiceTypeReplicas KeeperServiceType = "replicas" + KeeperServiceTypeReplicas KeeperServiceType = "Replicas" // KeeperServiceTypeService uses the CR-level headless service as a single ZK node entry. - KeeperServiceTypeService KeeperServiceType = "service" + KeeperServiceTypeService KeeperServiceType = "Service" ) // IsEmpty returns true if no service type is set. @@ -36,9 +38,9 @@ type KeeperRef struct { // Namespace is the namespace of the CHK resource. Defaults to the CHI namespace if omitted. // +optional Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` - // ServiceType controls how keeper endpoints are discovered: - // "replicas" (default) — enumerate per-host services, one ZK node per keeper replica - // "service" — use the CR-level headless service as a single ZK node entry + // ServiceType controls how keeper endpoints are discovered (case-insensitive): + // "Replicas" (default) — enumerate per-host services, one ZK node per keeper replica + // "Service" — use the CR-level headless service as a single ZK node entry // +optional ServiceType KeeperServiceType `json:"serviceType,omitempty" yaml:"serviceType,omitempty"` } @@ -61,10 +63,18 @@ func (r *KeeperRef) GetNamespace(defaultNamespace string) string { return r.Namespace } -// GetServiceType returns the service type, defaulting to replicas if empty. +// GetServiceType returns the service type, defaulting to Replicas if empty. The raw value +// is read un-normalized from the spec, so fold any accepted casing to the canonical const; +// an unrecognized value passes through unchanged so the resolver can report it as invalid. func (r *KeeperRef) GetServiceType() KeeperServiceType { if r == nil || r.ServiceType.IsEmpty() { return KeeperServiceTypeReplicas } - return r.ServiceType + return KeeperServiceType( + util.FoldEnum( + string(r.ServiceType), + string(KeeperServiceTypeReplicas), + string(KeeperServiceTypeService), + ), + ) } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go new file mode 100644 index 000000000..7091ca58a --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestKeeperRefGetServiceType verifies the accessor defaults to Replicas, folds any +// accepted casing to the canonical const, and passes unrecognized values through so the +// resolver can flag them as invalid. +func TestKeeperRefGetServiceType(t *testing.T) { + tests := []struct { + name string + ref *KeeperRef + want KeeperServiceType + }{ + {"nil defaults to Replicas", nil, KeeperServiceTypeReplicas}, + {"empty defaults to Replicas", &KeeperRef{}, KeeperServiceTypeReplicas}, + {"canonical Replicas", &KeeperRef{ServiceType: "Replicas"}, KeeperServiceTypeReplicas}, + {"lowercase replicas folds", &KeeperRef{ServiceType: "replicas"}, KeeperServiceTypeReplicas}, + {"canonical Service", &KeeperRef{ServiceType: "Service"}, KeeperServiceTypeService}, + {"uppercase SERVICE folds", &KeeperRef{ServiceType: "SERVICE"}, KeeperServiceTypeService}, + {"unrecognized passes through", &KeeperRef{ServiceType: "bogus"}, KeeperServiceType("bogus")}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.ref.GetServiceType()) + }) + } +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go b/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go index b0da8faf2..035e1d36b 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go @@ -235,21 +235,23 @@ func (r *ChiReconcile) GetConfigMapPropagationTimeoutDuration() time.Duration { return time.Duration(r.GetConfigMapPropagationTimeout()) * time.Second } -// Possible reconcile policy values +// Possible reconcile policy values (canonical humped form; CRD also accepts all-lowercase) const ( - ReconcilingPolicyUnspecified = "unspecified" - ReconcilingPolicyWait = "wait" - ReconcilingPolicyNoWait = "nowait" + ReconcilingPolicyUnspecified = "Unspecified" + ReconcilingPolicyWait = "Wait" + ReconcilingPolicyNoWait = "NoWait" ) -// IsReconcilingPolicyWait checks whether reconcile policy is "wait" +// IsReconcilingPolicyWait checks whether reconcile policy is "Wait". +// EqualFold so both humped and all-lowercase inputs match regardless of normalization order. func (r *ChiReconcile) IsReconcilingPolicyWait() bool { - return strings.ToLower(r.GetPolicy()) == ReconcilingPolicyWait + return strings.EqualFold(r.GetPolicy(), ReconcilingPolicyWait) } -// IsReconcilingPolicyNoWait checks whether reconcile policy is "no wait" +// IsReconcilingPolicyNoWait checks whether reconcile policy is "NoWait". +// EqualFold so both humped and all-lowercase inputs match regardless of normalization order. func (r *ChiReconcile) IsReconcilingPolicyNoWait() bool { - return strings.ToLower(r.GetPolicy()) == ReconcilingPolicyNoWait + return strings.EqualFold(r.GetPolicy(), ReconcilingPolicyNoWait) } // GetCleanup gets cleanup diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go index e898a6794..011e1c69c 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go @@ -15,6 +15,8 @@ package v1 import ( + "strings" + core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -42,6 +44,19 @@ func NewPVCProvisionerFromString(s string) PVCProvisioner { return PVCProvisioner(s) } +// Normalize folds any letter-casing of a recognized value to its canonical +// PVCProvisioner const, so the CRD can accept both humped and all-lowercase forms +// (e.g. "operator" -> "Operator"). Unrecognized values are returned unchanged for +// the caller's IsValid()/reset to handle. +func (v PVCProvisioner) Normalize() PVCProvisioner { + for _, known := range []PVCProvisioner{PVCProvisionerStatefulSet, PVCProvisionerOperator} { + if strings.EqualFold(string(v), string(known)) { + return known + } + } + return v +} + // IsValid checks whether PVCProvisioner is valid func (v PVCProvisioner) IsValid() bool { switch v { @@ -84,6 +99,19 @@ func NewPVCReclaimPolicyFromString(s string) PVCReclaimPolicy { return PVCReclaimPolicy(s) } +// Normalize folds any letter-casing of a recognized value to its canonical +// PVCReclaimPolicy const, so the CRD can accept both humped and all-lowercase forms +// (e.g. "delete" -> "Delete"). Unrecognized values are returned unchanged for the +// caller's IsValid()/reset to handle. +func (v PVCReclaimPolicy) Normalize() PVCReclaimPolicy { + for _, known := range []PVCReclaimPolicy{PVCReclaimPolicyRetain, PVCReclaimPolicyDelete} { + if strings.EqualFold(string(v), string(known)) { + return known + } + } + return v +} + // IsValid checks whether PVCReclaimPolicy is valid func (v PVCReclaimPolicy) IsValid() bool { switch v { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go new file mode 100644 index 000000000..38437de8e --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go @@ -0,0 +1,65 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPVCReclaimPolicyNormalize verifies casing-folding to the canonical humped const, +// so the CRD can accept both humped and all-lowercase forms. +func TestPVCReclaimPolicyNormalize(t *testing.T) { + tests := []struct { + in string + expected PVCReclaimPolicy + }{ + {"Retain", PVCReclaimPolicyRetain}, + {"retain", PVCReclaimPolicyRetain}, + {"RETAIN", PVCReclaimPolicyRetain}, + {"Delete", PVCReclaimPolicyDelete}, + {"delete", PVCReclaimPolicyDelete}, + {"DELETE", PVCReclaimPolicyDelete}, + {"", PVCReclaimPolicyUnspecified}, + {"bogus", "bogus"}, // unrecognized: returned unchanged (caller's IsValid resets) + } + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + require.Equal(t, tc.expected, PVCReclaimPolicy(tc.in).Normalize()) + }) + } +} + +// TestPVCProvisionerNormalize verifies casing-folding to the canonical humped const. +func TestPVCProvisionerNormalize(t *testing.T) { + tests := []struct { + in string + expected PVCProvisioner + }{ + {"StatefulSet", PVCProvisionerStatefulSet}, + {"statefulset", PVCProvisionerStatefulSet}, + {"STATEFULSET", PVCProvisionerStatefulSet}, + {"Operator", PVCProvisionerOperator}, + {"operator", PVCProvisionerOperator}, + {"", PVCProvisionerUnspecified}, + {"bogus", "bogus"}, + } + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + require.Equal(t, tc.expected, PVCProvisioner(tc.in).Normalize()) + }) + } +} diff --git a/pkg/apis/deployment/affinity.go b/pkg/apis/deployment/affinity.go index f5039c414..cc47f84db 100644 --- a/pkg/apis/deployment/affinity.go +++ b/pkg/apis/deployment/affinity.go @@ -14,6 +14,8 @@ package deployment +import "github.com/altinity/clickhouse-operator/pkg/util" + // Possible pod distributions const ( PodDistributionUnspecified = "Unspecified" @@ -60,3 +62,55 @@ const ( PortDistributionUnspecified = "Unspecified" PortDistributionClusterScopeIndex = "ClusterScopeIndex" ) + +// podDistributionTypes enumerates every recognized PodDistribution.Type value in canonical (humped) form. +var podDistributionTypes = []string{ + PodDistributionUnspecified, + PodDistributionClickHouseAntiAffinity, + PodDistributionShardAntiAffinity, + PodDistributionReplicaAntiAffinity, + PodDistributionAnotherNamespaceAntiAffinity, + PodDistributionAnotherClickHouseInstallationAntiAffinity, + PodDistributionAnotherClusterAntiAffinity, + PodDistributionNamespaceAffinity, + PodDistributionClickHouseInstallationAffinity, + PodDistributionClusterAffinity, + PodDistributionShardAffinity, + PodDistributionReplicaAffinity, + PodDistributionPreviousTailAffinity, + PodDistributionMaxNumberPerNode, + PodDistributionCircularReplication, + PodDistributionOnePerHost, +} + +// podDistributionScopes enumerates every recognized PodDistribution.Scope value in canonical (humped) form. +var podDistributionScopes = []string{ + PodDistributionScopeUnspecified, + PodDistributionScopeShard, + PodDistributionScopeReplica, + PodDistributionScopeCluster, + PodDistributionScopeClickHouseInstallation, + PodDistributionScopeNamespace, + PodDistributionScopeGlobal, +} + +// portDistributionTypes enumerates every recognized PortDistribution.Type value in canonical (humped) form. +var portDistributionTypes = []string{ + PortDistributionUnspecified, + PortDistributionClusterScopeIndex, +} + +// NormalizePodDistributionType folds any accepted casing of a PodDistribution type to its canonical const. +func NormalizePodDistributionType(value string) string { + return util.FoldEnum(value, podDistributionTypes...) +} + +// NormalizePodDistributionScope folds any accepted casing of a PodDistribution scope to its canonical const. +func NormalizePodDistributionScope(value string) string { + return util.FoldEnum(value, podDistributionScopes...) +} + +// NormalizePortDistributionType folds any accepted casing of a PortDistribution type to its canonical const. +func NormalizePortDistributionType(value string) string { + return util.FoldEnum(value, portDistributionTypes...) +} diff --git a/pkg/apis/deployment/affinity_test.go b/pkg/apis/deployment/affinity_test.go new file mode 100644 index 000000000..1235c18e3 --- /dev/null +++ b/pkg/apis/deployment/affinity_test.go @@ -0,0 +1,47 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deployment + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNormalizePodDistributionType verifies casing folds to the canonical const +// while unrecognized values pass through (the normalizer maps those to Unspecified). +func TestNormalizePodDistributionType(t *testing.T) { + require.Equal(t, PodDistributionClickHouseAntiAffinity, NormalizePodDistributionType("clickhouseantiaffinity")) + require.Equal(t, PodDistributionClickHouseAntiAffinity, NormalizePodDistributionType("ClickHouseAntiAffinity")) + require.Equal(t, PodDistributionCircularReplication, NormalizePodDistributionType("CIRCULARREPLICATION")) + require.Equal(t, PodDistributionMaxNumberPerNode, NormalizePodDistributionType("maxnumberpernode")) + require.Equal(t, "bogus", NormalizePodDistributionType("bogus")) + require.Equal(t, "", NormalizePodDistributionType("")) +} + +// TestNormalizePodDistributionScope verifies scope casing folds to the canonical const. +func TestNormalizePodDistributionScope(t *testing.T) { + require.Equal(t, PodDistributionScopeShard, NormalizePodDistributionScope("shard")) + require.Equal(t, PodDistributionScopeCluster, NormalizePodDistributionScope("Cluster")) + require.Equal(t, PodDistributionScopeClickHouseInstallation, NormalizePodDistributionScope("clickhouseinstallation")) + require.Equal(t, "bogus", NormalizePodDistributionScope("bogus")) +} + +// TestNormalizePortDistributionType verifies port-distribution casing folds to the canonical const. +func TestNormalizePortDistributionType(t *testing.T) { + require.Equal(t, PortDistributionClusterScopeIndex, NormalizePortDistributionType("clusterscopeindex")) + require.Equal(t, PortDistributionUnspecified, NormalizePortDistributionType("UNSPECIFIED")) + require.Equal(t, "bogus", NormalizePortDistributionType("bogus")) +} From 8fb51b861fdf51341f72dc4b632343406161b685 Mon Sep 17 00:00:00 2001 From: saba Date: Fri, 19 Jun 2026 20:21:59 +0200 Subject: [PATCH 069/164] updated expired certificates --- .../e2e/manifests/secret/test-058-secret.yaml | 122 +++++++++--------- 1 file changed, 62 insertions(+), 60 deletions(-) diff --git a/tests/e2e/manifests/secret/test-058-secret.yaml b/tests/e2e/manifests/secret/test-058-secret.yaml index 917154caa..1c79f5991 100644 --- a/tests/e2e/manifests/secret/test-058-secret.yaml +++ b/tests/e2e/manifests/secret/test-058-secret.yaml @@ -7,70 +7,72 @@ type: Opaque stringData: ca.crt: |- -----BEGIN CERTIFICATE----- - MIIDFzCCAf+gAwIBAgIUKPAilo3+YvoeiIkvieiBp5GaXYUwDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE1MDda - Fw0yODA2MTgxMzE1MDdaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi - MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv5k1vd7s1KrPnENFB9Tw0dtYT - wlIzpulIKuXmbEGNXIB9SEV69A7UxUZrwF585kFX91LVq+SOb3WD0/KWjs7N+hXq - RLiLuObBVrehoFFRHUca/JZS3Gz9Wsrlsr8twnX5pxavfmnhXOdw/+3P47e22kQ8 - zeAKaSfJ2wF9U9gic/uQWZmLUohrUaT0AejSQpXMm2dlk4ZhBos5BnDbQ+R+rsXu - WiTR4aS3W5Not/mV9neVJZpv2k5+02G0Wdhwy93Q44kEezSVBnSz3hvzEI5RmA72 - LhqTR/Z3y7wZf57vxkqKjqaampwzIhFxtcpAl3j+UVHPV7WkrJ59OkfaxLdtAgMB - AAGjUzBRMB0GA1UdDgQWBBSPxwYV5VGN3/J5N8ipfzefxQyvAjAfBgNVHSMEGDAW - gBSPxwYV5VGN3/J5N8ipfzefxQyvAjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 - DQEBCwUAA4IBAQCAKisQ/Ez9t88CzbrKkC4MA7rqLvHwV33sC9ttbsILk5kwvlyQ - yeeVYme6+KyK28UzuvUQrge6+JY4a33ki4G+gcltXHUaCxSWMGl20Kx++533uH4W - bLXmEPDsR1iPh+sl+3zJBs/aH3HSovBeaLu0pFRupKW5HWDDxgBz92JLVHfqHfY5 - U4bHqOiLopbcRKOQTRAqP9IQsbCmVr4PU8/LWvdMWrYTpn5IIcq1CD0GpunQBxv1 - N7YosHN5QijMvdHVTdR1B7m3ylJa5cVUPaR6HDrDkXJibaFdZBa0eIXZc3KwTon3 - q/+BIPQNS7JiXY82j8OC2HcPFSuS7t5wqcQN + MIIDFzCCAf+gAwIBAgIUedodKbILyx2B9pb1EZgpr+1pzCcwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNjA2MTkxODE2MTRa + Fw0zNjA2MTYxODE2MTRaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi + MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1ubZ1rjGb+oRp5Uwl7W5ZFPuH + y1r0Q/+kzm/hp45xWP/oSvt6ltPTqdZsqg0EB7M0Nv5Rdhunv/W0qhUB4QaWNWt6 + zVrIMjPtCZ/iTEinhNjWA0Ib9kTkCwzecvhz+P3rZXdu70jTdlBnnwIy0MXCnfnY + dCWfJFX6glgbiElFYjfcHli+1+BFL0XS+Ah5U0nNaY+Kx2gK8Mg5fYBbladszi21 + k84kPphb6jWpTQaEqYePSHryQWsWzOtmyyYLGtrnaepOU7OjO+ATtfm7BIPztewT + Cheo/xSDRDJjHak1DgvPvcSGIFSxrMO+u3Ob55H109olBFAlMt6wPCKgCxBJAgMB + AAGjUzBRMB0GA1UdDgQWBBRBsK0WTIVB0By9aKbmOzaFMwkaGzAfBgNVHSMEGDAW + gBRBsK0WTIVB0By9aKbmOzaFMwkaGzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 + DQEBCwUAA4IBAQAtDF1q3C2HVzsxjcCwAFDJ8o6Gb91fLOMqtJaiyJd4loWiTd2g + TAZWKAgt8d4SoKdlB72L3igWDHiVf+bA2fB2/CkrVyFG3Z/B3D3p+lqP29SE6zEX + luL7uFcSWcer9SAv/sQL6iw8djOlAxGvqvF4UhDdg7mMgSa+0bugdRe+q5IhWbEy + iD+yHZMBcFglDuaIK2I5Vq7TRjcAe0rdhzY5aaPm8brbk8F/dd9PRKA9TSe12BlA + jlziq3YlrKZ4CI3PpcXbFWVNOEMYUniIjs67YA/mDcEQkg1m7aHSteQ3MynALcOj + 9ER8pWfYNorCogEPg7qTIaRDrMmH8axSaVkJ -----END CERTIFICATE----- server.crt: |- -----BEGIN CERTIFICATE----- - MIIDJTCCAg2gAwIBAgIUFZBt5OVfoThrbao6LQ+CA1c2538wDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE2MTNa - Fw0yNjA2MTkxMzE2MTNaMBIxEDAOBgNVBAMMB2Nobm9kZTEwggEiMA0GCSqGSIb3 - DQEBAQUAA4IBDwAwggEKAoIBAQDMqNisnlgEiRTYCk1dEvgcRN7FQiNDxwWKSGjo - zhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTWgBFMPbbkrlGgYisoD8GDRKN0 - Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF5t/NuetMVMqxwe4mA+5RB4hW - rlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs6BkE1uVxFg+UQ38R1wnsE/up - Yj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1HgENnP6n/nN0FG8N6z3yolKk2 - xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2Qs3ntY3/AgMBAAGjajBoMCYG - A1UdEQQfMB2CFWNobm9kZTEubWFyc25ldC5sb2NhbIcEwKgB3TAdBgNVHQ4EFgQU - 3E9tBAiCqqJLSVQKi5sXzHLC9PkwHwYDVR0jBBgwFoAUj8cGFeVRjd/yeTfIqX83 - n8UMrwIwDQYJKoZIhvcNAQELBQADggEBABQIXDzlc/wINnkSfcncEAfIY5WvVTdl - Nilr6nVd1Fgq7JAlVD1WrbZ52xZLK3xg+T99Wezks/Js9x243DWt+qwlCKe/xlrC - ezzI3BunnhRxw/7IRm0soTvPNNImcZ2Fuwhn/ojlOg+37NttdTLKlJu2+RguRWLf - 95sXdxhhfTWrkhe1gWFmmDyl02hFpWRO1A/Ogy+hJ+yuruTrlokcM5zJ1L50kyi5 - iL3/ZXMZxWw1BDSSXlbUUZQnwXJuHQbVTd4NwRO6tLVlQyNj45gRf3WeygVzbQmq - sAjL7iOqF7iCwHEri5FWyDJ0Sj5b2YJKV1LLIEMiE8dJesN0O+OlH9U= + MIIDXjCCAkagAwIBAgIUdEhy/CkXE5rcDiG78vqri1g5RiUwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNjA2MTkxODE2MTRa + Fw0yOTA2MTgxODE2MTRaMBIxEDAOBgNVBAMMB2Nobm9kZTEwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQDPf9HxzkRKAtnL5YbLzyn3tzkbyeKeJuyGptOC + +0oSS17Ssj72fR5FBcIvsPFg7LTtSf+6lEO9uMrWhib8/dygvDv85fBUTVkSj2ol + vUnJrDPrhapfAsbGjhLDr/gjXMSnWp+E9gLo+GB3gHhLqDpAWfYZMEaW9jW+7TO5 + 7EliRCY/devUJPN4JMF91OGDmdqsLkYKdkUxmLkFxCCKfHwfV1adO9zwG1NFz6l4 + KSN8VF8rAXfw6mB9IrXoqW3Wub2YcikhXhq7FqE5yXcHBnR6DkypUVpns8+sfWEe + N/uoDunvaPk4Pibe7r9aOsonYzqUyOma0sHqCMg5Ze2Zj4N9AgMBAAGjgaIwgZ8w + SAYDVR0RBEEwP4IVY2hub2RlMS5tYXJzbmV0LmxvY2FsgiBjaGktdGVzdC0wNTgt + cm9vdC1jYS1kZWZhdWx0LTAtMIcEwKgB3TATBgNVHSUEDDAKBggrBgEFBQcDATAd + BgNVHQ4EFgQUEutarWBu1ZsicSVA4j3SftBlsfIwHwYDVR0jBBgwFoAUQbCtFkyF + QdAcvWim5js2hTMJGhswDQYJKoZIhvcNAQELBQADggEBAHmG8J/HsdOHbZ8L28WS + wNJcAfC3g6rVAqm7qbYUzAUeBDB4SroFaydLecBN8DupaKT0JzCm5HetxLiqQ+M3 + kQ8qij6hTtafvNMJzsRnJLw7zRV5fSiNAOcZWEEM7sOMKlKtPMq3dkoUTLddWJ9J + GuJ3oWUf0csgMmuzmhurzKb1TlbF3elv4u5g1K9bpwny5kSRyDLVcpFW39JzKwmR + NmtTVrEGLcAwfD6hRpl8ITN5pfDHfNg3BYkawB4obl1QI7K+1sPbwyliguojiuXo + a157F6fQCAN2WoQxDbmueenqFRnapZu4nBm0HSZqxU7CYSpCWBidlLZebP1oW6WW + D6I= -----END CERTIFICATE----- server.key: |- -----BEGIN PRIVATE KEY----- - MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDMqNisnlgEiRTY - Ck1dEvgcRN7FQiNDxwWKSGjozhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTW - gBFMPbbkrlGgYisoD8GDRKN0Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF - 5t/NuetMVMqxwe4mA+5RB4hWrlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs - 6BkE1uVxFg+UQ38R1wnsE/upYj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1H - gENnP6n/nN0FG8N6z3yolKk2xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2 - Qs3ntY3/AgMBAAECggEAL9/Nc6/Esibo79KVH1EZJe+8VtNuUWQEeUEP/Wl9MMaH - ao3WeUC7xPXdC+MM0D1xAVuz0NW5MMMBuT2TDk0fc+YNJSHhq4joAQ901ubxzfTR - zD9nEXMQQDDiCM8ok8IjT4T1ga59XpXwn8SulL7Jen0ZPzS4wz7HKEBFVdWKvRct - 24mW0H45ea5M6V1XUaMvzWSHbuFpT6C1MhvjdyEoNGvwbvw+xwwEc8W3tnEAP63d - 8wWPQf9kecVWbvg6ck8ufIfJnEMOR7u20V0sNki/JtD9mRCWEEn2wLunzflYW23x - WHrhPtz0wwoZLLGB2mn8+E9BNgXf06V/2Iitvrwz3QKBgQD2xq4/9H9wjsCAqK/0 - sth4JOIpmCcMT77Vz2iwjkrkJehpS6j1By3FOzQSQkaG8G2j/f9Zr8ofiscU1V/n - B+4ghGl21gZcCzxi67Ygp8rSTUN+ocpHODGN26jIUk8Jtf6c0BsiHJU5CrdcDwm5 - voR5Y1Ixaq5PWpegatUc1GUPKwKBgQDUTyqYyuOA4ul/bt8xYlO+puQ3G8ITTLW2 - JB8a7jdjED7+XMOwWh4CW9M81dnIVP6UyfCdPlSC3W6q3p/DgMPbtV1w93L9AXpS - HN5wrFqPUKAuXqxIWrIwDCk6oRmvQMXui/jUGZUHDK47h5EeU/PbDOWvWSSKzlNW - q3985tRyfQKBgQCVuMFryBmx3spoxO/MlN3FNwuIlOnMDG4KJwaraAmEFoPFrsPZ - tftNGLhlA5TqteCviKFudrs5G+fhefvvnd4aGHwsP3ooSiDfG4eqlGL36Sy0HdEu - GKfoG4dx0o5lo+fQmGp97b2TmC7bSbxq125kf6AUn1cWii5Ig8i87xhJdQKBgEmB - RzwzMmUTKshl+Hw+kMP3QBgcUisgaeEvzF0kkKSJoWWrdE0ARleGtzHe0FHdq26U - I+wtAlF0nLYn8aRcVnMg7cMIyRTziAgZ2qGj6o6n2W10da1vSTX9X+DemeflQyH9 - 8B5u5PvV1hTiMMoRQuJaKsN014P/PzdIlREHUhJ5AoGBAOODAt0nM4Jz4u8DykI/ - sPjMZ0yY62FWdAvzbIZAiZ/OPPE+fV1OGAsakot01mAym9xWM7kgIt3oWDPfaLN1 - vyt/evXV9YfnyVbhLfxbonhCWHRCiLE7CTulbQpdDkYx6D87SmEdtKbmJa4lwpVK - zBy5BhOMzamzojiV/4d1B30k + MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDPf9HxzkRKAtnL + 5YbLzyn3tzkbyeKeJuyGptOC+0oSS17Ssj72fR5FBcIvsPFg7LTtSf+6lEO9uMrW + hib8/dygvDv85fBUTVkSj2olvUnJrDPrhapfAsbGjhLDr/gjXMSnWp+E9gLo+GB3 + gHhLqDpAWfYZMEaW9jW+7TO57EliRCY/devUJPN4JMF91OGDmdqsLkYKdkUxmLkF + xCCKfHwfV1adO9zwG1NFz6l4KSN8VF8rAXfw6mB9IrXoqW3Wub2YcikhXhq7FqE5 + yXcHBnR6DkypUVpns8+sfWEeN/uoDunvaPk4Pibe7r9aOsonYzqUyOma0sHqCMg5 + Ze2Zj4N9AgMBAAECggEAH4beubwq0Qn02I+DtxS/XiVnFmtKtNfaKS1Qxr5yhywP + eLjfaykgf8E7u25Jhn4AkWskYU9kqX9ZxlV0uAzESGHBRZAUP2LPLUxABwLnIq64 + 5siU1cHAvOtaO4RRkeHm+TyCLzwHvij60z1gSbKCQ0PH6hTVLNu019EHgnC9x6Go + GoCsaCeVghE7FjhM7L0UsP8/9sI7QUPJ7tgi8wMVzFGmx3Yg0pE/DyOIqkHCo5YQ + /87KAhXbFMkSM7ZuDxuXbw98Kdf7cxuFfH5zvYlt7X4S/U6EBtHJZNLYkoRcMu5/ + DzYk5AgPqz8ws0UKubDLDRQbsRtj7EQTX2zz3s0fEQKBgQDm5m/SACxp5sLeF30E + KQ+TRMPs8sNnMDTpb80fDmmspdH7T/Lzn07trhaYOUXz7DsNJiI0czlgy7mZ+fu9 + /lNFlCuXM6BH3ryFFPSEsMlZbJHvil7XdtBsQ31KCU1RMDDNMhhMZroIAdM17Tha + MjxKYNeGt5BMqMCFtswU6ZMGmQKBgQDmDi1SnIDbR+l5mDWQlbW6uIriVBJnCTEH + rhQQBPl0AOuJHOcA0lvGL3GZxGkRyFu88nT151WShyC2TyYql927p2PivL+uGB0p + 6SmGKlk+0PdU5aDc91iXZDZW4f0ESfVuSLFTkG8wJS/mYPIWHFHXQV81lR068T/W + IfoomSGGhQKBgGhJa/fNXEH4l8r3kN9wLDi3tkYu31Kb00ob1OlR+SihPFXlpjWi + nmN7XXkjZRTmfUVTE9/cqu2tFgcVgT4uwC2M3qNMDfhoAX2aGVZVApwBDWjDdlKa + t0gus0fdK66ftWr4VUEspJk5OcwBeJJEXja5xp5yChqVnV7HUSWg2WcBAoGAXsys + xIQVXUcO4LpmKERo9J6jLIy0YJ0bBS0ou0kxein+5StxzZbhlpNqXpDfMyK7GutW + YxaG3rCsPDP1aEvYZUGfGYnp+tWY/vJD2DOPPyFhKizN0wBiE1CKS9coKiJH7sAp + wmOcTdylFmwQlifpWahokW3285kCUz0BFsWzWPkCgYAm+ESRUwlqsybGf4yTyOLO + DDamA6WL35Je5HjzS8HzjBNRWdPtuQGyVxg+ZB/oVYBF5ga3EcL8MBPTuNFEtYM6 + OMLIJjJW/DNEAuXgv47ZFo5EtqY0M9C5dCS+MyiCRjMNPUbw8OpzrGgJnLkchCdN + toco0TrtKpHDTpJZCPMiIg== -----END PRIVATE KEY----- From dbce47d06a0c2554ae51ace6047ce9039e59d9f2 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Sat, 20 Jun 2026 16:58:41 +0500 Subject: [PATCH 070/164] dev: normalizer --- pkg/model/chi/normalizer/normalizer.go | 76 ++++++++----------- .../chi/normalizer/templates_cr/const.go | 4 +- .../chi/normalizer/templates_cr/normalizer.go | 8 +- pkg/model/chk/normalizer/normalizer.go | 31 ++++---- 4 files changed, 57 insertions(+), 62 deletions(-) diff --git a/pkg/model/chi/normalizer/normalizer.go b/pkg/model/chi/normalizer/normalizer.go index 4c975e282..a7b75c8d5 100644 --- a/pkg/model/chi/normalizer/normalizer.go +++ b/pkg/model/chi/normalizer/normalizer.go @@ -238,14 +238,15 @@ func (n *Normalizer) normalizeStop(stop *types.StringBool) *types.StringBool { // normalizeRestart normalizes .spec.restart func (n *Normalizer) normalizeRestart(restart *types.String) *types.String { - switch strings.ToLower(restart.Value()) { - case strings.ToLower(chi.RestartRollingUpdate): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; anything else becomes empty. + // Kept as a switch so future restart policies are just new cases. + switch util.FoldEnum(restart.Value(), chi.RestartRollingUpdate) { + case chi.RestartRollingUpdate: return types.NewString(chi.RestartRollingUpdate) + default: + // Unknown value - just use empty + return nil } - - // In case it is unknown value - just use empty - return nil } // normalizeTroubleshoot normalizes .spec.stop @@ -302,6 +303,10 @@ func (n *Normalizer) normalizeDefaults(defaults *chi.Defaults) *chi.Defaults { if defaults.StorageManagement == nil { defaults.StorageManagement = chi.NewStorageManagement() } + // Fold casing + validate the default StorageManagement (provisioner/reclaimPolicy). + // This path was previously left un-normalized, so a lowercase reclaimPolicy at the + // defaults level was written verbatim into the PVC label. + templates.NormalizeStorageManagement(defaults.StorageManagement) // Ensure field if defaults.Templates == nil { //defaults.Templates = api.NewChiTemplateNames() @@ -359,15 +364,11 @@ func (n *Normalizer) normalizeTemplating(templating *chi.ChiTemplating) *chi.Chi if templating == nil { templating = chi.NewChiTemplating() } - switch strings.ToLower(templating.GetPolicy()) { - case strings.ToLower(chi.TemplatingPolicyAuto): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the default. + switch util.FoldEnum(templating.GetPolicy(), chi.TemplatingPolicyAuto, chi.TemplatingPolicyManual) { + case chi.TemplatingPolicyAuto: templating.SetPolicy(chi.TemplatingPolicyAuto) - case strings.ToLower(chi.TemplatingPolicyManual): - // Known value, overwrite it to ensure case-ness - templating.SetPolicy(chi.TemplatingPolicyManual) default: - // Unknown value, fallback to default templating.SetPolicy(chi.TemplatingPolicyManual) } return templating @@ -399,16 +400,13 @@ func (n *Normalizer) normalizeReconcile(reconcile *chi.ChiReconcile) *chi.ChiRec reconcile = chi.NewChiReconcile().SetDefaults() } - // Policy - switch strings.ToLower(reconcile.GetPolicy()) { - case strings.ToLower(chi.ReconcilingPolicyWait): - // Known value, overwrite it to ensure case-ness + // Policy — fold any accepted casing to the canonical const; unknown values fall back to default. + switch util.FoldEnum(reconcile.GetPolicy(), chi.ReconcilingPolicyWait, chi.ReconcilingPolicyNoWait) { + case chi.ReconcilingPolicyWait: reconcile.SetPolicy(chi.ReconcilingPolicyWait) - case strings.ToLower(chi.ReconcilingPolicyNoWait): - // Known value, overwrite it to ensure case-ness + case chi.ReconcilingPolicyNoWait: reconcile.SetPolicy(chi.ReconcilingPolicyNoWait) default: - // Unknown value, fallback to default reconcile.SetPolicy(chi.ReconcilingPolicyUnspecified) } @@ -450,7 +448,8 @@ func (n *Normalizer) normalizeReconcileRuntime(runtime chi.ReconcileRuntime) chi } func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) chi.ReconcileStatefulSet { - // Create + // Create — fold casing to canonical const, then default if empty. + sts.Create.OnFailure = chi.NormalizeOnStatefulSetCreateFailureAction(sts.Create.OnFailure) if sts.Create.OnFailure == "" { sts.Create.OnFailure = chi.OnStatefulSetCreateFailureActionDelete } @@ -461,13 +460,16 @@ func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) if sts.Update.PollInterval == 0 { sts.Update.PollInterval = defaultStatefulSetUpdatePollInterval } + sts.Update.OnFailure = chi.NormalizeOnStatefulSetUpdateFailureAction(sts.Update.OnFailure) if sts.Update.OnFailure == "" { sts.Update.OnFailure = chi.OnStatefulSetUpdateFailureActionRollback } // Recreate + sts.Recreate.OnDataLoss = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnDataLoss) if sts.Recreate.OnDataLoss == "" { sts.Recreate.OnDataLoss = chi.OnStatefulSetRecreateOnDataLossActionRecreate } + sts.Recreate.OnUpdateFailure = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnUpdateFailure) if sts.Recreate.OnUpdateFailure == "" { sts.Recreate.OnUpdateFailure = chi.OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -505,15 +507,13 @@ func (n *Normalizer) normalizeCleanup(str *string, value string) { if str == nil { return } - switch strings.ToLower(*str) { - case strings.ToLower(chi.ObjectsCleanupRetain): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the supplied default. + switch util.FoldEnum(*str, chi.ObjectsCleanupRetain, chi.ObjectsCleanupDelete) { + case chi.ObjectsCleanupRetain: *str = chi.ObjectsCleanupRetain - case strings.ToLower(chi.ObjectsCleanupDelete): - // Known value, overwrite it to ensure case-ness + case chi.ObjectsCleanupDelete: *str = chi.ObjectsCleanupDelete default: - // Unknown value, fallback to default *str = value } } @@ -973,30 +973,20 @@ func (n *Normalizer) normalizeClusterSchemaPolicy(policy *chi.SchemaPolicy) *chi policy = chi.NewClusterSchemaPolicy() } - switch strings.ToLower(policy.Replica) { - case strings.ToLower(schemer.SchemaPolicyReplicaNone): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the default. + switch util.FoldEnum(policy.Replica, schemer.SchemaPolicyReplicaNone, schemer.SchemaPolicyReplicaAll) { + case schemer.SchemaPolicyReplicaNone: policy.Replica = schemer.SchemaPolicyReplicaNone - case strings.ToLower(schemer.SchemaPolicyReplicaAll): - // Known value, overwrite it to ensure case-ness - policy.Replica = schemer.SchemaPolicyReplicaAll default: - // Unknown value, fallback to default policy.Replica = schemer.SchemaPolicyReplicaAll } - switch strings.ToLower(policy.Shard) { - case strings.ToLower(schemer.SchemaPolicyShardNone): - // Known value, overwrite it to ensure case-ness + switch util.FoldEnum(policy.Shard, schemer.SchemaPolicyShardNone, schemer.SchemaPolicyShardAll, schemer.SchemaPolicyShardDistributedTablesOnly) { + case schemer.SchemaPolicyShardNone: policy.Shard = schemer.SchemaPolicyShardNone - case strings.ToLower(schemer.SchemaPolicyShardAll): - // Known value, overwrite it to ensure case-ness - policy.Shard = schemer.SchemaPolicyShardAll - case strings.ToLower(schemer.SchemaPolicyShardDistributedTablesOnly): - // Known value, overwrite it to ensure case-ness + case schemer.SchemaPolicyShardDistributedTablesOnly: policy.Shard = schemer.SchemaPolicyShardDistributedTablesOnly default: - // unknown value, fallback to default policy.Shard = schemer.SchemaPolicyShardAll } diff --git a/pkg/model/chi/normalizer/templates_cr/const.go b/pkg/model/chi/normalizer/templates_cr/const.go index 9d2b2c6d0..5c617747a 100644 --- a/pkg/model/chi/normalizer/templates_cr/const.go +++ b/pkg/model/chi/normalizer/templates_cr/const.go @@ -15,6 +15,6 @@ package templates_cr const ( - // .spec.useTemplate.useType - UseTypeMerge = "merge" + // .spec.useTemplate.useType (canonical humped form; CRD also accepts all-lowercase) + UseTypeMerge = "Merge" ) diff --git a/pkg/model/chi/normalizer/templates_cr/normalizer.go b/pkg/model/chi/normalizer/templates_cr/normalizer.go index 2968cbbb7..de6167f8d 100644 --- a/pkg/model/chi/normalizer/templates_cr/normalizer.go +++ b/pkg/model/chi/normalizer/templates_cr/normalizer.go @@ -16,6 +16,7 @@ package templates_cr import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/util" ) // NormalizeTemplateRefList normalizes list of templates use specifications @@ -38,10 +39,11 @@ func normalizeTemplateRef(templateRef *api.TemplateRef) *api.TemplateRef { // So far do nothing with empty namespace } - // Ensure UseType - switch templateRef.UseType { + // Ensure UseType — fold any accepted casing to the canonical const; unknown values + // fall back to the default. Kept as a switch so future use types are just new cases. + switch util.FoldEnum(templateRef.UseType, UseTypeMerge) { case UseTypeMerge: - // Known use type, all is fine, do nothing + templateRef.UseType = UseTypeMerge default: // Unknown use type - overwrite with default value templateRef.UseType = UseTypeMerge diff --git a/pkg/model/chk/normalizer/normalizer.go b/pkg/model/chk/normalizer/normalizer.go index 0a820eb7a..40194bf60 100644 --- a/pkg/model/chk/normalizer/normalizer.go +++ b/pkg/model/chk/normalizer/normalizer.go @@ -35,6 +35,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer/subst" "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer/templates" "github.com/altinity/clickhouse-operator/pkg/model/managers" + "github.com/altinity/clickhouse-operator/pkg/util" ) // Normalizer specifies structures normalizer @@ -268,6 +269,9 @@ func (n *Normalizer) normalizeDefaults(defaults *chi.Defaults) *chi.Defaults { if defaults.StorageManagement == nil { defaults.StorageManagement = chi.NewStorageManagement() } + // Fold casing + validate the default StorageManagement (provisioner/reclaimPolicy), + // matching the CHI normalizer — previously left un-normalized at the defaults level. + templates.NormalizeStorageManagement(defaults.StorageManagement) // Ensure field if defaults.Templates == nil { //defaults.Templates = api.NewChiTemplateNames() @@ -342,16 +346,13 @@ func (n *Normalizer) normalizeReconcile(reconcile *chi.ChiReconcile) *chi.ChiRec reconcile = chi.NewChiReconcile().SetDefaults() } - // Policy - switch strings.ToLower(reconcile.GetPolicy()) { - case strings.ToLower(chi.ReconcilingPolicyWait): - // Known value, overwrite it to ensure case-ness + // Policy — fold any accepted casing to the canonical const; unknown values fall back to default. + switch util.FoldEnum(reconcile.GetPolicy(), chi.ReconcilingPolicyWait, chi.ReconcilingPolicyNoWait) { + case chi.ReconcilingPolicyWait: reconcile.SetPolicy(chi.ReconcilingPolicyWait) - case strings.ToLower(chi.ReconcilingPolicyNoWait): - // Known value, overwrite it to ensure case-ness + case chi.ReconcilingPolicyNoWait: reconcile.SetPolicy(chi.ReconcilingPolicyNoWait) default: - // Unknown value, fallback to default reconcile.SetPolicy(chi.ReconcilingPolicyUnspecified) } @@ -393,7 +394,8 @@ func (n *Normalizer) normalizeReconcileRuntime(runtime chi.ReconcileRuntime) chi } func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) chi.ReconcileStatefulSet { - // Create + // Create — fold casing to canonical const, then default if empty. + sts.Create.OnFailure = chi.NormalizeOnStatefulSetCreateFailureAction(sts.Create.OnFailure) if sts.Create.OnFailure == "" { sts.Create.OnFailure = chi.OnStatefulSetCreateFailureActionDelete } @@ -404,13 +406,16 @@ func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) if sts.Update.PollInterval == 0 { sts.Update.PollInterval = defaultStatefulSetUpdatePollInterval } + sts.Update.OnFailure = chi.NormalizeOnStatefulSetUpdateFailureAction(sts.Update.OnFailure) if sts.Update.OnFailure == "" { sts.Update.OnFailure = chi.OnStatefulSetUpdateFailureActionRollback } // Recreate + sts.Recreate.OnDataLoss = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnDataLoss) if sts.Recreate.OnDataLoss == "" { sts.Recreate.OnDataLoss = chi.OnStatefulSetRecreateOnDataLossActionRecreate } + sts.Recreate.OnUpdateFailure = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnUpdateFailure) if sts.Recreate.OnUpdateFailure == "" { sts.Recreate.OnUpdateFailure = chi.OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -460,15 +465,13 @@ func (n *Normalizer) normalizeCleanup(str *string, value string) { if str == nil { return } - switch strings.ToLower(*str) { - case strings.ToLower(chi.ObjectsCleanupRetain): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the supplied default. + switch util.FoldEnum(*str, chi.ObjectsCleanupRetain, chi.ObjectsCleanupDelete) { + case chi.ObjectsCleanupRetain: *str = chi.ObjectsCleanupRetain - case strings.ToLower(chi.ObjectsCleanupDelete): - // Known value, overwrite it to ensure case-ness + case chi.ObjectsCleanupDelete: *str = chi.ObjectsCleanupDelete default: - // Unknown value, fallback to default *str = value } } From 5deef8bd3267e91dbd0e493327b615648b4b6ddf Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Sat, 20 Jun 2026 17:03:27 +0500 Subject: [PATCH 071/164] dev: common wrappers --- pkg/model/common/normalizer/templates/host.go | 2 ++ pkg/model/common/normalizer/templates/pod.go | 5 +++++ .../common/normalizer/templates/volume_claim.go | 16 +++++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/pkg/model/common/normalizer/templates/host.go b/pkg/model/common/normalizer/templates/host.go index dafe9fcba..ce6fa843b 100644 --- a/pkg/model/common/normalizer/templates/host.go +++ b/pkg/model/common/normalizer/templates/host.go @@ -37,6 +37,8 @@ func NormalizeHostTemplate(template *api.HostTemplate) { // Normalize PortDistribution for i := range template.PortDistribution { portDistribution := &template.PortDistribution[i] + // Fold any accepted casing to the canonical const before validating. + portDistribution.Type = deployment.NormalizePortDistributionType(portDistribution.Type) switch portDistribution.Type { case deployment.PortDistributionUnspecified, diff --git a/pkg/model/common/normalizer/templates/pod.go b/pkg/model/common/normalizer/templates/pod.go index 1a484041e..0541d5c20 100644 --- a/pkg/model/common/normalizer/templates/pod.go +++ b/pkg/model/common/normalizer/templates/pod.go @@ -125,6 +125,11 @@ func normalizePodDistribution(replicasCount int, podDistribution *api.PodDistrib podDistribution.TopologyKey = defaultTopologyKey } + // Fold any accepted casing to the canonical const so the switch below (and downstream + // affinity builders) can compare with plain ==. + podDistribution.Type = deployment.NormalizePodDistributionType(podDistribution.Type) + podDistribution.Scope = deployment.NormalizePodDistributionScope(podDistribution.Scope) + switch podDistribution.Type { case deployment.PodDistributionUnspecified, diff --git a/pkg/model/common/normalizer/templates/volume_claim.go b/pkg/model/common/normalizer/templates/volume_claim.go index ffd6b1f05..066089a3f 100644 --- a/pkg/model/common/normalizer/templates/volume_claim.go +++ b/pkg/model/common/normalizer/templates/volume_claim.go @@ -22,20 +22,26 @@ func NormalizeVolumeClaimTemplate(template *api.VolumeClaimTemplate) { // Skip for now // StorageManagement - normalizeStorageManagement(&template.StorageManagement) + NormalizeStorageManagement(&template.StorageManagement) // Check Spec // Skip for now } -// normalizeStorageManagement normalizes StorageManagement -func normalizeStorageManagement(storage *api.StorageManagement) { - // Check PVCProvisioner +// NormalizeStorageManagement normalizes StorageManagement: it folds the letter-casing +// of PVCProvisioner / PVCReclaimPolicy to their canonical consts (so both humped and +// all-lowercase CRD inputs are accepted), then resets any unrecognized value to +// Unspecified. Exported so callers normalizing a bare StorageManagement (e.g. +// spec.defaults.storageManagement) reuse the same folding+validation. +func NormalizeStorageManagement(storage *api.StorageManagement) { + // PVCProvisioner — fold casing to canonical, then validate. + storage.PVCProvisioner = storage.PVCProvisioner.Normalize() if !storage.PVCProvisioner.IsValid() { storage.PVCProvisioner = api.PVCProvisionerUnspecified } - // Check PVCReclaimPolicy + // PVCReclaimPolicy — fold casing to canonical, then validate. + storage.PVCReclaimPolicy = storage.PVCReclaimPolicy.Normalize() if !storage.PVCReclaimPolicy.IsValid() { storage.PVCReclaimPolicy = api.PVCReclaimPolicyUnspecified } From 834c0cc5fc8a1ba3b73c5628deca00722dbb51de Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Sat, 20 Jun 2026 17:04:21 +0500 Subject: [PATCH 072/164] dev: switch to const --- pkg/controller/chi/controller-chk-watcher.go | 4 +++- pkg/metrics/clickhouse/exporter.go | 4 ++-- pkg/model/clickhouse/connection.go | 2 +- pkg/model/clickhouse/credentials_endpoint.go | 6 ++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/chi/controller-chk-watcher.go b/pkg/controller/chi/controller-chk-watcher.go index bb45c2f7a..5c9100dfe 100644 --- a/pkg/controller/chi/controller-chk-watcher.go +++ b/pkg/controller/chi/controller-chk-watcher.go @@ -83,7 +83,9 @@ func (c *Controller) StartCHKWatcher(ctx context.Context) { // isKeeperWatchEnabled checks if the CHK watch is configured. func (c *Controller) isKeeperWatchEnabled() bool { policy := chop.Config().Reconcile.Coordination.Keeper.OnKeeperResourceUpdate - return policy.HasValue() && policy.Value() == api.KeeperOnResourceUpdateReconcile + // EqualFoldString: the CRD accepts both "Reconcile" and "reconcile"; the chopconf value + // is not re-folded after load, so compare case-insensitively here. + return policy.HasValue() && policy.EqualFoldString(api.KeeperOnResourceUpdateReconcile) } // onCHKUpdate handles CHK update events. Triggers CHI reconcile only when CHK transitions to Completed. diff --git a/pkg/metrics/clickhouse/exporter.go b/pkg/metrics/clickhouse/exporter.go index ac494ab02..caeae08ab 100644 --- a/pkg/metrics/clickhouse/exporter.go +++ b/pkg/metrics/clickhouse/exporter.go @@ -120,10 +120,10 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher { case api.ChSchemeAuto: switch { case types.IsPortAssigned(host.HTTPPort): - clusterConnectionParams.Scheme = "http" + clusterConnectionParams.Scheme = api.ChSchemeHTTP clusterConnectionParams.Port = int(host.HTTPPort) case types.IsPortAssigned(host.HTTPSPort): - clusterConnectionParams.Scheme = "https" + clusterConnectionParams.Scheme = api.ChSchemeHTTPS clusterConnectionParams.Port = int(host.HTTPSPort) } case api.ChSchemeHTTP: diff --git a/pkg/model/clickhouse/connection.go b/pkg/model/clickhouse/connection.go index 88df39e47..05bdd56be 100644 --- a/pkg/model/clickhouse/connection.go +++ b/pkg/model/clickhouse/connection.go @@ -177,7 +177,7 @@ func legacyVerifiedTLSConfig() *tls.Config { // users opting into TLS hardening should not silently get InsecureSkipVerify=true. func (c *Connection) setupTLSAdvanced() { // Nothing to do for HTTP DSNs. - if c.params.scheme != httpsScheme { + if c.params.scheme != api.ChSchemeHTTPS { return } diff --git a/pkg/model/clickhouse/credentials_endpoint.go b/pkg/model/clickhouse/credentials_endpoint.go index 0d2a411ac..112094df1 100644 --- a/pkg/model/clickhouse/credentials_endpoint.go +++ b/pkg/model/clickhouse/credentials_endpoint.go @@ -33,8 +33,6 @@ const ( dsnUsernamePasswordPairPattern = "%s:%s@" dsnUsernamePasswordPairUsernameOnlyPattern = "%s@" - httpsScheme = "https" - // tlsSettingsLegacy is the registry key used when no per-endpoint TLS knobs // are configured (the legacy path). Identical knobs across endpoints share // this key. Endpoints with explicit Verify/MinVersion/ServerName/rootCA get @@ -212,7 +210,7 @@ func (c *EndpointCredentials) makeDSN(hideCredentials bool) string { c.hostname, strconv.Itoa(c.port), ) - if c.scheme == httpsScheme { + if c.scheme == api.ChSchemeHTTPS { baseUrl += "?tls_config=" + c.tlsConfigKey } return baseUrl @@ -228,7 +226,7 @@ func (c *EndpointCredentials) makeDSNLogQueries(hideCredentials bool) string { strconv.Itoa(c.port), ) baseUrl += "?log_queries=1" - if c.scheme == httpsScheme { + if c.scheme == api.ChSchemeHTTPS { baseUrl += "&tls_config=" + c.tlsConfigKey } return baseUrl From 443868a77f90f99d0c25a2ae352f96edec4f000d Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Sat, 20 Jun 2026 17:04:44 +0500 Subject: [PATCH 073/164] dev: config --- config/config-dev.yaml | 4 ++-- config/config.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config-dev.yaml b/config/config-dev.yaml index d403aa9d8..072a9b9ca 100644 --- a/config/config-dev.yaml +++ b/config/config-dev.yaml @@ -152,8 +152,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests diff --git a/config/config.yaml b/config/config.yaml index f63d20486..b29a88d42 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -152,8 +152,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests From dee54eba4072e3f808c6c5fb3ad0af0c747478de Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 15:16:17 +0500 Subject: [PATCH 074/164] dev: new service type --- pkg/interfaces/label_type.go | 9 +++++---- pkg/interfaces/name_type.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/interfaces/label_type.go b/pkg/interfaces/label_type.go index f152328e3..2596cc55c 100644 --- a/pkg/interfaces/label_type.go +++ b/pkg/interfaces/label_type.go @@ -22,10 +22,11 @@ const ( LabelConfigMapHost LabelType = "Label cm host" LabelConfigMapStorage LabelType = "Label cm storage" - LabelServiceCR LabelType = "Label svc chi" - LabelServiceCluster LabelType = "Label svc cluster" - LabelServiceShard LabelType = "Label svc shard" - LabelServiceHost LabelType = "Label svc host" + LabelServiceCR LabelType = "Label svc chi" + LabelServiceCluster LabelType = "Label svc cluster" + LabelServiceShard LabelType = "Label svc shard" + LabelServiceHost LabelType = "Label svc host" + LabelServiceHostClient LabelType = "Label svc host client" LabelExistingPV LabelType = "Label existing pv" LabelNewPVC LabelType = "Label new pvc" diff --git a/pkg/interfaces/name_type.go b/pkg/interfaces/name_type.go index b98acba1d..b277b00f0 100644 --- a/pkg/interfaces/name_type.go +++ b/pkg/interfaces/name_type.go @@ -33,6 +33,7 @@ const ( NameInstanceHostname NameType = "NameInstanceHostname" NameStatefulSet NameType = "NameStatefulSet" NameStatefulSetService NameType = "NameStatefulSetService" + NameStatefulSetServiceClient NameType = "NameStatefulSetServiceClient" NamePodHostname NameType = "NamePodHostname" NameFQDN NameType = "NameFQDN" NameFQDNs NameType = "NameFQDNs" From 87cc91b0619003e00b47eb17a4b83f79c4c7bfc4 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 15:21:55 +0500 Subject: [PATCH 075/164] dev: service model --- pkg/model/chi/tags/labeler/list.go | 1 + pkg/model/chk/creator/service.go | 72 ++++++--- .../chk/creator/service_host_ports_test.go | 65 ++++++++ .../chk/creator/service_host_split_test.go | 150 ++++++++++++++++++ pkg/model/chk/namer/const.go | 4 + pkg/model/chk/namer/name.go | 8 + pkg/model/chk/namer/name_test.go | 40 +++++ pkg/model/chk/namer/namer.go | 3 + pkg/model/chk/tags/labeler/list.go | 1 + pkg/model/chk/tags/labeler/list_test.go | 42 +++++ pkg/model/common/tags/labeler/labeler.go | 2 + pkg/model/common/tags/labeler/labels.go | 21 +++ pkg/model/common/tags/labeler/list.go | 1 + 13 files changed, 389 insertions(+), 21 deletions(-) create mode 100644 pkg/model/chk/creator/service_host_ports_test.go create mode 100644 pkg/model/chk/creator/service_host_split_test.go create mode 100644 pkg/model/chk/namer/name_test.go create mode 100644 pkg/model/chk/tags/labeler/list_test.go diff --git a/pkg/model/chi/tags/labeler/list.go b/pkg/model/chi/tags/labeler/list.go index 2bf27b5e7..b138cd542 100644 --- a/pkg/model/chi/tags/labeler/list.go +++ b/pkg/model/chi/tags/labeler/list.go @@ -47,6 +47,7 @@ var list = types.List{ labeler.LabelServiceValueCluster: "cluster", labeler.LabelServiceValueShard: "shard", labeler.LabelServiceValueHost: "host", + labeler.LabelServiceValueHostClient: "host-client", labeler.LabelPVCReclaimPolicyName: clickhouse_altinity_com.APIGroupName + "/" + "reclaimPolicy", // Supplementary service labels - used to cooperate with k8s diff --git a/pkg/model/chk/creator/service.go b/pkg/model/chk/creator/service.go index 46d50e8e0..c823c2486 100644 --- a/pkg/model/chk/creator/service.go +++ b/pkg/model/chk/creator/service.go @@ -72,7 +72,7 @@ func (m *ServiceManager) CreateService(what interfaces.ServiceType, params ...an var host *chi.Host if len(params) > 0 { host = params[0].(*chi.Host) - return []*core.Service{m.createServiceHost(host)} + return m.createServiceHost(host) } } panic("unknown service type") @@ -272,33 +272,59 @@ func (m *ServiceManager) createServiceShard(shard chi.IShard) *core.Service { return nil } -// createServiceHost creates new core.Service for specified host -func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { +// createServiceHost builds the per-host (replica-level) Services for a Keeper host. +// +// When the user supplies a replicaServiceTemplate it is honored as-is (single Service) — the +// user is in full control and back-compat is preserved. +// +// Otherwise two operator-managed headless Services are emitted (issue #1982): +// - peer : intra-keeper Raft + StatefulSet pod DNS. publishNotReadyAddresses=true so Raft +// peers can reach each other BEFORE pods are Ready and bootstrap quorum. Keeps the existing +// NameStatefulSetService name + LabelServiceHost label → byte-identical to the pre-split +// layout, no pod re-roll, and the Raft /STS serviceName binding is unchanged. +// - client : ClickHouse-facing. publishNotReadyAddresses=false so DNS only resolves Ready +// Keeper nodes; raft port omitted. The keeper-ref resolver selects this tier via +// LabelServiceHostClient so clients never connect to a not-yet-Ready Keeper. +func (m *ServiceManager) createServiceHost(host *chi.Host) []*core.Service { if host.IsZero() { return nil } if template, ok := host.GetServiceTemplate(); ok { // .templates.ServiceTemplate specified - return creator.CreateServiceFromTemplate( - template, - host.GetRuntime().GetAddress().GetNamespace(), - m.namer.Name(interfaces.NameStatefulSetService, host), - m.tagger.Label(interfaces.LabelServiceHost, host), - m.tagger.Annotate(interfaces.AnnotateServiceHost, host), - m.tagger.Selector(interfaces.SelectorHostScope, host), - m.or.CreateOwnerReferences(m.cr), - m.macro.Scope(host), - m.labeler, - ) + return []*core.Service{ + creator.CreateServiceFromTemplate( + template, + host.GetRuntime().GetAddress().GetNamespace(), + m.namer.Name(interfaces.NameStatefulSetService, host), + m.tagger.Label(interfaces.LabelServiceHost, host), + m.tagger.Annotate(interfaces.AnnotateServiceHost, host), + m.tagger.Selector(interfaces.SelectorHostScope, host), + m.or.CreateOwnerReferences(m.cr), + m.macro.Scope(host), + m.labeler, + ), + } } - // Create default Service - // We do not have .templates.ServiceTemplate specified or it is incorrect + // No user template - emit the two default per-host Services. + peer := m.buildDefaultHostService(host, + m.namer.Name(interfaces.NameStatefulSetService, host), interfaces.LabelServiceHost, + true /* publishNotReady */, true /* includeRaftPort */) + client := m.buildDefaultHostService(host, + m.namer.Name(interfaces.NameStatefulSetServiceClient, host), interfaces.LabelServiceHostClient, + false /* publishNotReady */, false /* includeRaftPort */) + return []*core.Service{peer, client} +} + +// buildDefaultHostService builds one operator-managed headless per-host Service. publishNotReady +// toggles ServiceSpec.PublishNotReadyAddresses; includeRaftPort keeps the Raft port (peer) or +// drops it (client, which only needs the ZK client ports). +func (m *ServiceManager) buildDefaultHostService(host *chi.Host, name string, label interfaces.LabelType, publishNotReady, includeRaftPort bool) *core.Service { svc := &core.Service{ ObjectMeta: meta.ObjectMeta{ - Name: m.namer.Name(interfaces.NameStatefulSetService, host), + Name: name, Namespace: host.GetRuntime().GetAddress().GetNamespace(), - Labels: m.macro.Scope(host).Map(m.tagger.Label(interfaces.LabelServiceHost, host)), + Labels: m.macro.Scope(host).Map(m.tagger.Label(label, host)), Annotations: m.macro.Scope(host).Map(m.tagger.Annotate(interfaces.AnnotateServiceHost, host)), OwnerReferences: m.or.CreateOwnerReferences(m.cr), }, @@ -306,10 +332,10 @@ func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { Selector: m.tagger.Selector(interfaces.SelectorHostScope, host), ClusterIP: TemplateDefaultsServiceClusterIP, Type: "ClusterIP", - PublishNotReadyAddresses: true, + PublishNotReadyAddresses: publishNotReady, }, } - appendHostExposedPorts(svc, host) + appendHostExposedPorts(svc, host, includeRaftPort) m.labeler.MakeObjectVersion(svc.GetObjectMeta(), svc) return svc } @@ -320,11 +346,15 @@ func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { // matching per-host XML overlay emits so the Keeper // process binds no plaintext listener at all; liveness probe falls back to // pgrep. Other ports (zk-secure, raft) flow through unchanged. -func appendHostExposedPorts(svc *core.Service, host *chi.Host) { +func appendHostExposedPorts(svc *core.Service, host *chi.Host, includeRaftPort bool) { host.WalkSpecifiedPorts(func(name string, port *types.Int32, protocol core.Protocol) bool { if (name == chi.KpDefaultZKPortName) && !host.IsInsecure() { return false } + // The client-facing Service omits the Raft port — clients only use the ZK client ports. + if (name == chi.KpDefaultRaftPortName) && !includeRaftPort { + return false + } svc.Spec.Ports = append(svc.Spec.Ports, core.ServicePort{ Name: name, Protocol: protocol, diff --git a/pkg/model/chk/creator/service_host_ports_test.go b/pkg/model/chk/creator/service_host_ports_test.go new file mode 100644 index 000000000..a17b2dd1c --- /dev/null +++ b/pkg/model/chk/creator/service_host_ports_test.go @@ -0,0 +1,65 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package creator + +import ( + "testing" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/apis/common/types" +) + +// portNames extracts the port names emitted onto a Service for assertion. +func portNames(svc *core.Service) []string { + names := make([]string, 0, len(svc.Spec.Ports)) + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + return names +} + +// TestAppendHostExposedPortsRaftToggle pins the per-host Service port partition (issue #1982): +// the peer/Raft Service keeps the raft port (includeRaftPort=true) while the client-facing +// Service drops it (includeRaftPort=false) and retains only the ZK client ports. +func TestAppendHostExposedPortsRaftToggle(t *testing.T) { + newHost := func() *chi.Host { + return &chi.Host{ + HostSecure: chi.HostSecure{Insecure: types.NewStringBool(true)}, + HostPorts: chi.HostPorts{ + ZKPort: types.NewInt32(2181), + RaftPort: types.NewInt32(9234), + }, + } + } + + t.Run("peer keeps raft port", func(t *testing.T) { + svc := &core.Service{} + appendHostExposedPorts(svc, newHost(), true) + names := portNames(svc) + require.Contains(t, names, chi.KpDefaultZKPortName) + require.Contains(t, names, chi.KpDefaultRaftPortName) + }) + + t.Run("client drops raft port, keeps zk", func(t *testing.T) { + svc := &core.Service{} + appendHostExposedPorts(svc, newHost(), false) + names := portNames(svc) + require.Contains(t, names, chi.KpDefaultZKPortName) + require.NotContains(t, names, chi.KpDefaultRaftPortName) + }) +} diff --git a/pkg/model/chk/creator/service_host_split_test.go b/pkg/model/chk/creator/service_host_split_test.go new file mode 100644 index 000000000..b014dcb00 --- /dev/null +++ b/pkg/model/chk/creator/service_host_split_test.go @@ -0,0 +1,150 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is an external test package (creator_test) so it can import the managers +// package to build a real tagger; managers imports creator, so an in-package test +// would form an import cycle. +package creator_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + + chk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/chop" + "github.com/altinity/clickhouse-operator/pkg/interfaces" + chkNormalizer "github.com/altinity/clickhouse-operator/pkg/model/chk/normalizer" + chkLabeler "github.com/altinity/clickhouse-operator/pkg/model/chk/tags/labeler" + commonNormalizer "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer" + commonLabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler" + "github.com/altinity/clickhouse-operator/pkg/model/managers" +) + +// The normalizer reads the global operator config (chop.Config()) via the labeler; +// initialize a default instance once for the package. +func init() { chop.New(nil, nil, "") } + +// serviceLabelKey is the fully-qualified label key that carries the Service tier value +// (host vs host-client), resolved through the CHK labeler so the test does not hard-code it. +func serviceLabelKey(t *testing.T) string { + t.Helper() + l := chkLabeler.New(chk.NewClickHouseKeeperInstallation("x", "ns")) + return l.Get(commonLabeler.LabelService) +} + +func portNames(svc *core.Service) []string { + names := make([]string, 0, len(svc.Spec.Ports)) + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + return names +} + +// normalizeSingleHostCHK builds a minimal one-cluster CHK and normalizes it, returning the +// ready-to-use CR plus a ServiceManager wired exactly as the controller wires it. +func normalizeSingleHostCHK(t *testing.T) (*chk.ClickHouseKeeperInstallation, interfaces.IServiceManager) { + t.Helper() + src := chk.NewClickHouseKeeperInstallation("kpr", "ns") + src.Spec.Configuration = &chk.Configuration{Clusters: []*chk.Cluster{{Name: "keeper"}}} + cr, err := chkNormalizer.New().CreateTemplated(src, commonNormalizer.NewOptions[chk.ClickHouseKeeperInstallation]()) + require.NoError(t, err) + require.NotNil(t, cr) + + sm := managers.NewServiceManager(managers.ServiceManagerTypeKeeper) + sm.SetCR(cr) + sm.SetTagger(managers.NewTagManager(managers.TagManagerTypeKeeper, cr)) + return cr, sm +} + +// TestCreateServiceHostEmitsPeerAndClient pins the issue #1982 contract: with no +// user-supplied replicaServiceTemplate, each Keeper host gets TWO headless Services — +// - a peer/Raft Service: publishNotReadyAddresses=true, keeps the Raft port, Service tier "host" +// - a client Service: publishNotReadyAddresses=false, no Raft port, Service tier "host-client" +// +// The peer keeps the bare StatefulSet Service name (Raft / pod DNS binding) and the +// client name is that + "-client". The distinct "host-client" tier label is what the keeper-ref +// resolver selects so ClickHouse clients never resolve a not-yet-Ready Keeper. +func TestCreateServiceHostEmitsPeerAndClient(t *testing.T) { + cr, sm := normalizeSingleHostCHK(t) + svcLabel := serviceLabelKey(t) + + hosts := 0 + cr.WalkHosts(func(host *chi.Host) error { + hosts++ + services := sm.CreateService(interfaces.ServiceHost, host) + require.Len(t, services, 2, "a default Keeper host must emit exactly two Services") + + peer, client := services[0], services[1] + + // Peer: bare name, publishNotReady=true, host tier, retains the Raft port. + require.Equal(t, "chk-kpr-keeper-0-0", peer.Name) + require.True(t, peer.Spec.PublishNotReadyAddresses, + "peer/Raft Service must publish not-ready addresses for quorum bootstrap") + require.Equal(t, "host", peer.Labels[svcLabel]) + require.Contains(t, portNames(peer), chi.KpDefaultRaftPortName) + + // Client: peer name + "-client", publishNotReady=false, host-client tier, no Raft port. + require.Equal(t, peer.Name+"-client", client.Name) + require.False(t, client.Spec.PublishNotReadyAddresses, + "client Service must resolve only Ready Keeper endpoints") + require.Equal(t, "host-client", client.Labels[svcLabel]) + require.NotContains(t, portNames(client), chi.KpDefaultRaftPortName, + "client Service must not expose the Raft port") + + // Both are headless and share the host selector (one pod, two readiness views). + require.Equal(t, "None", peer.Spec.ClusterIP) + require.Equal(t, "None", client.Spec.ClusterIP) + return nil + }) + require.Equal(t, 1, hosts, "minimal one-cluster CHK must normalize to exactly one host") +} + +// TestCreateServiceHostHonorsUserTemplate verifies the back-compat escape hatch: when the host +// carries a replicaServiceTemplate the operator emits the single user-controlled Service and does +// NOT inject the second client Service. +func TestCreateServiceHostHonorsUserTemplate(t *testing.T) { + src := chk.NewClickHouseKeeperInstallation("kpr", "ns") + src.Spec.Configuration = &chk.Configuration{ + Clusters: []*chk.Cluster{{ + Name: "keeper", + Templates: &chi.TemplatesList{ReplicaServiceTemplate: "svc-tpl"}, + }}, + } + src.Spec.Templates = &chi.Templates{ + ServiceTemplates: []chi.ServiceTemplate{{ + Name: "svc-tpl", + Spec: core.ServiceSpec{Type: core.ServiceTypeClusterIP}, + }}, + } + cr, err := chkNormalizer.New().CreateTemplated(src, commonNormalizer.NewOptions[chk.ClickHouseKeeperInstallation]()) + require.NoError(t, err) + + sm := managers.NewServiceManager(managers.ServiceManagerTypeKeeper) + sm.SetCR(cr) + sm.SetTagger(managers.NewTagManager(managers.TagManagerTypeKeeper, cr)) + + cr.WalkHosts(func(host *chi.Host) error { + if _, ok := host.GetServiceTemplate(); !ok { + // Template did not attach (normalization specifics) — skip rather than assert a + // false negative; the two-Service default path is covered by the test above. + t.Skip("replicaServiceTemplate did not attach to host; template wiring not exercised") + } + services := sm.CreateService(interfaces.ServiceHost, host) + require.Len(t, services, 1, "a templated host must emit exactly one (user-controlled) Service") + return nil + }) +} diff --git a/pkg/model/chk/namer/const.go b/pkg/model/chk/namer/const.go index 277502b14..415116cf2 100644 --- a/pkg/model/chk/namer/const.go +++ b/pkg/model/chk/namer/const.go @@ -44,4 +44,8 @@ const ( // patternClusterPDBName is a template of cluster scope PDB. "chi-{chi}-{cluster}" patternClusterPDBName = "pdb chk- + macrosList.Get().Get(macro.MacrosCRName) + - + macrosList.Get().Get(macro.MacrosClusterName)" + + // statefulSetClientServiceNameSuffix is appended to the per-host StatefulSet Service name to + // form the client-facing Service name (see createStatefulSetServiceClientName). + statefulSetClientServiceNameSuffix = "-client" ) diff --git a/pkg/model/chk/namer/name.go b/pkg/model/chk/namer/name.go index 605d1c7d0..dbad76a1e 100644 --- a/pkg/model/chk/namer/name.go +++ b/pkg/model/chk/namer/name.go @@ -174,6 +174,14 @@ func (n *Namer) createStatefulSetServiceName(host *api.Host) string { return n.macro.Scope(host).Line(pattern) } +// createStatefulSetServiceClientName returns the name of the client-facing per-host Service — +// the StatefulSet Service name plus a "-client" suffix. The client Service publishes only Ready +// Keeper endpoints (publishNotReadyAddresses=false) so ClickHouse never resolves a not-yet-Ready +// node; the peer Service (createStatefulSetServiceName) keeps the bare name for Raft and pod DNS. +func (n *Namer) createStatefulSetServiceClientName(host *api.Host) string { + return n.createStatefulSetServiceName(host) + statefulSetClientServiceNameSuffix +} + // createPodHostname returns a hostname of a Pod of a ClickHouse instance. // Is supposed to be used where network connection to a Pod is required. // NB: right now Pod's hostname points to a Service, through which Pod can be accessed. diff --git a/pkg/model/chk/namer/name_test.go b/pkg/model/chk/namer/name_test.go new file mode 100644 index 000000000..6a35481d9 --- /dev/null +++ b/pkg/model/chk/namer/name_test.go @@ -0,0 +1,40 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package namer + +import ( + "testing" + + "github.com/stretchr/testify/require" + + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/interfaces" +) + +// TestStatefulSetServiceClientName pins the client Service naming contract (issue #1982): +// the client-facing Service name is the peer/StatefulSet Service name plus a "-client" suffix. +// The peer name MUST stay the bare StatefulSet Service name — Raft and the pod DNS +// (StatefulSet.serviceName) bind to it, so any drift would break quorum on existing clusters. +func TestStatefulSetServiceClientName(t *testing.T) { + n := New() + host := &chi.Host{} + + peer := n.Name(interfaces.NameStatefulSetService, host) + client := n.Name(interfaces.NameStatefulSetServiceClient, host) + + require.Equal(t, peer+"-client", client, + "client Service name must be the peer Service name plus the -client suffix") + require.NotEqual(t, peer, client, "peer and client Service names must differ") +} diff --git a/pkg/model/chk/namer/namer.go b/pkg/model/chk/namer/namer.go index 4912cc524..7af38e315 100644 --- a/pkg/model/chk/namer/namer.go +++ b/pkg/model/chk/namer/namer.go @@ -83,6 +83,9 @@ func (n *Namer) Name(what interfaces.NameType, params ...any) string { case interfaces.NameStatefulSetService: host := params[0].(*api.Host) return n.createStatefulSetServiceName(host) + case interfaces.NameStatefulSetServiceClient: + host := params[0].(*api.Host) + return n.createStatefulSetServiceClientName(host) case interfaces.NamePodHostname: host := params[0].(*api.Host) return n.createPodHostname(host) diff --git a/pkg/model/chk/tags/labeler/list.go b/pkg/model/chk/tags/labeler/list.go index d3ee92022..497646107 100644 --- a/pkg/model/chk/tags/labeler/list.go +++ b/pkg/model/chk/tags/labeler/list.go @@ -46,6 +46,7 @@ var list = types.List{ labeler.LabelServiceValueCluster: "cluster", labeler.LabelServiceValueShard: "shard", labeler.LabelServiceValueHost: "host", + labeler.LabelServiceValueHostClient: "host-client", labeler.LabelPVCReclaimPolicyName: clickhouse_keeper_altinity_com.APIGroupName + "/" + "reclaimPolicy", // Supplementary service labels - used to cooperate with k8s diff --git a/pkg/model/chk/tags/labeler/list_test.go b/pkg/model/chk/tags/labeler/list_test.go new file mode 100644 index 000000000..afe4326f0 --- /dev/null +++ b/pkg/model/chk/tags/labeler/list_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labeler + +import ( + "testing" + + "github.com/stretchr/testify/require" + + chk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/chop" + commonLabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler" +) + +// The labeler reads the global operator config (chop.Config()) during construction. +func init() { chop.New(nil, nil, "") } + +// TestServiceTierLabelValues guards the Service-tier label-value mapping (issue #1982). The +// client Service tier MUST resolve to a concrete, distinct value: a missing entry in the +// labeler `list` map silently yields an empty value, which makes the client Service unselectable +// by the keeper-ref resolver and leaves clients pointed at the not-ready peer tier. +func TestServiceTierLabelValues(t *testing.T) { + l := New(chk.NewClickHouseKeeperInstallation("kpr", "ns")) + + require.Equal(t, "host", l.Get(commonLabeler.LabelServiceValueHost)) + require.Equal(t, "host-client", l.Get(commonLabeler.LabelServiceValueHostClient), + "host-client tier must map to a concrete value; empty means the resolver can't select it") + require.NotEqual(t, l.Get(commonLabeler.LabelServiceValueHost), l.Get(commonLabeler.LabelServiceValueHostClient), + "peer and client Service tiers must be distinguishable by label") +} diff --git a/pkg/model/common/tags/labeler/labeler.go b/pkg/model/common/tags/labeler/labeler.go index 6ca7903c8..06c1589cf 100644 --- a/pkg/model/common/tags/labeler/labeler.go +++ b/pkg/model/common/tags/labeler/labeler.go @@ -56,6 +56,8 @@ func (l *Labeler) Label(what interfaces.LabelType, params ...any) map[string]str return l.labelServiceShard(params...) case interfaces.LabelServiceHost: return l.labelServiceHost(params...) + case interfaces.LabelServiceHostClient: + return l.labelServiceHostClient(params...) case interfaces.LabelExistingPV: return l.labelExistingPV(params...) diff --git a/pkg/model/common/tags/labeler/labels.go b/pkg/model/common/tags/labeler/labels.go index a5f48383c..6f7bb9ea1 100644 --- a/pkg/model/common/tags/labeler/labels.go +++ b/pkg/model/common/tags/labeler/labels.go @@ -87,6 +87,27 @@ func (l *Labeler) _labelServiceHost(host *api.Host) map[string]string { }) } +// labelServiceHostClient +func (l *Labeler) labelServiceHostClient(params ...any) map[string]string { + var host *api.Host + if len(params) > 0 { + host = params[0].(*api.Host) + return l._labelServiceHostClient(host) + } + panic("not enough params for labeler") +} + +// _labelServiceHostClient labels the client-facing per-host Service (publishNotReadyAddresses=false). +// Mirrors _labelServiceHost but carries the host-client Service value so the keeper-ref resolver +// can select the ready-only client tier distinctly from the Raft/peer tier. +func (l *Labeler) _labelServiceHostClient(host *api.Host) map[string]string { + return util.MergeStringMapsOverwrite( + l.GetHostScope(host, false), + map[string]string{ + l.Get(LabelService): l.Get(LabelServiceValueHostClient), + }) +} + func (l *Labeler) labelExistingPV(params ...any) map[string]string { var pv *core.PersistentVolume var host *api.Host diff --git a/pkg/model/common/tags/labeler/list.go b/pkg/model/common/tags/labeler/list.go index 250d1bfb4..ce0e83c74 100644 --- a/pkg/model/common/tags/labeler/list.go +++ b/pkg/model/common/tags/labeler/list.go @@ -41,6 +41,7 @@ const ( LabelServiceValueCluster = "cluster" LabelServiceValueShard = "shard" LabelServiceValueHost = "host" + LabelServiceValueHostClient = "host-client" LabelPVCReclaimPolicyName = "APIGroupName" + "/" + "reclaimPolicy" // Supplementary service labels - used to cooperate with k8s From c5a8f17e72d421362d05f22f92fdb986c0c08314 Mon Sep 17 00:00:00 2001 From: saba Date: Tue, 23 Jun 2026 14:26:51 +0200 Subject: [PATCH 076/164] minor adjustments to the FIPS test plan and requirements --- tests/requirements/fips.md | 4 +--- tests/requirements/fips_test_plan.md | 11 ++++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/requirements/fips.md b/tests/requirements/fips.md index b57ae00a1..f9b98baa8 100644 --- a/tests/requirements/fips.md +++ b/tests/requirements/fips.md @@ -99,8 +99,6 @@ plaintext HTTP regardless of the secure/insecure knobs and is outside the FIPS T ## Configuration Requirements -Plain HTTP/TCP on any external connection is a configuration error for FIPS compliance. - ### RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts version: 1.0 @@ -115,7 +113,7 @@ version: 1.0 Each shipped pod binary — `clickhouse-operator` and `metrics-exporter` — SHALL satisfy all of the following: * Both binaries SHALL be built with `GOFIPS140=v1.0.0` (or `certified`); `go version -m` on each binary SHALL show the `GOFIPS140` build setting when the binary is inspectable. -* Each binary SHALL identify itself as a FIPS build via `--version` output, `--fips-info`, or startup logs containing a FIPS indicator. +* Each binary SHALL identify itself as a FIPS build via `--fips-info` and startup logs containing a FIPS indicator. * Each binary SHALL report `crypto/fips140.Version()` equal to `v1.0.0` (for example via `--fips-info` or in-process inspection). * Each binary SHALL report `crypto/fips140.Enabled()` equal to `true` when FIPS mode is active per `GODEBUG=fips140`. diff --git a/tests/requirements/fips_test_plan.md b/tests/requirements/fips_test_plan.md index a459c85ff..edf02b493 100644 --- a/tests/requirements/fips_test_plan.md +++ b/tests/requirements/fips_test_plan.md @@ -27,7 +27,7 @@ ## Introduction This test plan covers FIPS 140-3 compatibility testing for the -**clickhouse-operator**, **metrics-exporter**, and **clickhouse-backup** +**clickhouse-operator** and **metrics-exporter** components used within ClickHouse deployments. The goal is to verify that FIPS-enabled components: @@ -234,7 +234,8 @@ Therefore: ## clickhouse-backup Sidecar -**Objective:** Verify clickhouse-backup sidecar operates correctly in FIPS mode and uses FIPS-compliant TLS for ClickHouse backup and restore operations. +**Objective:** Verify clickhouse-backup sidecar operates correctly alongside the FIPS enforced ClickHouse server and the +keeper deployed by the operator. **Connection Overview:** @@ -242,7 +243,6 @@ Therefore: | --------- | ------------------------------------ | ---------------- | ------------ | ------------------------------ | | Outbound | ClickHouse Server | HTTPS/native TLS | 8443/9440 | Yes, via ClickHouse TLS config | | Inbound | Backup API | HTTPS | 7171 | Yes | -| Storage | Local mounted ClickHouse data volume | filesystem | N/A | N/A | | Test Assertion | Description | Expected Result | |------------------------------|-----------------------------------------------------------------------------| ----------------------------------------------- | @@ -377,9 +377,6 @@ Therefore: **Exporter to ClickHouse Server** -> TLS supported via `chop.Config()`, but `ChSchemeAuto` prefers HTTP if both ports available. -> Must configure `scheme: https` explicitly for FIPS compliance. - | Test Assertion | Description | Expected Result | |----------------|-------------|-----------------| | Exporter FIPS cipher to CH | Exporter queries with FIPS-approved cipher | Connection succeeds | @@ -463,7 +460,7 @@ This scenario validates that both containers in the operator pod can negotiate a | Excluded Target | Reason | | -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ClickHouse Keeper / CHK | The operator does not normally establish a runtime TLS client session to ClickHouse Keeper. CHK is only deployed because the CHI manifest depends on Keeper. Keeper TLS is covered by real CHK listener/configuration checks. | +| ClickHouse Keeper | The operator does not normally establish a runtime TLS client session to ClickHouse Keeper. CHK is only deployed because the CHI manifest depends on Keeper. Keeper TLS is covered by real CHK listener/configuration checks. | | Operator metrics `:9999` | Plain HTTP Prometheus endpoint; outside FIPS TLS scope by documented boundary. | | Exporter metrics `:8888` | Plain HTTP Prometheus/IPC endpoint; outside FIPS TLS scope by documented boundary. | From 4280c0aff5bcc9eab0db85a3568b84c746d897af Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 17:28:27 +0500 Subject: [PATCH 077/164] dev: two services --- .../chi/controller-keeper-resolver.go | 29 +++++++++++--- pkg/controller/chk/worker-reconciler-chk.go | 39 +++++++++++++------ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/pkg/controller/chi/controller-keeper-resolver.go b/pkg/controller/chi/controller-keeper-resolver.go index 73dbd1f3d..26b4126f2 100644 --- a/pkg/controller/chi/controller-keeper-resolver.go +++ b/pkg/controller/chi/controller-keeper-resolver.go @@ -115,11 +115,19 @@ func (c *Controller) resolveKeeperByService(ctx context.Context, namespace, name } // resolveKeeperByReplicas resolves using per-host services (one node per replica). -// CHK host services are discovered by CHK labeler's LabelCRName and LabelService/LabelServiceValueHost labels. +// +// A CHK host exposes a ready-only client Service (LabelServiceValueHostClient, +// publishNotReadyAddresses=false) alongside the peer/Raft Service (LabelServiceValueHost, +// publishNotReadyAddresses=true). ClickHouse clients must resolve only Ready Keeper nodes, so +// the client tier is selected first. The peer tier is the fallback for CHK clusters that predate +// the split (single host Service) or use a user-supplied replicaServiceTemplate. func (c *Controller) resolveKeeperByReplicas(ctx context.Context, namespace, name string, domainPattern *types.String) (api.ZookeeperNodes, error) { - // Discover CHK host services by label selector - opts := chkHostServiceListOptions(name, namespace) - services, err := c.kubeClient.CoreV1().Services(namespace).List(ctx, opts) + // Prefer the ready-only client services + services, err := c.kubeClient.CoreV1().Services(namespace).List(ctx, chkClientServiceListOptions(name, namespace)) + if (err == nil) && (len(services.Items) == 0) { + // No client-tier services - fall back to the peer/host services (pre-split / templated CRs) + services, err = c.kubeClient.CoreV1().Services(namespace).List(ctx, chkHostServiceListOptions(name, namespace)) + } // Handle errors and empty list switch { @@ -156,7 +164,7 @@ func chkListOptions(name, namespace string) meta.ListOptions { }) } -// chkHostServiceListOptions builds meta.ListOptions to select CHK per-host services. +// chkHostServiceListOptions builds meta.ListOptions to select CHK per-host peer/Raft services. func chkHostServiceListOptions(name, namespace string) meta.ListOptions { chk := chkApi.NewClickHouseKeeperInstallation(name, namespace) l := chkLabeler.New(chk) @@ -165,3 +173,14 @@ func chkHostServiceListOptions(name, namespace string) meta.ListOptions { l.Get(commonLabeler.LabelService): l.Get(commonLabeler.LabelServiceValueHost), }) } + +// chkClientServiceListOptions builds meta.ListOptions to select CHK per-host client-facing +// services (publishNotReadyAddresses=false), the tier ClickHouse clients should resolve. +func chkClientServiceListOptions(name, namespace string) meta.ListOptions { + chk := chkApi.NewClickHouseKeeperInstallation(name, namespace) + l := chkLabeler.New(chk) + return controller.NewListOptions(map[string]string{ + l.Get(commonLabeler.LabelCRName): name, + l.Get(commonLabeler.LabelService): l.Get(commonLabeler.LabelServiceValueHostClient), + }) +} diff --git a/pkg/controller/chk/worker-reconciler-chk.go b/pkg/controller/chk/worker-reconciler-chk.go index e696979d0..1bd95a4e7 100644 --- a/pkg/controller/chk/worker-reconciler-chk.go +++ b/pkg/controller/chk/worker-reconciler-chk.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" log "github.com/altinity/clickhouse-operator/pkg/announcer" @@ -493,23 +494,37 @@ func (w *worker) hostScaleDown(ctx context.Context, host *api.Host, opts *statef return nil } -// reconcileHostService reconciles host's Service +// reconcileHostService reconciles host's Service(s). A host may expose more than one Service +// (the peer/Raft Service and the client-facing Service — see creator.createServiceHost), so all +// of them are reconciled and registered; failing to register every reconciled Service would let +// the cleanup pass purge it as an unknown object. func (w *worker) reconcileHostService(ctx context.Context, host *api.Host) error { - service := w.task.Creator().CreateService(interfaces.ServiceHost, host).First() - if service == nil { + services := w.task.Creator().CreateService(interfaces.ServiceHost, host) + if len(services) == 0 { // This is not a problem, service may be omitted return nil } - prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()).First() - err := w.reconcileService(ctx, host.GetCR(), service, prevService) - if err == nil { - w.a.V(1).M(host).F().Info("DONE Reconcile service of the host: %s", host.GetName()) - w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) - } else { - w.a.V(1).M(host).F().Warning("FAILED Reconcile service of the host: %s", host.GetName()) - w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) + prevServices := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()) + for i, service := range services { + if service == nil { + continue + } + // Pair with the previous-generation Service at the same index (creators emit a stable order). + var prevService *core.Service + if i < len(prevServices) { + prevService = prevServices[i] + } + err := w.reconcileService(ctx, host.GetCR(), service, prevService) + if err == nil { + w.a.V(1).M(host).F().Info("DONE Reconcile service %s of the host: %s", service.GetName(), host.GetName()) + w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) + } else { + w.a.V(1).M(host).F().Warning("FAILED Reconcile service %s of the host: %s", service.GetName(), host.GetName()) + w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) + return err + } } - return err + return nil } // reconcileCluster reconciles cluster, excluding nested shards From 6b6f173b3cd09270951b9ed5514bff84931cfa12 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 17:28:48 +0500 Subject: [PATCH 078/164] test: manifests --- .../chk/test-020017-chk-two-services.yaml | 28 ++++ .../manifests/chopconf/test-058-chopconf.yaml | 35 +++-- .../e2e/manifests/secret/test-058-secret.yaml | 123 +++++++++--------- 3 files changed, 106 insertions(+), 80 deletions(-) create mode 100644 tests/e2e/manifests/chk/test-020017-chk-two-services.yaml diff --git a/tests/e2e/manifests/chk/test-020017-chk-two-services.yaml b/tests/e2e/manifests/chk/test-020017-chk-two-services.yaml new file mode 100644 index 000000000..2d571782f --- /dev/null +++ b/tests/e2e/manifests/chk/test-020017-chk-two-services.yaml @@ -0,0 +1,28 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: test-020017 +spec: + # No replicaServiceTemplate is declared, so the operator emits the two default + # per-host Services (issue #1982): a peer/Raft Service (publishNotReadyAddresses=true) + # and a client-facing Service (publishNotReadyAddresses=false, "-client" suffix). + defaults: + templates: + podTemplate: default + configuration: + clusters: + - name: "keeper" + layout: + replicasCount: 1 + settings: + logger/level: "information" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/four_letter_word_white_list: "*" + templates: + podTemplates: + - name: default + spec: + containers: + - name: clickhouse-keeper + image: "clickhouse/clickhouse-keeper:25.8" diff --git a/tests/e2e/manifests/chopconf/test-058-chopconf.yaml b/tests/e2e/manifests/chopconf/test-058-chopconf.yaml index 82df14e9c..7d3065474 100644 --- a/tests/e2e/manifests/chopconf/test-058-chopconf.yaml +++ b/tests/e2e/manifests/chopconf/test-058-chopconf.yaml @@ -8,22 +8,21 @@ spec: scheme: https # Use HTTPS for connecting to ClickHouse rootCA: |- -----BEGIN CERTIFICATE----- - MIIDFzCCAf+gAwIBAgIUKPAilo3+YvoeiIkvieiBp5GaXYUwDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE1MDda - Fw0yODA2MTgxMzE1MDdaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi - MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv5k1vd7s1KrPnENFB9Tw0dtYT - wlIzpulIKuXmbEGNXIB9SEV69A7UxUZrwF585kFX91LVq+SOb3WD0/KWjs7N+hXq - RLiLuObBVrehoFFRHUca/JZS3Gz9Wsrlsr8twnX5pxavfmnhXOdw/+3P47e22kQ8 - zeAKaSfJ2wF9U9gic/uQWZmLUohrUaT0AejSQpXMm2dlk4ZhBos5BnDbQ+R+rsXu - WiTR4aS3W5Not/mV9neVJZpv2k5+02G0Wdhwy93Q44kEezSVBnSz3hvzEI5RmA72 - LhqTR/Z3y7wZf57vxkqKjqaampwzIhFxtcpAl3j+UVHPV7WkrJ59OkfaxLdtAgMB - AAGjUzBRMB0GA1UdDgQWBBSPxwYV5VGN3/J5N8ipfzefxQyvAjAfBgNVHSMEGDAW - gBSPxwYV5VGN3/J5N8ipfzefxQyvAjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 - DQEBCwUAA4IBAQCAKisQ/Ez9t88CzbrKkC4MA7rqLvHwV33sC9ttbsILk5kwvlyQ - yeeVYme6+KyK28UzuvUQrge6+JY4a33ki4G+gcltXHUaCxSWMGl20Kx++533uH4W - bLXmEPDsR1iPh+sl+3zJBs/aH3HSovBeaLu0pFRupKW5HWDDxgBz92JLVHfqHfY5 - U4bHqOiLopbcRKOQTRAqP9IQsbCmVr4PU8/LWvdMWrYTpn5IIcq1CD0GpunQBxv1 - N7YosHN5QijMvdHVTdR1B7m3ylJa5cVUPaR6HDrDkXJibaFdZBa0eIXZc3KwTon3 - q/+BIPQNS7JiXY82j8OC2HcPFSuS7t5wqcQN + MIIDGTCCAgGgAwIBAgIUXXjvd9XZQrs8ZAlYd7dCmT3FTnAwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTCC + ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANiokuw/31gySsIwAirYaHGW + 5WaW+qTf3FttUH7hX6nf5K5FoDD6vGEjDIt/nufbgeQDx8x7tNT0kuJW7ocCohaY + rhXHVwpaTRMZPGbfOpX1cPI0HRAEmWEkM7dkUg4kbtKrfcybynMVnN4Ieu4s2ruo + GKETKieb8TWuH3YNjAOMIRZK7foiuqDSyW5O62N9pAH6Y1WFg30kzkAgkrLwHFRU + LeC6Uup7/KB5sqiQMJo8EMo/VxLHfgZW0IgB7Ck0H0mNpEuaWFwXKa6zB0RYTqjY + M8FUCWruT0JxlUF+53HfrCVLFMiHq0vEtK0w1YoygCcgFKyypw2k91SmpMFaFKcC + AwEAAaNTMFEwHQYDVR0OBBYEFFe+dG2Y4vcOX7BtHZDBALSqwb7dMB8GA1UdIwQY + MBaAFFe+dG2Y4vcOX7BtHZDBALSqwb7dMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI + hvcNAQELBQADggEBAFZo6eOhHdgJQll3Nyvn7uXg3oA/Vof4rADESdhxeyY5tSUR + e/JWElXOKMDWAzzOIlTyvc0yzmC27ksYYqQA+/WNq88Q2kT7NHHyERnCPf8dklFl + OPAcAsg5LmH4l52m37RWkN+CIr6Gj6r5zF5uX8RfEMQPOFsj3QdgqLvA/Y3+b9EZ + BSg4GWOHpm4EktdvNVfzrYazt0Ithv4mtS4k6Q2mjUUesilwhAx/7x+3xCtnT7I6 + SeElKpvA3J7JcMjGICT0ke6jxgWpZ2gqyf9C4wQgfawZgt4Y9ZOkSFTSmykJfi+b + 2lekI4vDuoCrKJ+39/8n0FUsN3O93WDnAUg+SHY= -----END CERTIFICATE----- - diff --git a/tests/e2e/manifests/secret/test-058-secret.yaml b/tests/e2e/manifests/secret/test-058-secret.yaml index 917154caa..34bfc011c 100644 --- a/tests/e2e/manifests/secret/test-058-secret.yaml +++ b/tests/e2e/manifests/secret/test-058-secret.yaml @@ -7,70 +7,69 @@ type: Opaque stringData: ca.crt: |- -----BEGIN CERTIFICATE----- - MIIDFzCCAf+gAwIBAgIUKPAilo3+YvoeiIkvieiBp5GaXYUwDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE1MDda - Fw0yODA2MTgxMzE1MDdaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi - MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv5k1vd7s1KrPnENFB9Tw0dtYT - wlIzpulIKuXmbEGNXIB9SEV69A7UxUZrwF585kFX91LVq+SOb3WD0/KWjs7N+hXq - RLiLuObBVrehoFFRHUca/JZS3Gz9Wsrlsr8twnX5pxavfmnhXOdw/+3P47e22kQ8 - zeAKaSfJ2wF9U9gic/uQWZmLUohrUaT0AejSQpXMm2dlk4ZhBos5BnDbQ+R+rsXu - WiTR4aS3W5Not/mV9neVJZpv2k5+02G0Wdhwy93Q44kEezSVBnSz3hvzEI5RmA72 - LhqTR/Z3y7wZf57vxkqKjqaampwzIhFxtcpAl3j+UVHPV7WkrJ59OkfaxLdtAgMB - AAGjUzBRMB0GA1UdDgQWBBSPxwYV5VGN3/J5N8ipfzefxQyvAjAfBgNVHSMEGDAW - gBSPxwYV5VGN3/J5N8ipfzefxQyvAjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 - DQEBCwUAA4IBAQCAKisQ/Ez9t88CzbrKkC4MA7rqLvHwV33sC9ttbsILk5kwvlyQ - yeeVYme6+KyK28UzuvUQrge6+JY4a33ki4G+gcltXHUaCxSWMGl20Kx++533uH4W - bLXmEPDsR1iPh+sl+3zJBs/aH3HSovBeaLu0pFRupKW5HWDDxgBz92JLVHfqHfY5 - U4bHqOiLopbcRKOQTRAqP9IQsbCmVr4PU8/LWvdMWrYTpn5IIcq1CD0GpunQBxv1 - N7YosHN5QijMvdHVTdR1B7m3ylJa5cVUPaR6HDrDkXJibaFdZBa0eIXZc3KwTon3 - q/+BIPQNS7JiXY82j8OC2HcPFSuS7t5wqcQN + MIIDGTCCAgGgAwIBAgIUXXjvd9XZQrs8ZAlYd7dCmT3FTnAwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTCC + ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANiokuw/31gySsIwAirYaHGW + 5WaW+qTf3FttUH7hX6nf5K5FoDD6vGEjDIt/nufbgeQDx8x7tNT0kuJW7ocCohaY + rhXHVwpaTRMZPGbfOpX1cPI0HRAEmWEkM7dkUg4kbtKrfcybynMVnN4Ieu4s2ruo + GKETKieb8TWuH3YNjAOMIRZK7foiuqDSyW5O62N9pAH6Y1WFg30kzkAgkrLwHFRU + LeC6Uup7/KB5sqiQMJo8EMo/VxLHfgZW0IgB7Ck0H0mNpEuaWFwXKa6zB0RYTqjY + M8FUCWruT0JxlUF+53HfrCVLFMiHq0vEtK0w1YoygCcgFKyypw2k91SmpMFaFKcC + AwEAAaNTMFEwHQYDVR0OBBYEFFe+dG2Y4vcOX7BtHZDBALSqwb7dMB8GA1UdIwQY + MBaAFFe+dG2Y4vcOX7BtHZDBALSqwb7dMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI + hvcNAQELBQADggEBAFZo6eOhHdgJQll3Nyvn7uXg3oA/Vof4rADESdhxeyY5tSUR + e/JWElXOKMDWAzzOIlTyvc0yzmC27ksYYqQA+/WNq88Q2kT7NHHyERnCPf8dklFl + OPAcAsg5LmH4l52m37RWkN+CIr6Gj6r5zF5uX8RfEMQPOFsj3QdgqLvA/Y3+b9EZ + BSg4GWOHpm4EktdvNVfzrYazt0Ithv4mtS4k6Q2mjUUesilwhAx/7x+3xCtnT7I6 + SeElKpvA3J7JcMjGICT0ke6jxgWpZ2gqyf9C4wQgfawZgt4Y9ZOkSFTSmykJfi+b + 2lekI4vDuoCrKJ+39/8n0FUsN3O93WDnAUg+SHY= -----END CERTIFICATE----- server.crt: |- -----BEGIN CERTIFICATE----- - MIIDJTCCAg2gAwIBAgIUFZBt5OVfoThrbao6LQ+CA1c2538wDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE2MTNa - Fw0yNjA2MTkxMzE2MTNaMBIxEDAOBgNVBAMMB2Nobm9kZTEwggEiMA0GCSqGSIb3 - DQEBAQUAA4IBDwAwggEKAoIBAQDMqNisnlgEiRTYCk1dEvgcRN7FQiNDxwWKSGjo - zhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTWgBFMPbbkrlGgYisoD8GDRKN0 - Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF5t/NuetMVMqxwe4mA+5RB4hW - rlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs6BkE1uVxFg+UQ38R1wnsE/up - Yj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1HgENnP6n/nN0FG8N6z3yolKk2 - xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2Qs3ntY3/AgMBAAGjajBoMCYG - A1UdEQQfMB2CFWNobm9kZTEubWFyc25ldC5sb2NhbIcEwKgB3TAdBgNVHQ4EFgQU - 3E9tBAiCqqJLSVQKi5sXzHLC9PkwHwYDVR0jBBgwFoAUj8cGFeVRjd/yeTfIqX83 - n8UMrwIwDQYJKoZIhvcNAQELBQADggEBABQIXDzlc/wINnkSfcncEAfIY5WvVTdl - Nilr6nVd1Fgq7JAlVD1WrbZ52xZLK3xg+T99Wezks/Js9x243DWt+qwlCKe/xlrC - ezzI3BunnhRxw/7IRm0soTvPNNImcZ2Fuwhn/ojlOg+37NttdTLKlJu2+RguRWLf - 95sXdxhhfTWrkhe1gWFmmDyl02hFpWRO1A/Ogy+hJ+yuruTrlokcM5zJ1L50kyi5 - iL3/ZXMZxWw1BDSSXlbUUZQnwXJuHQbVTd4NwRO6tLVlQyNj45gRf3WeygVzbQmq - sAjL7iOqF7iCwHEri5FWyDJ0Sj5b2YJKV1LLIEMiE8dJesN0O+OlH9U= + MIIDIzCCAgugAwIBAgIUNO0av7W3Je8qhH5yoNm2HkIxoO4wDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowEjEQMA4GA1UEAwwHY2hub2RlMTCCASIwDQYJKoZI + hvcNAQEBBQADggEPADCCAQoCggEBAKWDZU6xTV2iwFP/bpWh6StKvEuAHYUgJwnm + bYGKoS7ssuj2BRO128JEjjJHCHqzgaIPzb0W3371b0XkJCl6EHFED6cxIvjF65Rv + hVO80rDzk4NOMxpb5naFX+Vb2l0zqwlmjvyvqUyE+sYj3pWZ4930uLddFU9mPxRs + mxJ9Xti2FPwpifL/lQLygxkqZWOnfM/gdxLaOexx/GeVrWfzVXPfxXkYuitUs6Le + RCxmsX+td/+DBmTla6fgStFmC6zt+XsiD1LjacFJ4UCeA3Y9AcNT8WyB2wigHbAm + Eth4Hdu77z0pL4VviTpi5IVmBZ/8tvcPU2vq44cTkNnAgEiNNPsCAwEAAaNmMGQw + YgYDVR0RBFswWYIVY2hub2RlMS5tYXJzbmV0LmxvY2FsgiBjaGktdGVzdC0wNTgt + cm9vdC1jYS1kZWZhdWx0LTAtMIIYKi50ZXN0LnN2Yy5jbHVzdGVyLmxvY2FshwTA + qAHdMA0GCSqGSIb3DQEBCwUAA4IBAQB+CnyOTWV7qoG5shpK/2YWCZF+wPiD5eRP + knVx4m3n+u86hpMoT3pjvDWjdO6KFHpijTO5JRBcnB14jSWPsNqTfnaZZrKTZ5e9 + t43uDUJ7urTTVY3RZZTntr1H/cRlRfgwTcUaNZ+Y50wpqAC/uD68m8imsctX7jhg + afM/f20T268xVYdWRL/+9EPW15lmUJQlbf/iPuBdjr/rJNkqDMcxu0DURogvEjWy + EmmY709MNK8icxlhkNWnI8NFzpe8dTqfxnznAO8Gd3xEHajXGw+EAbL9c2/ZYgEF + 4Nry8iGAqXDULGuaRNgx/458ZvNeDN8P1+LEVsB8pv18GQj8/MLK -----END CERTIFICATE----- server.key: |- - -----BEGIN PRIVATE KEY----- - MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDMqNisnlgEiRTY - Ck1dEvgcRN7FQiNDxwWKSGjozhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTW - gBFMPbbkrlGgYisoD8GDRKN0Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF - 5t/NuetMVMqxwe4mA+5RB4hWrlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs - 6BkE1uVxFg+UQ38R1wnsE/upYj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1H - gENnP6n/nN0FG8N6z3yolKk2xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2 - Qs3ntY3/AgMBAAECggEAL9/Nc6/Esibo79KVH1EZJe+8VtNuUWQEeUEP/Wl9MMaH - ao3WeUC7xPXdC+MM0D1xAVuz0NW5MMMBuT2TDk0fc+YNJSHhq4joAQ901ubxzfTR - zD9nEXMQQDDiCM8ok8IjT4T1ga59XpXwn8SulL7Jen0ZPzS4wz7HKEBFVdWKvRct - 24mW0H45ea5M6V1XUaMvzWSHbuFpT6C1MhvjdyEoNGvwbvw+xwwEc8W3tnEAP63d - 8wWPQf9kecVWbvg6ck8ufIfJnEMOR7u20V0sNki/JtD9mRCWEEn2wLunzflYW23x - WHrhPtz0wwoZLLGB2mn8+E9BNgXf06V/2Iitvrwz3QKBgQD2xq4/9H9wjsCAqK/0 - sth4JOIpmCcMT77Vz2iwjkrkJehpS6j1By3FOzQSQkaG8G2j/f9Zr8ofiscU1V/n - B+4ghGl21gZcCzxi67Ygp8rSTUN+ocpHODGN26jIUk8Jtf6c0BsiHJU5CrdcDwm5 - voR5Y1Ixaq5PWpegatUc1GUPKwKBgQDUTyqYyuOA4ul/bt8xYlO+puQ3G8ITTLW2 - JB8a7jdjED7+XMOwWh4CW9M81dnIVP6UyfCdPlSC3W6q3p/DgMPbtV1w93L9AXpS - HN5wrFqPUKAuXqxIWrIwDCk6oRmvQMXui/jUGZUHDK47h5EeU/PbDOWvWSSKzlNW - q3985tRyfQKBgQCVuMFryBmx3spoxO/MlN3FNwuIlOnMDG4KJwaraAmEFoPFrsPZ - tftNGLhlA5TqteCviKFudrs5G+fhefvvnd4aGHwsP3ooSiDfG4eqlGL36Sy0HdEu - GKfoG4dx0o5lo+fQmGp97b2TmC7bSbxq125kf6AUn1cWii5Ig8i87xhJdQKBgEmB - RzwzMmUTKshl+Hw+kMP3QBgcUisgaeEvzF0kkKSJoWWrdE0ARleGtzHe0FHdq26U - I+wtAlF0nLYn8aRcVnMg7cMIyRTziAgZ2qGj6o6n2W10da1vSTX9X+DemeflQyH9 - 8B5u5PvV1hTiMMoRQuJaKsN014P/PzdIlREHUhJ5AoGBAOODAt0nM4Jz4u8DykI/ - sPjMZ0yY62FWdAvzbIZAiZ/OPPE+fV1OGAsakot01mAym9xWM7kgIt3oWDPfaLN1 - vyt/evXV9YfnyVbhLfxbonhCWHRCiLE7CTulbQpdDkYx6D87SmEdtKbmJa4lwpVK - zBy5BhOMzamzojiV/4d1B30k - -----END PRIVATE KEY----- + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEApYNlTrFNXaLAU/9ulaHpK0q8S4AdhSAnCeZtgYqhLuyy6PYF + E7XbwkSOMkcIerOBog/NvRbffvVvReQkKXoQcUQPpzEi+MXrlG+FU7zSsPOTg04z + GlvmdoVf5VvaXTOrCWaO/K+pTIT6xiPelZnj3fS4t10VT2Y/FGybEn1e2LYU/CmJ + 8v+VAvKDGSplY6d8z+B3Eto57HH8Z5WtZ/NVc9/FeRi6K1Szot5ELGaxf613/4MG + ZOVrp+BK0WYLrO35eyIPUuNpwUnhQJ4Ddj0Bw1PxbIHbCKAdsCYS2Hgd27vvPSkv + hW+JOmLkhWYFn/y29w9Ta+rjhxOQ2cCASI00+wIDAQABAoIBAErDw+t8I9p8Piyz + YZyt+snXhJ8GTE9qargKIsU1fgHYYijhmQGCULa8iQ8lDyt+ErzGLsWPo32SGKWV + nNAvl2XSvM9lXsrJfNUcWzmsPfA41xWlKWhqwvwe22aby1P2lvg0H7r9DpjGKRF/ + +nfRgCEu/pG1tn6bTTtIo/QCNenl/DUTS5Qi5oXKIKWE5t+N2xAKENZyktMwjfze + zygPPGO8/zYvPdciN65rdLLGEVScPpTfxg8MxEALK6p3bPi75hQq/iH7stB3v9YK + wmPym3nUu2TwCtc4xr9uibhHr3W8Qp5WqtuZQNpCKS3BRlvlnOgWpoJwDocza3Pg + DEBKiAECgYEAzeXkBC3oHflpx202CM4mkcoCIwaqD9AcEa2IfdFyzikDGGVBHZmY + J44PBJdS7LeAmdefE1Ujq/SQ6X7/QbYfH8/3OGMAYD3Kac7SeV8Sb9BGUwjwZSR6 + aHYS07GoiyrB/VISg1+luUEJKohQhZxCBtaTBgpTGDDzvU+akJ61ZfsCgYEAzcnM + LOnOAfHD5TedgMLYPCeMnozDk2McqRAR2vTGBAhVZ2ulE1SdmXZKmojX0s1ifTmm + Nza/9Mn4s0ksgn/77mMtK/w4OalRLsFu52QENwI8gxVC8XzC5u5HDWvoypOK8WCC + ngPsgI9Oa6Smo2ys4TH97D5J/kJbTEyVG2qxPQECgYAFJjWwoRFIBp/Nm/6Y88bl + KH8rLxR7tsGs84ERXHaZj08DgizBt8ClZJkdjUdGokQ2FL1mt19gAorJPCLYGtzm + Z8YQA/HTdlgkk0aSQH1ujG/lzbhtXx8sk59e6feEG3qkgjPyUycK3gSDqssQvFqu + XxloMkPnu/msh1wfN8jjlwKBgQCXfaetpIyAB+9S7UcoQ8eVOPQev7c15+9wUaEj + U6/1xgDA+pByE4dVMqym6Hgg+gs37ll7KfXTiV9o9EQs6XSXwDC/wZPOduOJjOJM + uucTa7UKNnuqdFKyV9S8f6TGhCjzmj1tf6v51AVB3trBUb5OpVOtNwmXgFffaj0W + CsvhAQKBgB4IYNSEie5b8orY/XO5ONkBEOCNxO0kuuv0xTjNx5lRaFePuCAUEQYa + HvzKUwNPvw/ezFkUXv/oWWX3ZhW7ILCF/IWi3mkX9ZJ74HJGbWFsy1pZyH+4tn8b + v9yJTkkLkMWaqiPsbsijAzMCg8Y07+gpcOPjHAoXe8ty6gfgKdAs + -----END RSA PRIVATE KEY----- From b3ce0f20191f578edbd0de259cb1a59fd57edf10 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 17:29:19 +0500 Subject: [PATCH 079/164] test: services --- tests/e2e/test_operator.py | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 0cc58c196..97f970287 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -6252,6 +6252,16 @@ def test_010063(self): out = clickhouse.query(chi, "SELECT path FROM system.zookeeper WHERE path = '/' limit 1", pod=pod_name) assert out == '/', error(f"ZooKeeper should be accessible from {pod_name}") + with And("CHI resolves the keeper CLIENT tier, not the not-ready peer tier (issue #1982)"): + # The CHK exposes a ready-only client Service (…-client, publishNotReadyAddresses=false) + # alongside the peer/Raft Service. The keeper-ref resolver MUST hand ClickHouse the client + # tier so queries never hit a not-yet-Ready Keeper. Proven here by the resolved + # chop-generated-zookeeper.xml referencing the -client Service FQDN. + zk_xml = kubectl.get("configmap", f"chi-{chi}-deploy-confd-default-0-0")["data"]["chop-generated-zookeeper.xml"] + assert "-client." in zk_xml, error( + f"CHI must resolve to the ready-only client Service (…-client); got:\n{zk_xml}" + ) + with When("Rescale Keeper to 3 nodes"): kubectl.create_and_check( manifest=chk_manifest_3nodes, @@ -7887,6 +7897,82 @@ def test_020016(self): delete_test_namespace() +@TestScenario +@Name("test_020017. CHK emits two per-host Services (peer + client) with split readiness") +@Requirements(RQ_SRS_026_ClickHouseOperator_Create("1.0")) +def test_020017(self): + """issue #1982. A CHK with no replicaServiceTemplate must emit TWO + per-host headless Services: + - peer/Raft Service `chk-{chk}-{cluster}-{host}`: publishNotReadyAddresses=true + (Raft peers must reach each other before pods are Ready to bootstrap quorum), + retains the raft port, carries the Service=host tier label. + - client Service `chk-{chk}-{cluster}-{host}-client`: publishNotReadyAddresses=false + (ClickHouse clients must resolve only Ready Keeper nodes), drops the raft port, + carries the Service=host-client tier label that the keeper-ref resolver selects. + The peer name/label are byte-identical to the pre-split layout so the Raft + and StatefulSet serviceName bindings are unchanged on upgrade. + """ + create_shell_namespace_clickhouse_template() + + chk_manifest = "manifests/chk/test-020017-chk-two-services.yaml" + chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) + cluster = "keeper" + host = "0-0" + service_label = "clickhouse-keeper.altinity.com/Service" + peer_name = f"chk-{chk}-{cluster}-{host}" + client_name = f"{peer_name}-client" + + with Given("Install CHK with no replicaServiceTemplate"): + kubectl.create_and_check( + manifest=chk_manifest, + kind="chk", + check={ + "pod_count": 1, + "chk_status": "Completed", + "do_not_delete": 1, + }, + ) + + with Then(f"Peer/Raft Service {peer_name} publishes not-ready addresses and keeps the raft port"): + peer = kubectl.get("service", peer_name) + assert peer["spec"]["clusterIP"] == "None", error(f"peer Service must be headless; got {peer['spec'].get('clusterIP')}") + assert peer["spec"].get("publishNotReadyAddresses") is True, error( + f"peer Service must set publishNotReadyAddresses=true; got {peer['spec'].get('publishNotReadyAddresses')}" + ) + peer_ports = {p["name"] for p in peer["spec"]["ports"]} + assert "raft" in peer_ports, error(f"peer Service must expose the raft port; got {peer_ports}") + assert peer["metadata"]["labels"].get(service_label) == "host", error( + f"peer Service must carry Service=host label; got {peer['metadata']['labels'].get(service_label)}" + ) + + with And(f"Client Service {client_name} resolves only Ready endpoints and drops the raft port"): + client = kubectl.get("service", client_name) + assert client["spec"]["clusterIP"] == "None", error(f"client Service must be headless; got {client['spec'].get('clusterIP')}") + # publishNotReadyAddresses=false is the JSON zero value and is omitted by the apiserver. + assert client["spec"].get("publishNotReadyAddresses", False) is False, error( + f"client Service must NOT publish not-ready addresses; got {client['spec'].get('publishNotReadyAddresses')}" + ) + client_ports = {p["name"] for p in client["spec"]["ports"]} + assert "raft" not in client_ports, error(f"client Service must NOT expose the raft port; got {client_ports}") + assert client["metadata"]["labels"].get(service_label) == "host-client", error( + f"client Service must carry Service=host-client label (resolver tier selector); " + f"got {client['metadata']['labels'].get(service_label)}" + ) + + with And("The keeper-ref resolver can select the client tier by label"): + # The CHI resolver lists per-host Services by the host-client label. + out = kubectl.launch( + f"get service -l clickhouse-keeper.altinity.com/chk={chk},{service_label}=host-client " + f"-o jsonpath='{{.items[*].metadata.name}}'" + ) + assert client_name in out, error(f"host-client label selector must return {client_name}; got {out}") + + with Then("Delete CHK"): + kubectl.delete_chk(chk) + + with Finally("I clean up"): + delete_test_namespace() + @TestScenario @Tags("HEAVY") From 6c0244458a54de76df5cb3866a9dad56d148553c Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 17:30:05 +0500 Subject: [PATCH 080/164] test: image preloader --- tests/e2e/run_tests_metrics_local.sh | 2 +- tests/e2e/run_tests_operator_local.sh | 2 +- tests/e2e/test_common.sh | 66 ++++++++++----------------- 3 files changed, 27 insertions(+), 43 deletions(-) diff --git a/tests/e2e/run_tests_metrics_local.sh b/tests/e2e/run_tests_metrics_local.sh index c85ae6e08..c67eecc39 100755 --- a/tests/e2e/run_tests_metrics_local.sh +++ b/tests/e2e/run_tests_metrics_local.sh @@ -9,6 +9,6 @@ MINIKUBE_PRELOAD_IMAGES="${MINIKUBE_PRELOAD_IMAGES:-"yes"}" export MINIKUBE_PRELOAD_IMAGES common_minikube_reset -common_preload_images "${PRELOAD_IMAGES_METRICS[@]}" +common_preload_images "${PRELOAD_IMAGES_ALL[@]}" common_build_and_load_images && \ common_run_test_script "run_tests_metrics.sh" diff --git a/tests/e2e/run_tests_operator_local.sh b/tests/e2e/run_tests_operator_local.sh index e8c889394..b37d21e3c 100755 --- a/tests/e2e/run_tests_operator_local.sh +++ b/tests/e2e/run_tests_operator_local.sh @@ -21,6 +21,6 @@ export MINIKUBE_PRELOAD_IMAGES export RETRY_COUNT common_minikube_reset -common_preload_images "${PRELOAD_IMAGES_OPERATOR[@]}" +common_preload_images "${PRELOAD_IMAGES_ALL[@]}" common_build_and_load_images && \ common_run_test_script "run_tests_operator.sh" diff --git a/tests/e2e/test_common.sh b/tests/e2e/test_common.sh index 90384b0a3..b0423945b 100755 --- a/tests/e2e/test_common.sh +++ b/tests/e2e/test_common.sh @@ -48,19 +48,31 @@ NO_CLEANUP="${NO_CLEANUP:-""}" # - tests/e2e/manifests/chit/tpl-clickhouse-stable.yaml (default CLICKHOUSE_TEMPLATE) # - tests/e2e/manifests/chit/tpl-clickhouse-23.3.yaml (clickhouse_template_old) # - tests/e2e/manifests/chk/*.yaml (keeper tests, incl. FIPS) -# Intentionally EXCLUDED from preload: -# - clickhouse/clickhouse-server:24.3-broken / :24.822 (meant-to-fail rollback tests) -# - yandex/clickhouse-server:* (opt-in via CLICKHOUSE_TEMPLATE) -# - altinity/clickhouse-server:22.8.15.25.altinitystable (only in optional tpl-clickhouse-22.8.yaml) -# Quick audit: -# grep -rhE "image:[[:space:]]+[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+" tests/e2e/manifests/ | sort -u - -PRELOAD_IMAGES_OPERATOR=( +# Intentionally EXCLUDED from preload (verified — do not re-add): +# - Meant-to-fail / decoy images (preloading them would defeat the test that rejects them): +# clickhouse/clickhouse-server:24.3-broken / :24.822 (rollback tests) +# altinity/clickhouse-server:*.altinityfips-decoy (test-030008-runtime-decoy) +# clickhouse/clickhouse-keeper:latest (test-020010 non-FIPS rejection) +# - Opt-in CLICKHOUSE_TEMPLATE overrides, not run by default (manifests/chit/tpl-clickhouse-*.yaml): +# clickhouse/clickhouse-server:21.3 / 21.8 / 22.1 / 22.2 / 22.3 / 22.6 / 22.7 +# altinity/clickhouse-server:22.8.15.25.altinitystable (tpl-clickhouse-22.8.yaml) +# yandex/clickhouse-server:* +# Audit coverage (every default-suite image is listed below): +# comm -23 <(grep -rhoE "(clickhouse|altinity)/clickhouse-(server|keeper):[A-Za-z0-9._-]+" tests/e2e/manifests/ | sort -u) \ +# <(grep -oE "(clickhouse|altinity)/clickhouse-(server|keeper):[A-Za-z0-9._-]+" tests/e2e/test_common.sh | sort -u) + +# Single canonical preload list shared by ALL suites (operator, metrics, keeper). +# One list, not per-suite lists: a per-suite list silently drifts from the manifests +# another suite uses (e.g. the metrics suite once omitted server:24.3/24.8 that +# test-017-multi-version needs, cold-pulling them on every fresh minikube and timing +# out). Preloading the full set everywhere is cheap — common_preload_images runs in +# parallel and skips images already present — and removes that whole class of flake. +PRELOAD_IMAGES_ALL=( # ClickHouse server versions used in manifests and templates "clickhouse/clickhouse-server:23.3" # clickhouse_template_old + older-version compat tests "clickhouse/clickhouse-server:23.8" - "clickhouse/clickhouse-server:24.3" # also base for 24.3-broken rollback tests - "clickhouse/clickhouse-server:24.8" + "clickhouse/clickhouse-server:24.3" # also base for 24.3-broken rollback tests; test-017-multi-version (metrics) + "clickhouse/clickhouse-server:24.8" # test-017-multi-version (metrics) "clickhouse/clickhouse-server:25.3" "clickhouse/clickhouse-server:25.8" "clickhouse/clickhouse-server:latest" @@ -68,9 +80,10 @@ PRELOAD_IMAGES_OPERATOR=( "altinity/clickhouse-server:25.8.16.10001.altinitystable" # default clickhouse_template "altinity/clickhouse-server:25.8.16.10002.altinitystable" # test_010035 auto-recovery upgrade target (manifests/chi/test-035-auto-recovery-2.yaml) "altinity/clickhouse-server:25.3.8.30001.altinityfips" # FIPS CHI (e.g. manifests/chk/test-020008-chi-fips.yaml) - # ClickHouse Keeper versions used in operator tests (test_010063, test_020008, ...) + # ClickHouse Keeper versions "clickhouse/clickhouse-keeper:25.3" "clickhouse/clickhouse-keeper:25.8" + "clickhouse/clickhouse-keeper:latest-alpine" # test_clickhouse_keeper_rescale (deploy/clickhouse-keeper/clickhouse-keeper-manually/...-for-test-only.yaml) "altinity/clickhouse-keeper:25.3.8.30001.altinityfips" # Zookeeper "docker.io/zookeeper:3.8.4" @@ -81,35 +94,6 @@ PRELOAD_IMAGES_OPERATOR=( "altinity/clickhouse-backup:2.4.15" ) -PRELOAD_IMAGES_KEEPER=( - # ClickHouse server versions - "clickhouse/clickhouse-server:23.3" - "clickhouse/clickhouse-server:23.8" - "clickhouse/clickhouse-server:24.3" - "clickhouse/clickhouse-server:24.8" - "clickhouse/clickhouse-server:25.3" - "clickhouse/clickhouse-server:25.8" - "clickhouse/clickhouse-server:latest" - # Altinity builds - "altinity/clickhouse-server:25.8.16.10001.altinitystable" # default clickhouse_template - "altinity/clickhouse-server:25.8.16.10002.altinitystable" # test_010035 auto-recovery upgrade target - "altinity/clickhouse-server:25.3.8.30001.altinityfips" # FIPS CHI (manifests/chk/test-020008-chi-fips.yaml) - # ClickHouse Keeper versions - "clickhouse/clickhouse-keeper:25.3" - "clickhouse/clickhouse-keeper:25.8" - "clickhouse/clickhouse-keeper:latest-alpine" # test_clickhouse_keeper_rescale (deploy/clickhouse-keeper/clickhouse-keeper-manually/...-for-test-only.yaml) - "altinity/clickhouse-keeper:25.3.8.30001.altinityfips" - # Zookeeper - "docker.io/zookeeper:3.8.4" -) - -PRELOAD_IMAGES_METRICS=( - "clickhouse/clickhouse-server:23.3" - "clickhouse/clickhouse-server:25.3" - "clickhouse/clickhouse-server:latest" - "altinity/clickhouse-server:25.8.16.10001.altinitystable" # default clickhouse_template -) - # ============================================================================= # Functions # ============================================================================= @@ -144,7 +128,7 @@ function common_minikube_reset() { # Pull images and load them into minikube in parallel. # Only runs if MINIKUBE_PRELOAD_IMAGES is set. -# Usage: common_preload_images "${PRELOAD_IMAGES_OPERATOR[@]}" +# Usage: common_preload_images "${PRELOAD_IMAGES_ALL[@]}" function common_preload_images() { if [[ -n "${MINIKUBE_PRELOAD_IMAGES}" ]]; then echo "pre-load images into minikube (parallel)" From 7b41d0cba240371af1a24d993735cbe6db5bc936 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 19:17:35 +0500 Subject: [PATCH 081/164] dev: minor --- pkg/model/chk/creator/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/chk/creator/service.go b/pkg/model/chk/creator/service.go index c823c2486..e2fb0856e 100644 --- a/pkg/model/chk/creator/service.go +++ b/pkg/model/chk/creator/service.go @@ -214,7 +214,7 @@ func crExposesSecureZK(cr chi.ICustomResource) bool { } exposed := false cr.WalkHosts(func(h *chi.Host) error { - if h != nil && h.ZKPortSecure.HasValue() { + if (h != nil) && h.ZKPortSecure.HasValue() { exposed = true } return nil From 597158ceff62ba93b0f7758c9a47cc11a1510641 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 19:17:45 +0500 Subject: [PATCH 082/164] dev: notes --- release_notes.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/release_notes.md b/release_notes.md index 60335ce9a..a5fd4172f 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,13 @@ ## Release 0.27.2 +### New Features +* **Keeper split client/peer Services** ([#1982](https://github.com/Altinity/clickhouse-operator/issues/1982)). Each `ClickHouseKeeper` host now exposes two headless Services instead of one: + * a **peer** Service (unchanged name) with `publishNotReadyAddresses: true` — used for intra-Keeper Raft traffic and the StatefulSet pod DNS, so Raft peers can reach each other before pods are `Ready` and bootstrap quorum; + * a **client** Service (`-client`) with `publishNotReadyAddresses: false` — used by ClickHouse, so clients only ever resolve `Ready` Keeper nodes and never connect to a node still starting up. + + A user-supplied `replicaServiceTemplate` still produces a single Service (the split applies only to the operator's default Services). + ### Behavior Changes +* **One-time ClickHouse rolling restart on upgrade for CHI→CHK keeper references** ([#1982](https://github.com/Altinity/clickhouse-operator/issues/1982)). A `ClickHouseInstallation` that references a `ClickHouseKeeper` via a `keeper:` ref (the default `serviceType: Replicas`) now resolves to the new ready-only Keeper **client** Service. The first reconcile after upgrade rewrites the CHI's `` endpoints from `chk-…` to `chk-…-client`, which the operator treats as a configuration change requiring a restart — so **every ClickHouse pod of an affected CHI restarts once**. This is a one-time event; the resolved endpoints are stable afterwards. No action required. * **Backward-incompatible config rename** in `ClickHouseOperatorConfiguration`: `reconcile.recovery.from.{aborted,completed}` → `reconcile.recovery.onStatus.{aborted,completed}`. The `from` grouping level is removed; the per-status scopes and their action keys (`onPodReady`/`onPodNotReady`/`onPodNotReadyThreshold`) are unchanged. The obsolete `from` key is silently ignored on load. **If you set `reconcile.recovery.from.aborted.onPodReady: none` on 0.27.0/0.27.1 to disable Aborted auto-recovery, re-apply it as `reconcile.recovery.onStatus.aborted.onPodReady: none`** — otherwise the default (`retry`) silently re-enables it. The `completed` scope (sustained-NotReady host recovery) is new in 0.27.2 and off by default. See [docs/operator_upgrade.md](docs/operator_upgrade.md). ## Release 0.27.1 From 36201dc12b348d95601f0f675a4b5bd2feadc5ec Mon Sep 17 00:00:00 2001 From: saba Date: Tue, 23 Jun 2026 18:00:03 +0200 Subject: [PATCH 083/164] added godebug set to =only for every operator deployment --- tests/e2e/steps_fips.py | 37 ------------------------------------- tests/e2e/test_operator.py | 36 ++++++++++++++++++++---------------- tests/e2e/util.py | 23 +++++++++++++++++++++++ tests/requirements/fips.py | 6 ++---- 4 files changed, 45 insertions(+), 57 deletions(-) diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index e5efc9f4b..4b4bc9b49 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -272,43 +272,6 @@ def fips_assert_fips_enforced_coercion_in_logs(self, logs): ) -@TestStep(Given) -def fips_apply_operator_godebug(self): - """Apply suite-configured GODEBUG=fips140= on the operator deployment.""" - mode = self.context.fips140_mode - - ns = current().context.operator_namespace - expected = f"fips140={mode}" - # One patch for both containers (kubectl set env defaults to --containers='*', - # and the operator Deployment has exactly clickhouse-operator + metrics-exporter). - # Two separate per-container edits produced two ReplicaSet revisions racing each - # other, so `rollout status` could latch onto an intermediate revision while the - # final pod was still cache-syncing -- one of the test_030008 restart-storm races. - kubectl.launch( - "set env deployment/clickhouse-operator " - f"--overwrite GODEBUG={expected}", - ns=ns, - ) - kubectl.launch( - "rollout status deployment/clickhouse-operator", - ns=ns, - timeout=600, - ) - -@TestStep(Given) -def fips_apply_operator_config(self, chopconf_path): - """Apply a ClickHouseOperatorConfiguration and restart the operator.""" - util.apply_operator_config(chopconf_path) - fips_apply_operator_godebug() - - -@TestStep(Given) -def fips_create_shell_namespace_clickhouse_template(self): - """Create test namespace and apply suite-configured operator GODEBUG.""" - create_shell_namespace_clickhouse_template() - fips_apply_operator_godebug() - - @TestStep(When) def fips_apply_manifest_raw(self, manifest_path): """Apply a CHI/CHK manifest without waiting for reconcile.""" diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index c4e99f62d..d4cf68721 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -661,6 +661,8 @@ def test_010009_1(self, version_from="0.26.3", version_to=None): if version_to is None: version_to = self.context.operator_version + self.context.skip_fips = True + with Check("Test simple chi for operator upgrade"): test_operator_upgrade( manifest="manifests/chi/test-009-operator-upgrade-1.yaml", @@ -677,6 +679,8 @@ def test_010009_2(self, version_from="0.26.3", version_to=None): if version_to is None: version_to = self.context.operator_version + self.context.skip_fips = True + with Check("Test advanced chi for operator upgrade"): test_operator_upgrade( manifest="manifests/chi/test-009-operator-upgrade-2.yaml", @@ -7776,7 +7780,7 @@ def test_030003(self): chk_manifest = "manifests/chk/test-030003.yaml" backup_template = "manifests/chit/test-030003-backup-template.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) @@ -7784,7 +7788,7 @@ def test_030003(self): chi_replica_count = 2 with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with Check("operator pod passes essential FIPS checks"): run_operator_fips_checks() @@ -7871,7 +7875,7 @@ def test_030004(self): chk_manifest = "manifests/chk/test-030003.yaml" backup_template = "manifests/chit/test-030003-backup-template.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) @@ -8018,7 +8022,7 @@ def test_030005(self): chk_manifest = "manifests/chk/test-030003.yaml" backup_template = "manifests/chit/test-030003-backup-template.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) @@ -8193,11 +8197,11 @@ def test_030006(self): "manifests/chopconf/test-030006-chopconf.yaml" ) - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() operator_namespace = self.context.operator_namespace with Given("FIPS chopconf with relaxed verify and IPC is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with When("operator startup logs are fetched"): operator_pod = kubectl.get_operator_pod(ns=operator_namespace) @@ -8237,10 +8241,10 @@ def test_030007(self): "manifests/chk/test-020009-chk-fips-bypass-rejected.yaml" ) - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with When("CHI references plain-text external ZooKeeper"): chi_zk = yaml_manifest.get_name(util.get_full_path(chi_zk_rejected)) @@ -8385,7 +8389,7 @@ def test_030008(self): ) - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi_non_fips = yaml_manifest.get_name( util.get_full_path(chi_non_fips_manifest) @@ -8420,7 +8424,7 @@ def test_030008(self): ) with Given("FIPS image policy Required is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with When("CHI with non-fips image tag is applied"): fips_apply_manifest_raw(manifest_path=chi_non_fips_manifest) @@ -8593,14 +8597,14 @@ def test_030009(self): chk_tls12_manifest = "manifests/chk/test-030009-chk-tls12-only.yaml" chk_manifest = "manifests/chk/test-030003.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi_tls12 = yaml_manifest.get_name(util.get_full_path(chi_tls12_manifest)) chk_tls12 = yaml_manifest.get_name(util.get_full_path(chk_tls12_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with And("test TLS secret is installed"): create_tls_secret_for_fips_hosts(chi=chi_tls12, chk=chk) @@ -8664,13 +8668,13 @@ def test_030010(self): chk_manifest = "manifests/chk/test-030003.yaml" backup_template = "manifests/chit/test-030003-backup-template.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk = yaml_manifest.get_name(util.get_full_path(chk_manifest)) with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with And("test TLS secret is installed"): create_tls_secret_for_fips_hosts(chi=chi, chk=chk) @@ -8783,13 +8787,13 @@ def test_030016(self): chopconf = "manifests/chopconf/test-030002-chopconf.yaml" chi_manifest = "manifests/chi/test-030016.yaml" - fips_create_shell_namespace_clickhouse_template() + create_shell_namespace_clickhouse_template() chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) chk_dummy = "unused" with Given("strict FIPS operator configuration is applied"): - fips_apply_operator_config(chopconf_path=chopconf) + util.apply_operator_config(chopconf) with And("test TLS secret is installed"): create_tls_secret_for_fips_hosts( diff --git a/tests/e2e/util.py b/tests/e2e/util.py index 426778abf..b2859d874 100644 --- a/tests/e2e/util.py +++ b/tests/e2e/util.py @@ -286,6 +286,26 @@ def get_metrics(operator_pod=None, operator_namespace=None, container="metrics-e ns=operator_namespace, ) +def _apply_operator_godebug(shell=None): + mode = getattr(current().context, "fips140_mode", None) + if not mode: + return + + ns = current().context.operator_namespace + expected = f"fips140={mode}" + + kubectl.launch( + "set env deployment/clickhouse-operator " + f"--overwrite GODEBUG={expected}", + ns=ns, + shell=shell, + ) + kubectl.launch( + "rollout status deployment/clickhouse-operator", + ns=ns, + timeout=600, + shell=shell, + ) def install_operator_if_not_exist( reinstall=False, @@ -324,6 +344,9 @@ def install_operator_if_not_exist( ) set_operator_version(current().context.operator_version, shell=shell) + if not hasattr(current().context, "skip_fips"): + _apply_operator_godebug(shell=shell) + def install_operator_version(version, shell=None): if version == current().context.operator_version or version == "dev": diff --git a/tests/requirements/fips.py b/tests/requirements/fips.py index c178bf6c3..8b5d0e4e7 100644 --- a/tests/requirements/fips.py +++ b/tests/requirements/fips.py @@ -36,7 +36,7 @@ 'Each shipped pod binary — `clickhouse-operator` and `metrics-exporter` — SHALL satisfy all of the following:\n' '\n' '* Both binaries SHALL be built with `GOFIPS140=v1.0.0` (or `certified`); `go version -m` on each binary SHALL show the `GOFIPS140` build setting when the binary is inspectable.\n' - '* Each binary SHALL identify itself as a FIPS build via `--version` output, `--fips-info`, or startup logs containing a FIPS indicator.\n' + '* Each binary SHALL identify itself as a FIPS build via `--fips-info` and startup logs containing a FIPS indicator.\n' '* Each binary SHALL report `crypto/fips140.Version()` equal to `v1.0.0` (for example via `--fips-info` or in-process inspection).\n' '* Each binary SHALL report `crypto/fips140.Enabled()` equal to `true` when FIPS mode is active per `GODEBUG=fips140`.\n' '\n' @@ -988,8 +988,6 @@ ## Configuration Requirements -Plain HTTP/TCP on any external connection is a configuration error for FIPS compliance. - ### RQ.SRS-026.ClickHouseOperator.FIPS.HTTPPorts version: 1.0 @@ -1004,7 +1002,7 @@ Each shipped pod binary — `clickhouse-operator` and `metrics-exporter` — SHALL satisfy all of the following: * Both binaries SHALL be built with `GOFIPS140=v1.0.0` (or `certified`); `go version -m` on each binary SHALL show the `GOFIPS140` build setting when the binary is inspectable. -* Each binary SHALL identify itself as a FIPS build via `--version` output, `--fips-info`, or startup logs containing a FIPS indicator. +* Each binary SHALL identify itself as a FIPS build via `--fips-info` and startup logs containing a FIPS indicator. * Each binary SHALL report `crypto/fips140.Version()` equal to `v1.0.0` (for example via `--fips-info` or in-process inspection). * Each binary SHALL report `crypto/fips140.Enabled()` equal to `true` when FIPS mode is active per `GODEBUG=fips140`. From da6e5a45b3c462cc4bf1c2fda0ee1d68b6950583 Mon Sep 17 00:00:00 2001 From: saba Date: Tue, 23 Jun 2026 18:01:40 +0200 Subject: [PATCH 084/164] added godebug set to =only for every operator deployment --- tests/e2e/test_operator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index d4cf68721..bfad78c15 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -661,7 +661,7 @@ def test_010009_1(self, version_from="0.26.3", version_to=None): if version_to is None: version_to = self.context.operator_version - self.context.skip_fips = True + self.context.skip_fips = True # avoids setting GODEBUG to fips enforced for this test with Check("Test simple chi for operator upgrade"): test_operator_upgrade( From 33eb3497b96816e0a5db4dd02c58a62db4f7df8b Mon Sep 17 00:00:00 2001 From: saba Date: Tue, 23 Jun 2026 18:09:50 +0200 Subject: [PATCH 085/164] fixed chopconf install --- tests/e2e/test_operator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index bfad78c15..29241a706 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -8419,9 +8419,6 @@ def test_030008(self): chi_permissive = yaml_manifest.get_name( util.get_full_path(chi_permissive_manifest) ) - chi_backup_non_fips = yaml_manifest.get_name( - util.get_full_path(chi_backup_non_fips_manifest) - ) with Given("FIPS image policy Required is applied"): util.apply_operator_config(chopconf) @@ -8568,7 +8565,7 @@ def test_030008(self): ok_to_fail=True, ) - fips_apply_operator_config(chopconf_path=chi_permissive_chopconf) + util.apply_operator_config(chi_permissive_chopconf) with And("a non-fips CHI is applied under Permissive policy"): fips_apply_manifest_raw(manifest_path=chi_permissive_manifest) From 22d816ca82cb47fbfa44cbed605751c36444c1b4 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 23 Jun 2026 21:11:29 +0500 Subject: [PATCH 086/164] Fix test_010035_2/_3 after PR #2011 merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #2011 added a livenessProbe to test-035-2-sustained-not-ready.yaml and reworked test_010035_2 to assert kubelet restartCount — a false green: the operator's sustained-NotReady pod recreation (PR #1998) was asserted nowhere, and the livenessProbe restored /tmp/ready (breaking test_010035_3's stays-NotReady premise). - Restore the manifest to readinessProbe-only (pod stays NotReady so the operator, not the kubelet, is what acts). - test_010035_2: assert the operator recreates the pod (UID change) within the sustained-NotReady window, per the actual recovery behavior. - test_010035_3 (opt-out) now holds: pod stays NotReady, UID unchanged. --- .../chi/test-035-2-sustained-not-ready.yaml | 9 ------ tests/e2e/test_operator.py | 31 ++++++++----------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml index 3d04d7d57..dd548b54a 100644 --- a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml +++ b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml @@ -31,15 +31,6 @@ spec: initialDelaySeconds: 1 periodSeconds: 2 failureThreshold: 1 - livenessProbe: - exec: - command: - - "/bin/bash" - - "-c" - - "test -f /tmp/ready" - initialDelaySeconds: 1 - periodSeconds: 2 - failureThreshold: 3 defaults: templates: podTemplate: readiness-flap diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index adf1bcfd8..904e8769e 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3833,34 +3833,29 @@ def test_010035_2(self): f"expected container {dummy_container} to be NotReady, got ready={dummy_ready}" ) - with Then("Kubernetes should restart the dummy container after liveness failure"): - - old_restart_count = kubectl.get_container_restart_count(pod, dummy_container) + with Then("Operator should recreate the pod after sustained NotReady timeout"): start_time = time.time() - new_restart_count = old_restart_count - dummy_ready = kubectl.get_container_status(pod, 1) - while time.time() - start_time < 120: - new_restart_count = kubectl.get_container_restart_count(pod, dummy_container) - dummy_ready = kubectl.get_container_status(pod, 1) + new_uid = old_uid + pod_ready = kubectl.get_condition_status(pod, "Ready") - if new_restart_count is not None and new_restart_count > old_restart_count and dummy_ready == "true": + while time.time() - start_time < 420: + new_uid = kubectl.get_field("pod", pod, ".metadata.uid") + pod_ready = kubectl.get_condition_status(pod, "Ready") + if new_uid != old_uid and pod_ready == "True": break - retry_sleep( int((time.time() - start_time) / 5) + 1, 5, - f"{dummy_container} restartCount={new_restart_count}, ready={dummy_ready}", + f"pod uid={new_uid}, Ready={pod_ready}", ) - assert new_restart_count is not None and new_restart_count > old_restart_count, error( - f"expected {dummy_container} to restart after liveness failure" - ) - assert dummy_ready == "true", error( - f"expected {dummy_container} to become Ready after restart, got {dummy_ready}" + assert new_uid != old_uid, error( + f"expected operator to recreate pod {pod} within sustained NotReady timeout" ) + assert pod_ready == "True", error(f"expected recreated pod {pod} to become Ready, got {pod_ready}") - with Finally("I clean up"): - delete_test_namespace() + with Finally("I clean up"): + delete_test_namespace() @TestScenario From d43abffad92c480e173c804a3d887bb1b3f5f6a5 Mon Sep 17 00:00:00 2001 From: saba Date: Wed, 24 Jun 2026 01:30:26 +0200 Subject: [PATCH 087/164] fixed gofips tag for failing scenario --- tests/e2e/test_operator.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 29241a706..64f6b9319 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3390,6 +3390,7 @@ def test_090099(self): @TestScenario @Name("test_010031. Test excludeFromPropagationAnnotations work") def test_010031(self): + self.context.skip_fips = True create_shell_namespace_clickhouse_template() chi_manifest = "manifests/chi/test-031-wo-tpl.yaml" @@ -7720,10 +7721,10 @@ def test_030001(self): op_bin = self.context.fips_op_bin me_bin = self.context.fips_me_bin - with When("check go version -m version for operator"): + with Then("check go version -m version for operator"): check_binary_go_version(binary_path=op_bin, version=gofips140_needle) - with Then("check runtime FIPS modes for operator"): + with And("check runtime FIPS modes for operator"): check_fips_runtime_modes( binary_path=op_bin, binary="clickhouse-operator", @@ -7732,7 +7733,7 @@ def test_030001(self): godebug_default=godebug_default, ) - with When("check go version -m version for metrics exporter"): + with Then("check go version -m version for metrics exporter"): check_binary_go_version(binary_path=me_bin, version=gofips140_needle) with And("check runtime FIPS modes for metrics-exporter"): @@ -7806,7 +7807,7 @@ def test_030003(self): kind="chk", ) - with Then("Keeper cluster passes essential FIPS checks"): + with Then("check Keeper cluster passes essential FIPS checks"): chk_pods = run_chk_fips_checks(workload=chk, replica_count=2) with When("FIPS ClickHouse is deployed with TLS settings"): @@ -7889,14 +7890,14 @@ def test_030004(self): with And("external ClickHouse client container is started"): start_external_ch_container() - with And("FIPS ClickHouse Keeper is deployed with TLS settings"): + with When("FIPS ClickHouse Keeper is deployed with TLS settings"): fips_apply_manifest( manifest_path=chk_manifest, replica_count=2, kind="chk", ) - with When("FIPS ClickHouse is deployed with 2 replicas"): + with And("FIPS ClickHouse is deployed with 2 replicas"): chi_manifest_2 = fips_edit_manifest( source_manifest=chi_manifest, replicas_count=2, @@ -7909,19 +7910,19 @@ def test_030004(self): apply_templates=[backup_template], ) - with Then("2-replica cluster passes essential FIPS checks"): + with Then("check 2-replica cluster passes essential FIPS checks"): chi_pods = run_chi_fips_checks( workload=chi, replica_count=2, ) - with And("clickhouse-backup sidecar passes essential FIPS checks"): + with And("check clickhouse-backup sidecar passes essential FIPS checks"): run_backup_fips_checks( workload=chi, replica_count=2, ) - with And("ReplicatedMergeTree data converges across 2 replicas"): + with And("check ReplicatedMergeTree data converges across 2 replicas"): fips_check_replication_across_replicas(chi_pods=chi_pods) with When("CHI is upscaled to 3 replicas"): @@ -7936,19 +7937,19 @@ def test_030004(self): kind="chi", ) - with Then("3-replica cluster passes essential FIPS checks"): + with Then("check 3-replica cluster passes essential FIPS checks"): chi_pods = run_chi_fips_checks( workload=chi, replica_count=3, ) - with And("clickhouse-backup sidecar passes essential FIPS checks"): + with And("check clickhouse-backup sidecar passes essential FIPS checks"): run_backup_fips_checks( workload=chi, replica_count=3, ) - with And("ReplicatedMergeTree data converges across 3 replicas"): + with And("check ReplicatedMergeTree data converges across 3 replicas"): fips_check_replication_across_replicas( chi_pods=chi_pods, table="repl_scale_test_3", @@ -8381,12 +8382,6 @@ def test_030008(self): chi_permissive_manifest = ( "manifests/chi/test-030008-permissive-non-fips.yaml" ) - backup_non_fips_template = ( - "manifests/chit/test-030008-backup-non-fips-template.yaml" - ) - chi_backup_non_fips_manifest = ( - "manifests/chi/test-030003.yaml" - ) create_shell_namespace_clickhouse_template() From d4d4c204a3a641abc30876267f9535564863a3d1 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 24 Jun 2026 19:43:44 +0500 Subject: [PATCH 088/164] test: preloader --- tests/e2e/test_common.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_common.sh b/tests/e2e/test_common.sh index b0423945b..e583e824c 100755 --- a/tests/e2e/test_common.sh +++ b/tests/e2e/test_common.sh @@ -156,11 +156,24 @@ function common_preload_images() { # Build operator + metrics-exporter docker images and load them into minikube function common_build_and_load_images() { + # settings.py resolves the operator version from the `release` file, so the + # e2e install path requests altinity/clickhouse-operator:. The build + # always tags images :dev, so without retagging an IfNotPresent install pulls + # the PUBLISHED image from the registry instead of the freshly-built + # local one -- silently testing the wrong binary. Retag the local :dev build + # to : and load that too, so the suite exercises local changes. + local release + release="$(tr -d ' \r\n\t' < "${COMMON_DIR}/../../release")" echo "Build" && \ VERBOSITY="${VERBOSITY}" "${COMMON_DIR}/../../dev/image_build_all_dev.sh" && \ + echo "Retag local :dev build as :${release} (match install version)" && \ + docker tag "${OPERATOR_DOCKER_REPO}:dev" "${OPERATOR_DOCKER_REPO}:${release}" && \ + docker tag "${METRICS_EXPORTER_DOCKER_REPO}:dev" "${METRICS_EXPORTER_DOCKER_REPO}:${release}" && \ echo "Load images" && \ - minikube image load "${OPERATOR_IMAGE}" && \ - minikube image load "${METRICS_EXPORTER_IMAGE}" && \ + minikube image load "${OPERATOR_DOCKER_REPO}:dev" && \ + minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:dev" && \ + minikube image load "${OPERATOR_DOCKER_REPO}:${release}" --overwrite=true && \ + minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:${release}" --overwrite=true && \ echo "Images prepared" } From 0daae88045fe718f6474dadb8f766efa0e3df40f Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 24 Jun 2026 19:43:55 +0500 Subject: [PATCH 089/164] test: polish --- tests/e2e/test_metrics_exporter.py | 4 ++++ tests/e2e/test_operator.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_metrics_exporter.py b/tests/e2e/test_metrics_exporter.py index 3ddfdcfbc..fbf6956a0 100644 --- a/tests/e2e/test_metrics_exporter.py +++ b/tests/e2e/test_metrics_exporter.py @@ -253,6 +253,10 @@ def test(self): set_settings() self.context.test_namespace = "test" self.context.operator_namespace = "test" + # Metrics-exporter functional suite is not a FIPS test — its FIPS posture is + # covered by test_acvp. Skip the global GODEBUG=fips140=only override so the + # operator/exporter deployment is not forced into strict FIPS mode here. + self.context.skip_fips = True with Given("I create shell"): shell = get_shell() self.context.shell = shell diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 904e8769e..efae4b0a8 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3419,8 +3419,25 @@ def test_010031(self): f.write(yaml.dump_all(manifest_yaml).encode()) util.install_operator_if_not_exist(reinstall=True, manifest=f.name) - with And("Restart operator"): - util.restart_operator(ns=current().context.operator_namespace) + with And("Restart operator until it actually loaded the custom annotation-exclude config"): + # The operator parses config.yaml only at startup. The reinstall above + # applies the custom ConfigMap (annotation.exclude=[excl]), but kubelet + # syncs the projected ConfigMap volume lazily (~60s), so a restart that + # races that sync reads the stale default (empty exclude) and wrongly + # propagates 'excl'. Restart until the operator's boot-time config dump + # shows the exclude actually loaded, instead of asserting on a coin-flip. + op_ns = current().context.operator_namespace + start = time.time() + loaded = False + while time.time() - start < 180: + util.restart_operator(ns=op_ns) + op_pod = kubectl.get_operator_pod(ns=op_ns) + logs = kubectl.launch(f"logs {op_pod} -c clickhouse-operator", ns=op_ns, ok_to_fail=True) + if re.search(r"^\s*-\s*excl\s*$", logs, re.M): + loaded = True + break + time.sleep(10) + assert loaded, error("operator never loaded annotation.exclude=[excl] from the custom config") with When("I apply chi"): kubectl.create_and_check(chi_manifest, check={"do_not_delete": 1}) From 40c388ade791ed7b7075689431301fe1351107ca Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Thu, 25 Jun 2026 14:22:58 +0500 Subject: [PATCH 090/164] dev: satisfy linter --- pkg/controller/chi/worker-statefulset-rollback.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/controller/chi/worker-statefulset-rollback.go b/pkg/controller/chi/worker-statefulset-rollback.go index 42cb8600e..b647886fa 100644 --- a/pkg/controller/chi/worker-statefulset-rollback.go +++ b/pkg/controller/chi/worker-statefulset-rollback.go @@ -58,7 +58,9 @@ func (c *Controller) OnStatefulSetCreateFailed(ctx context.Context, host *api.Ho return common.ErrCRUDIgnore } - return common.ErrCRUDUnexpectedFlow + // This is unexpected flow + // Keep it commented out for not to have linter complain + // return common.ErrCRUDUnexpectedFlow } // OnStatefulSetUpdateFailed handles situation when StatefulSet update failed in k8s level @@ -102,7 +104,9 @@ func (c *Controller) OnStatefulSetUpdateFailed(ctx context.Context, rollbackStat return common.ErrCRUDIgnore } - return common.ErrCRUDUnexpectedFlow + // This is unexpected flow + // Keep it commented out for not to have linter complain + // return common.ErrCRUDUnexpectedFlow } // shouldContinueOnCreateFailed return nil in case 'continue' or error in case 'do not continue' From cf7749cdde8a02df9dc3b5a4389a94c470e8a147 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:52:31 +0500 Subject: [PATCH 091/164] test: delete outdated templates --- .../manifests/chit/tpl-clickhouse-19.11.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.1.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.3.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.4.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.5.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.6.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.7.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-20.8.yaml | 16 ---------------- .../manifests/chit/tpl-clickhouse-21.11.yaml | 16 ---------------- .../manifests/chit/tpl-clickhouse-21.12.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-21.3.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-21.8.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-22.1.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-22.2.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-22.3.yaml | 16 ---------------- .../e2e/manifests/chit/tpl-clickhouse-22.6.yaml | 17 ----------------- .../e2e/manifests/chit/tpl-clickhouse-22.7.yaml | 17 ----------------- .../e2e/manifests/chit/tpl-clickhouse-22.8.yaml | 17 ----------------- 18 files changed, 291 deletions(-) delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml delete mode 100644 tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml deleted file mode 100644 index 5df512e53..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:19.11.12.69 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml deleted file mode 100644 index a21442346..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.1 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml deleted file mode 100644 index b139a30a2..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml deleted file mode 100644 index 0141a880a..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.4 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml deleted file mode 100644 index f2869d66d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.5 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml deleted file mode 100644 index e1b2527a0..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.6 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml deleted file mode 100644 index af13cbd38..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.7 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml deleted file mode 100644 index 0b6448e9c..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.8 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml deleted file mode 100644 index c0ad7a1c1..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:21.11 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml deleted file mode 100644 index 58bd4bdcb..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:21.12 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml deleted file mode 100644 index 7c82d105d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:21.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml deleted file mode 100644 index 9a1f371b0..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:21.8 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml deleted file mode 100644 index b057df097..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml deleted file mode 100644 index 9e06352dd..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml deleted file mode 100644 index 79b94c7a8..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml deleted file mode 100644 index 619c6ea7d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.6 - imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml deleted file mode 100644 index c04ecbc93..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.7 - imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml deleted file mode 100644 index a414a07cb..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: altinity/clickhouse-server:22.8.15.25.altinitystable - imagePullPolicy: IfNotPresent From 8c0791a74b17f1aba2d6a56c337105ea4db803b2 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:53:23 +0500 Subject: [PATCH 092/164] test: kubectl ctx in tests --- tests/e2e/merge_dual_results.py | 135 ++++++++++++++++++++++++++++++++ tests/e2e/settings.py | 7 -- tests/e2e/steps.py | 30 +++++-- tests/e2e/steps_fips.py | 60 ++++++++------ tests/e2e/test_operator.py | 60 ++++++++------ tests/e2e/util.py | 9 ++- 6 files changed, 238 insertions(+), 63 deletions(-) create mode 100755 tests/e2e/merge_dual_results.py diff --git a/tests/e2e/merge_dual_results.py b/tests/e2e/merge_dual_results.py new file mode 100755 index 000000000..0443cd0a3 --- /dev/null +++ b/tests/e2e/merge_dual_results.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Fuse the result lists of dual-cluster e2e raw logs into ONE unified report. + +Each raw log is a complete, independent TestFlows run, so the logs cannot be +concatenated and replayed: their test-id namespaces both root at the same path, the +ids collide, and the later run's scenarios are silently dropped. Instead we render +each log's ``tfs transform short`` separately, then fuse the per-scenario result lines +into a single Passing/Failing listing and re-tally the scenario/step totals. + +Usage: merge_dual_results.py [ ...] +Prints the unified report; exits 1 if any scenario failed/errored, else 0. +""" +import re +import subprocess +import sys + +ANSI = re.compile(r"\x1b\[[0-9;]*m") +# Leaf scenario result line, e.g. "[ OK ] /regression/.../test_010072. name (8m 58s)". +# Rollup lines (/regression, /regression/e2e.test_operator) lack "/test_" -> excluded. +RESULT = re.compile(r"\[\s*(OK|Fail|Error|Skip)\s*\]\s+(/regression/\S*/test_\S.*)$") +# Scenario counts are derived from deduped RESULT lines (retry-safe), not this summary, +# so there is no SCEN regex. STEP is still read from the summary for the informational tally. +STEP = re.compile(r"(\d+)\s+steps?\s+\(([^)]*)\)") +TOTAL = re.compile(r"Total time\s+(.+)") +BREAKDOWN = re.compile(r"(\d+)\s+(ok|failed|skipped|errored)") +PASS_SYMBOL, FAIL_SYMBOL = "✔", "✘" # ✔ ✘ + + +def transform(raw): + with open(raw, "rb") as fh: + proc = subprocess.run( + ["tfs", "--no-colors", "transform", "short"], + stdin=fh, capture_output=True, text=True, + ) + return ANSI.sub("", proc.stdout) + + +def breakdown(counts): + parts = [f"{v} {k}" for k, v in counts.items() if v] + return ", ".join(parts) if parts else "0" + + +_STATUS_KEY = {"OK": "ok", "Fail": "failed", "Skip": "skipped", "Error": "errored"} +_TIME_SUFFIX = re.compile(r"\s*\([0-9hms .]+\)\s*$") # trailing " (10m 13s)" / " (16s 903ms)" + + +def parse(raw, label): + # Dedup by scenario (path+name, minus the trailing "(time)") so a test that was + # retried fail->...->ok collapses to ONE entry, with OK/Skip winning over Fail/Error + # — i.e. a scenario that eventually passed counts as passing. Scenario counts are + # derived from these deduped results, so --retry never double-counts. Step counts + # come from the transform summary (informational; may include retry attempts). + by_scenario = {} # scenario-key -> (status, formatted entry line) + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + n_steps = 0 + total = "?" + for line in transform(raw).splitlines(): + m = RESULT.search(line) + if m: + status, rest = m.group(1), m.group(2).strip() + key = _TIME_SUFFIX.sub("", rest) + passed = status in ("OK", "Skip") + prev = by_scenario.get(key) + # First sighting, or a later pass that supersedes an earlier fail (retry won). + if prev is None or (passed and prev[0] in ("Fail", "Error")): + symbol = PASS_SYMBOL if passed else FAIL_SYMBOL + by_scenario[key] = (status, f"{symbol} [ {status} ] [{label}] {rest}") + continue + st = STEP.search(line) + if st: + n_steps = int(st.group(1)) + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + steps.update({w: int(n) for n, w in BREAKDOWN.findall(st.group(2)) if w in steps}) + tm = TOTAL.search(line) + if tm: + total = tm.group(1).strip() + passing = [e for s, e in by_scenario.values() if s in ("OK", "Skip")] + failing = [e for s, e in by_scenario.values() if s in ("Fail", "Error")] + scen = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + for s, _ in by_scenario.values(): + scen[_STATUS_KEY[s]] += 1 + return {"label": label, "passing": passing, "failing": failing, + "n_scen": len(by_scenario), "scen": scen, "n_steps": n_steps, "steps": steps, "total": total} + + +def main(argv): + args = argv[1:] + if len(args) < 2 or len(args) % 2: + sys.exit("usage: merge_dual_results.py [ ...]") + runs = [parse(args[i], args[i + 1]) for i in range(0, len(args), 2)] + + passing = [e for r in runs for e in r["passing"]] + failing = [e for r in runs for e in r["failing"]] + scen = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + n_scen = n_steps = 0 + for r in runs: + n_scen += r["n_scen"] + n_steps += r["n_steps"] + for k in scen: + scen[k] += r["scen"][k] + for k in steps: + steps[k] += r["steps"][k] + + print() + print("==================== COMBINED dual-cluster results ====================") + if passing: + print("\nPassing\n") + print("\n".join(passing)) + if failing: + print("\nFailing\n") + print("\n".join(failing)) + print() + print(f"{n_scen} scenarios ({breakdown(scen)})") + print(f"{n_steps} steps ({breakdown(steps)})") + for r in runs: + print(f" {r['label']}: {r['n_scen']} scenarios ({breakdown(r['scen'])}), total time {r['total']}") + print("=======================================================================") + + # Prominent, scannable headline — the one line to read for the run's outcome. + failed_total = scen["failed"] + scen["errored"] + bar = "#" * 71 + print() + print(bar) + if failed_total == 0: + print(f"### RESULT: ALL OK — {scen['ok']}/{n_scen} scenarios passed, 0 failed") + else: + print(f"### RESULT: FAILED TESTS: {failed_total} " + f"({scen['failed']} failed, {scen['errored']} errored of {n_scen}) — see 'Failing' above") + print(bar) + return 1 if failed_total else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/e2e/settings.py b/tests/e2e/settings.py index c1aa19bfe..0687b36a9 100644 --- a/tests/e2e/settings.py +++ b/tests/e2e/settings.py @@ -57,13 +57,6 @@ def get_docker_compose_path(): os.getenv("CLICKHOUSE_TEMPLATE") if "CLICKHOUSE_TEMPLATE" in os.environ else "manifests/chit/tpl-clickhouse-stable.yaml" - # "manifests/chit/tpl-clickhouse-19.17.yaml" - # "manifests/chit/tpl-clickhouse-20.3.yaml" - # "manifests/chit/tpl-clickhouse-20.8.yaml" - # "manifests/chit/tpl-clickhouse-21.3.yaml" - # "manifests/chit/tpl-clickhouse-21.8.yaml" - # "manifests/chit/tpl-clickhouse-22.3.yaml" - # "manifests/chit/tpl-clickhouse-22.8.yaml" # "manifests/chit/tpl-clickhouse-23.3.yaml" # "manifests/chit/tpl-clickhouse-23.8.yaml" ) diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py index 30bba785c..13635b30c 100644 --- a/tests/e2e/steps.py +++ b/tests/e2e/steps.py @@ -5,6 +5,7 @@ import uuid import os import re +import shlex import yaml import time import inspect @@ -92,6 +93,28 @@ def set_settings(self): self.context.kubectl_cmd = define("kubectl_cmd", os.getenv("KUBECTL_CMD") if "KUBECTL_CMD" in os.environ else self.context.kubectl_cmd) + # Dual-cluster e2e: extract the --context / --kubeconfig flags from kubectl_cmd so + # direct subprocess kubectl calls (e.g. the port-forward helpers in steps_fips.py) + # hit the SAME cluster as kubectl.launch(). Empty list for single-cluster runs. + # Only the --flag=value form is recognized (the dual wrapper emits exactly that); + # space-separated "--context foo" would drop the value and is not used. + self.context.kubectl_context_args = [ + arg for arg in shlex.split(self.context.kubectl_cmd) + if arg.startswith(("--context=", "--kubeconfig=")) + ] + # minikube profile for direct `minikube` invocations (e.g. decoy image load). + self.context.minikube_profile = define( + "minikube_profile", os.getenv("MINIKUBE_PROFILE") if "MINIKUBE_PROFILE" in os.environ else "minikube" + ) + # Direct-subprocess kube calls invoke the `kubectl` binary natively, so they can + # only carry --context/--kubeconfig when the suite runs --native. A dual-cluster + # run (context flags present) via the docker-compose runner path cannot route them. + if self.context.kubectl_context_args and not current().context.native: + raise ValueError( + "KUBECTL_CMD carries --context/--kubeconfig but the suite is not --native; " + "dual-cluster runs require --native so port-forward/minikube calls reach the right cluster" + ) + self.context.test_namespace = define("test_namespace", os.getenv("TEST_NAMESPACE") if "TEST_NAMESPACE" in os.environ else "test") self.context.operator_version = define("operator_version", ( os.getenv("OPERATOR_VERSION") @@ -124,13 +147,6 @@ def set_settings(self): self.context.image_pull_policy = define("image_pull_policy", os.getenv("IMAGE_PULL_POLICY") if "IMAGE_PULL_POLICY" in os.environ else "Always") # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-stable.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-19.17.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-20.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-20.8.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-21.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-21.8.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-22.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-22.8.yaml" # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-23.3.yaml" # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-23.8.yaml" self.context.clickhouse_template = define("clickhouse_template", os.getenv("CLICKHOUSE_TEMPLATE") if "CLICKHOUSE_TEMPLATE" in os.environ else "manifests/chit/tpl-clickhouse-stable.yaml") diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index 4b4bc9b49..9ca427782 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -62,29 +62,42 @@ def fips_extract_shipped_binaries(self): f"{self.context.operator_version}" ) - extract_dir = tempfile.mkdtemp(prefix="fips-shipped-bin-") - op_bin = os.path.join(extract_dir, "clickhouse-operator") - me_bin = os.path.join(extract_dir, "metrics-exporter") - suffix = uuid.uuid1().hex[:8] - - for image, image_path, dest, label in ( - (operator_image, "/clickhouse-operator", op_bin, f"cho-verify-{suffix}"), - ( - metrics_exporter_image, - "/metrics-exporter", - me_bin, - f"me-verify-{suffix}", - ), - ): - container_name = shlex.quote(label) - kubectl.run_shell(f"docker create --name {container_name} {shlex.quote(image)}") + # Concurrent FIPS tests all extract via the host docker daemon at once; under that + # contention the docker create/cp calls intermittently fail mid-extraction (seen as + # a transient IndexError under POOL_SIZE=25). Retry the whole extraction — each + # attempt uses a fresh tempdir + uuid-suffixed containers, so it is idempotent. + attempts = 4 + for attempt in range(1, attempts + 1): + extract_dir = tempfile.mkdtemp(prefix="fips-shipped-bin-") + op_bin = os.path.join(extract_dir, "clickhouse-operator") + me_bin = os.path.join(extract_dir, "metrics-exporter") + suffix = uuid.uuid1().hex[:8] try: - kubectl.run_shell( - f"docker cp {container_name}:{shlex.quote(image_path)} {shlex.quote(dest)}" - ) - finally: - kubectl.run_shell(f"docker rm {container_name}", ok_to_fail=True) - os.chmod(dest, 0o755) + for image, image_path, dest, label in ( + (operator_image, "/clickhouse-operator", op_bin, f"cho-verify-{suffix}"), + ( + metrics_exporter_image, + "/metrics-exporter", + me_bin, + f"me-verify-{suffix}", + ), + ): + container_name = shlex.quote(label) + kubectl.run_shell(f"docker create --name {container_name} {shlex.quote(image)}") + try: + kubectl.run_shell( + f"docker cp {container_name}:{shlex.quote(image_path)} {shlex.quote(dest)}" + ) + finally: + kubectl.run_shell(f"docker rm {container_name}", ok_to_fail=True) + os.chmod(dest, 0o755) + break + except Exception as exc: + shutil.rmtree(extract_dir, ignore_errors=True) + if attempt == attempts: + raise + note(f"FIPS binary extraction failed ({type(exc).__name__}: {exc}); retry {attempt}/{attempts - 1}") + time.sleep(attempt * 3) self.context.fips_extract_dir = extract_dir self.context.fips_op_bin = op_bin @@ -564,6 +577,7 @@ def fips_ch_external_secure_query(self, pod, sql, ns=None): pf = subprocess.Popen( [ "kubectl", + *self.context.kubectl_context_args, "-n", ns, "port-forward", f"pod/{pod}", @@ -899,6 +913,7 @@ def fips_run_openssl_s_client_on_pod_port( pf = subprocess.Popen( [ "kubectl", + *self.context.kubectl_context_args, "-n", ns, "port-forward", f"pod/{pod}", @@ -1086,6 +1101,7 @@ def fips_curl_pod_port(self, pod, port, path="/", ns=None): pf = subprocess.Popen( [ "kubectl", + *self.context.kubectl_context_args, "-n", ns, "port-forward", f"pod/{pod}", diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 5ccc492b3..41e21ff4e 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -8845,7 +8845,7 @@ def test_030008(self): load_result = None if tag_result.returncode == 0: load_result = subprocess.run( - ["minikube", "image", "load", decoy_tag], + ["minikube", "image", "load", decoy_tag, "-p", self.context.minikube_profile], capture_output=True, text=True, check=False, @@ -9202,28 +9202,36 @@ def test(self): # define values for Operator upgrade test (test_009) - with Pool(int(os.environ.get("POOL_SIZE", 3))) as pool: - # Front-load HEAVY scenarios (chopconf->operator restart, scaling, - # multi-deploy). sorted() is stable, so the key (0 for HEAVY, else 1) - # moves heavy tests to the front while preserving the natural per-group - # (0100xx/0200xx/0300xx) source order within each bucket — slow tests - # start early AND lead within each group, grabbing pool workers instead - # of trickling in at the tail (where retries serialize). Order-only: - # membership and the NO_PARALLEL exclusion are unaffected. - parallel_scenarios = [ - scenario - for scenario in loads(current_module(), Scenario, Suite) - if not (hasattr(scenario, "tags") and ("NO_PARALLEL" in scenario.tags)) - ] - parallel_scenarios.sort( - key=lambda s: 0 if (hasattr(s, "tags") and ("HEAVY" in s.tags)) else 1 - ) - for scenario in parallel_scenarios: - Scenario(run=scenario, parallel=True, executor=pool) - join() - - # Sequential pass: intentionally NOT reordered — test_090099 (CRD deletion) - # must run last, after test_010036 / the upgrade tests. - for scenario in loads(current_module(), Scenario, Suite): - if hasattr(scenario, "tags") and ("NO_PARALLEL" in scenario.tags): - Scenario(run=scenario) + # E2E_PHASE routes the two test groups to separate clusters for a dual-cluster + # run: "parallel" runs only the concurrent pool, "serial" runs only the + # NO_PARALLEL scenarios. Unset/"" runs BOTH passes — the single-cluster default, + # byte-identical to before this gate existed. + e2e_phase = os.environ.get("E2E_PHASE", "") + + if e2e_phase in ("", "parallel"): + with Pool(int(os.environ.get("POOL_SIZE", 3))) as pool: + # Front-load HEAVY scenarios (chopconf->operator restart, scaling, + # multi-deploy). sorted() is stable, so the key (0 for HEAVY, else 1) + # moves heavy tests to the front while preserving the natural per-group + # (0100xx/0200xx/0300xx) source order within each bucket — slow tests + # start early AND lead within each group, grabbing pool workers instead + # of trickling in at the tail (where retries serialize). Order-only: + # membership and the NO_PARALLEL exclusion are unaffected. + parallel_scenarios = [ + scenario + for scenario in loads(current_module(), Scenario, Suite) + if not (hasattr(scenario, "tags") and ("NO_PARALLEL" in scenario.tags)) + ] + parallel_scenarios.sort( + key=lambda s: 0 if (hasattr(s, "tags") and ("HEAVY" in s.tags)) else 1 + ) + for scenario in parallel_scenarios: + Scenario(run=scenario, parallel=True, executor=pool) + join() + + if e2e_phase in ("", "serial"): + # Sequential pass: intentionally NOT reordered — test_090099 (CRD deletion) + # must run last, after test_010036 / the upgrade tests. + for scenario in loads(current_module(), Scenario, Suite): + if hasattr(scenario, "tags") and ("NO_PARALLEL" in scenario.tags): + Scenario(run=scenario) diff --git a/tests/e2e/util.py b/tests/e2e/util.py index b2859d874..d90413cc0 100644 --- a/tests/e2e/util.py +++ b/tests/e2e/util.py @@ -64,7 +64,14 @@ def restart_operator(ns=None, timeout=600, shell=None): pod = kubectl.get("pod", name="", ns=ns, label=operator_label, shell=shell)["items"][0] old_pod_name = pod["metadata"]["name"] old_pod_ip = pod["status"]["podIP"] - kubectl.launch(f"delete pod {old_pod_name}", ns=ns, timeout=timeout, shell=shell) + # --ignore-not-found: when called via apply_operator_config(), the just-applied + # chopconf makes the operator self-restart (watch.configuration.onChange=restart), + # replacing this pod before we delete it -> `delete pod ` would hit NotFound. + # That is the desired end state (the restart already happened), so a missing pod is + # success. Using --ignore-not-found rather than ok_to_fail tolerates ONLY that race + # while still surfacing a genuine delete failure (RBAC, wrong namespace) as an error. + # The rollout-status wait below remains the real readiness gate. + kubectl.launch(f"delete pod {old_pod_name} --ignore-not-found", ns=ns, timeout=timeout, shell=shell) kubectl.wait_object("pod", name="", ns=ns, label=operator_label, shell=shell) pod = kubectl.get("pod", name="", ns=ns, label=operator_label, shell=shell)["items"][0] new_pod_name = pod["metadata"]["name"] From 78de59ef8b10171fa5b0d0dfc1b50a2bba351d63 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:54:10 +0500 Subject: [PATCH 093/164] test: use minikube profile --- tests/e2e/run_minikube_reset.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/e2e/run_minikube_reset.sh b/tests/e2e/run_minikube_reset.sh index bc5621b2c..9b43aa060 100755 --- a/tests/e2e/run_minikube_reset.sh +++ b/tests/e2e/run_minikube_reset.sh @@ -21,6 +21,11 @@ DOCKER_VERSION="${DOCKER_VERSION:-""}" # Whether to prune minikube during reset process MINIKUBE_PRUNE="${MINIKUBE_PRUNE:-""}" +# Minikube profile to operate on. Defaults to "minikube" (single-cluster, unchanged). +# Set to an isolated name for dual-cluster e2e; for a non-default profile the +# destructive cross-profile prune is skipped so a sibling cluster is never nuked. +MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" + echo "Reset kubernetes cluster." echo "k8s version: ${KUBERNETES_VERSION}" echo "nodes: ${NODES}" @@ -28,9 +33,10 @@ echo "cpus: ${CPUS}" echo "memory: ${MEMORY}" echo "docker prune: ${DOCKER_PRUNE}" echo "minikube prune:${MINIKUBE_PRUNE}" +echo "minikube profile:${MINIKUBE_PROFILE}" echo "Delete cluster" -minikube delete +minikube delete -p "${MINIKUBE_PROFILE}" if [[ ! -z "${DOCKER_PRUNE}" ]]; then echo "Docker system prune" docker system prune -f @@ -39,7 +45,10 @@ if [[ ! -z "${DOCKER_PRUNE_ALL}" ]]; then echo "Docker system prune all" docker system prune -f --all fi -if [[ ! -z "${MINIKUBE_PRUNE}" ]]; then +if [[ ! -z "${MINIKUBE_PRUNE}" && "${MINIKUBE_PROFILE}" == "minikube" ]]; then + # `--all --purge` + `rm -rf ~/.minikube` destroy EVERY profile, so this only + # runs for the default profile. A named (dual-cluster) profile skips it to + # avoid nuking the sibling cluster running concurrently. echo "Minikube prune" minikube stop minikube delete --all --purge @@ -76,13 +85,13 @@ echo "-----------------------" echo "-- Starting minikube --" echo "-----------------------" -minikube start --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" -#minikube start --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" --cache-images=false +minikube start -p "${MINIKUBE_PROFILE}" --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" +#minikube start -p "${MINIKUBE_PROFILE}" --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" --cache-images=false echo "Enabling metrics-server addon" -minikube addons enable metrics-server +minikube addons enable metrics-server -p "${MINIKUBE_PROFILE}" -if [[ -z "${SKIP_K9S}" ]]; then +if [[ -z "${SKIP_K9S}" && "${MINIKUBE_PROFILE}" == "minikube" ]]; then echo "Launching k9s" k9s -c ns fi From 7f55009a27495591227272c03f66cfd7deb24820 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:54:47 +0500 Subject: [PATCH 094/164] test: introduce dual entry in local script --- tests/e2e/run_tests_local.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/e2e/run_tests_local.sh b/tests/e2e/run_tests_local.sh index 1d43898f8..feb93163a 100755 --- a/tests/e2e/run_tests_local.sh +++ b/tests/e2e/run_tests_local.sh @@ -87,6 +87,19 @@ case "${WHAT}" in ;; esac +# Dual-cluster opt-in: route the operator suite to the two-cluster orchestrator +# (PARALLEL pool on one minikube, NO_PARALLEL set on another, merged into one +# table). Only the operator suite is split; acvp/metrics stay single-cluster, so +# DUAL_CLUSTER is most useful with WHAT=operator. Default (unset) is unchanged. +if [[ "${DUAL_CLUSTER:-}" == "yes" ]]; then + for i in "${!LOCAL_SCRIPTS[@]}"; do + if [[ "${LOCAL_SCRIPTS[$i]}" == "run_tests_operator_local.sh" ]]; then + LOCAL_SCRIPTS[$i]="run_tests_operator_dual.sh" + fi + done + echo "DUAL_CLUSTER=yes -> operator suite uses run_tests_operator_dual.sh" +fi + # Only wait for confirmation when running interactively (stdin is a terminal) if [ -t 0 ]; then TIMEOUT=30 From e6dd0962d307c271ec14d0e78aa973c1f5c06818 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:55:21 +0500 Subject: [PATCH 095/164] test: enhanse log --- tests/e2e/run_tests_operator.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/e2e/run_tests_operator.sh b/tests/e2e/run_tests_operator.sh index 4126e5c5f..ceaa5c9b9 100755 --- a/tests/e2e/run_tests_operator.sh +++ b/tests/e2e/run_tests_operator.sh @@ -10,15 +10,27 @@ common_export_test_env RUN_ALL_FLAG=$(common_convert_run_all) RETRY_ARGS=() +# Retry failing scenarios in-process (TestFlows native). Applies to single-cluster AND +# dual: a retried fail->pass writes both attempts to the raw log, but merge_dual_results.py +# dedups by scenario (OK wins over Fail), so a rescued test is counted once as passing. if [[ -n "${RETRY_COUNT}" ]]; then RETRY_ARGS+=(--retry "/regression/e2e.test_operator/test_0:,${RETRY_COUNT},,${RETRY_DELAY:-30}") fi +# Optional untrimmed native raw log for result aggregation. When TF_LOG is set, write +# the full TestFlows log so two concurrent runs can be merged into ONE combined table +# via `tfs transform short`. Unset (single-cluster/CI) -> no --log, argv unchanged. +LOG_ARGS=() +if [[ -n "${TF_LOG:-}" ]]; then + LOG_ARGS+=(--log "${TF_LOG}") +fi + python3 "${COMMON_DIR}/../regression.py" \ --only="/regression/e2e.test_operator/${ONLY}" \ ${RUN_ALL_FLAG} \ "${RETRY_ARGS[@]}" \ + "${LOG_ARGS[@]}" \ -o short \ - --trim-results on \ + --trim-results "${TRIM_RESULTS:-on}" \ --debug \ --native From 42c72d4dbf7955a127ce1aac9a60ef3c3c371dc1 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:56:09 +0500 Subject: [PATCH 096/164] test: preloader --- tests/e2e/test_common.sh | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/tests/e2e/test_common.sh b/tests/e2e/test_common.sh index e583e824c..282b467b5 100755 --- a/tests/e2e/test_common.sh +++ b/tests/e2e/test_common.sh @@ -53,10 +53,6 @@ NO_CLEANUP="${NO_CLEANUP:-""}" # clickhouse/clickhouse-server:24.3-broken / :24.822 (rollback tests) # altinity/clickhouse-server:*.altinityfips-decoy (test-030008-runtime-decoy) # clickhouse/clickhouse-keeper:latest (test-020010 non-FIPS rejection) -# - Opt-in CLICKHOUSE_TEMPLATE overrides, not run by default (manifests/chit/tpl-clickhouse-*.yaml): -# clickhouse/clickhouse-server:21.3 / 21.8 / 22.1 / 22.2 / 22.3 / 22.6 / 22.7 -# altinity/clickhouse-server:22.8.15.25.altinitystable (tpl-clickhouse-22.8.yaml) -# yandex/clickhouse-server:* # Audit coverage (every default-suite image is listed below): # comm -23 <(grep -rhoE "(clickhouse|altinity)/clickhouse-(server|keeper):[A-Za-z0-9._-]+" tests/e2e/manifests/ | sort -u) \ # <(grep -oE "(clickhouse|altinity)/clickhouse-(server|keeper):[A-Za-z0-9._-]+" tests/e2e/test_common.sh | sort -u) @@ -92,6 +88,7 @@ PRELOAD_IMAGES_ALL=( "nginx:latest" "altinity/clickhouse-backup:stable" "altinity/clickhouse-backup:2.4.15" + "altinity/clickhouse-backup:2.7.0-fips" # FIPS backup sidecar (manifests/chit/test-030003-backup-template.yaml; test_030003/030004 run_backup_fips_checks) ) # ============================================================================= @@ -135,9 +132,11 @@ function common_preload_images() { local pids=() for image in "$@"; do ( - docker pull -q "${image}" && \ - echo "pushing ${image} to minikube" && \ - minikube image load "${image}" --overwrite=false --daemon=true && \ + docker pull -q "${image}" && echo "pushing ${image} to minikube" || exit 1 + # Load into each target profile (default: single "minikube"). + for p in ${MINIKUBE_PROFILES:-minikube}; do + minikube image load "${image}" -p "${p}" --overwrite=false --daemon=true || exit 1 + done echo "done: ${image}" ) & pids+=($!) @@ -168,12 +167,18 @@ function common_build_and_load_images() { VERBOSITY="${VERBOSITY}" "${COMMON_DIR}/../../dev/image_build_all_dev.sh" && \ echo "Retag local :dev build as :${release} (match install version)" && \ docker tag "${OPERATOR_DOCKER_REPO}:dev" "${OPERATOR_DOCKER_REPO}:${release}" && \ - docker tag "${METRICS_EXPORTER_DOCKER_REPO}:dev" "${METRICS_EXPORTER_DOCKER_REPO}:${release}" && \ - echo "Load images" && \ - minikube image load "${OPERATOR_DOCKER_REPO}:dev" && \ - minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:dev" && \ - minikube image load "${OPERATOR_DOCKER_REPO}:${release}" --overwrite=true && \ - minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:${release}" --overwrite=true && \ + docker tag "${METRICS_EXPORTER_DOCKER_REPO}:dev" "${METRICS_EXPORTER_DOCKER_REPO}:${release}" || return 1 + # Load into each target minikube profile. MINIKUBE_PROFILES defaults to the + # single "minikube" profile, so `-p minikube` is a no-op equivalent to the + # prior un-profiled load; dual-cluster sets MINIKUBE_PROFILES="k8s-par k8s-seq". + local p + for p in ${MINIKUBE_PROFILES:-minikube}; do + echo "Load images into minikube profile ${p}" && \ + minikube image load "${OPERATOR_DOCKER_REPO}:dev" -p "${p}" && \ + minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:dev" -p "${p}" && \ + minikube image load "${OPERATOR_DOCKER_REPO}:${release}" -p "${p}" --overwrite=true && \ + minikube image load "${METRICS_EXPORTER_DOCKER_REPO}:${release}" -p "${p}" --overwrite=true || return 1 + done echo "Images prepared" } @@ -181,6 +186,18 @@ function common_build_and_load_images() { # Usage: common_run_test_script "run_tests_operator.sh" function common_run_test_script() { local script="${1}" + # Forward the dual-cluster envs ONLY when set & non-empty. settings.py reads + # KUBECTL_CMD via `"KUBECTL_CMD" in os.environ` (not `:-default`), so forwarding + # an empty value would wrongly blank the kubectl command for a single-cluster run. + # These go through `env`, NOT the assignment-prefix below: bash recognizes + # `VAR=val` prefixes only as literal parse-time tokens, so a `"${arr[@]}"` that + # expands to `POOL_SIZE=25 ...` would be treated as the COMMAND, not assignments. + # `env` interprets its `VAR=val` arguments correctly (and is a no-op when empty). + local -a extra_env=() + local v + for v in KUBECTL_CMD POOL_SIZE MINIKUBE_PROFILE MINIKUBE_PROFILES E2E_PHASE TF_LOG TRIM_RESULTS; do + [[ -n "${!v:-}" ]] && extra_env+=("${v}=${!v}") + done OPERATOR_DOCKER_REPO="${OPERATOR_DOCKER_REPO}" \ METRICS_EXPORTER_DOCKER_REPO="${METRICS_EXPORTER_DOCKER_REPO}" \ OPERATOR_VERSION="${OPERATOR_VERSION}" \ @@ -191,5 +208,5 @@ function common_run_test_script() { KUBECTL_MODE="${KUBECTL_MODE}" \ RUN_ALL="${RUN_ALL}" \ NO_CLEANUP="${NO_CLEANUP}" \ - "${COMMON_DIR}/${script}" + env "${extra_env[@]}" "${COMMON_DIR}/${script}" } From 00fd5d84508615cc2b1cb56c05313d48857661b5 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:56:49 +0500 Subject: [PATCH 097/164] test: dual resetter --- tests/e2e/run_minikube_dual_reset.sh | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 tests/e2e/run_minikube_dual_reset.sh diff --git a/tests/e2e/run_minikube_dual_reset.sh b/tests/e2e/run_minikube_dual_reset.sh new file mode 100755 index 000000000..ad77a689c --- /dev/null +++ b/tests/e2e/run_minikube_dual_reset.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Reset TWO isolated minikube profiles concurrently for dual-cluster e2e. +# +# k8s-par hosts the PARALLEL workload, k8s-seq the SEQUENTIAL (NO_PARALLEL) workload. +# Each profile gets its own kubeconfig file so the two concurrent `minikube start` +# calls never race on a shared ~/.kube/config, and a split of host CPU/RAM. The +# single-cluster path (run_minikube_reset.sh with no MINIKUBE_PROFILE) is untouched; +# this wrapper just drives it twice with distinct profiles. Used by +# run_tests_operator_dual.sh. +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +PROFILE_PAR="${MINIKUBE_PROFILE_PAR:-k8s-par}" +PROFILE_SEQ="${MINIKUBE_PROFILE_SEQ:-k8s-seq}" +KUBECONFIG_PAR="${KUBECONFIG_PAR:-${HOME}/.kube/${PROFILE_PAR}.config}" +KUBECONFIG_SEQ="${KUBECONFIG_SEQ:-${HOME}/.kube/${PROFILE_SEQ}.config}" +# ASYMMETRIC resource split (override per host). k8s-par runs the full PARALLEL pool +# (POOL_SIZE scenarios at once, each spinning its own operator + ClickHouse pods), so +# it needs the BULK of the host. k8s-seq runs NO_PARALLEL tests SERIALLY (one at a +# time) and needs only a small slice. An equal split starves k8s-par: its control +# plane and operator-restart paths race under load (apiserver transients, +# pod-not-found), which fails operator-mutating tests (e.g. test_010055 chopconf +# restart) and times out others as collateral. Defaults below suit a ~12-CPU / ~31g +# host; tune for yours, and lower POOL_SIZE if k8s-par is still CPU-bound (it can't +# have the whole host like a single-cluster run does). +CPUS_PAR="${CPUS_PAR:-8}" +MEMORY_PAR="${MEMORY_PAR:-16g}" +CPUS_SEQ="${CPUS_SEQ:-4}" +MEMORY_SEQ="${MEMORY_SEQ:-8g}" + +reset_one() { + local profile="$1" kubeconfig="$2" cpus="$3" memory="$4" + # MINIKUBE_PROFILE (non-default) makes run_minikube_reset.sh target this profile + # AND skip the destructive cross-profile prune + k9s; a dedicated KUBECONFIG + # isolates this cluster's context so the concurrent start does not corrupt the + # sibling's kubeconfig. + SKIP_K9S=yes \ + MINIKUBE_PROFILE="${profile}" \ + KUBECONFIG="${kubeconfig}" \ + CPUS="${cpus}" MEMORY="${memory}" \ + "${CUR_DIR}/run_minikube_reset.sh" +} + +echo "Resetting dual minikube clusters concurrently: ${PROFILE_PAR} (${CPUS_PAR} CPU / ${MEMORY_PAR}) + ${PROFILE_SEQ} (${CPUS_SEQ} CPU / ${MEMORY_SEQ})" +reset_one "${PROFILE_PAR}" "${KUBECONFIG_PAR}" "${CPUS_PAR}" "${MEMORY_PAR}" > "/tmp/minikube_reset_${PROFILE_PAR}.log" 2>&1 & +PID_PAR=$! +reset_one "${PROFILE_SEQ}" "${KUBECONFIG_SEQ}" "${CPUS_SEQ}" "${MEMORY_SEQ}" > "/tmp/minikube_reset_${PROFILE_SEQ}.log" 2>&1 & +PID_SEQ=$! + +wait "${PID_PAR}"; RC_PAR=$? +wait "${PID_SEQ}"; RC_SEQ=$? + +echo "=== ${PROFILE_PAR} reset log tail ==="; tail -8 "/tmp/minikube_reset_${PROFILE_PAR}.log" +echo "=== ${PROFILE_SEQ} reset log tail ==="; tail -8 "/tmp/minikube_reset_${PROFILE_SEQ}.log" +echo "dual reset exit codes: ${PROFILE_PAR}=${RC_PAR} ${PROFILE_SEQ}=${RC_SEQ}" + +[[ "${RC_PAR}" -eq 0 && "${RC_SEQ}" -eq 0 ]] From 5e6689611e5ceb9b6332f022e1cfb7addf5d6760 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Tue, 30 Jun 2026 17:57:26 +0500 Subject: [PATCH 098/164] test: operator dual entrypoint --- tests/e2e/run_tests_operator_dual.sh | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 tests/e2e/run_tests_operator_dual.sh diff --git a/tests/e2e/run_tests_operator_dual.sh b/tests/e2e/run_tests_operator_dual.sh new file mode 100755 index 000000000..45da4156a --- /dev/null +++ b/tests/e2e/run_tests_operator_dual.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Dual-cluster operator e2e: run the PARALLEL-safe scenarios and the NO_PARALLEL +# scenarios SIMULTANEOUSLY against two independent minikube clusters, then merge +# both result sets into ONE combined table. +# +# k8s-par (parallel cluster): E2E_PHASE=parallel, POOL_SIZE threads -> the concurrent pool +# k8s-seq (sequential cluster): E2E_PHASE=serial, POOL_SIZE=1 -> NO_PARALLEL scenarios +# +# PAR_ONLY=yes runs ONLY k8s-par (no k8s-seq) so the parallel cluster gets the whole +# host — used to test high parallelism (e.g. POOL_SIZE=25 on all 12 CPU). +# +# Each suite is a normal run_tests_operator.sh process whose kube comms are pinned to +# its cluster via KUBECTL_CMD=--context/--kubeconfig, emitting an untrimmed raw log; +# the logs are rendered into one combined table by merge_dual_results.py. +# The single-cluster scripts are untouched; this is a separate opt-in entry point. +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source "${CUR_DIR}/test_common.sh" + +PROFILE_PAR="${MINIKUBE_PROFILE_PAR:-k8s-par}" +PROFILE_SEQ="${MINIKUBE_PROFILE_SEQ:-k8s-seq}" +KUBECONFIG_PAR="${KUBECONFIG_PAR:-${HOME}/.kube/${PROFILE_PAR}.config}" +KUBECONFIG_SEQ="${KUBECONFIG_SEQ:-${HOME}/.kube/${PROFILE_SEQ}.config}" +RAW_PAR="${RAW_PAR:-/tmp/e2e_dual_${PROFILE_PAR}.raw}" +RAW_SEQ="${RAW_SEQ:-/tmp/e2e_dual_${PROFILE_SEQ}.raw}" +OUT_PAR="${OUT_PAR:-/tmp/e2e_dual_${PROFILE_PAR}.out}" +OUT_SEQ="${OUT_SEQ:-/tmp/e2e_dual_${PROFILE_SEQ}.out}" + +# PAR_ONLY=yes: run only the parallel cluster. With k8s-seq not competing for the host, +# k8s-par gets the whole machine, so its CPU/RAM defaults jump (override as needed). +PAR_ONLY="${PAR_ONLY:-}" +if [[ -n "${PAR_ONLY}" ]]; then + export CPUS_PAR="${CPUS_PAR:-12}" + export MEMORY_PAR="${MEMORY_PAR:-28g}" +fi + +# Active profiles drive both image preload/load and (for normal mode) the dual reset. +if [[ -n "${PAR_ONLY}" ]]; then + ACTIVE_PROFILES="${PROFILE_PAR}" +else + ACTIVE_PROFILES="${PROFILE_PAR} ${PROFILE_SEQ}" +fi + +# Tear down the profile(s) on ANY exit (normal, early FATAL, Ctrl-C) so a failed +# reset/build or interrupt never leaks clusters. KEEP_CLUSTERS=yes opts out. Deleting +# a non-existent profile is a harmless no-op (safe on preflight exit / PAR_ONLY). +teardown_clusters() { + [[ -n "${KEEP_CLUSTERS:-}" ]] && return + echo "Tearing down ${ACTIVE_PROFILES} (set KEEP_CLUSTERS=yes to keep)" + local p + for p in ${ACTIVE_PROFILES}; do minikube delete -p "${p}" >/dev/null 2>&1; done +} +trap teardown_clusters EXIT + +# PREFLIGHT: the result merge depends on the `tfs` CLI. Fail LOUD now rather than +# after ~an hour of testing. +command -v tfs >/dev/null 2>&1 || { echo "FATAL: tfs CLI not found (needed to render results)"; exit 3; } +tfs transform short --help >/dev/null 2>&1 || { echo "FATAL: 'tfs transform short' unavailable"; exit 3; } + +# Reset cluster(s) unless explicitly opted out (MINIKUBE_RESET=no). +if [[ "${MINIKUBE_RESET:-yes}" != "no" ]]; then + if [[ -n "${PAR_ONLY}" ]]; then + SKIP_K9S=yes MINIKUBE_PROFILE="${PROFILE_PAR}" KUBECONFIG="${KUBECONFIG_PAR}" \ + CPUS="${CPUS_PAR}" MEMORY="${MEMORY_PAR}" \ + "${CUR_DIR}/run_minikube_reset.sh" || { echo "FATAL: ${PROFILE_PAR} reset failed"; exit 2; } + else + MINIKUBE_PROFILE_PAR="${PROFILE_PAR}" MINIKUBE_PROFILE_SEQ="${PROFILE_SEQ}" \ + KUBECONFIG_PAR="${KUBECONFIG_PAR}" KUBECONFIG_SEQ="${KUBECONFIG_SEQ}" \ + "${CUR_DIR}/run_minikube_dual_reset.sh" || { echo "FATAL: dual minikube reset failed"; exit 2; } + fi +fi + +# Preload the ClickHouse/Keeper/Zookeeper images into the cluster(s) BEFORE the run. +# Without this, ~POOL_SIZE parallel tests each pull large images from the registry +# concurrently -> network contention -> pods stuck ContainerCreating/InProgress for +# minutes (the single-cluster runner preloads these; the dual path must too). +MINIKUBE_PRELOAD_IMAGES=yes MINIKUBE_PROFILES="${ACTIVE_PROFILES}" \ + common_preload_images "${PRELOAD_IMAGES_ALL[@]}" || echo "WARNING: image preload had failures (continuing)" + +# Build operator+metrics images ONCE, load into the active profile(s). +MINIKUBE_PROFILES="${ACTIVE_PROFILES}" common_build_and_load_images || { echo "FATAL: image build/load failed"; exit 2; } + +# k8s-par's pool size. Defaults to 25 threads. Image preload (above) removes the +# per-test image-pull stalls that previously made high parallelism flake; the +# remaining limit is host CPU (~1 CH-server-test per core), so 25 needs ~12 cores — +# which PAR_ONLY gives by handing k8s-par the whole host. Override POOL_SIZE to tune. +POOL_SIZE_PAR="${POOL_SIZE:-25}" + +# Retry failing scenarios in-process. Dual default is 5 (matches single-cluster full runs); +# merge_dual_results.py collapses a retried fail->pass to one passing entry, so retries never +# double-count. Exported so both child run_tests_operator.sh suites inherit it. RETRY_DELAY is +# seconds between attempts (run_tests_operator.sh defaults it to 30). Note: retry masks +# transient flakes but cannot rescue a deterministic resource shortage on an undersized cluster. +export RETRY_COUNT="${RETRY_COUNT:-5}" +export RETRY_DELAY="${RETRY_DELAY:-30}" + +# Each suite streams to the console LIVE, line-prefixed by cluster, AND to a per-suite +# file. `> >(sed | tee file)` is process substitution: the sed|tee runs concurrently +# but is NOT $!, so `wait "${PID_PAR}"` still captures run_tests_operator.sh's status. +# IMAGE_PULL_POLICY=IfNotPresent uses the locally built/preloaded images. +KUBECTL_CMD="kubectl --context=${PROFILE_PAR} --kubeconfig=${KUBECONFIG_PAR}" \ +E2E_PHASE=parallel POOL_SIZE="${POOL_SIZE_PAR}" MINIKUBE_PROFILE="${PROFILE_PAR}" \ +IMAGE_PULL_POLICY="${IMAGE_PULL_POLICY:-IfNotPresent}" \ +TF_LOG="${RAW_PAR}" TRIM_RESULTS=off ONLY="${ONLY:-*}" \ + "${CUR_DIR}/run_tests_operator.sh" > >(sed -u "s/^/[${PROFILE_PAR}] /" | tee "${OUT_PAR}") 2>&1 & +PID_PAR=$! + +if [[ -z "${PAR_ONLY}" ]]; then + KUBECTL_CMD="kubectl --context=${PROFILE_SEQ} --kubeconfig=${KUBECONFIG_SEQ}" \ + E2E_PHASE=serial POOL_SIZE=1 MINIKUBE_PROFILE="${PROFILE_SEQ}" \ + IMAGE_PULL_POLICY="${IMAGE_PULL_POLICY:-IfNotPresent}" \ + TF_LOG="${RAW_SEQ}" TRIM_RESULTS=off ONLY="${ONLY:-*}" \ + "${CUR_DIR}/run_tests_operator.sh" > >(sed -u "s/^/[${PROFILE_SEQ}] /" | tee "${OUT_SEQ}") 2>&1 & + PID_SEQ=$! +fi + +wait "${PID_PAR}"; RC_PAR=$? +RC_SEQ=0 +[[ -z "${PAR_ONLY}" ]] && { wait "${PID_SEQ}"; RC_SEQ=$?; } +# Drain the process-substitution tee/sed pipelines before printing the summary. +wait 2>/dev/null + +# Build the (raw, label) pairs for the merge — only the clusters that actually ran. +MERGE_ARGS=("${RAW_PAR}" "${PROFILE_PAR}") +[[ -z "${PAR_ONLY}" ]] && MERGE_ARGS+=("${RAW_SEQ}" "${PROFILE_SEQ}") +for ((i = 0; i < ${#MERGE_ARGS[@]}; i += 2)); do + f="${MERGE_ARGS[$i]}" + [[ -s "${f}" ]] || { echo "FATAL: raw log ${f} missing/empty — cannot render results"; exit 4; } +done + +# Fuse the run(s) into ONE unified report (single Passing/Failing listing + tally). +# Not `cat *.raw | transform`: each raw log is a complete TestFlows stream whose +# test-id namespace roots at the same path, so concatenation collides ids and drops a +# run's scenarios. The merge helper renders each log separately and fuses result lines. +python3 "${CUR_DIR}/merge_dual_results.py" "${MERGE_ARGS[@]}" || true + +# Combined verdict: fail if EITHER cluster failed. Teardown runs via the EXIT trap. +if [[ "${RC_PAR}" -eq 0 && "${RC_SEQ}" -eq 0 ]]; then COMBINED=0; VERDICT="PASS"; else COMBINED=1; VERDICT="FAIL"; fi +echo +echo "==================== COMBINED dual-cluster verdict: ${VERDICT} ====================" +echo " ${PROFILE_PAR} (parallel) exit=${RC_PAR}" +[[ -z "${PAR_ONLY}" ]] && echo " ${PROFILE_SEQ} (no-parallel) exit=${RC_SEQ}" +echo "===================================================================================" + +exit "${COMBINED}" From 4856f1ce63d368ee4502140607a06d10c52ca22d Mon Sep 17 00:00:00 2001 From: saba Date: Tue, 30 Jun 2026 16:37:04 +0200 Subject: [PATCH 099/164] wire K8s client TLS minVersion into rest.Config transport --- cmd/metrics_exporter/app/metrics_exporter.go | 2 +- cmd/operator/app/thread_chi.go | 2 +- cmd/operator/app/thread_keeper.go | 2 +- .../v1/type_configuration_chop.go | 14 ++++++ .../v1/type_security.go | 17 ++----- .../v1/type_security_fips_test.go | 30 ++++++++++++ pkg/chop/kube_machinery.go | 48 ++++++++++++++++++- 7 files changed, 98 insertions(+), 17 deletions(-) diff --git a/cmd/metrics_exporter/app/metrics_exporter.go b/cmd/metrics_exporter/app/metrics_exporter.go index 37be8eef7..de16f0272 100644 --- a/cmd/metrics_exporter/app/metrics_exporter.go +++ b/cmd/metrics_exporter/app/metrics_exporter.go @@ -124,7 +124,7 @@ func Run() { log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt) // Initialize k8s API clients - kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL) + kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) // Create operator instance chop.New(kubeClient, chopClient, chopConfigFile) diff --git a/cmd/operator/app/thread_chi.go b/cmd/operator/app/thread_chi.go index bee87ca13..c2ac3e326 100644 --- a/cmd/operator/app/thread_chi.go +++ b/cmd/operator/app/thread_chi.go @@ -56,7 +56,7 @@ func initClickHouse(ctx context.Context) { } // Initialize k8s API clients - kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL) + kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) // Create operator instance. The chopconf load inside chop.New gates on // clickhouse.security.kubernetes.allowInsecure BEFORE the first network call, diff --git a/cmd/operator/app/thread_keeper.go b/cmd/operator/app/thread_keeper.go index 053cd6cd4..1095833b2 100644 --- a/cmd/operator/app/thread_keeper.go +++ b/cmd/operator/app/thread_keeper.go @@ -75,7 +75,7 @@ func initKeeper(ctx context.Context) error { // Build the apiextensions client for CRD deletion checks during CHK cleanup. // Uses the same kubeConfigFile/masterURL package vars as the CHI thread. - _, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL) + _, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) err = ctrlRuntime. NewControllerManagedBy(manager). diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index f9610f1ab..4b5764bd7 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -1530,6 +1530,20 @@ func (c *OperatorConfig) RequiresStrictK8sTLS() bool { c.Security.GetKubernetes().GetTLS().GetVerify() == TLSVerifyStrict } +// ResolveK8sTLSMinVersion returns the K8s-API TLS floor for GetClientset. Under +// hardened posture (policy=Enforced or fips.enforced) returns TLSMinVersion13; +// otherwise security.kubernetes.tls.minVersion. Callable on raw file-based config +// before applyEnforcedHardening runs. Empty = Go stdlib default. +func (c *OperatorConfig) ResolveK8sTLSMinVersion() TLSMinVersion { + if c == nil { + return TLSMinVersion("") + } + if c.Security.RequiresHardening() { + return TLSMinVersion13 + } + return c.Security.GetKubernetes().GetTLS().GetMinVersion() +} + // coerceTypedString one-way coerces a *types.String-valued config field (TLSVerify, // TLSMinVersion, IPCMode — all type aliases of types.String) to the FIPS-strict // target value and logs the change. Caller passes the address of the struct field diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_security.go b/pkg/apis/clickhouse.altinity.com/v1/type_security.go index d501eda92..45b24ad29 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_security.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_security.go @@ -63,10 +63,8 @@ type ClusterSecurityKubernetes struct { } // ClusterSecurityKubernetesTLS holds knobs for the operator's outbound -// Kubernetes API client. The k8s client-go respects whatever's in the -// kubeconfig; the operator never builds the kubeconfig's tls.Config itself, -// so these knobs are evaluated as a LOAD-TIME GATE — the operator refuses -// to start with a kubeconfig that doesn't meet the requested posture. +// Kubernetes API client. Verify is a load-time gate against the kubeconfig's +// Insecure flag; MinVersion is applied to the client transport by GetClientset. type ClusterSecurityKubernetesTLS struct { // Verify gates startup against the kubeconfig's TLSClientConfig.Insecure // field. Valid values are TLSVerifyStrict and TLSVerifyNone. @@ -77,12 +75,8 @@ type ClusterSecurityKubernetesTLS struct { // override the kubeconfig; it only refuses to load an insecure one. Verify *types.String `json:"verify,omitempty" yaml:"verify,omitempty"` // MinVersion floors TLS at the named protocol version. Valid values are - // TLSMinVersion12 and TLSMinVersion13. Nil = Go stdlib default. - // - // Declared for shape symmetry with ClickHouse/Zookeeper and coerced under - // FIPS, but not yet enforced on the operator's K8s API transport — a future - // enhancement will wire it into rest.Config when the operator wraps the - // kubeconfig with stricter TLS settings. + // TLSMinVersion12 and TLSMinVersion13. Nil = Go stdlib default. Coerced to + // 1.3 under FIPS/Enforced; applied on the K8s API transport by GetClientset. MinVersion *types.String `json:"minVersion,omitempty" yaml:"minVersion,omitempty"` } @@ -420,8 +414,7 @@ func (t *ClusterSecurityKubernetesTLS) GetVerify() TLSVerify { } // GetMinVersion returns the resolved TLSMinVersion for the operator's K8s client. -// Nil-safe; returns empty value when unset. Declared for shape consistency and -// FIPS coercion uniformity — not yet wired into the K8s API transport. +// Nil-safe; returns empty value when unset. func (t *ClusterSecurityKubernetesTLS) GetMinVersion() TLSMinVersion { if (t == nil) || (t.MinVersion == nil) || !t.MinVersion.HasValue() { return TLSMinVersion("") diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go index 71f013252..c294a4903 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go @@ -51,6 +51,36 @@ func TestSecurity_GetFIPS_IsEnforced_NilSafe(t *testing.T) { require.True(t, (&OperatorConfigSecurity{FIPS: &OperatorConfigSecurityFIPS{Enforced: types.NewStringBool(true)}}).GetFIPS().IsEnforced()) } +// TestResolveK8sTLSMinVersion verifies hardened posture forces TLS 1.3 and that +// explicit security.kubernetes.tls.minVersion is honored when not hardened. +func TestResolveK8sTLSMinVersion(t *testing.T) { + require.Equal(t, TLSMinVersion(""), (*OperatorConfig)(nil).ResolveK8sTLSMinVersion()) + require.Equal(t, TLSMinVersion(""), (&OperatorConfig{}).ResolveK8sTLSMinVersion()) + + explicit12 := &OperatorConfig{} + explicit12.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion12))}, + } + require.Equal(t, TLSMinVersion12, explicit12.ResolveK8sTLSMinVersion()) + + explicit13 := &OperatorConfig{} + explicit13.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion13))}, + } + require.Equal(t, TLSMinVersion13, explicit13.ResolveK8sTLSMinVersion()) + + enforcedOver12 := &OperatorConfig{} + enforcedOver12.Security.Policy = types.NewString(string(SecurityPolicyEnforced)) + enforcedOver12.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion12))}, + } + require.Equal(t, TLSMinVersion13, enforcedOver12.ResolveK8sTLSMinVersion()) + + fipsForced := &OperatorConfig{} + fipsForced.Security.FIPS = &OperatorConfigSecurityFIPS{Enforced: types.NewStringBool(true)} + require.Equal(t, TLSMinVersion13, fipsForced.ResolveK8sTLSMinVersion()) +} + // TestSecurity_RequiresHardening_NilSafe verifies the union accessor used to // gate per-CR security checks (plain-text ZK rejection, FIPS-bypass rejection, // ZK digest-auth rejection). Fires when EITHER security.policy=Enforced OR diff --git a/pkg/chop/kube_machinery.go b/pkg/chop/kube_machinery.go index 282a27a92..7cce3cb85 100644 --- a/pkg/chop/kube_machinery.go +++ b/pkg/chop/kube_machinery.go @@ -15,7 +15,9 @@ package chop import ( + "crypto/tls" "fmt" + "net/http" "os" "os/user" "path/filepath" @@ -30,6 +32,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/apis/deployment" chopclientset "github.com/altinity/clickhouse-operator/pkg/client/clientset/versioned" + "github.com/altinity/clickhouse-operator/pkg/util/tlsutil" ) // lastKubeConfigInsecure records whether the most recently loaded kubeconfig @@ -80,8 +83,11 @@ func getKubeConfig(kubeConfigFile, masterURL string) (*kuberest.Config, error) { return captureInsecure(conf, nil) } -// GetClientset gets k8s API clients - both kube native client and our custom client -func GetClientset(kubeConfigFile, masterURL string) ( +// GetClientset gets k8s API clients - both kube native client and our custom client. +// chopConfigFile supplies the file-based chopconf path used to resolve the K8s-API +// TLS minVersion floor before the first network call (same timing as the +// insecure-kubeconfig gate in ConfigManager.Init). +func GetClientset(kubeConfigFile, masterURL, chopConfigFile string) ( *kube.Clientset, *apiextensions.Clientset, *chopclientset.Clientset, @@ -93,6 +99,10 @@ func GetClientset(kubeConfigFile, masterURL string) ( os.Exit(1) } + if minVer := tlsutil.VersionUint16(string(resolveK8sTLSMinVersion(chopConfigFile))); minVer != 0 { + applyK8sClientTLSMinVersion(kubeConfig, minVer) + } + // Layer on k8s client rate limiting overrides if specified in CHOP config. if maybeQps := os.Getenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT); maybeQps != "" { parsedQps, err := strconv.ParseFloat(maybeQps, 32) @@ -139,3 +149,37 @@ func GetClientset(kubeConfigFile, masterURL string) ( return kubeClientset, apiextensionsClientset, chopClientset, dynamicClientset } + +// resolveK8sTLSMinVersion reads the file-based chopconf and returns the effective +// K8s-API TLS floor ("1.2"|"1.3"|""). Uses a nil-client ConfigManager because +// file loading never touches the API. Errors yield "" (no floor). +func resolveK8sTLSMinVersion(chopConfigFile string) string { + cm := newConfigManager(nil, nil, chopConfigFile) + fileConfig, err := cm.getFileBasedConfig(chopConfigFile) + if err != nil || fileConfig == nil { + return "" + } + return string(fileConfig.ResolveK8sTLSMinVersion()) +} + +// applyK8sClientTLSMinVersion stamps MinVersion onto the rest.Config transport +// via rest.Config.Wrap, preserving client-go's TLS/proxy/HTTP2 setup. Warns if +// the built RoundTripper is not *http.Transport. +func applyK8sClientTLSMinVersion(cfg *kuberest.Config, minVer uint16) { + cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper { + t, ok := rt.(*http.Transport) + if !ok { + log.F().Warning( + "k8s client TLS minVersion floor requested (0x%04x) but transport is %T, not *http.Transport — floor NOT applied", + minVer, rt, + ) + return rt + } + if t.TLSClientConfig == nil { + t.TLSClientConfig = &tls.Config{} + } + t.TLSClientConfig.MinVersion = minVer + log.F().Info("k8s client TLS minVersion floor applied: 0x%04x", minVer) + return rt + }) +} From 17acea705d188f33ae0b1c2cf94b76c8e992b1f7 Mon Sep 17 00:00:00 2001 From: sachidananda Date: Wed, 1 Jul 2026 17:12:03 +0530 Subject: [PATCH 100/164] add replica restart fix for operator when remote_servers configmap is pushed Restart hosts created during the current reconcile once, right after the final remote_servers (with all hosts) has been published and propagated, so the new replica's boot loads with the full cluster defined. Scoped to the new host only, one-shot (host becomes Same on next reconcile), and skipped for single-host clusters. Closes the scale-up CLUSTER_DOESNT_EXIST race (Altinity/clickhouse-operator#2013). --- pkg/controller/chi/worker-reconciler-chi.go | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pkg/controller/chi/worker-reconciler-chi.go b/pkg/controller/chi/worker-reconciler-chi.go index 34123eaa5..d88173571 100644 --- a/pkg/controller/chi/worker-reconciler-chi.go +++ b/pkg/controller/chi/worker-reconciler-chi.go @@ -360,6 +360,8 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *api.ClickHo cr.GetRuntime().UnlockCommonConfig() w.includeAllHostsIntoCluster(ctx, cr) + w.restartCreatedHosts(ctx, cr) + return err } @@ -373,6 +375,34 @@ func (w *worker) includeAllHostsIntoCluster(ctx context.Context, cr *api.ClickHo }) } +// restartCreatedHosts restarts, once, each host that was created during the current reconcile. +func (w *worker) restartCreatedHosts(ctx context.Context, cr *api.ClickHouseInstallation) { + if util.IsContextDone(ctx) { + log.V(1).Info("Reconcile is aborted. Restart created hosts: %s ", cr.GetName()) + return + } + + cr.WalkHosts(func(host *api.Host) error { + // Only hosts created during this reconcile + if !host.GetReconcileAttributes().GetStatus().Is(types.ObjectStatusCreated) { + return nil + } + // Skip single-host clusters - remote_servers references only localhost, which always resolves + if host.GetCluster().HostsCount() < 2 { + w.a.V(1).M(host).F().Info("Skip post-include restart of created host in single-host cluster. Host: %s", host.GetName()) + return nil + } + + w.a.V(1).M(host).F().Info("Restart created host after final remote_servers propagation. Host: %s", host.GetName()) + w.task.WaitForConfigMapPropagation(ctx, host) + + if err := w.hostSoftwareRestart(ctx, host); err != nil { + w.a.V(1).M(host).F().Warning("Failed to restart created host after final remote_servers propagation. Host: %s err: %v", host.GetName(), err) + } + return nil + }) +} + // reconcileConfigMapCommon reconciles common ConfigMap func (w *worker) reconcileConfigMapCommon( ctx context.Context, From 2258433398d9e8b4e9c849b3eab03835dfcf6ea6 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 1 Jul 2026 22:26:13 +0500 Subject: [PATCH 101/164] dev: switch to olm.targtNamespaces in template --- ...se-operator.vVERSION.clusterserviceversion-template.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml index 7b3d41736..ea76f87da 100644 --- a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml +++ b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml @@ -1291,10 +1291,12 @@ spec: containerName: clickhouse-operator resource: limits.memory divisor: "1Mi" - - name: WATCH_NAMESPACE + # Honor the OperatorGroup's target namespaces so every advertised + # installMode works (AllNamespaces sends an empty string = watch all). + - name: WATCH_NAMESPACES valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['olm.targetNamespaces'] image: docker.io/altinity/clickhouse-operator:${OPERATOR_VERSION} imagePullPolicy: Always name: clickhouse-operator From 8b727ef6c552f2afd291a0785d2eb19da08f7615 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 1 Jul 2026 22:26:45 +0500 Subject: [PATCH 102/164] dev: special case for olm amespaces --- .../v1/type_configuration_chop.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index f9610f1ab..76ad114a9 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -1565,11 +1565,17 @@ func (c *OperatorConfig) applyEnvVarParams() { c.Watch.Namespaces.Include = types.NewStrings([]string{ns}) } - if nss := os.Getenv(deployment.WATCH_NAMESPACES); len(nss) > 0 { - // We have WATCH_NAMESPACES explicitly specified - if namespaces := c.splitNamespaces(nss); len(namespaces) > 0 { - c.Watch.Namespaces.Include = types.NewStrings(namespaces) + // LookupEnv, not Getenv+len: OLM's AllNamespaces mode sets WATCH_NAMESPACES to an empty + // string (present-but-empty), which must mean watch-all - distinct from a non-OLM deploy + // that leaves it unset. Present-and-empty is coerced to the watch-all pattern so it does + // not fall through to applyDefaultWatchNamespace()'s own-namespace inference. Supersedes + // the singular WATCH_NAMESPACE above. + if nss, ok := os.LookupEnv(deployment.WATCH_NAMESPACES); ok { + namespaces := c.splitNamespaces(nss) + if len(namespaces) == 0 { + namespaces = []string{".*"} } + c.Watch.Namespaces.Include = types.NewStrings(namespaces) } if nss := os.Getenv(deployment.WATCH_NAMESPACES_EXCLUDE); len(nss) > 0 { From 7318c91b04ca2e504fd3675c5d514c31c152a615 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 1 Jul 2026 22:26:58 +0500 Subject: [PATCH 103/164] test: minor --- tests/e2e/merge_dual_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/merge_dual_results.py b/tests/e2e/merge_dual_results.py index 0443cd0a3..a2f1a532f 100755 --- a/tests/e2e/merge_dual_results.py +++ b/tests/e2e/merge_dual_results.py @@ -23,7 +23,7 @@ STEP = re.compile(r"(\d+)\s+steps?\s+\(([^)]*)\)") TOTAL = re.compile(r"Total time\s+(.+)") BREAKDOWN = re.compile(r"(\d+)\s+(ok|failed|skipped|errored)") -PASS_SYMBOL, FAIL_SYMBOL = "✔", "✘" # ✔ ✘ +PASS_SYMBOL, FAIL_SYMBOL = "✔", "✘" def transform(raw): From 4ce250b7d23ff0d03434bf79bffdb789eacedb0a Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 1 Jul 2026 22:27:09 +0500 Subject: [PATCH 104/164] dev: notes --- release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release_notes.md b/release_notes.md index a5fd4172f..d93cc9bb2 100644 --- a/release_notes.md +++ b/release_notes.md @@ -7,6 +7,7 @@ A user-supplied `replicaServiceTemplate` still produces a single Service (the split applies only to the operator's default Services). ### Behavior Changes +* **OLM install modes now honor the OperatorGroup** ([#2008](https://github.com/Altinity/clickhouse-operator/issues/2008)). The OperatorHub bundle previously hard-wired `WATCH_NAMESPACE` to the operator's own namespace, so `SingleNamespace`, `MultiNamespace`, and `AllNamespaces` installs were all silently scoped to the operator's namespace only. The CSV now reads `metadata.annotations['olm.targetNamespaces']`, so each advertised install mode watches the namespaces the OperatorGroup actually selects. **On upgrade, an OLM install configured for Single/Multi/AllNamespaces will start watching the intended namespaces for the first time** — if you relied on the old own-namespace-only behavior, scope the OperatorGroup (or a `ClickHouseOperatorConfiguration` `watch.namespaces`) accordingly. Affects OLM/OperatorHub installs only; plain manifest and Helm installs are unchanged. * **One-time ClickHouse rolling restart on upgrade for CHI→CHK keeper references** ([#1982](https://github.com/Altinity/clickhouse-operator/issues/1982)). A `ClickHouseInstallation` that references a `ClickHouseKeeper` via a `keeper:` ref (the default `serviceType: Replicas`) now resolves to the new ready-only Keeper **client** Service. The first reconcile after upgrade rewrites the CHI's `` endpoints from `chk-…` to `chk-…-client`, which the operator treats as a configuration change requiring a restart — so **every ClickHouse pod of an affected CHI restarts once**. This is a one-time event; the resolved endpoints are stable afterwards. No action required. * **Backward-incompatible config rename** in `ClickHouseOperatorConfiguration`: `reconcile.recovery.from.{aborted,completed}` → `reconcile.recovery.onStatus.{aborted,completed}`. The `from` grouping level is removed; the per-status scopes and their action keys (`onPodReady`/`onPodNotReady`/`onPodNotReadyThreshold`) are unchanged. The obsolete `from` key is silently ignored on load. **If you set `reconcile.recovery.from.aborted.onPodReady: none` on 0.27.0/0.27.1 to disable Aborted auto-recovery, re-apply it as `reconcile.recovery.onStatus.aborted.onPodReady: none`** — otherwise the default (`retry`) silently re-enables it. The `completed` scope (sustained-NotReady host recovery) is new in 0.27.2 and off by default. See [docs/operator_upgrade.md](docs/operator_upgrade.md). From f2c3fbf8e82bf90956f39802036eac696363d9d3 Mon Sep 17 00:00:00 2001 From: Vladislav Klimenko Date: Wed, 1 Jul 2026 22:27:23 +0500 Subject: [PATCH 105/164] dev: unit test --- .../v1/type_configuration_chop_watch_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go new file mode 100644 index 000000000..99ded8924 --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go @@ -0,0 +1,55 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/altinity/clickhouse-operator/pkg/apis/deployment" +) + +// TestApplyEnvVarParamsWatchNamespaces verifies that WATCH_NAMESPACES (wired by the OLM CSV +// to the OperatorGroup's olm.targetNamespaces annotation) maps every advertised OLM install +// mode to the right watch set. The decisive case is AllNamespaces: OLM sets the var to an +// empty string, which must mean "watch all namespaces", not "watch own namespace". +func TestApplyEnvVarParamsWatchNamespaces(t *testing.T) { + tests := []struct { + name string // OLM install mode under test + value string // WATCH_NAMESPACES as OLM sets it from olm.targetNamespaces + expected []string + }{ + {"OwnNamespace", "openshift-operators", []string{"openshift-operators"}}, + {"SingleNamespace", "team-a", []string{"team-a"}}, + {"MultiNamespace comma", "team-a,team-b", []string{"team-a", "team-b"}}, + {"MultiNamespace colon", "team-a:team-b", []string{"team-a", "team-b"}}, + {"AllNamespaces empty -> watch all", "", []string{".*"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(deployment.WATCH_NAMESPACES, tt.value) + + c := &OperatorConfig{} + c.applyEnvVarParams() + + // ElementsMatch, not Equal: the include set is order-insensitive (NewStrings + // dedups via a map), and watch.namespaces is consumed as a set downstream. + require.ElementsMatch(t, tt.expected, c.Watch.Namespaces.Include.Value(), + "WATCH_NAMESPACES=%q", tt.value) + }) + } +} From 4e726f7471bb3de403505af7bf035e601d50a41e Mon Sep 17 00:00:00 2001 From: Javier Tomas Zon Date: Thu, 2 Jul 2026 11:43:17 +0000 Subject: [PATCH 106/164] fix(helm): generate watchNamespaces ConfigMap wiring from the generator Address review: the previous change hand-edited the generated ConfigMap-etc-clickhouse-operator-files.yaml. Instead, special-case the etc-clickhouse-operator-files ConfigMap in update_configmap_resource() so generate_helm_chart.sh emits the configmap-files helper (which wires watchNamespaces into watch.namespaces.include). Re-running the generator now reproduces the committed template. README values table regenerated by helm-docs. --- deploy/helm/clickhouse-operator/README.md | 1 + dev/generate_helm_chart.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md index 8e9cfa4ca..a18123bc0 100644 --- a/deploy/helm/clickhouse-operator/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -138,4 +138,5 @@ crdHook: | serviceMonitor.operatorMetrics.scrapeTimeout | string | `""` | | | tolerations | list | `[]` | tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details | | topologySpreadConstraints | list | `[]` | | +| watchNamespaces | list | `[]` | namespaces where the operator watches for ClickHouseInstallation resources. If empty, the operator watches only its own namespace (or all namespaces when running in kube-system). Use [".*"] to watch all namespaces. Example: watchNamespaces: ["clickhouse", "my-other-namespace"] | diff --git a/dev/generate_helm_chart.sh b/dev/generate_helm_chart.sh index 8a0664efa..f519a210a 100755 --- a/dev/generate_helm_chart.sh +++ b/dev/generate_helm_chart.sh @@ -290,7 +290,14 @@ function update_configmap_resource() { yq e -i '.metadata.namespace |= "{{ include \"altinity-clickhouse-operator.namespace\" . }}"' "${file}" yq e -i '.metadata.labels |= "{{ include \"altinity-clickhouse-operator.labels\" . | nindent 4 }}"' "${file}" yq e -i '.metadata.annotations |= "{{ include \"altinity-clickhouse-operator.annotations\" . | nindent 4 }}"' "${file}" - yq e -i '.data |= "{{ include \"altinity-clickhouse-operator.configmap-data\" (list . .Values.configs.'"${camel_cased_name}"') | nindent 2 }}"' "${file}" + if [ "${name}" = "etc-clickhouse-operator-files" ]; then + # The operator config ConfigMap needs watchNamespaces wired in. Use the + # configmap-files helper (which patches watch.namespaces.include from the + # top-level watchNamespaces value) instead of the generic configmap-data. + yq e -i '.data |= "{{ include \"altinity-clickhouse-operator.configmap-files\" (list . .Values.configs.files .Values.watchNamespaces) | nindent 2 }}"' "${file}" + else + yq e -i '.data |= "{{ include \"altinity-clickhouse-operator.configmap-data\" (list . .Values.configs.'"${camel_cased_name}"') | nindent 2 }}"' "${file}" + fi if [ -z "${data}" ]; then yq e -i '.configs.'"${camel_cased_name}"' |= null' "${values_yaml}" From 159281a04d96099f79cab96b2a38960138bd7507 Mon Sep 17 00:00:00 2001 From: alz Date: Fri, 3 Jul 2026 12:53:34 +0300 Subject: [PATCH 107/164] Improve Keeper rescale test --- tests/e2e/test_operator.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 5ccc492b3..533c8d1e3 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -7538,34 +7538,12 @@ def test_020005(self): }, ) - check_replication(chi, {0, 1}, 2) - - # TODO: This does not work now - # with Then("Kill first pod to switch the leader"): - # kubectl.launch(f"delete pod chk-test-052-chk-keeper-0-0-0") - # time.sleep(10) - - # with Then("Force leader to be on the first node only"): - # kubectl.create_and_check( - # manifest="manifests/chk/test-052-chk-rescale-1.1.yaml", kind="chk", - # check={ - # "pod_count": 3, - # "do_not_delete": 1, - # }, - # ) - - # check_replication(chi, {0,1}, 3) + with Then("Confirm all CHK pods are ready"): + kubectl.wait_field('pod', 'chk-test-052-chk-keeper-0-0-0', '.status.containerStatuses[0].ready', 'true', retries=10) + kubectl.wait_field('pod', 'chk-test-052-chk-keeper-0-1-0', '.status.containerStatuses[0].ready', 'true', retries=10) + kubectl.wait_field('pod', 'chk-test-052-chk-keeper-0-2-0', '.status.containerStatuses[0].ready', 'true', retries=10) - - # with Then("Remove other nodes from the raft configuration"): - # kubectl.create_and_check( - # manifest="manifests/chk/test-052-chk-rescale-1.2.yaml", kind="chk", - # check={ - # "do_not_delete": 1, - # }, - # ) - - # check_replication(chi, {0,1}, 4) + check_replication(chi, {0, 1}, 2) with Then("Rescale CHK back to 1 replica"): kubectl.create_and_check( From 9e78d09c1388c10fe08fb36b8050af4fc7e219bb Mon Sep 17 00:00:00 2001 From: dentinyhao Date: Fri, 3 Jul 2026 04:03:02 -0700 Subject: [PATCH 108/164] Apply filter when scan SQL results Signed-off-by: dentinyhao --- config/config.yaml | 2 +- deploy/builder/templates-config/config.yaml | 2 +- deploy/helm/clickhouse-operator/values.yaml | 2 +- docs/chi-examples/70-chop-config.yaml | 2 +- .../clickhouse/clickhouse_metrics_fetcher.go | 21 +++++++++++++------ pkg/metrics/clickhouse/exporter.go | 1 + 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 4bfc9d015..38bcc39d2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -256,7 +256,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 46740bd15..ff2b77ecb 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -250,7 +250,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..2ffe5a7dc 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -531,7 +531,7 @@ configs: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index b9f573ec1..8c68d300c 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -98,7 +98,7 @@ spec: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: diff --git a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go index 9f0bc0d80..613bd5023 100644 --- a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go +++ b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go @@ -142,16 +142,20 @@ const ( type MetricsFetcher struct { connectionParams *clickhouse.EndpointConnectionParams tablesRegexp string + // Used to filter system-metric names while fetching metrics. Nil means keep all. + metricsFilter MetricsFilter } // NewMetricsFetcher creates new clickhouse fetcher object func NewMetricsFetcher( endpointConnectionParams *clickhouse.EndpointConnectionParams, tablesRegexp string, + metricsFilter MetricsFilter, ) *MetricsFetcher { return &MetricsFetcher{ connectionParams: endpointConnectionParams, tablesRegexp: tablesRegexp, + metricsFilter: metricsFilter, } } @@ -179,11 +183,9 @@ func (f *MetricsFetcher) buildMetricsSQL() string { } // getClickHouseQueryMetrics requests metrics data from ClickHouse. -// Exclusion of "noisy" metrics is enforced solely by the writer-side filter -// (see CHIPrometheusWriter.metricsFilter). A SQL-side filter was tried and -// dropped: wrapping the UNION-ALL chain in `FROM (...) WHERE NOT (...)` left -// the metrics query returning zero rows across restart-then-scrape windows; -// the writer-side filter is sufficient and avoids that fragility. +// Excluded names are dropped during row scan so they never enter the in-memory buffer. +// SQL-side filtering was tried and abandoned: wrapping the UNION-ALL in +// `FROM (...) WHERE NOT (...)` caused zero rows on restart-then-scrape windows. func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( ctx, @@ -191,13 +193,20 @@ func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, func(rows *sql.Rows, data *Table) error { var metric, value, description, _type string if err := rows.Scan(&metric, &value, &description, &_type); err == nil { - *data = append(*data, []string{metric, value, description, _type}) + f.appendMetricRow(data, metric, value, description, _type) } return nil }, ) } +func (f *MetricsFetcher) appendMetricRow(data *Table, metric, value, description, _type string) { + if f.metricsFilter != nil && f.metricsFilter.IsExcluded(metric) { + return + } + *data = append(*data, []string{metric, value, description, _type}) +} + // getClickHouseSystemParts requests data sizes from ClickHouse func (f *MetricsFetcher) getClickHouseSystemParts(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( diff --git a/pkg/metrics/clickhouse/exporter.go b/pkg/metrics/clickhouse/exporter.go index ac494ab02..f2acf64ed 100644 --- a/pkg/metrics/clickhouse/exporter.go +++ b/pkg/metrics/clickhouse/exporter.go @@ -135,6 +135,7 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher { return NewMetricsFetcher( clusterConnectionParams.NewEndpointConnectionParams(host.Hostname), chop.Config().ClickHouse.Metrics.TablesRegexp, + e.metricsFilter, ) } From 205cd1b4bc8a013d39cce0d310756adb998e477b Mon Sep 17 00:00:00 2001 From: saba Date: Mon, 6 Jul 2026 15:27:47 +0200 Subject: [PATCH 109/164] Expand FIPS TLS e2e coverage and add host-run cipher probes (test_030017) - Add host-run openssl/fake-k8s TLS probe helpers in steps_fips.py - Add test_030017 + CHI manifests for local operator TLS testing - Broaden test_030003 rejected-TLS checks; disable TLS 1.2 in manifests - Fix namespace derivation in steps.py; kubectl delete wait flag - Drop ACVP test suite from regression - Tidy FIPS test wording in test_operator.py --- tests/e2e/kubectl.py | 9 +- tests/e2e/manifests/chi/test-030003.yaml | 4 +- tests/e2e/manifests/chi/test-030017-chi.yaml | 33 + .../chi/test-030017-phase0-chi-tls12.yaml | 87 + tests/e2e/manifests/chk/test-030003.yaml | 4 +- tests/e2e/run_tests_acvp_local.sh | 30 - tests/e2e/steps.py | 13 +- tests/e2e/steps_fips.py | 2993 ++++++++++++++--- tests/e2e/test_acvp.py | 295 -- tests/e2e/test_operator.py | 409 ++- tests/regression.py | 1 - 11 files changed, 2927 insertions(+), 951 deletions(-) create mode 100644 tests/e2e/manifests/chi/test-030017-chi.yaml create mode 100644 tests/e2e/manifests/chi/test-030017-phase0-chi-tls12.yaml delete mode 100755 tests/e2e/run_tests_acvp_local.sh delete mode 100644 tests/e2e/test_acvp.py diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py index b07fdd20c..72c08311d 100644 --- a/tests/e2e/kubectl.py +++ b/tests/e2e/kubectl.py @@ -111,10 +111,11 @@ def run_shell(cmd, timeout=600, ok_to_fail=False, shell=None, retry_transient=Fa assert code == 0, error() -def delete_kind(kind, name, ns=None, ok_to_fail=False, shell=None): +def delete_kind(kind, name, ns=None, ok_to_fail=False, shell=None, wait=True): with When(f"Delete {kind} {name}"): + wait_flag = "" if wait else "--wait=false" launch( - f"delete {kind} {name} -v 5 --now --timeout=600s", + f"delete {kind} {name} -v 5 --now --timeout=600s {wait_flag}".strip(), ns=ns, timeout=600, ok_to_fail=ok_to_fail, @@ -233,7 +234,7 @@ def delete_all(kind, ns=None): # OR mid-restart — e.g. chopconf onChange=restart in test_030008) # makes `kubectl delete --timeout` exit non-zero; we recover via # the force-clear loop below, so this must not raise here. - delete_kind(kind, name, ns=ns, ok_to_fail=True) + delete_kind(kind, name, ns=ns, ok_to_fail=True, wait=False) # Stuck/re-attached finalizer recovery. The operator can RE-ATTACH # a finalizer after a clear while it is restarting, so a single # wait_object would race the restart and raise. Re-clear + re-delete @@ -244,7 +245,7 @@ def delete_all(kind, ns=None): if get_count(kind, name=name, ns=ns) == 0: break force_clear_finalizers(kind, name, ns=ns) - delete_kind(kind, name, ns=ns, ok_to_fail=True) + delete_kind(kind, name, ns=ns, ok_to_fail=True, wait=False) # Only sleep if another re-check follows; skip on the last # attempt so a genuinely-stuck CR hits the final wait_object # (the authoritative leak assertion) without an extra wait. diff --git a/tests/e2e/manifests/chi/test-030003.yaml b/tests/e2e/manifests/chi/test-030003.yaml index c61b0fffc..f7fcc00f7 100644 --- a/tests/e2e/manifests/chi/test-030003.yaml +++ b/tests/e2e/manifests/chi/test-030003.yaml @@ -43,14 +43,14 @@ spec: /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem none - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt false strict - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 diff --git a/tests/e2e/manifests/chi/test-030017-chi.yaml b/tests/e2e/manifests/chi/test-030017-chi.yaml new file mode 100644 index 000000000..d1bf7b0d1 --- /dev/null +++ b/tests/e2e/manifests/chi/test-030017-chi.yaml @@ -0,0 +1,33 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030017-chi +spec: + # chi-...-0-0..127.0.0.1.nip.io → 127.0.0.1 on the host; operator Ping(:8443) + # hits local openssl s_server, not a real ClickHouse HTTPS listener. + namespaceDomainPattern: "%s.127.0.0.1.nip.io" + reconcile: + host: + wait: + exclude: "no" + queries: "no" + probes: + startup: "no" + readiness: "no" + templates: + podTemplates: + - name: clickhouse + spec: + containers: + - name: clickhouse + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + templates: + podTemplate: clickhouse + layout: + shardsCount: 1 + replicasCount: 1 diff --git a/tests/e2e/manifests/chi/test-030017-phase0-chi-tls12.yaml b/tests/e2e/manifests/chi/test-030017-phase0-chi-tls12.yaml new file mode 100644 index 000000000..7891f69bb --- /dev/null +++ b/tests/e2e/manifests/chi/test-030017-phase0-chi-tls12.yaml @@ -0,0 +1,87 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030017-phase0 +spec: + # "%s" → namespace; FQDN is service..127.0.0.1.nip.io (resolves to 127.0.0.1 for + # host-run + kubectl port-forward; avoids *.localhost → ::1 on Linux). + namespaceDomainPattern: "%s.127.0.0.1.nip.io" + defaults: + templates: + podTemplate: test-030017-phase0 + templates: + podTemplates: + - name: test-030017-phase0 + spec: + containers: + - name: clickhouse + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + env: + - name: CLICKHOUSE_USER + value: clickhouse_operator + - name: CLICKHOUSE_PASSWORD + value: clickhouse_operator_password + security: + clickhouse: + tls: + rootCASecretRef: + name: clickhouse-certs + key: ca.crt + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + layout: + shardsCount: 1 + replicasCount: 1 + settings: + http_port: _removed_ + tcp_port: _removed_ + interserver_http_port: _removed_ + mysql_port: _removed_ + postgresql_port: _removed_ + https_port: 8443 + tcp_port_secure: 9440 + interserver_https_port: 9010 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + none + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_3 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 + + + + server.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.crt + server.key: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.key + dhparam.pem: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: dhparam.pem + ca.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: ca.crt diff --git a/tests/e2e/manifests/chk/test-030003.yaml b/tests/e2e/manifests/chk/test-030003.yaml index e1c0bc736..0b2d288ae 100644 --- a/tests/e2e/manifests/chk/test-030003.yaml +++ b/tests/e2e/manifests/chk/test-030003.yaml @@ -26,14 +26,14 @@ spec: /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key none - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt false strict - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 diff --git a/tests/e2e/run_tests_acvp_local.sh b/tests/e2e/run_tests_acvp_local.sh deleted file mode 100755 index f33175ead..000000000 --- a/tests/e2e/run_tests_acvp_local.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Runs the e2e ACVP responder smoke tests only. -# -# The test module (tests/e2e/test_acvp.py) builds the operator and -# metrics-exporter binaries with `-tags acvp_wrapper`, invokes them via argv0 -# dispatch (binary symlinked as `-acvp`), and round-trips ACVP requests -# over stdin/stdout. NO minikube cluster, NO operator image is required — -# the host's Go toolchain and `GOFIPS140=v1.0.0` build env are all the test -# needs. -# -# Full BoringSSL acvptool reproducibility (vector-by-vector comparison -# against geomys/acvp-testdata) lives in pkg/util/fips/acvp/run.sh and is -# reproduced locally per release — this script is the fast pre-flight that -# catches build-tag / argv0-dispatch / FIPS-mode regressions before the -# heavier vector-roundtrip run. -CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -source "${CUR_DIR}/test_common.sh" - -common_install_pip_requirements -common_export_test_env - -RUN_ALL_FLAG=$(common_convert_run_all) - -python3 "${COMMON_DIR}/../regression.py" \ - --only="/regression/e2e.test_acvp/${ONLY}" \ - ${RUN_ALL_FLAG} \ - -o short \ - --trim-results on \ - --debug \ - --native diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py index 30bba785c..f386279d5 100644 --- a/tests/e2e/steps.py +++ b/tests/e2e/steps.py @@ -31,9 +31,18 @@ def get_shell(self, timeout=600): def create_test_namespace(self, force=False): """Create unique test namespace for test.""" - random_namespace = self.name[self.name.find('test_0'):self.name.find('. ')].replace("_", "-") + "-" + str(uuid.uuid1()) + # Derive prefix from scenario id: test_010001, test_010006_2, test_039, etc. + # Nested TestStep names may not contain ". ", so slicing on ". " is unsafe. + match = re.search(r"test_\d+(?:_\d+)?", self.name) + assert match, error(f"cannot derive namespace prefix from test name: {self.name!r}") + + random_namespace = ( + match.group(0).replace("_", "-") + + "-" + + str(uuid.uuid1()) + ) - if not force: # (self.cflags & PARALLEL) and not force: + if not force: # (self.cflags & PARALLEL) and not force: self.context.test_namespace = random_namespace self.context.operator_namespace = self.context.test_namespace diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index 4b4bc9b49..51418b975 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -12,28 +12,235 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy +import json import os import re +import select import shlex import shutil import socket +import ssl import subprocess +import sys import tempfile +import threading import time import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qs, urlparse import yaml import e2e.util as util -from e2e.steps import create_shell_namespace_clickhouse_template +from e2e.steps import create_shell_namespace_clickhouse_template, delete_test_namespace, get_shell from testflows.asserts import error from testflows.core import * import e2e.kubectl as kubectl +import struct +import hashlib +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) FAKE_OPENSSL_SERVER = "fake-openssl-server" +TLS_REJECT_MARKERS = ( + "Cipher is (NONE)", + "Cipher : 0000", + "handshake failure", + "alert handshake failure", + "no shared cipher", + "no protocols available", + "unsupported protocol", + "wrong version number", + "tlsv1 alert protocol version", + "no peer certificate available", +) + +approved_tls1_3_ciphers = [ + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", +] +ciphers_by_protocol = { + "TLSv1.3": [ + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_GCM_SHA256", + ], + "TLSv1.2": [ + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "DHE-DSS-AES256-GCM-SHA384", + "DHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-RSA-CHACHA20-POLY1305", + "DHE-RSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES256-CCM", + "DHE-RSA-AES256-CCM", + "ECDHE-ECDSA-ARIA256-GCM-SHA384", + "ECDHE-ARIA256-GCM-SHA384", + "DHE-DSS-ARIA256-GCM-SHA384", + "DHE-RSA-ARIA256-GCM-SHA384", + "ADH-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "DHE-DSS-AES128-GCM-SHA256", + "DHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-CCM", + "DHE-RSA-AES128-CCM", + "ECDHE-ECDSA-ARIA128-GCM-SHA256", + "ECDHE-ARIA128-GCM-SHA256", + "DHE-DSS-ARIA128-GCM-SHA256", + "DHE-RSA-ARIA128-GCM-SHA256", + "ADH-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-CCM8", + "ECDHE-ECDSA-AES128-CCM8", + "DHE-RSA-AES256-CCM8", + "DHE-RSA-AES128-CCM8", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "DHE-RSA-AES256-SHA256", + "DHE-DSS-AES256-SHA256", + "ECDHE-ECDSA-CAMELLIA256-SHA384", + "ECDHE-RSA-CAMELLIA256-SHA384", + "DHE-RSA-CAMELLIA256-SHA256", + "DHE-DSS-CAMELLIA256-SHA256", + "ADH-AES256-SHA256", + "ADH-CAMELLIA256-SHA256", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "DHE-RSA-AES128-SHA256", + "DHE-DSS-AES128-SHA256", + "ECDHE-ECDSA-CAMELLIA128-SHA256", + "ECDHE-RSA-CAMELLIA128-SHA256", + "DHE-RSA-CAMELLIA128-SHA256", + "DHE-DSS-CAMELLIA128-SHA256", + "ADH-AES128-SHA256", + "ADH-CAMELLIA128-SHA256", + "RSA-PSK-AES256-GCM-SHA384", + "DHE-PSK-AES256-GCM-SHA384", + "RSA-PSK-CHACHA20-POLY1305", + "DHE-PSK-CHACHA20-POLY1305", + "ECDHE-PSK-CHACHA20-POLY1305", + "DHE-PSK-AES256-CCM", + "RSA-PSK-ARIA256-GCM-SHA384", + "DHE-PSK-ARIA256-GCM-SHA384", + "AES256-GCM-SHA384", + "AES256-CCM", + "ARIA256-GCM-SHA384", + "PSK-AES256-GCM-SHA384", + "PSK-CHACHA20-POLY1305", + "PSK-AES256-CCM", + "PSK-ARIA256-GCM-SHA384", + "RSA-PSK-AES128-GCM-SHA256", + "DHE-PSK-AES128-GCM-SHA256", + "DHE-PSK-AES128-CCM", + "RSA-PSK-ARIA128-GCM-SHA256", + "DHE-PSK-ARIA128-GCM-SHA256", + "AES128-GCM-SHA256", + "AES128-CCM", + "ARIA128-GCM-SHA256", + "PSK-AES128-GCM-SHA256", + "PSK-AES128-CCM", + "PSK-ARIA128-GCM-SHA256", + "DHE-PSK-AES256-CCM8", + "DHE-PSK-AES128-CCM8", + "AES256-CCM8", + "AES128-CCM8", + "PSK-AES256-CCM8", + "PSK-AES128-CCM8", + "AES256-SHA256", + "CAMELLIA256-SHA256", + "AES128-SHA256", + "CAMELLIA128-SHA256", + ], + "TLSv1": [ + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA", + "AECDH-AES256-SHA", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "AECDH-AES128-SHA", + "ECDHE-PSK-AES256-CBC-SHA384", + "ECDHE-PSK-AES256-CBC-SHA", + "RSA-PSK-AES256-CBC-SHA384", + "DHE-PSK-AES256-CBC-SHA384", + "ECDHE-PSK-CAMELLIA256-SHA384", + "RSA-PSK-CAMELLIA256-SHA384", + "DHE-PSK-CAMELLIA256-SHA384", + "PSK-AES256-CBC-SHA384", + "PSK-CAMELLIA256-SHA384", + "ECDHE-PSK-AES128-CBC-SHA256", + "ECDHE-PSK-AES128-CBC-SHA", + "RSA-PSK-AES128-CBC-SHA256", + "DHE-PSK-AES128-CBC-SHA256", + "ECDHE-PSK-CAMELLIA128-SHA256", + "RSA-PSK-CAMELLIA128-SHA256", + "DHE-PSK-CAMELLIA128-SHA256", + "PSK-AES128-CBC-SHA256", + "PSK-CAMELLIA128-SHA256", + ], +} + +_OPENSSL_NEGOTIATED_CIPHER = re.compile( + r"(?:^|\n)Cipher is (?!\(NONE\))(?P\S+)", + re.IGNORECASE, +) + +CIPHERS_PROTOCOL_TLS_VERSION = { + "TLSv1.3": "1.3", + "TLSv1.2": "1.2", + "TLSv1": "1.0", +} + +FIPS_REJECTED_PROTOCOL_CASES = ( + {"name": "TLS 1.0 protocol", "tls_version": "1.0", "cipher_suite": None}, + {"name": "TLS 1.1 protocol", "tls_version": "1.1", "cipher_suite": None}, + {"name": "TLS 1.2 protocol", "tls_version": "1.2", "cipher_suite": None}, +) + + +def fips_rejected_cipher_cases_from_ciphers_by_protocol(): + """Every cipher in ciphers_by_protocol except approved TLS 1.3 suites.""" + cases = [] + for protocol, tls_version in CIPHERS_PROTOCOL_TLS_VERSION.items(): + for cipher in ciphers_by_protocol[protocol]: + if protocol == "TLSv1.3" and cipher in approved_tls1_3_ciphers: + continue + cases.append({ + "name": f"TLS {tls_version} {cipher}", + "tls_version": tls_version, + "cipher_suite": cipher, + }) + return tuple(cases) + + +FIPS_LISTENER_REJECTED_TLS_CASES = ( + *FIPS_REJECTED_PROTOCOL_CASES, + *fips_rejected_cipher_cases_from_ciphers_by_protocol(), +) + +FIPS_APPROVED_TLS13_CIPHER_CASES = tuple( + { + "name": f"TLS 1.3 {cipher}", + "tls_version": "1.3", + "cipher_suite": cipher, + } + for cipher in approved_tls1_3_ciphers +) + +FIPS_OPERATOR_APPROVED_TLS13_CIPHER = "TLS_AES_256_GCM_SHA384" +FIPS_OPERATOR_APPROVED_TLS13_CIPHER_SUITES = ":".join(approved_tls1_3_ciphers) + +OPERATOR_CONTAINER_TLS_FAILURE_NEEDLES = ( + "handshake failure", + "no shared cipher", + "alert handshake failure", + "TLS connect error", + "HTTP:000", +) # --------------------------------------------------------------------------- # Build verification @@ -353,10 +560,12 @@ def fips_assert_chi_admitted(self, chi, reason="FIPSImagePolicyViolation"): @TestStep(Given) def create_tls_secret_for_fips_hosts( self, - chi, - chk, + chi=None, + chk=None, secret_name="clickhouse-certs", replicas=2, + pod_hostnames=None, + extra_dns_names=None, ): """Create a TLS secret whose SANs match this test namespace's pod DNS names.""" ns = self.context.test_namespace @@ -378,13 +587,25 @@ def create_tls_secret_for_fips_hosts( dns_suffixes = ("", f".{ns}", f".{ns}.svc", f".{ns}.svc.cluster.local") dns_names = ["localhost", "clickhouse", "clickhouse1", f"*.{ns}.svc.cluster.local"] - for replica in range(replicas): - for host in ( - f"chi-{chi}-default-0-{replica}", - f"chk-{chk}-keeper-0-{replica}", - ): + if pod_hostnames: + for host in pod_hostnames: for suffix in dns_suffixes: dns_names.append(f"{host}{suffix}") + else: + assert chi and chk, error( + "create_tls_secret_for_fips_hosts requires chi and chk " + "when pod_hostnames is not set" + ) + for replica in range(replicas): + for host in ( + f"chi-{chi}-default-0-{replica}", + f"chk-{chk}-keeper-0-{replica}", + ): + for suffix in dns_suffixes: + dns_names.append(f"{host}{suffix}") + + if extra_dns_names: + dns_names.extend(extra_dns_names) san_entries = ["IP.1 = 127.0.0.1"] san_entries.extend( @@ -881,6 +1102,42 @@ def fips_wait_cluster_topology( note(f"{pod} sees {replica_count} hosts in cluster {cluster_name!r}") +def openssl_tls_version_args(tls_version): + if tls_version == "1.3": + return ["-tls1_3"] + if tls_version == "1.2": + return ["-tls1_2"] + if tls_version == "1.1": + return ["-tls1_1"] + if tls_version == "1.0": + return ["-tls1"] + if tls_version == "ssl3": + return ["-ssl3"] + if tls_version == "ssl2": + return ["-ssl2"] + + raise ValueError(f"unsupported TLS/SSL version: {tls_version}") + + +def openssl_cipher_args(tls_version, cipher_suite): + if not cipher_suite: + return [] + + if tls_version == "1.3": + return ["-ciphersuites", cipher_suite] + + return ["-cipher", cipher_suite] + + + +def openssl_s_client_negotiated_cipher(output): + """Return the negotiated cipher name when s_client completed a handshake.""" + match = _OPENSSL_NEGOTIATED_CIPHER.search(output) + if match: + return match.group("cipher") + return None + + @TestStep(When) def fips_run_openssl_s_client_on_pod_port( self, @@ -896,83 +1153,75 @@ def fips_run_openssl_s_client_on_pod_port( ca_crt = self.context.tls["ca_crt"] local_port = _free_local_port() - pf = subprocess.Popen( - [ - "kubectl", - "-n", ns, - "port-forward", - f"pod/{pod}", - f"{local_port}:{port}", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) + with Given(f"port-forward from localhost:{local_port} to {pod}:{port}"): + pf = subprocess.Popen( + [ + "kubectl", + "-n", ns, + "port-forward", + f"pod/{pod}", + f"{local_port}:{port}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) try: - deadline = time.time() + 10 - while time.time() < deadline: - if pf.poll() is not None: - out, err = pf.communicate() + with When("port-forward is ready on localhost"): + deadline = time.time() + 10 + while time.time() < deadline: + if pf.poll() is not None: + out, err = pf.communicate() + assert False, error( + "kubectl port-forward exited early\n" + f"stdout:\n{out}\n" + f"stderr:\n{err}" + ) + + try: + socket.create_connection( + ("127.0.0.1", int(local_port)), timeout=0.5 + ).close() + break + except OSError: + time.sleep(0.2) + else: assert False, error( - "kubectl port-forward exited early\n" - f"stdout:\n{out}\n" - f"stderr:\n{err}" + f"kubectl port-forward to {pod}:{port} " + f"did not become ready on 127.0.0.1:{local_port}" ) - try: - socket.create_connection( - ("127.0.0.1", int(local_port)), timeout=0.5 - ).close() - break - except OSError: - time.sleep(0.2) - else: - assert False, error( - f"kubectl port-forward to {pod}:{port} " - f"did not become ready on 127.0.0.1:{local_port}" + with And("openssl s_client runs against the forwarded port"): + command = [ + "openssl", "s_client", + "-connect", f"127.0.0.1:{local_port}", + "-servername", "localhost", + "-CAfile", ca_crt, + "-verify_return_error", + ] + + command.extend(openssl_tls_version_args(tls_version)) + command.extend(openssl_cipher_args(tls_version, cipher_suite)) + + result = subprocess.run( + command, + input="Q\n", + text=True, + capture_output=True, + check=False, ) - command = [ - "openssl", "s_client", - "-connect", f"127.0.0.1:{local_port}", - "-servername", "localhost", - "-CAfile", ca_crt, - "-verify_return_error", - ] - - if tls_version == "1.3": - command.append("-tls1_3") - if cipher_suite: - command.extend(["-ciphersuites", cipher_suite]) - elif tls_version == "1.2": - command.append("-tls1_2") - if cipher_suite: - command.extend(["-cipher", cipher_suite]) - elif tls_version == "1.1": - command.append("-tls1_1") - if cipher_suite: - command.extend(["-cipher", cipher_suite]) - else: - raise ValueError(f"unsupported TLS version: {tls_version}") - - result = subprocess.run( - command, - input="Q\n", - text=True, - capture_output=True, - check=False, - ) - - output = f"{result.stdout}\n{result.stderr}" + output = f"{result.stdout}\n{result.stderr}" if not ok_to_fail: - assert result.returncode == 0, error( - f"{pod}:{port}: openssl s_client failed for " - f"tls={tls_version}, cipher={cipher_suite}\n" - f"exit code: {result.returncode}\n" - f"output:\n{output}" - ) + with Then("openssl s_client handshake succeeds"): + assert result.returncode == 0, error( + f"{pod}:{port}: openssl s_client failed for " + f"tls={tls_version}, cipher={cipher_suite}\n" + f"exit code: {result.returncode}\n" + f"output:\n{output}" + ) return output @@ -983,64 +1232,78 @@ def fips_run_openssl_s_client_on_pod_port( except subprocess.TimeoutExpired: pf.kill() -@TestStep(Then) -def fips_assert_rejected_tls_probes( +@TestStep(Check) +def fips_assert_rejected_tls_cases_on_endpoint( + self, + label, + pod, + port, + rejected_cases, + ns=None, +): + """Assert rejected TLS protocol/cipher cases fail on one endpoint.""" + ns = ns or self.context.test_namespace + + for case in rejected_cases: + with Check(f"{label} {pod}:{port} rejects {case['name']}"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, + port=port, + tls_version=case["tls_version"], + cipher_suite=case["cipher_suite"], + ok_to_fail=True, + ns=ns, + ) + + output_lower = output.lower() + + negotiated_cipher = openssl_s_client_negotiated_cipher(output) + assert negotiated_cipher is None, error( + f"{label} {pod}:{port}: server negotiated disallowed {case['name']}\n" + f"negotiated cipher: {negotiated_cipher}\n" + f"tls_version={case['tls_version']}\n" + f"cipher_suite={case['cipher_suite']}\n" + f"output:\n{output}" + ) + + assert any( + marker.lower() in output_lower + for marker in TLS_REJECT_MARKERS + ), error( + f"{label} {pod}:{port}: expected TLS rejection for {case['name']}\n" + f"tls_version={case['tls_version']}\n" + f"cipher_suite={case['cipher_suite']}\n" + f"output:\n{output}" + ) + + +@TestStep(Check) +def fips_assert_all_rejected_tls_cases_on_all_endpoints( self, chi_pods, chk_pods, ns=None, ): - """Assert rejected TLS protocol/cipher combinations fail handshake.""" + """Assert all rejected TLS cases fail on every FIPS TLS endpoint.""" ns = ns or self.context.test_namespace - - rejected_cases = ( - { - "name": "TLS 1.3 ChaCha20-Poly1305", - "tls_version": "1.3", - "cipher_suite": "TLS_CHACHA20_POLY1305_SHA256", - }, - { - "name": "TLS 1.1 protocol", - "tls_version": "1.1", - "cipher_suite": None, - }, - ) + rejected_cases = FIPS_LISTENER_REJECTED_TLS_CASES endpoints = ( ("ClickHouse HTTPS", chi_pods[0], 8443), ("ClickHouse native TLS", chi_pods[0], 9440), ("ClickHouse interserver HTTPS", chi_pods[0], 9010), ("Keeper secure client", chk_pods[0], 2281), - ("Backup API HTTPS", chi_pods[0], 7171), - ) - - rejected_markers = ( - "Cipher is (NONE)", - "handshake failure", - "no protocols available", - "no shared cipher", ) - for case in rejected_cases: - for label, pod, port in endpoints: - with Then(f"{label} {pod}:{port} rejects {case['name']}"): - output = fips_run_openssl_s_client_on_pod_port( - pod=pod, - port=port, - tls_version=case["tls_version"], - cipher_suite=case["cipher_suite"], - ok_to_fail=True, - ns=ns, - ) + for label, pod, port in endpoints: + fips_assert_rejected_tls_cases_on_endpoint( + label=label, + pod=pod, + port=port, + rejected_cases=rejected_cases, + ns=ns, + ) - assert any( - marker.lower() in output.lower() - for marker in rejected_markers - ), error( - f"{label} {pod}:{port}: expected rejected TLS probe to fail " - f"for {case['name']}\n" - f"output:\n{output}" - ) @TestStep(Then) def fips_assert_aes256_tls13_probes( @@ -1694,72 +1957,283 @@ def check_external_clickhouse_reports_fips_version(self, pod): f"expected FIPS in ClickHouse version(), got {version!r}" ) +def _fips_tls_rejection_present_in_logs(logs, min_version, rejection): + """Return True when logs contain coerced TLS setup and a connect rejection.""" + expected_setup_parts = ( + "setupTLSAdvanced():TLS setup OK", + f"minVersion={min_version}", + ) + setup_found = any( + all(part in line for part in expected_setup_parts) + for line in logs.splitlines() + ) + rejection_found = any( + "connect():FAILED" in line and rejection in line + for line in logs.splitlines() + ) + return setup_found and rejection_found + + +def _fips_tls_rejection_log_excerpt(logs): + return "\n".join( + line for line in logs.splitlines() + if ( + "setupTLSAdvanced()" in line + or "connect():FAILED" in line + or "tls:" in line + or "minVersion" in line + ) + ) + + +# Distroless operator/exporter images ship sh/curl only (no cat/base64). Read the +# IPC token with POSIX shell builtins — same file both containers mount. +_IPC_TOKEN_READ_SHELL = ( + 'TOKEN=""; ' + 'while IFS= read -r line || [ -n "$line" ]; do TOKEN="${TOKEN}${line}"; done ' + "< /etc/clickhouse-operator-ipc/token" +) + + +def _kubectl_pod_exec_stdin(ns, pod, container, shell_script, stdin=None, timeout=120): + """kubectl exec -i … sh -c