0.27.2

.github/workflows/run_tests.yaml

@@ -22,6 +22,41 @@ jobs:
     timeout-minutes: 300
     steps:
       - uses: actions/checkout@v4
+
+      # The e2e suite (notably test_010025's 100M-row insert + its replica) fills the
+      # runner's small root partition (~14 GB): with the minikube docker driver the k8s
+      # node runs as a container under the docker data-root (/var/lib/docker on /), so
+      # ClickHouse pod ephemeral storage lands there and exhausts it — the runner then
+      # loses communication mid-run. Move the docker data-root onto the large /mnt
+      # ephemeral SSD (~65 GB) before minikube starts, and trim unused preinstalled
+      # toolchains from / for headroom. MUST run before setup-minikube (no docker is
+      # used before this step, so restarting the daemon here is safe).
+      - name: Free disk and relocate docker data-root to /mnt
+        run: |
+          set -euxo pipefail
+          # If docker fails to restart on the moved data-root, surface disk + daemon logs
+          # so the failure is triageable rather than a bare non-zero exit at this early step.
+          trap 'ec=$?; echo "::error::disk/docker prep failed (exit $ec)"; df -h / /mnt || true; sudo journalctl -u docker --no-pager -n 50 || true' ERR
+          df -h /
+          # Reclaim space on / — none of these toolchains are used by this Go+Python+k8s job.
+          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/local/lib/android /opt/hostedtoolcache/CodeQL
+          # Relocate docker data-root to /mnt so the minikube node container (and thus the
+          # ClickHouse pod storage inside it) uses the 65 GB SSD, not the 14 GB root.
+          sudo systemctl stop docker docker.socket
+          sudo mkdir -p /mnt/docker
+          # Merge data-root into any existing daemon.json — do NOT clobber runner defaults
+          # (cgroup driver, registry mirrors) or the daemon may fail to restart.
+          if [ -s /etc/docker/daemon.json ]; then
+            sudo jq '. + {"data-root":"/mnt/docker"}' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp >/dev/null
+            sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
+          else
+            echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json >/dev/null
+          fi
+          sudo systemctl start docker
+          # Fail loudly if the relocation did not take effect.
+          test "$(docker info -f '{{.DockerRootDir}}')" = "/mnt/docker"
+          df -h /mnt /
+
       - name: Cache python
         uses: actions/cache@v4
         id: cache-python

cmd/metrics_exporter/app/metrics_exporter.go

@@ -124,7 +124,7 @@ func Run() {
 	log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt)
 
 	// Initialize k8s API clients
-	kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL)
+	kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
 
 	// Create operator instance
 	chop.New(kubeClient, chopClient, chopConfigFile)

cmd/operator/app/thread_chi.go

@@ -56,7 +56,7 @@ func initClickHouse(ctx context.Context) {
 	}
 
 	// Initialize k8s API clients
-	kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL)
+	kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
 
 	// Create operator instance. The chopconf load inside chop.New gates on
 	// clickhouse.security.kubernetes.allowInsecure BEFORE the first network call,

cmd/operator/app/thread_keeper.go

@@ -17,7 +17,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
-	//	ctrl "sigs.k8s.io/controller-runtime/pkg/controller"
+	ctrlController "sigs.k8s.io/controller-runtime/pkg/controller"
 
 	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1"
 	"github.com/altinity/clickhouse-operator/pkg/chop"
@@ -75,7 +75,10 @@ func initKeeper(ctx context.Context) error {
 
 	// Build the apiextensions client for CRD deletion checks during CHK cleanup.
 	// Uses the same kubeConfigFile/masterURL package vars as the CHI thread.
-	_, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL)
+	_, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
+
+	maxConcurrentReconciles := chop.Config().Reconcile.Runtime.ReconcileCHKsThreadsNumber
+	logger.Info("init keeper - CHK controller concurrency", "maxConcurrentReconciles", maxConcurrentReconciles)
 
 	err = ctrlRuntime.
 		NewControllerManagedBy(manager).
@@ -84,13 +87,16 @@ func initKeeper(ctx context.Context) error {
 			builder.WithPredicates(keeperPredicate()),
 		).
 		Owns(&apps.StatefulSet{}).
+		WithOptions(ctrlController.Options{
+			MaxConcurrentReconciles: maxConcurrentReconciles,
+		}).
 		Complete(
-			&controller.Controller{
-				Client:    manager.GetClient(),
-				APIReader: manager.GetAPIReader(),
-				Scheme:    manager.GetScheme(),
-				ExtClient: extClient,
-			},
+			controller.NewController(
+				manager.GetClient(),
+				manager.GetAPIReader(),
+				manager.GetScheme(),
+				extClient,
+			),
 		)
 	if err != nil {
 		logger.Error(err, "init keeper - unable to ctrlRuntime.NewControllerManagedBy")

config/config-dev.yaml

@@ -152,8 +152,8 @@ clickhouse:
     # Possible values for 'scheme' are:
     #   1. http - force http to be used to connect to ClickHouse instances
     #   2. https - force https to be used to connect to ClickHouse instances
-    #   3. auto - either http or https is selected based on open ports
-    scheme: "auto"
+    #   3. Auto - either http or https is selected based on open ports
+    scheme: "Auto"
     # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.
     # These credentials are used for:
     #   1. Metrics requests
@@ -316,6 +316,8 @@ reconcile:
   runtime:
     # Max number of concurrent CHI reconciles in progress
     reconcileCHIsThreadsNumber: 1
+    # Max number of concurrent CHK reconciles in progress
+    reconcileCHKsThreadsNumber: 1
 
     # The operator reconciles shards concurrently in each CHI with the following limitations:
     #   1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
@@ -437,12 +439,12 @@ reconcile:
   ##
   ################################################
   recovery:
-    # Recovery scopes keyed by CHI state being recovered from.
+    # Recovery scopes keyed by the CHI .status.status they apply to.
     # Each scope contains on<Event>: <action> mappings that apply while the CHI
-    # is in that state. Multi-scope design anticipates future states beyond Aborted
+    # is in that status. Multi-scope design anticipates future states beyond Aborted
     # (e.g. Failed, Broken).
-    from:
-      # Recovery from Status=Aborted
+    onStatus:
+      # Recovery while Status=Aborted
       aborted:
         # Action when a pod belonging to an Aborted CHI transitions to Ready:
         #   retry (default) — re-enqueue the CHI for reconcile
@@ -457,7 +459,7 @@ reconcile:
       # broken:
       #   onPodReady: retry
 
-    # Future global policy knobs (not yet implemented) — flat peers of `from`,
+    # Future global policy knobs (not yet implemented) — flat peers of `onStatus`,
     # apply across all recovery scopes:
     #
     # Global kill-switch for auto-recovery:

config/config.yaml

@@ -152,8 +152,8 @@ clickhouse:
     # Possible values for 'scheme' are:
     #   1. http - force http to be used to connect to ClickHouse instances
     #   2. https - force https to be used to connect to ClickHouse instances
-    #   3. auto - either http or https is selected based on open ports
-    scheme: "auto"
+    #   3. Auto - either http or https is selected based on open ports
+    scheme: "Auto"
     # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.
     # These credentials are used for:
     #   1. Metrics requests
@@ -162,7 +162,16 @@ clickhouse:
     # located in 'clickhouse.configuration.file.path.user' folder
     username: ""
     password: ""
+    # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS.
     rootCA: ""
+    # Alternate source for rootCA — a Secret in the operator's namespace.
+    # Mutually exclusive with the inline rootCA above (inline wins). Empty
+    # `name` = not used (no-op). When `key` is empty, the operator tries
+    # "ca.crt" then "tls.crt". Resolved once at config load (an operator
+    # restart picks up a rotated Secret).
+    rootCASecretRef:
+      name: ""
+      key: ""
 
     # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances.
     # Can be used instead of explicitly specified username and password available in sections:
@@ -256,7 +265,7 @@ clickhouse:
     # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
     # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
     tablesRegexp: "^(metrics|custom_metrics)$"
-    # List of regexps to match ClickHouse metrics to exclude from export.
+    # List of regexps to match ClickHouse metrics to exclude from collection/export.
     # Regexps match internal metric names before Prometheus normalization and prefixing.
     # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
     excludeRegexp:
@@ -326,7 +335,7 @@ security:
     tls:
       # Strict refuses an insecure kubeconfig at startup
       verify: ""
-      # Reserved — not yet enforced on K8s API transport
+      # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced
       minVersion: ""
   ipc:
     # Plain (default) | Secure (loopback + X-CHOP-Token)
@@ -410,6 +419,8 @@ reconcile:
   runtime:
     # Max number of concurrent CHI reconciles in progress
     reconcileCHIsThreadsNumber: 10
+    # Max number of concurrent CHK reconciles in progress
+    reconcileCHKsThreadsNumber: 1
 
     # The operator reconciles shards concurrently in each CHI with the following limitations:
     #   1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
@@ -531,12 +542,13 @@ reconcile:
   ##
   ################################################
   recovery:
-    # Recovery scopes keyed by CHI state being recovered from.
+    # Recovery scopes keyed by the CHI .status.status they apply to.
     # Each scope contains on<Event>: <action> mappings that apply while the CHI
-    # is in that state. Multi-scope design anticipates future states beyond Aborted
+    # is in that status. Multi-scope design anticipates future states beyond Aborted
     # (e.g. Failed, Broken).
-    from:
-      # Recovery from Status=Aborted
+    onStatus:
+      # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete)
+      # when one of its host pods transitions to Ready — auto-resumes the reconcile.
       aborted:
         # Action when a pod belonging to an Aborted CHI transitions to Ready:
         #   retry (default) — re-enqueue the CHI for reconcile
@@ -545,13 +557,28 @@ reconcile:
         # Future events (not yet implemented):
         #   onKeeperReady: retry       — retry when a referenced CHK becomes ready
         #   onOperatorRestart: retry   — sweep Aborted CHIs on operator startup
+      # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one
+      # of its host pods regresses to Ready=False and stays NotReady (sustained) without
+      # crashing — auto-heals stuck hosts.
+      completed:
+        # Action when a Completed CHI's pod flips Ready=True -> Ready=False and
+        # stays NotReady for at least onPodNotReadyThreshold:
+        #   none (default) — do nothing
+        #   retry          — re-enqueue the CHI so the stuck host is force-restarted
+        # OFF by default: force-recreating a Completed CHI's pod is destructive — it can
+        # interrupt a replica's in-progress recovery and means hard downtime for a
+        # single-replica shard. Opt in with `retry` only where that trade-off is acceptable.
+        onPodNotReady: none
+        # Minimum duration a pod must stay Ready=False before recovery fires, once enabled
+        # (Go duration string; default 5m). Raise it for slow-recovering replicas.
+        onPodNotReadyThreshold: 5m
       # Future scopes (not yet implemented):
       # failed:
       #   onPodReady: retry
       # broken:
       #   onPodReady: retry
 
-    # Future global policy knobs (not yet implemented) — flat peers of `from`,
+    # Future global policy knobs (not yet implemented) — flat peers of `onStatus`,
     # apply across all recovery scopes:
     #
     # Global kill-switch for auto-recovery:

deploy/builder/templates-config/config.yaml

@@ -146,8 +146,8 @@ clickhouse:
     # Possible values for 'scheme' are:
     #   1. http - force http to be used to connect to ClickHouse instances
     #   2. https - force https to be used to connect to ClickHouse instances
-    #   3. auto - either http or https is selected based on open ports
-    scheme: "auto"
+    #   3. Auto - either http or https is selected based on open ports
+    scheme: "Auto"
     # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.
     # These credentials are used for:
     #   1. Metrics requests
@@ -156,7 +156,16 @@ clickhouse:
     # located in 'clickhouse.configuration.file.path.user' folder
     username: "${CH_USERNAME_PLAIN}"
     password: "${CH_PASSWORD_PLAIN}"
+    # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS.
     rootCA: ""
+    # Alternate source for rootCA — a Secret in the operator's namespace.
+    # Mutually exclusive with the inline rootCA above (inline wins). Empty
+    # `name` = not used (no-op). When `key` is empty, the operator tries
+    # "ca.crt" then "tls.crt". Resolved once at config load (an operator
+    # restart picks up a rotated Secret).
+    rootCASecretRef:
+      name: ""
+      key: ""
 
     # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances.
     # Can be used instead of explicitly specified username and password available in sections:
@@ -250,7 +259,7 @@ clickhouse:
     # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
     # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
     tablesRegexp: "^(metrics|custom_metrics)$"
-    # List of regexps to match ClickHouse metrics to exclude from export.
+    # List of regexps to match ClickHouse metrics to exclude from collection/export.
     # Regexps match internal metric names before Prometheus normalization and prefixing.
     # Default is the per-CPU OS metrics filter shown below; set to [] to disable.
     excludeRegexp:
@@ -320,7 +329,7 @@ security:
     tls:
       # Strict refuses an insecure kubeconfig at startup
       verify: ""
-      # Reserved — not yet enforced on K8s API transport
+      # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced
       minVersion: ""
   ipc:
     # Plain (default) | Secure (loopback + X-CHOP-Token)
@@ -404,6 +413,8 @@ reconcile:
   runtime:
     # Max number of concurrent CHI reconciles in progress
     reconcileCHIsThreadsNumber: 10
+    # Max number of concurrent CHK reconciles in progress
+    reconcileCHKsThreadsNumber: 1
 
     # The operator reconciles shards concurrently in each CHI with the following limitations:
     #   1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently
@@ -525,12 +536,13 @@ reconcile:
   ##
   ################################################
   recovery:
-    # Recovery scopes keyed by CHI state being recovered from.
+    # Recovery scopes keyed by the CHI .status.status they apply to.
     # Each scope contains on<Event>: <action> mappings that apply while the CHI
-    # is in that state. Multi-scope design anticipates future states beyond Aborted
+    # is in that status. Multi-scope design anticipates future states beyond Aborted
     # (e.g. Failed, Broken).
-    from:
-      # Recovery from Status=Aborted
+    onStatus:
+      # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete)
+      # when one of its host pods transitions to Ready — auto-resumes the reconcile.
       aborted:
         # Action when a pod belonging to an Aborted CHI transitions to Ready:
         #   retry (default) — re-enqueue the CHI for reconcile
@@ -539,13 +551,28 @@ reconcile:
         # Future events (not yet implemented):
         #   onKeeperReady: retry       — retry when a referenced CHK becomes ready
         #   onOperatorRestart: retry   — sweep Aborted CHIs on operator startup
+      # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one
+      # of its host pods regresses to Ready=False and stays NotReady (sustained) without
+      # crashing — auto-heals stuck hosts.
+      completed:
+        # Action when a Completed CHI's pod flips Ready=True -> Ready=False and
+        # stays NotReady for at least onPodNotReadyThreshold:
+        #   none (default) — do nothing
+        #   retry          — re-enqueue the CHI so the stuck host is force-restarted
+        # OFF by default: force-recreating a Completed CHI's pod is destructive — it can
+        # interrupt a replica's in-progress recovery and means hard downtime for a
+        # single-replica shard. Opt in with `retry` only where that trade-off is acceptable.
+        onPodNotReady: none
+        # Minimum duration a pod must stay Ready=False before recovery fires, once enabled
+        # (Go duration string; default 5m). Raise it for slow-recovering replicas.
+        onPodNotReadyThreshold: 5m
       # Future scopes (not yet implemented):
       # failed:
       #   onPodReady: retry
       # broken:
       #   onPodReady: retry
 
-    # Future global policy knobs (not yet implemented) — flat peers of `from`,
+    # Future global policy knobs (not yet implemented) — flat peers of `onStatus`,
     # apply across all recovery scopes:
     #
     # Global kill-switch for auto-recovery: