diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 8f7dc216a..033201859 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -22,6 +22,41 @@ jobs: timeout-minutes: 300 steps: - uses: actions/checkout@v4 + + # The e2e suite (notably test_010025's 100M-row insert + its replica) fills the + # runner's small root partition (~14 GB): with the minikube docker driver the k8s + # node runs as a container under the docker data-root (/var/lib/docker on /), so + # ClickHouse pod ephemeral storage lands there and exhausts it — the runner then + # loses communication mid-run. Move the docker data-root onto the large /mnt + # ephemeral SSD (~65 GB) before minikube starts, and trim unused preinstalled + # toolchains from / for headroom. MUST run before setup-minikube (no docker is + # used before this step, so restarting the daemon here is safe). + - name: Free disk and relocate docker data-root to /mnt + run: | + set -euxo pipefail + # If docker fails to restart on the moved data-root, surface disk + daemon logs + # so the failure is triageable rather than a bare non-zero exit at this early step. + trap 'ec=$?; echo "::error::disk/docker prep failed (exit $ec)"; df -h / /mnt || true; sudo journalctl -u docker --no-pager -n 50 || true' ERR + df -h / + # Reclaim space on / — none of these toolchains are used by this Go+Python+k8s job. + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/local/lib/android /opt/hostedtoolcache/CodeQL + # Relocate docker data-root to /mnt so the minikube node container (and thus the + # ClickHouse pod storage inside it) uses the 65 GB SSD, not the 14 GB root. + sudo systemctl stop docker docker.socket + sudo mkdir -p /mnt/docker + # Merge data-root into any existing daemon.json — do NOT clobber runner defaults + # (cgroup driver, registry mirrors) or the daemon may fail to restart. + if [ -s /etc/docker/daemon.json ]; then + sudo jq '. + {"data-root":"/mnt/docker"}' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp >/dev/null + sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json + else + echo '{"data-root":"/mnt/docker"}' | sudo tee /etc/docker/daemon.json >/dev/null + fi + sudo systemctl start docker + # Fail loudly if the relocation did not take effect. + test "$(docker info -f '{{.DockerRootDir}}')" = "/mnt/docker" + df -h /mnt / + - name: Cache python uses: actions/cache@v4 id: cache-python diff --git a/cmd/metrics_exporter/app/metrics_exporter.go b/cmd/metrics_exporter/app/metrics_exporter.go index 37be8eef7..de16f0272 100644 --- a/cmd/metrics_exporter/app/metrics_exporter.go +++ b/cmd/metrics_exporter/app/metrics_exporter.go @@ -124,7 +124,7 @@ func Run() { log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt) // Initialize k8s API clients - kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL) + kubeClient, _, chopClient, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) // Create operator instance chop.New(kubeClient, chopClient, chopConfigFile) diff --git a/cmd/operator/app/thread_chi.go b/cmd/operator/app/thread_chi.go index bee87ca13..c2ac3e326 100644 --- a/cmd/operator/app/thread_chi.go +++ b/cmd/operator/app/thread_chi.go @@ -56,7 +56,7 @@ func initClickHouse(ctx context.Context) { } // Initialize k8s API clients - kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL) + kubeClient, extClient, chopClient, dynamicClient := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) // Create operator instance. The chopconf load inside chop.New gates on // clickhouse.security.kubernetes.allowInsecure BEFORE the first network call, diff --git a/cmd/operator/app/thread_keeper.go b/cmd/operator/app/thread_keeper.go index 053cd6cd4..bcd61e254 100644 --- a/cmd/operator/app/thread_keeper.go +++ b/cmd/operator/app/thread_keeper.go @@ -17,7 +17,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/predicate" - // ctrl "sigs.k8s.io/controller-runtime/pkg/controller" + ctrlController "sigs.k8s.io/controller-runtime/pkg/controller" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/chop" @@ -75,7 +75,10 @@ func initKeeper(ctx context.Context) error { // Build the apiextensions client for CRD deletion checks during CHK cleanup. // Uses the same kubeConfigFile/masterURL package vars as the CHI thread. - _, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL) + _, extClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile) + + maxConcurrentReconciles := chop.Config().Reconcile.Runtime.ReconcileCHKsThreadsNumber + logger.Info("init keeper - CHK controller concurrency", "maxConcurrentReconciles", maxConcurrentReconciles) err = ctrlRuntime. NewControllerManagedBy(manager). @@ -84,13 +87,16 @@ func initKeeper(ctx context.Context) error { builder.WithPredicates(keeperPredicate()), ). Owns(&apps.StatefulSet{}). + WithOptions(ctrlController.Options{ + MaxConcurrentReconciles: maxConcurrentReconciles, + }). Complete( - &controller.Controller{ - Client: manager.GetClient(), - APIReader: manager.GetAPIReader(), - Scheme: manager.GetScheme(), - ExtClient: extClient, - }, + controller.NewController( + manager.GetClient(), + manager.GetAPIReader(), + manager.GetScheme(), + extClient, + ), ) if err != nil { logger.Error(err, "init keeper - unable to ctrlRuntime.NewControllerManagedBy") diff --git a/config/config-dev.yaml b/config/config-dev.yaml index 6ca12d944..4edb872d2 100644 --- a/config/config-dev.yaml +++ b/config/config-dev.yaml @@ -152,8 +152,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -316,6 +316,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 1 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -437,12 +439,12 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery while Status=Aborted aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -457,7 +459,7 @@ reconcile: # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/config/config.yaml b/config/config.yaml index 4bfc9d015..c932c84d1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -152,8 +152,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -162,7 +162,16 @@ clickhouse: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -256,7 +265,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -326,7 +335,7 @@ security: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -410,6 +419,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -531,12 +542,13 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -545,13 +557,28 @@ reconcile: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 46740bd15..de55a41d8 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -146,8 +146,8 @@ clickhouse: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -156,7 +156,16 @@ clickhouse: # located in 'clickhouse.configuration.file.path.user' folder username: "${CH_USERNAME_PLAIN}" password: "${CH_PASSWORD_PLAIN}" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -250,7 +259,7 @@ clickhouse: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -320,7 +329,7 @@ security: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -404,6 +413,8 @@ reconcile: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -525,12 +536,13 @@ reconcile: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -539,13 +551,28 @@ reconcile: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml index dd8e7b4a8..c44c5243d 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,8 +1844,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index 044685794..e2e49703c 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -167,6 +167,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -265,7 +275,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -273,7 +283,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -307,9 +317,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -326,6 +339,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -546,15 +564,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -568,6 +586,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml index a1f4b3c42..a8aa40612 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml @@ -312,9 +312,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -338,12 +341,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -402,20 +407,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -680,10 +689,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -757,9 +768,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -773,35 +787,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" diff --git a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml index 7b3d41736..ea76f87da 100644 --- a/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml +++ b/deploy/builder/templates-operatorhub/clickhouse-operator.vVERSION.clusterserviceversion-template.yaml @@ -1291,10 +1291,12 @@ spec: containerName: clickhouse-operator resource: limits.memory divisor: "1Mi" - - name: WATCH_NAMESPACE + # Honor the OperatorGroup's target namespaces so every advertised + # installMode works (AllNamespaces sends an empty string = watch all). + - name: WATCH_NAMESPACES valueFrom: fieldRef: - fieldPath: metadata.namespace + fieldPath: metadata.annotations['olm.targetNamespaces'] image: docker.io/altinity/clickhouse-operator:${OPERATOR_VERSION} imagePullPolicy: Always name: clickhouse-operator diff --git a/deploy/helm/clickhouse-operator/Chart.yaml b/deploy/helm/clickhouse-operator/Chart.yaml index ca82bf48a..a6631c54e 100644 --- a/deploy/helm/clickhouse-operator/Chart.yaml +++ b/deploy/helm/clickhouse-operator/Chart.yaml @@ -17,8 +17,8 @@ description: |- kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ``` type: application -version: 0.27.1 -appVersion: 0.27.1 +version: 0.27.2 +appVersion: 0.27.2 home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg maintainers: diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md index 8e9cfa4ca..87d7a259d 100644 --- a/deploy/helm/clickhouse-operator/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.27.1](https://img.shields.io/badge/Version-0.27.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.27.1](https://img.shields.io/badge/AppVersion-0.27.1-informational?style=flat-square) +![Version: 0.27.2](https://img.shields.io/badge/Version-0.27.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.27.2](https://img.shields.io/badge/AppVersion-0.27.2-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). @@ -83,6 +83,8 @@ crdHook: | crdHook.image.tag | string | `"latest"` | image tag for CRD installation job | | crdHook.imagePullSecrets | list | `[]` | image pull secrets for CRD installation job possible value format `[{"name":"your-secret-name"}]`, check `kubectl explain pod.spec.imagePullSecrets` for details | | crdHook.nodeSelector | object | `{}` | node selector for CRD installation job | +| crdHook.podAnnotations | object | `{}` | additional annotations for CRD installation job pod template useful to opt out of service mesh injection, e.g. `sidecar.istio.io/inject: "false"` | +| crdHook.podSecurityContext | object | `{}` | pod-level security context for CRD installation job required by some admission policies (e.g. Kyverno `restrict-seccomp-strict`) check `kubectl explain pod.spec.securityContext` for details | | crdHook.resources | object | `{}` | resource limits and requests for CRD installation job | | crdHook.tolerations | list | `[]` | tolerations for CRD installation job | | dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards | @@ -138,4 +140,5 @@ crdHook: | serviceMonitor.operatorMetrics.scrapeTimeout | string | `""` | | | tolerations | list | `[]` | tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details | | topologySpreadConstraints | list | `[]` | | +| watchNamespaces | list | `[]` | namespaces where the operator watches for ClickHouseInstallation resources. Sets config.yaml watch.namespaces.include (the exclude list is not exposed here). If empty, the operator watches only its own namespace (or all namespaces when running in kube-system). Use [".*"] to watch all namespaces. Entries are regexps and are Helm-templated, so avoid a literal "{{" in a namespace/regexp. Example: watchNamespaces: ["clickhouse", "my-other-namespace"] | diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index 267d11645..7ad285b82 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1127,21 +1150,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1511,10 +1539,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1614,9 +1644,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1630,35 +1663,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1785,8 +1839,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index bf5909b09..d33ca14f7 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1127,21 +1150,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1511,10 +1539,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1614,9 +1644,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1630,35 +1663,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1785,8 +1839,9 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 6ce0865ac..26c3ee738 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -312,9 +312,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -338,12 +341,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -402,20 +407,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -679,10 +688,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -755,9 +766,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -771,35 +785,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 257915bde..ccd472390 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -167,6 +167,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -265,7 +275,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -273,7 +283,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -307,9 +317,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -326,6 +339,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -546,15 +564,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -568,6 +586,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" diff --git a/deploy/helm/clickhouse-operator/templates/_helpers.tpl b/deploy/helm/clickhouse-operator/templates/_helpers.tpl index f2d1aee2b..50170d3b9 100644 --- a/deploy/helm/clickhouse-operator/templates/_helpers.tpl +++ b/deploy/helm/clickhouse-operator/templates/_helpers.tpl @@ -122,3 +122,29 @@ null {{- tpl (toYaml (dict $k $v)) $root }} {{ end }} {{- end }} + +{{/* +altinity-clickhouse-operator.configmap-files merges watchNamespaces into the +operator config before rendering the ConfigMap data block. + +This exists because configs.files.config.yaml.watch.namespaces.include is +deep inside a nested structure — Helm's values merge cannot target it +directly. Instead we deepCopy the files map, patch the nested value in-place, +and pass the result to configmap-data. + +Arguments (list): root context, configs.files, watchNamespaces list +*/}} +{{- define "altinity-clickhouse-operator.configmap-files" -}} +{{- $root := index . 0 -}} +{{- $files := deepCopy (index . 1) -}} +{{- $watchNamespaces := index . 2 -}} +{{- if $watchNamespaces -}} + {{- $namespaces := dig "watch" "namespaces" "" (index $files "config.yaml") -}} + {{- if kindIs "map" $namespaces -}} + {{- $_ := set $namespaces "include" $watchNamespaces -}} + {{- else -}} + {{- fail "watchNamespaces is set but configs.files.\"config.yaml\".watch.namespaces is missing or null; cannot apply the namespace filter" -}} + {{- end -}} +{{- end -}} +{{- include "altinity-clickhouse-operator.configmap-data" (list $root $files) -}} +{{- end -}} diff --git a/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml index bc6d21dd1..df1886ac3 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml @@ -11,4 +11,4 @@ metadata: namespace: {{ include "altinity-clickhouse-operator.namespace" . }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }} -data: {{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.files) | nindent 2 }} +data: {{ include "altinity-clickhouse-operator.configmap-files" (list . .Values.configs.files .Values.watchNamespaces) | nindent 2 }} diff --git a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml index 825679095..5cfc08e3c 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator diff --git a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml index 358c495d6..43d4be33f 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # diff --git a/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml b/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml index df980785f..196d5af0e 100644 --- a/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml +++ b/deploy/helm/clickhouse-operator/templates/hooks/crd-install-job.yaml @@ -21,9 +21,17 @@ spec: labels: {{- include "altinity-clickhouse-operator.labels" . | nindent 8 }} app.kubernetes.io/component: crd-install-hook + {{- with .Values.crdHook.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "altinity-clickhouse-operator.fullname" . }}-crd-install restartPolicy: OnFailure + {{- with .Values.crdHook.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.crdHook.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json index 5fbee64d1..f51eda897 100644 --- a/deploy/helm/clickhouse-operator/values.schema.json +++ b/deploy/helm/clickhouse-operator/values.schema.json @@ -480,6 +480,9 @@ "reconcileCHIsThreadsNumber": { "type": "integer" }, + "reconcileCHKsThreadsNumber": { + "type": "integer" + }, "reconcileShardsMaxConcurrencyPercent": { "type": "integer" }, @@ -518,7 +521,7 @@ "recovery": { "type": "object", "properties": { - "from": { + "onStatus": { "type": "object", "properties": { "aborted": { @@ -529,6 +532,18 @@ "enum": ["", "none", "None", "retry", "Retry"] } } + }, + "completed": { + "type": "object", + "properties": { + "onPodNotReady": { + "type": "string", + "enum": ["", "none", "None", "retry", "Retry"] + }, + "onPodNotReadyThreshold": { + "type": "string" + } + } } } } @@ -710,6 +725,12 @@ "annotations": { "type": "object" }, + "podAnnotations": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, "containerSecurityContext": { "type": "object" } diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index b7579fce9..a0b60c024 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -38,6 +38,17 @@ crdHook: affinity: {} # crdHook.annotations -- additional annotations for CRD installation job annotations: {} + # crdHook.podAnnotations -- additional annotations for CRD installation job pod template + # useful to opt out of service mesh injection, e.g. `sidecar.istio.io/inject: "false"` + podAnnotations: {} + # crdHook.podSecurityContext -- pod-level security context for CRD installation job + # required by some admission policies (e.g. Kyverno `restrict-seccomp-strict`) + # check `kubectl explain pod.spec.securityContext` for details + podSecurityContext: {} + # runAsNonRoot: true + # runAsUser: 1000 + # seccompProfile: + # type: RuntimeDefault # crdHook.containerSecurityContext -- container security context for CRD installation job # check `kubectl explain pod.spec.containers.securityContext` for details containerSecurityContext: {} @@ -147,6 +158,13 @@ podAnnotations: prometheus.io/scrape: 'true' clickhouse-operator-metrics/port: '9999' clickhouse-operator-metrics/scrape: 'true' +# watchNamespaces -- namespaces where the operator watches for ClickHouseInstallation resources. +# Sets config.yaml watch.namespaces.include (the exclude list is not exposed here). If empty, the +# operator watches only its own namespace (or all namespaces when running in kube-system). +# Use [".*"] to watch all namespaces. Entries are regexps and are Helm-templated, so avoid a literal +# "{{" in a namespace/regexp. +# Example: watchNamespaces: ["clickhouse", "my-other-namespace"] +watchNamespaces: [] # nameOverride -- override name of the chart nameOverride: "" # fullnameOverride -- full name of the chart. @@ -432,8 +450,8 @@ configs: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -442,7 +460,16 @@ configs: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: # - clickhouse.access.username @@ -531,7 +558,7 @@ configs: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -599,7 +626,7 @@ configs: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -679,6 +706,8 @@ configs: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently # can not be greater than 'reconcileShardsThreadsNumber'. @@ -793,12 +822,13 @@ configs: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -807,12 +837,27 @@ configs: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 46a8b2125..b9acc061e 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -311,8 +311,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -349,7 +351,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -370,9 +374,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -396,12 +403,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -506,8 +515,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -533,8 +545,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -549,7 +564,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -560,7 +577,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -885,20 +904,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -989,9 +1012,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1137,21 +1160,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1521,10 +1549,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1625,9 +1655,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1641,35 +1674,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1797,10 +1851,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1809,14 +1864,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2109,8 +2164,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2147,7 +2204,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2168,9 +2227,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2194,12 +2256,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2304,8 +2368,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2331,8 +2398,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2347,7 +2417,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2358,7 +2430,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2683,20 +2757,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2787,9 +2865,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2935,21 +3013,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3319,10 +3402,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3423,9 +3508,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3439,35 +3527,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3595,10 +3704,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3610,7 +3720,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3770,6 +3880,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3868,7 +3988,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3876,7 +3996,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3910,9 +4030,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3929,6 +4052,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4149,15 +4277,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4171,6 +4299,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4303,14 +4453,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4615,9 +4765,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4641,12 +4794,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4705,20 +4860,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4983,10 +5142,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5060,9 +5221,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5076,35 +5240,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5220,7 +5405,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5246,7 +5431,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5478,7 +5663,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5500,7 +5685,7 @@ metadata: name: etc-clickhouse-operator-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5658,8 +5843,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5668,7 +5853,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -5762,7 +5956,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -5832,7 +6026,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -5916,6 +6110,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -6037,12 +6233,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6051,13 +6248,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6186,7 +6398,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6202,7 +6414,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6301,7 +6513,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6401,7 +6613,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6464,7 +6676,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6480,7 +6692,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6577,7 +6789,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6595,7 +6807,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6603,7 +6815,7 @@ data: # Template parameters available: # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN={{ password }} # @@ -6613,7 +6825,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6624,9 +6836,9 @@ stringData: # # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6637,7 +6849,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6693,7 +6905,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6775,7 +6987,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6872,7 +7084,7 @@ metadata: name: clickhouse-operator-metrics namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index a294e39af..9c34c79ad 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -299,8 +299,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -337,7 +339,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -358,9 +362,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -384,12 +391,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -494,8 +503,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -521,8 +533,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -537,7 +552,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -548,7 +565,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -873,20 +892,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -977,9 +1000,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1122,21 +1145,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1506,10 +1534,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1609,9 +1639,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1625,35 +1658,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1780,10 +1834,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1792,14 +1847,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2085,8 +2140,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -2123,7 +2180,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2144,9 +2203,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2170,12 +2232,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -2280,8 +2344,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2307,8 +2374,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2323,7 +2393,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2334,7 +2406,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2659,20 +2733,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2763,9 +2841,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2908,21 +2986,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3292,10 +3375,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3395,9 +3480,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3411,35 +3499,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3566,10 +3675,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3581,7 +3691,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3737,6 +3847,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3835,7 +3955,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3843,7 +3963,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3877,9 +3997,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3896,6 +4019,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4116,15 +4244,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4138,6 +4266,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4265,14 +4415,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4577,9 +4727,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4603,12 +4756,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -4667,20 +4822,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4944,10 +5103,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5020,9 +5181,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5036,35 +5200,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5179,7 +5364,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 # Template Parameters: # @@ -5204,7 +5389,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5424,7 +5609,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5457,7 +5642,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5677,7 +5862,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5699,7 +5884,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5857,8 +6042,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5867,7 +6052,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -5961,7 +6155,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -6031,7 +6225,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -6115,6 +6309,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -6236,12 +6432,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6250,13 +6447,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6384,7 +6596,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6400,7 +6612,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6494,7 +6706,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6592,7 +6804,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6654,7 +6866,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6670,7 +6882,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6764,7 +6976,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6782,7 +6994,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6790,7 +7002,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6800,7 +7012,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6811,9 +7023,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6824,7 +7036,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6880,7 +7092,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6960,7 +7172,7 @@ spec: - containerPort: 9999 name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7056,7 +7268,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 623d8e115..68b9ebe21 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,10 +1844,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1802,14 +1857,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2102,8 +2157,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2140,7 +2197,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2161,9 +2220,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2187,12 +2249,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2297,8 +2361,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2324,8 +2391,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2340,7 +2410,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2351,7 +2423,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2676,20 +2750,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2780,9 +2858,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2928,21 +3006,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3312,10 +3395,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3416,9 +3501,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3432,35 +3520,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3588,10 +3697,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3603,7 +3713,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3763,6 +3873,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3861,7 +3981,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3869,7 +3989,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3903,9 +4023,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3922,6 +4045,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4142,15 +4270,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4164,6 +4292,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4296,14 +4446,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4608,9 +4758,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4634,12 +4787,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4698,20 +4853,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4976,10 +5135,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5053,9 +5214,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5069,35 +5233,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5213,7 +5398,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5239,7 +5424,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5471,7 +5656,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5505,7 +5690,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5737,7 +5922,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5759,7 +5944,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5917,8 +6102,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5927,7 +6112,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -6021,7 +6215,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -6091,7 +6285,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -6175,6 +6369,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -6296,12 +6492,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6310,13 +6507,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6445,7 +6657,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6461,7 +6673,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6560,7 +6772,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6660,7 +6872,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6723,7 +6935,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6739,7 +6951,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6836,7 +7048,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6854,7 +7066,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6862,7 +7074,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6872,7 +7084,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6883,9 +7095,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6896,7 +7108,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6952,7 +7164,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7034,7 +7246,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -7131,7 +7343,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index 4e027223e..49f48f701 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -299,8 +299,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -337,7 +339,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -358,9 +362,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -384,12 +391,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -494,8 +503,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -521,8 +533,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -537,7 +552,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -548,7 +565,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -873,20 +892,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -977,9 +1000,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1122,21 +1145,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1506,10 +1534,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1609,9 +1639,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1625,35 +1658,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1780,10 +1834,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1792,14 +1847,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2085,8 +2140,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: !!merge <<: *TypeStringBool description: | @@ -2123,7 +2180,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2144,9 +2203,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2170,12 +2232,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -2280,8 +2344,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2307,8 +2374,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2323,7 +2393,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2334,7 +2406,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2659,20 +2733,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2763,9 +2841,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2908,21 +2986,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: !!merge <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3292,10 +3375,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3395,9 +3480,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3411,35 +3499,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3566,10 +3675,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3581,7 +3691,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3737,6 +3847,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3835,7 +3955,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3843,7 +3963,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3877,9 +3997,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3896,6 +4019,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4116,15 +4244,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4138,6 +4266,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4265,14 +4415,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4577,9 +4727,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4603,12 +4756,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string !!merge <<: *TypeObjectsCleanup @@ -4667,20 +4822,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4944,10 +5103,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5020,9 +5181,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5036,35 +5200,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5179,7 +5364,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 # Template Parameters: # @@ -5204,7 +5389,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # # Core API group @@ -5424,7 +5609,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5446,7 +5631,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5604,8 +5789,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5614,7 +5799,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -5708,7 +5902,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -5778,7 +5972,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -5862,6 +6056,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -5983,12 +6179,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -5997,13 +6194,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6131,7 +6343,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6147,7 +6359,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6241,7 +6453,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6339,7 +6551,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6401,7 +6613,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6417,7 +6629,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6511,7 +6723,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6529,7 +6741,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6537,7 +6749,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6547,7 +6759,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6571,7 +6783,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6803,7 +7015,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index 97bdd7d6f..4cfda86bf 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -342,7 +344,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -363,9 +367,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -389,12 +396,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -499,8 +508,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -526,8 +538,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -542,7 +557,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -553,7 +570,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -878,20 +897,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -982,9 +1005,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1130,21 +1153,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1514,10 +1542,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1618,9 +1648,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1634,35 +1667,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1790,10 +1844,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1802,14 +1857,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2102,8 +2157,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2140,7 +2197,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2161,9 +2220,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2187,12 +2249,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2297,8 +2361,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2324,8 +2391,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2340,7 +2410,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2351,7 +2423,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2676,20 +2750,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2780,9 +2858,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2928,21 +3006,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3312,10 +3395,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3416,9 +3501,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3432,35 +3520,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3588,10 +3697,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3603,7 +3713,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3763,6 +3873,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3861,7 +3981,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3869,7 +3989,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3903,9 +4023,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3922,6 +4045,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4142,15 +4270,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4164,6 +4292,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4296,14 +4446,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4608,9 +4758,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4634,12 +4787,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4698,20 +4853,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4976,10 +5135,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5053,9 +5214,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5069,35 +5233,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5213,7 +5398,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5239,7 +5424,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5471,7 +5656,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -5493,7 +5678,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5651,8 +5836,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5661,7 +5846,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -5755,7 +5949,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -5825,7 +6019,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -5909,6 +6103,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -6030,12 +6226,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6044,13 +6241,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6179,7 +6391,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6195,7 +6407,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6294,7 +6506,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6394,7 +6606,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6457,7 +6669,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6473,7 +6685,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6570,7 +6782,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6588,7 +6800,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6596,7 +6808,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -6606,7 +6818,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6630,7 +6842,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6865,7 +7077,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 4c1ed23b5..729a8ab6a 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -311,8 +311,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -349,7 +351,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -370,9 +374,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -396,12 +403,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -506,8 +515,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -533,8 +545,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -549,7 +564,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -560,7 +577,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -885,20 +904,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -989,9 +1012,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -1137,21 +1160,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -1521,10 +1549,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -1625,9 +1655,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -1641,35 +1674,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -1797,10 +1851,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -1809,14 +1864,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2109,8 +2164,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: <<: *TypeStringBool description: | @@ -2147,7 +2204,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -2168,9 +2227,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -2194,12 +2256,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -2304,8 +2368,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -2331,8 +2398,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -2347,7 +2417,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -2358,7 +2430,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: &TypeReconcileHost type: object @@ -2683,20 +2757,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2787,9 +2865,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2935,21 +3013,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: <<: *TypeStringBool description: optional, open insecure ports for cluster, defaults to "yes" @@ -3319,10 +3402,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3423,9 +3508,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3439,35 +3527,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3595,10 +3704,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -3610,7 +3720,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3770,6 +3880,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -3868,7 +3988,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -3876,7 +3996,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -3910,9 +4030,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -3929,6 +4052,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -4149,15 +4277,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -4171,6 +4299,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -4303,14 +4453,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -4615,9 +4765,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4641,12 +4794,14 @@ spec: properties: statefulSet: &TypeObjectsCleanup type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string <<: *TypeObjectsCleanup @@ -4705,20 +4860,24 @@ spec: properties: provisioner: &TypePVCProvisioner type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: &TypePVCReclaimPolicy type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: &TypeTemplateNames type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -4983,10 +5142,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -5060,9 +5221,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -5076,35 +5240,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -5220,7 +5405,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 --- # Template Parameters: # @@ -5246,7 +5431,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 rules: # @@ -5478,7 +5663,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -5500,7 +5685,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: config.yaml: | @@ -5658,8 +5843,8 @@ data: # Possible values for 'scheme' are: # 1. http - force http to be used to connect to ClickHouse instances # 2. https - force https to be used to connect to ClickHouse instances - # 3. auto - either http or https is selected based on open ports - scheme: "auto" + # 3. Auto - either http or https is selected based on open ports + scheme: "Auto" # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. # These credentials are used for: # 1. Metrics requests @@ -5668,7 +5853,16 @@ data: # located in 'clickhouse.configuration.file.path.user' folder username: "" password: "" + # Inline PEM CA bundle the operator uses to verify ClickHouse server TLS. rootCA: "" + # Alternate source for rootCA — a Secret in the operator's namespace. + # Mutually exclusive with the inline rootCA above (inline wins). Empty + # `name` = not used (no-op). When `key` is empty, the operator tries + # "ca.crt" then "tls.crt". Resolved once at config load (an operator + # restart picks up a rotated Secret). + rootCASecretRef: + name: "" + key: "" # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. # Can be used instead of explicitly specified username and password available in sections: @@ -5762,7 +5956,7 @@ data: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -5832,7 +6026,7 @@ data: tls: # Strict refuses an insecure kubeconfig at startup verify: "" - # Reserved — not yet enforced on K8s API transport + # Floors the K8s API client transport TLS version; coerced to 1.3 under FIPS/Enforced minVersion: "" ipc: # Plain (default) | Secure (loopback + X-CHOP-Token) @@ -5916,6 +6110,8 @@ data: runtime: # Max number of concurrent CHI reconciles in progress reconcileCHIsThreadsNumber: 10 + # Max number of concurrent CHK reconciles in progress + reconcileCHKsThreadsNumber: 1 # The operator reconciles shards concurrently in each CHI with the following limitations: # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently @@ -6037,12 +6233,13 @@ data: ## ################################################ recovery: - # Recovery scopes keyed by CHI state being recovered from. + # Recovery scopes keyed by the CHI .status.status they apply to. # Each scope contains on: mappings that apply while the CHI - # is in that state. Multi-scope design anticipates future states beyond Aborted + # is in that status. Multi-scope design anticipates future states beyond Aborted # (e.g. Failed, Broken). - from: - # Recovery from Status=Aborted + onStatus: + # Recovery for a CHI whose .status.status is Aborted (reconcile did not complete) + # when one of its host pods transitions to Ready — auto-resumes the reconcile. aborted: # Action when a pod belonging to an Aborted CHI transitions to Ready: # retry (default) — re-enqueue the CHI for reconcile @@ -6051,13 +6248,28 @@ data: # Future events (not yet implemented): # onKeeperReady: retry — retry when a referenced CHK becomes ready # onOperatorRestart: retry — sweep Aborted CHIs on operator startup + # Recovery for a CHI whose .status.status is Completed (fully reconciled) when one + # of its host pods regresses to Ready=False and stays NotReady (sustained) without + # crashing — auto-heals stuck hosts. + completed: + # Action when a Completed CHI's pod flips Ready=True -> Ready=False and + # stays NotReady for at least onPodNotReadyThreshold: + # none (default) — do nothing + # retry — re-enqueue the CHI so the stuck host is force-restarted + # OFF by default: force-recreating a Completed CHI's pod is destructive — it can + # interrupt a replica's in-progress recovery and means hard downtime for a + # single-replica shard. Opt in with `retry` only where that trade-off is acceptable. + onPodNotReady: none + # Minimum duration a pod must stay Ready=False before recovery fires, once enabled + # (Go duration string; default 5m). Raise it for slow-recovering replicas. + onPodNotReadyThreshold: 5m # Future scopes (not yet implemented): # failed: # onPodReady: retry # broken: # onPodReady: retry - # Future global policy knobs (not yet implemented) — flat peers of `from`, + # Future global policy knobs (not yet implemented) — flat peers of `onStatus`, # apply across all recovery scopes: # # Global kill-switch for auto-recovery: @@ -6186,7 +6398,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6202,7 +6414,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -6301,7 +6513,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 001-templates.json.example: | @@ -6401,7 +6613,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -6464,7 +6676,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6480,7 +6692,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -6577,7 +6789,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: readme: | @@ -6595,7 +6807,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator data: --- @@ -6603,7 +6815,7 @@ data: # Template parameters available: # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=${password} # @@ -6613,7 +6825,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator type: Opaque stringData: @@ -6624,9 +6836,9 @@ stringData: # # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.1 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.27.2 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.1 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.27.2 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -6637,7 +6849,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: replicas: 1 @@ -6693,7 +6905,7 @@ spec: sizeLimit: 1Mi containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.27.1 + image: altinity/clickhouse-operator:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6775,7 +6987,7 @@ spec: name: op-metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.27.1 + image: altinity/metrics-exporter:0.27.2 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -6872,7 +7084,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 03b5dbebd..c5f1f5a48 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -304,8 +304,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -360,7 +362,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -381,9 +385,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -407,35 +414,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -447,34 +462,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -596,8 +619,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -623,8 +649,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -639,7 +668,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -650,7 +681,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -1232,9 +1265,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -1258,35 +1294,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -1298,34 +1342,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -1447,8 +1499,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -1474,8 +1529,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -1490,7 +1548,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -1501,7 +1561,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -2109,20 +2171,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -2222,9 +2288,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2399,9 +2465,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -2497,21 +2563,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -3670,10 +3741,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -3830,9 +3903,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -3846,35 +3922,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -3919,20 +4016,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | @@ -4015,10 +4116,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -4027,14 +4129,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4327,8 +4429,10 @@ spec: In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. enum: + # both humped and all-lowercase accepted - "" - "RollingUpdate" + - "rollingupdate" suspend: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -4383,7 +4487,9 @@ spec: Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. enum: - "" + - "Auto" - "auto" + - "Manual" - "manual" chiSelector: type: object @@ -4404,9 +4510,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -4430,35 +4539,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -4470,34 +4587,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -4619,8 +4744,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -4646,8 +4774,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -4662,7 +4793,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -4673,7 +4806,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -5255,9 +5390,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -5281,35 +5419,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -5321,34 +5467,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" macros: type: object @@ -5470,8 +5624,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Delete" - "delete" + - "Ignore" - "ignore" update: type: object @@ -5497,8 +5654,11 @@ spec: 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. enum: - "" + - "Abort" - "abort" + - "Rollback" - "rollback" + - "Ignore" - "ignore" recreate: type: object @@ -5513,7 +5673,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" onUpdateFailure: type: string @@ -5524,7 +5686,9 @@ spec: 2. recreate - proceed and recreate StatefulSet. enum: - "" + - "Abort" - "abort" + - "Recreate" - "recreate" host: type: object @@ -6132,20 +6296,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -6245,9 +6413,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -6422,9 +6590,9 @@ spec: serviceType: type: string description: | - how to discover keeper endpoints: - replicas (default) — enumerate per-host services, one ZK node per keeper replica - service — use the CR-level headless service as a single ZK node entry + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry enum: - "" - "Replicas" @@ -6520,21 +6688,26 @@ spec: properties: replica: type: string - description: "how schema is propagated within a replica" + description: "how schema is propagated within a replica (case-insensitive)" enum: - # List SchemaPolicyReplicaXXX constants from model + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" shard: type: string - description: "how schema is propagated between shards" + description: "how schema is propagated between shards (case-insensitive)" enum: - # List SchemaPolicyShardXXX constants from model + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) - "" - "None" + - "none" - "All" + - "all" - "DistributedTablesOnly" + - "distributedtablesonly" insecure: # StringBool is polymorphic — accepts native YAML bool (true/false), # integer (0/1), or string from the recognized vocabulary @@ -7693,10 +7866,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -7853,9 +8028,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -7869,35 +8047,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -7942,20 +8141,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | @@ -8038,10 +8241,11 @@ spec: description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" useType: type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" enum: - # List useTypeXXX constants from model + # List useTypeXXX constants from model (both humped and all-lowercase accepted) - "" + - "Merge" - "merge" --- # Template Parameters: @@ -8053,7 +8257,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.27.1 + clickhouse.altinity.com/chop: 0.27.2 spec: group: clickhouse.altinity.com scope: Namespaced @@ -8213,6 +8417,16 @@ spec: rootCA: type: string description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" secret: type: object properties: @@ -8311,7 +8525,7 @@ spec: items: type: string description: | - List of regexps to match ClickHouse metrics to exclude from export. + List of regexps to match ClickHouse metrics to exclude from collection/export. Regexps match internal metric names before Prometheus normalization and prefixing. security: type: object @@ -8319,7 +8533,7 @@ spec: Per-component security toggles for outbound connections the operator establishes: ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), - Kubernetes-client startup gate (kubernetes.tls.verify=Strict refuses an insecure kubeconfig; kubernetes.tls.minVersion is declared for shape uniformity but not yet enforced on the K8s API transport), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 behavior; Enforced coerces all per-component knobs above to their Strict positions @@ -8353,9 +8567,12 @@ spec: - ReadOnStart. Accept CHIT updates on the operators start only. - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI enum: + # both humped and all-lowercase accepted - "" - "ReadOnStart" + - "readonstart" - "ApplyOnNextReconcile" + - "applyonnextreconcile" path: type: string description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." @@ -8372,6 +8589,11 @@ spec: minimum: 1 maximum: 65535 description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" reconcileShardsThreadsNumber: type: integer minimum: 1 @@ -8692,15 +8914,15 @@ spec: - "reconcile" recovery: type: object - description: "Auto-recovery from reconcile failures, scoped by CHI state" + description: "Auto-recovery from reconcile failures, scoped by CHI status" properties: - from: + onStatus: type: object - description: "Recovery scopes keyed by CHI state being recovered from" + description: "Recovery scopes keyed by the CHI .status.status they apply to" properties: aborted: type: object - description: "Recovery from Status=Aborted" + description: "Recovery while Status=Aborted" properties: onPodReady: type: string @@ -8714,6 +8936,28 @@ spec: - "none" - "Retry" - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" annotation: type: object description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" @@ -8876,14 +9120,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.27.1 +# OPERATOR_VERSION=0.27.2 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.27.1 + clickhouse-keeper.altinity.com/chop: 0.27.2 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -9193,9 +9437,12 @@ spec: Possible values: - wait - should wait to exclude host, complete queries and include host back into the cluster - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) enum: - "" + - "Wait" - "wait" + - "NoWait" - "nowait" configMapPropagationTimeout: type: integer @@ -9219,35 +9466,43 @@ spec: properties: statefulSet: type: string - description: "Behavior policy for unknown StatefulSet, `Delete` by default" + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown PVC, `Delete` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown ConfigMap, `Delete` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for unknown Service, `Delete` by default" reconcileFailedObjects: type: object @@ -9259,34 +9514,42 @@ spec: statefulSet: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed StatefulSet, `Retain` by default" pvc: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed PVC, `Retain` by default" configMap: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed ConfigMap, `Retain` by default" service: type: string enum: - # List ObjectsCleanupXXX constants from model + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) - "" - "Retain" + - "retain" - "Delete" + - "delete" description: "Behavior policy for failed Service, `Retain` by default" defaults: type: object @@ -9322,20 +9585,24 @@ spec: properties: provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" templates: type: object description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" @@ -9731,10 +9998,12 @@ spec: type: string description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" enum: - # List PortDistributionXXX constants + # List PortDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClusterScopeIndex" + - "clusterscopeindex" spec: # Host type: object @@ -9846,9 +10115,12 @@ spec: type: string description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" enum: + # both humped and all-lowercase accepted - "" - "Unspecified" + - "unspecified" - "OnePerHost" + - "oneperhost" podDistribution: type: array description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" @@ -9862,35 +10134,56 @@ spec: type: string description: "you can define multiple affinity policy types" enum: - # List PodDistributionXXX constants + # List PodDistributionXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" - "ShardAntiAffinity" + - "shardantiaffinity" - "ReplicaAntiAffinity" + - "replicaantiaffinity" - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" - "MaxNumberPerNode" + - "maxnumberpernode" - "NamespaceAffinity" + - "namespaceaffinity" - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" - "ClusterAffinity" + - "clusteraffinity" - "ShardAffinity" + - "shardaffinity" - "ReplicaAffinity" + - "replicaaffinity" - "PreviousTailAffinity" + - "previoustailaffinity" - "CircularReplication" + - "circularreplication" scope: type: string description: "scope for apply each podDistribution" enum: - # list PodDistributionScopeXXX constants + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) - "" - "Unspecified" + - "unspecified" - "Shard" + - "shard" - "Replica" + - "replica" - "Cluster" + - "cluster" - "ClickHouseInstallation" + - "clickhouseinstallation" - "Namespace" + - "namespace" number: type: integer description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" @@ -9935,20 +10228,24 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` provisioner: type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" enum: - "" - "StatefulSet" + - "statefulset" - "Operator" + - "operator" reclaimPolicy: type: string description: | - defines behavior of `PVC` deletion. + defines behavior of `PVC` deletion (case-insensitive). `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet enum: - "" - "Retain" + - "retain" - "Delete" + - "delete" metadata: type: object description: | diff --git a/deploy/operatorhub/0.27.2/clickhouse-operator.v0.27.2.clusterserviceversion.yaml b/deploy/operatorhub/0.27.2/clickhouse-operator.v0.27.2.clusterserviceversion.yaml new file mode 100644 index 000000000..c63bdac9d --- /dev/null +++ b/deploy/operatorhub/0.27.2/clickhouse-operator.v0.27.2.clusterserviceversion.yaml @@ -0,0 +1,1768 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: clickhouse-operator.v0.27.2 + namespace: placeholder + annotations: + features.operators.openshift.io/disconnected: "true" + features.operators.openshift.io/fips-compliant: "false" + features.operators.openshift.io/proxy-aware: "false" + features.operators.openshift.io/tls-profiles: "false" + features.operators.openshift.io/token-auth-aws: "false" + features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-gcp: "false" + capabilities: Full Lifecycle + categories: Database + containerImage: docker.io/altinity/clickhouse-operator:0.27.2 + createdAt: '2026-07-21T18:02:53Z' + support: Altinity Ltd. https://altinity.com + description: The Altinity® Kubernetes Operator for ClickHouse® manages the full lifecycle of ClickHouse clusters. + repository: https://github.com/altinity/clickhouse-operator + certified: 'false' + alm-examples: | + [ + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "simple-01" + }, + "spec": { + "configuration": { + "users": { + "test_user/password_sha256_hex": "10a6e6cc8311a3e2bcc09bf6c199adecd5dd59408c343e926b129c4914f3cb01", + "test_user/password": "test_password", + "test_user/networks/ip": [ + "0.0.0.0/0" + ] + }, + "clusters": [ + { + "name": "simple" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "chit-02", + "annotations": { + "chit-02-annotation": "chit-02-annotation-value" + } + }, + "spec": { + "templating": { + "policy": "manual", + "chiSelector": { + "target-chi-label-manual": "target-chi-label-manual-value" + } + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseOperatorConfiguration", + "metadata": { + "name": "chop-config-01" + }, + "spec": { + "watch": { + "namespaces": { + "include": [], + "exclude": [] + } + }, + "clickhouse": { + "configuration": { + "file": { + "path": { + "common": "config.d", + "host": "conf.d", + "user": "users.d" + } + }, + "user": { + "default": { + "profile": "default", + "quota": "default", + "networksIP": [ + "::1", + "127.0.0.1" + ], + "password": "default" + } + }, + "network": { + "hostRegexpTemplate": "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + } + }, + "access": { + "username": "clickhouse_operator", + "password": "clickhouse_operator_password", + "secret": { + "namespace": "", + "name": "" + }, + "port": 8123, + "rootCA": "", + "rootCASecretRef": { + "name": "", + "key": "" + } + }, + "metrics": { + "timeouts": { + "collect": 9 + }, + "tablesRegexp": "^(metrics|custom_metrics)$", + "excludeRegexp": [ + "^metric\\.(OS.*CPU[0-9]+|CPUFrequencyMHz_[0-9]+)$" + ] + } + }, + "security": { + "clickhouse": { + "tls": { + "verify": "", + "minVersion": "", + "serverName": "", + "rootCA": "", + "rootCASecretRef": { + "name": "my-ca-secret", + "key": "ca.crt" + } + } + }, + "zookeeper": { + "tls": { + "verify": "", + "minVersion": "" + } + }, + "kubernetes": { + "tls": { + "verify": "", + "minVersion": "" + } + }, + "ipc": { + "mode": "Plain", + "bindHost": "", + "tokenPath": "" + }, + "policy": "Permissive", + "fips": { + "enforced": false + }, + "images": { + "policy": "Permissive" + } + }, + "template": { + "chi": { + "path": "templates.d" + } + }, + "reconcile": { + "runtime": { + "reconcileCHIsThreadsNumber": 10, + "reconcileShardsThreadsNumber": 5, + "reconcileShardsMaxConcurrencyPercent": 50 + }, + "statefulSet": { + "create": { + "onFailure": "ignore" + }, + "update": { + "timeout": 300, + "pollInterval": 5, + "onFailure": "abort" + } + }, + "host": { + "wait": { + "exclude": "true", + "queries": "true", + "include": "false", + "replicas": { + "all": "no", + "new": "yes", + "delay": 10 + }, + "probes": { + "startup": "no", + "readiness": "yes" + } + } + }, + "coordination": { + "keeper": { + "readyTimeout": 120, + "onKeeperResourceUpdate": "none" + } + }, + "recovery": { + "onStatus": { + "aborted": { + "onPodReady": "retry" + }, + "completed": { + "onPodNotReady": "none", + "onPodNotReadyThreshold": "5m" + } + } + } + }, + "annotation": { + "include": [], + "exclude": [] + }, + "label": { + "include": [], + "exclude": [], + "appendScope": "no" + }, + "statefulSet": { + "revisionHistoryLimit": 0 + }, + "pod": { + "terminationGracePeriod": 30 + }, + "logger": { + "logtostderr": "true", + "alsologtostderr": "false", + "v": "1", + "stderrthreshold": "", + "vmodule": "", + "log_backtrace_at": "" + } + } + }, + { + "apiVersion": "clickhouse-keeper.altinity.com/v1", + "kind": "ClickHouseKeeperInstallation", + "metadata": { + "name": "simple-1" + }, + "spec": { + "configuration": { + "clusters": [ + { + "name": "cluster1" + } + ] + } + } + } + ] +spec: + version: 0.27.2 + minKubeVersion: 1.16.0 + maturity: alpha + replaces: clickhouse-operator.v0.26.0 + maintainers: + - email: support@altinity.com + name: Altinity + provider: + name: Altinity + displayName: Altinity® Kubernetes Operator for ClickHouse® + keywords: + - "clickhouse" + - "database" + - "oltp" + - "timeseries" + - "time series" + - "altinity" + customresourcedefinitions: + owned: + - description: ClickHouse Installation - set of ClickHouse Clusters + displayName: ClickHouseInstallation + group: clickhouse.altinity.com + kind: ClickHouseInstallation + name: clickhouseinstallations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Installation Template - template for ClickHouse Installation + displayName: ClickHouseInstallationTemplate + group: clickhouse.altinity.com + kind: ClickHouseInstallationTemplate + name: clickhouseinstallationtemplates.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Operator Configuration - configuration of ClickHouse operator + displayName: ClickHouseOperatorConfiguration + group: clickhouse.altinity.com + kind: ClickHouseOperatorConfiguration + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Keeper Installation - ClickHouse Keeper cluster instance + displayName: ClickHouseKeeperInstallation + group: clickhouse-keeper.altinity.com + kind: ClickHouseKeeperInstallation + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + description: |- + ## ClickHouse + [ClickHouse](https://clickhouse.yandex) is an open source column-oriented database management system capable of real time generation of analytical data reports. + Check [ClickHouse documentation](https://clickhouse.yandex/docs/en) for more complete details. + ## The Altinity Operator for ClickHouse + The [Altinity Operator for ClickHouse](https://github.com/altinity/clickhouse-operator) automates the creation, alteration, or deletion of nodes in your ClickHouse cluster environment. + Check [operator documentation](https://github.com/Altinity/clickhouse-operator/tree/master/docs) for complete details and examples. + links: + - name: Altinity + url: https://altinity.com/ + - name: Operator homepage + url: https://www.altinity.com/kubernetes-operator + - name: Github + url: https://github.com/altinity/clickhouse-operator + - name: Documentation + url: https://github.com/Altinity/clickhouse-operator/tree/master/docs + icon: + - mediatype: image/png + base64data: |- + iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAYAAAB5fY51AAAAAXNSR0IArs4c6QAAQABJREFUeAHs + vQmgZ2lVH3j/r6p676abpSNLE2TrRlwSQBoVtVFHiUGFLKOEKCQTTUzGmTExhmhiSJBoTMZEs2KQ + BsWNREWdyCTOGOMEQZbI0i0NyCaIxAB203tXvfef33LO+c697/+qqqu3V1Xvvq773fOd3/md5Tvf + 9+57/erVajq4DiqwoQLnvf4PH3Pe4ekx29P0oNVqfTEgl6xWE8eLV6uVxgnjCjLHab0TI+ZW00PW + 6zWgq0+up/XNkG+edjiubt6Zppu31tOnd1brm7fwvL1e3by1Bf165+b1+vDN02r7Jhh+6MZnXfYh + jAfXQQVmFUC/HVxnbQV+aX3Bg875xJOOracrD21NV00705U4XK6aVtMTVqut81WX6BAcNpOOIIye + x4iJFMfDhmry8CIwRh5mZBHfppH61XT7znp672pavQeH3g07W1s3rLbX77npQQ+6YXra6rYNXg6m + zoIKRPedBZmerSmu16vzfvnGRx9a3XXlar2+CscEDqXVVTgSrtzaWj2SZRmHURRJh0uf9/EycNE2 + cVpxPg+jLLMOJczPRiiPd0j1Q634eMgtr/X693CIvWe9OnQD+G/goXbXzs4Nt3/FZR8BxwaDJcGB + fLpW4ODAOl1Xbq+4cUBd+Mv/4/Om9bFrcF5cs1qvvgSvM5fpMIDN8tCxzDcaHjoCiFkyn+psur/f + sOaHneLfdHgxRs7zcNxZfwrjryOPX5u2pl+78VmXvgsyvgo9uM6UChwcWKf7Sq7XWxf+8u997s72 + 1jWHpvU169X6S/Dl3GVKi4cQrjp8JO11aGnPGxGHlw+ztPeh5jOtTjHhfdj50AgX8zcrHib8Mg9K + 2W8a49DJw2c2Jmkf85Aib/LnGPxw9ik8/joyODjAeu1O4+eDA+t0WzwcUBf84keeslof4uF0DRbw + mTgJ8I1xHArIRYdHjD4c4piAntdm3JnxhjU75BaHF7PH98Q+hfHgAFMnnJ43d/HpGfvZEXUcUOtt + fHm3tb4Gp9Izp62tBzF5HU4+pVQLnkn90AIg5ufLvPnQIp/gfgDRHHf6vWExHR/abWxvcsgIB9hO + HGBbB19CxvLv5yFbdD/HeFbGdu7PffSJ07T9l7ZWqxegAI/YdJioMFLsPkzqsMkvxNrh1Q81486O + N6xNh5fyzy8rd3Y+tl5NP74znfPKm7/ikveelY23z5M+OLD20wL9Xx++7Nw7tl+wNW29EBvnadxM + vOrwydVq8/FKFbiDN6xT+l6Zqje/gectWINXr849/JM3ffGlfzjXHkgPVAVyCzxQ/g/8vnZ9zjmr + D/0JLMQ34bh5ztbW1jkqSjuU/EYUpeI8JvIw89dxB29YqkP7co/yyRxeszcs2vcLMip7Fwr+Szs7 + 61fffM5DXz89a3WsQw6e798KHBxY92+9y9uRf//Bq7d2dl6IDfH1+HmoB0uhw+g4h0+uVjvMDt6w + ol44XOrwQTF3ffmHOZaPh9iuw03FX9wC1w89PP8BiH9ie3341bc++7J3LCwOxPuhArkF7gdXBy6m + 137wM86Zdl6EHfNN+N7Uk+JVaVaY+ZuT36S0+XKlDt6wZvWSsOkQupfesDa+qcEfjsd3Y0lefezI + zqtvfdblH98d1MHMfVGB3Ab3BfcBZ1bgtR958Dnru/43NP//ii9UHqI3AehYfB9GsQwHb1h8Bbr7 + b0B5OOWYdd00ngp/8GBwfHrwbWe988nVtPXPp/U5//zTz34Qf+7r4LoPK3BwYN2HxZ1+9n2POnJs + 62+gyN+MQ+pCHk/jsIrjiodUuw7esHgmtC/v4hCqL+Narepx0yF0koeX1qP5K04+BG//slCrxvn6 + dBO4abp1Z1r/yLHtwz94+1c/5KM0P7ju/QrMd8u9z39WMp772g9cubOz/bfwd9z+PPr6SB1C0eTj + 0OIR5i/7VCgeXrl52nzhc7XikBOvCYZ5s9Mm77JQ9tf9buQHYMxrmy5kEYvRccggPGw+dMwytvpM + jsMhD4nZWKztoR8meTjlCJjy2zRu8tNo67HzB490nG+XDzP+0C4OWczvrNdHkeGPT+vVD9z8Jx72 + ngY9eLwXKsAaH1z3UgWO/NT78aMI67+Nln4uvkeF357SN7G3Zx0C+Rk6Dp8MQZufQjuUtPlypTgv + 2pgQrr25Le0Wfsr/DGd78na/iqnczH+SXrhZehmgrOa3xSGx642FbvFH7jkCrzjbaH9EbLgW/HnY + nZKfTh+8u3g4XxHjMeQ8tCjiz860Wv/89tbW99/2lQ97a6c9eD71Chyny06d9GyzPPxT7//y1Wr7 + b+OM+vI8o9zS8Zk3Dods8jo0UCjhUs8RnV762aFSZ0k9EDc/ZFKMZW32fU1Oih+BzXG749IhAmLH + IYNys+nQYVSuy4aRuzzy3zUWa3sI/L3ip9HWY+fHJOPWxfl2+TAbb1iUkQj+GBc0/w9+LOL7bvnq + z/jVZnrweAoViM4+Bcuz3eQl661DV33guVvrHRxU09O8yWLPoTb4chB3NG0cGtnEdQjs0rug2vx8 + bIeNtkCulDY11TGhcfdhspefmp/x296niXkH/4jLcTS/s/QyQONn99i19+jNR3kzgg3Xgv8e+en0 + wetDqR2ynM/1Iz7k/oZlmog34Ov1zlumaev7bn725b+ABTz4LRIu0t26H6fL7hbPWQU+8tPv+Tz8 + FpeX48u+q3OvqADaVD5r3KMHb1izNyAUqW91Nl/JWchN46buCtyMH/XfdbjA9oR+TsQfcQpGv+2y + vxO+YUVcjg8B/cb26tC33fbsh/23RnXweBIVODiwTqJIBcGPJxzZvvNl+Ez5Lfhhz63abQRsOKy0 + HTmvy9um3nByG5U+UCnHWPiiwQP2zHgDWvAu7RZ+Bp8JLR+8YakOi8Nozzc14Vx3rVrIJ37DQp3x + wUMOF355xPrlq3Mu/Lv4e4uf9Oof3E9UAXftiVBnux5f/h258n3fggZ7GX4h3oN5JujNAA/svTgj + Nh5aauIBQCXbl2+SFocPCDcfKgs/sCUuAtEKDTGWNfwKJ4RvJ8Ufh2LmOYs78+n8s0IAnXn0Ee7F + t2lM+01ji70eA3ev+CnS9tD5Mc24dXG+Xaf4hsUCgQXrtLPzyenQ9F03P/vhr8CCHnyZ2Gq76TE6 + e5PqYI4VOPJT770azfVyNBN+i6cPiTqEoudUqTgtYtBnUrV5bm42JwjqsAgZEzLPWx0uMV/4hIWD + Oa7xLu0WfgafCS3b3qfJmHdejmxpp7hVj4h8kUfmozE2f57u3uQ+BOgty1gj8PLXRvsjYsO14L9H + fjp98O6Kl/NZV+JDVl+kyHllFgMSrcMNeC1j8mDE7zb7b9s7h77t9uf8kd+Q6cFtYwXcnRtVZ/nk + j/3O5UcOb/9jvLd/I75XxXbjadWH2FSeVrWWejR1HW4G4N4OF0k+BId905MP1zgsJJbDDEtxCafw + hBey2YdlTDOu4Xcjv9LtuN1xDb+sS9QnHGlzwv9shE5+N41pv2kMztkQuBl/+tvEjzlWk3jF3ccZ + cQidn3aJ4Xy75D/XGfPib4fZcIP6EacJAXHLutFOpFCvf2xaHX7xrX/y4K/7qCKLm3fEYvKsFv/z + +vDWx977bfghqpegOJd4U2aTe5PXIcQmywrycBgwTFMREyqo5TocdulddR1CfGyHjdzs8hMTwu0+ + TPbyU/Mzftv7NDHviGPE5Tia31l6GaDxs/vYtcqLm5Zo8W0aqUd8wsVYh8yMOIQFv3Z/2m/ix5z8 + b/LT+YN3V7ycrwzowPI9ecMindyRblp/Gs9/79YLH/4vpoPfDtFXRFWfTZzNwuGffN+XTjvb/wab + Bf+qTG4a7rHYXhxjk6pFtSm0B122pR7lTZ4AYAhePAVr8HPCXavNKpEI+7c/ieVQcTVFuJ/zhX1Y + ajgpfuXJ+O1/Fjcd8YrRccjA87j3w0b+sAMrX+pp3kftVsxsGoHbdQXuXvGzixwTnZ9iYjjfLh9m + sc6YpzwyKxrXg/0gXgGNC7mm05Pd/Pb2tP7m25/zyIMvE1EtXtF5Fs7a+2vwd/5W2z+Ipvmz3sw+ + VGK3oizonjgNduujaqV3cx+8YbVu0m4ch5E3edZpwwh8HXKoqzd52Dfaelzww0DrdUp+ihQPe/Fw + vo7bwPEwwgc3lNQYnVkMCp9656N2SR75CXeCDx54orODLxNRBF6s79l9/eT1Tz60fegX0ECPc7ex + 16P5tFksq2/UZZTdRd5UllXEpZ7NySbmvAG4W+4tX3rZN33YOZ6FHzDJTkTmD/fDX7O3f98HX9ox + zgU/Jua43XEBIELHIYNyo8MC+tkIrfxsGrVpwbdpLNb2ELgZf/rbxI85Rku84u5jo63Hzk+7VHC+ + XfKf64x58WcjSU53qB9x4g0FcSHXdHoqN3yA3c76evxWiK+/5Wsffj0mztoL36o5e69DP/7ubzm0 + c+gt6PPHae+hN7xJvTnZO9qMflDXzjax9FE/EoSMQc3JCdsTo+0S/KlP/uBA1ya+j+KjOa/0o2YP + WX7kfre9cGTwNYsfU5bpF4JgmYcdaj7ycBwRSMVBO2gMtJPgpaA8FnmJB7rZGPYzfLNb8qe8F955 + wf/J+Mk4Mdal/LweGSd18idQ1scedBht1GNS5eEnhVjfkEPRB8SbvHJCRhGstlZPXk/bb7nglz72 + zak5G0dX52zL/LXXX3ToztWPo/me610TZXBvqCmzubwXSzFvLjT1bK+qydnUgqn5ksclNs+uzUQD + XjJsmyTmCx8w4QRPR1aU386XOPLHNfjSjpvJ7gUJojlud1zzOLQL04XeJGQfh47fRLIuG8Ys5Kax + WNtD4MSLeGcjYMpn03gq/MEj77Rvl/OKwwjzlPOQIWy4Q/3wIT3LnjgBAsdpRa4H3HiZz36sB8/r + brnoyDdOz7r8FmPOnrs79uzJd5p+4t1P3dpe/wx+Uv1x7oXctNE0bH58cLMNPZpmtom7PopX+mwx + dWU/BQBsmz4+c+amzyWQXwrk08B4SpzFEQAMjXdpt/AzP4RItylfz5tf98D1edcn3DkuQ3ffx64V + bmw+iED3LS4ZeMXVRtWDPJuuBX+eEqfkp/MH7y4exZGRwyBk9U2K4ol4I0Hz+NBi3SirAvJjGrMi + /108MghW6d+/s9r6+tue84i3afIsuZ09XxLiL/9t/fi7//rWsZ034ueqHpe94marnlMT9c+Eu5sq + mwnNNnqoui16D5vQzWh7dtOQsyk1q0Ci23h4hNzHWfOGA/GXgnax+Zf29FunCsNs8TMqybbXLhHe + 846P99hkgedMOmRWmw6THj/1XXb+ES/NRScm4xKfI31EXnzU1fNMXI4AVJ54nvnbZBd48eaNuOBL + e6oyDzwJaRn54aPnMfSAQeF4og4hh6IP4rEf0eNGP61+8kMZ33Pd3n7j+b/40W9P5NkwssZn/vUT + H77s0M5tP41V/srxhoG01QsuAXqTTYAejebTBDAcCQx5tz7KV/pssWYniOXyn9tI/MGBgXrDPRY+ + pnscA4fNAjuns7Bb+NmMs/28HhlHhjPnH3FYLzkedw2x+aPAGGJzA0gviruP1DOfNtofkRuuBf89 + 8tPpg3dXvJyvyPEY8ji0kiTiDXg/tLjMlMUjP6ZxPaI+YV4VEp4S9a4PPUF+3W1bF//F6Tln/j9H + Fl2ZBT4Dxx+7/plb6+knsQGuyB4bm1arzVPCibtbvFnYFNo0VJdCvaNmg8XQR91CUXo2VfB0B+Uf + k2pZ8YsQE+FXouMqfIQJx6JTXCYIeQNf4xEo5O53Iz8AY975z2URidJxyCBdaDMKn/lwhFZ+N43c + jCrchrFY20Pg+6FW/jbxY07+N/lptPXY+YNPOs63qw4hrjPm6xBSplrOSIv6OGQMBBoP8lMDrIK/ + 3PAhDm/y46N4IOeF37f1kdXhQ887079EPHO/JOSXgK+6/nu2ptWvoZGv8KbinuAhwhG3ehiy9WgK + bR7jBWyyzdhsYceuKX3QshnZXHIkQMmylz75qceF5k18H1uYww/tS0G7FOl38LK5tSk063mbDZwP + VQCoCN7hn3OOq9ulQ7HE5iYyr2Fv/10WD4CzMeokXNYhR5JWHcPDCfDkmfGnv012onc9gt3+wn/y + UJd5qE4loz74EK7izPoCBIXjifUNORR9EI/98M6LPGEniX6GTAQvfE/2iunYMX+JiN737Jl3PzMT + ++X3nbv1B3e+arW19Q3+DBZpeu2jedhEWFB3mVd2pndTeC+WYt5c3BRqvmgMEoYsWjVxby7z7NpM + 2eSyD7+gzM1ReJrzCrz0Lf5wX3YznC3DfL65NvIrj47bHdeMf1YIlCE3ex/h3XXZMKb9prHFXo+B + u1f8FGl76PyYZty6ON+u/fKG5Tqw7jzi1j9965ErXjR99erOFuoZ8Zhb4IxIRkn82DsuXK0P/RJ+ + XdWz+mmiTYnVnG1O7R6XIM6K0MchU3p2AXAauIm7PkpX+tyM5A07QSyX//jMmZs+WOTf8IwrD41A + lB/rbd/zWtgt/JR/8uCy3PMZ8wboHjjjPQ/cLD3bGb24x+bP9fAmzzptGHOd2qg8ybPpWvDfIz+d + P3h3xas4mG/EE7K/XMNshZl6QFkuKPLLOS1j8nCUXgNYiVvyBEBeBw/Ecc15/vOt5x35mumrPuPW + ATj9n86sLwnxGxZWO4d+FV8GPovN0ZvccjZNNEPritE7rRnUI2y6aJaQbTaaTG0wbxY1He3k1wDx + uGnhHx+8hp7qgc/5whvuwBkH7R0IDbVJKq7Gaxw1vgbf8O9NBL1g5h3+aee4On86FIt3nx3EfdiT + b56XeMTKPOx2I77ZyV/3oAWL+iUuR+Aqz+TPcZNd4DGMS+vZ8g5NxunIwz/rg49ZfZQZ9TCEwvFE + HUIORR/EQ1cRJp9EkH7tZ9STWF7Si7fq+azzbz/6q9PPffxyI86MO2t8ZlyvfNcVq0MrHlaP1xKj + ebXqHHHpTQJdMN4wMGlg6BMezSd7GRoY8u43EpnTgfjKzO3reXsQT/nfpTcP9Ya3uDGR08NPx/W8 + FnYLP+U/CHfnM+wdR4bDuriOngcOE7O4DN19j82f6+HNG+UHWrx9zHVqo/MmcsO14L9Hfjp98O6K + l/NZV+JDHodWkkS8ATePDxvWTYcMeeSnBtQjD7/OEzi6k373obWLx/H/znq99WW3P/eRH0m203k8 + M96wrn3356AB3oA3hcdXc3mx3AVc5GqK1gzoAS1yDtFE+druXqpuU4/O+cvcD6P31FQ0cFOyRcxT + 9pJpFk1NCJJIuY8tzOGH9qWgXYrmK3vhCPTFeZsN3MEb1mIdWCoV1OuherX6RSU1uM7A4aPXFQSh + xwCF6x7rG3Io+iAeGo62IE/YcV5+hkwsL8UhXtGFrInHT9P2G8553Uc+x8jT+46MTvPr1dc/fbXe + +Y/4ntWltZhISW8q0XTMUG8S3Kw8FCTjVg/Uu0k2v3EAKEDwsGlCVpfagfjGNEsbdtS7nYf/kMUr + vW/iFdxLM4s350UbS0eHEY/TWdgt/Ay+juv5jHlFZHHELX/QRKIpLvOQbd5yHWKsQ5M0+KO4+5jr + 1Eb7I3LDteDXbkdgp+Sn0+8VL+crcjyGPA6tJIl4A94PLZUveeTHNK5HHn6dBxrhWS/qdx9azrvx + AK/1lt36xqPrra86+rwr3pysp+N4er9hvfpdz8Fh9atYvEv1GSYWUYuuJmiLLJmL2ZrBQK2b1lTw + 1gwhqzl32aMLB311ScDUVAQ4Lrpw15Z/yY6nGic2mdAKaBGvFfZL+4qfflKkn8HL5u6HSfmf8dte + fMGbcRvmPOzO/OlQknef/OZt2Nt/l8UD4Gxs8TBe4XMkaeiTP+UZruE5P+NPf5v8iD7ySgfEBV/y + UJV54ElIy4gXH8JVnKkHDArHk3kljxShz3o0O3sQQfrd67CSftA5zlk9pksPTzu/et7PfvRZoj1N + b6fvgXXtdd+wWk8/j8PqQtbebyZoGi5SytF0pYdi6DFrINXsTcnWm4f65BVgZs/uCLskCBkDetSE + thdADsq/9MlPPS7Em/g+io90vBgo/dC+FLRL0cCyF45AX+VfCdu/NgHNBDNR2YtuUVdSVRy0Y0D2 + SxWvYU++eV4KG5jZ2OIpfLNb8qcsP4nLMfzP+NPfJj8tXjz6Un4t75rOPD06T+DwIX9Vh9TDEAri + XGfWI+ujh5CzHs1OPim3+snPkCMs13vQhTxwsR4XrraOvf78n/vgN6Td6TaengfWq971HVj+n0ST + HM6C12c6Ni0m/RmHzYFVLNnz1mOyHgImOO3BTruQ46Fkm6Ve9CYQPmjZlckjyJBlL73jCQbAwy8m + nI9H+wtUxqWuFxA32oXfni/VwpHBF3nNx3iS3/ZSBO/wT5Tj6nbpUCyxuYnMa9gzsHle4gFwNva4 + E58jSUOf/CnLT+JyFDzyxPMJ/QS+uPlAf8GX9p523VjZISM/fAhXcaYeKChc96hDyKHog3jMK3rc + yNPqJz9DLpTijbBp1eInxv75sDoXP1L9k+f93IdPy19Tc9odWFuvetc/xv8J/Mf4jMH+qMsimoZN + hlnJsWgEWfZoPSbrgfOWbW8e6mXnB+Gt73xkx0UC4YM2tontBcCtxSd98lOPC/Emvo8tzOFHuwA2 + EXi4320vHBl8zeLHlGX6hSBY5sGJoO9xCxgK5ktQqzNFXj3+ZV7KB5jZGLyyyzrkaELex3UCfOUJ + ixP6ASbjLQfkD/9pT93AZX04on74EC7r48pUXR1PrC+A5tGDApQ78nc7OpTc+gJyvalJ75v4Bp35 + W/3sn37xBz8Jj+FHzvvZD31XozgtHk+rA2vr2uu+H2vwHaosHvqlzyhcTC4SFPkZhk3Hy7JH6zFZ + D5y3bHvzUC87PwhvfecjOy4SCB+0sU1sLwBuLT7pk596XGijxPexhTn80L4UtEuRiQzegzesqAvL + 2+uigu2uN2unq+llV9OuLyocMI7gwUfnH3qpta7Hf3PPOJM3HMpPi1N+hlyoZf+1+IkZfQuBxxV5 + VtPLzvvZD34/9afLxchPj+va6/4P/FPL/9TF3h1yfQZT83BNkFosGtHSQ+6fadwLLsGAR/NpQoa4 + oYlCLj+lJzuu0guNKNwUmjcA9+Z/l14gxWd4xsV4TF/zCsd68zdexiF82C38jPw7zvbdEXEmigGy + 48h5jFEW42Le8Pk916FtIqLFt2nMdWqj60uLDdeCP9e9NilMNvpr/KoLefoVvLt4hEtGGIQ8Dq0k + Cb5I1DyoI/Bql+ThCLoYUBfok1ZURRAS9eZJT1YseORn4CqP9KsV4D9Bvf72O//0Z/6zGdc+FViX + /X+96rrno6qv4T8P78XO1V2E7y7QptcSh+xNjTTVBdz8ufkwZ6BrMNN7E6uJyEOg+GowT0yLYKln + U+GjNn8EUP7LfeQh+/ALXdoVPmCKI/Ut/nBfdjOcAvRt8JlwyAGqPLJOzn/gWrwVRy8EN1/Wr43A + KtxNowtNwyhwGyOs2RC4e8XPjDiEzo8pxq2L8+3yIRDrjHnKSIBPQgWN68F+0ISAxoVc0+mp3JjP + fsiKeiaPPPgmPdwWjx5iQu5oZ/14oAz+Nf796Wn9wrv+zGNf0yj35SNz2N/Xte+6Bl9z/wr+IvNh + r8bmkL2ZYjGREeWOlx6LMzYdQOoF8w14O2RKj4fZJjaP7aN8pQ9ad4XtBKGf5n+X3jyKm4/k05CH + hsRZHAHA0HiXdgs/I//O3/MZ8+a336XdqIf1GW9I86FvHsQ3Nh/SAVJl7mOuUxvtj8gN14I/1/2U + /HT64N3Fw/mKHI8hb34zot5w80T+SJyyeOTHNK4H1iNpMTaCkKg3jybytuSBrHWTH/sz7zx+x7E+ + Nq22vuqOP/WYX026/Tju7+9hXXvd52Oxf5GHlYoaza5CxiJkUXPxOdaicPO2xaJi6GFp4KArfTRD + yPOm6vxF7wfhs5XdFI6LLoYst5IdjwLgreWXdrN4iWE+9EN7ESVvipQHr3EE+hp8A+fmh14w8w7/ + tGuHTNa94qCaAZmPaF7Dnnxjc5V/YBR+jmEvu8TnaELex3UC/N3yA9aMtxwov5Z3KAZu1E/1wXoo + n6pD6mGo8lB/vL7KejQ7+aTc6ic/Q46wHH8sQ8XR6lf1EJ0QZke8lEB7eL2z87ojr/3A05NzP46K + dT8GNr3ynY9Fo78Zv874IdoMKH6NGwL2G0A0BfSUOz4/0xhHPUC1eJbdo2gGNoXsRWRgyLbveqh5 + lT5o1QZwIEcC4GY7t2AEUHpiCOe8HmJgPJ2GeS1xjTfsK89FHDU/w/V87H/EoTAUl+No8SkO60ee + IfchNn+uR20eYMgm3j5yEyE+4WIUP3k2XQv+e+Sn8wfvrngVR0YOg5BP5zcsZaM8pk+uDx2++s7n + Pfr9vRT75Xl/vmH96A2PQIF+BcE9hM3CZp2NrJ6LW3WUHu1fzRV2ibM9zbAZ0rwequdCn/6M17YS + X7dnXGGX8YQs2tiGjksA3Fp80ic/9bgyTzym3SxeYjIOnVohi1fhzOwEF44R+Rp83vyWGRf0gmVe + qaddi1tATFUcVEfihMbV41/m5fpEvMBLDl7ZZR1yJGf6Df6U98I7L5glf46b/Ije+Sa982t5hyLz + MjPDsodxWCWPx1ALt5/fsCqP9fSQ6ejR/zj9wu9yD+67Sy26r6L60Rsunrbuwj8UsfVkNV1uhj5u + CNhvDjyM0GTQ6w2B3UK7kJPPekzWg2GG0375xhE8GpZ60QdBDYpDDsI/NHLoOClZzviCxXFT6HEP + sRwov8LFG4lwLV/KCz/lf8a/zJduzCPz4HW5ch5jlIVhZLx6Xt5yHWLUOgAjvk0jcIqzjeKn/aYr + ePOQyfGU/HT+4N3FozhYh4gn5HFoJUnqIbNcwOWhxfJSDkUfwEoc9Z1HBpqw3odpIqyAQcBsP+pI + feWRfiP++TyBwl5357mXfOH0dQ+7Wdz75La/3rCu/eB509bRX8QPhvqwiiKzWbW4ObJ4YzVVylz8 + WfHdFUOvxWzN4FUNPYbSpz8vcu+COX8Lg/HIXoOajk+Oiy7YfZbltjULtbpafmk3/AUm/dC+4idv + ivSTcdMrFQT6GnwD500EvWAmGv5p1+LOulccVMMw5wnHNeztv8sKG5jZGPbCZR1yNCHv4zoBnjwz + flhK3mQHXcZXDogL/8lD3cCN+pF5HDKeB1JUcgcCx+M6MhDz6EGByR35u50ZcA+70OehJ3XcxDfo + zN/qZ//0SzoAy0/uBymsJ+dq+uxz7/z06ybuyX107Z8D6yVr/JjVLf8OAV3jmrIJWFuvwmxkAVX0 + UUm/EcRnlGaXONubb7ZmwZNrSJyajxMAJq94Qh5xtTAK71bINxvbK2DcWnzsWlxDT7X1fd7xbPCj + rhcBDTO84ktexaEuJav9Vfwl2575AqEEyl5htrhVKBkaRxPtNudDkdewJ988L/kHZjYGr+wSn6MJ + eR/XCfDkmfHDUvImO+gy3nJAXPhPHuoGzvlaRn746Pyqo/C4QeF4og4hh6IP4rEf3nnRT6uf/AxZ + EKIUL0fRhTxw9m+91ku8thtxE29G9+/0ZedcuP0zE/fmPrn2TSDTFe/8l1ur6TkqHoozPiNodSWz + mv7MJMCshJ63XkuMZvPqkdF8XKPBi0kDQ2/Z+vATeAHF1+3JT1nmfghZtG7fEW+Th958waB4M78+ + Cr/0o10ASwXAeDMdA8teODL4cn7Dr2Xbu4dNVPaiW9SVVFkPPmuX2C9FXsOefFHPmHf+ES/mnJ/t + ZZf4HE3I+7ii8HvhOX/SfsCa8ZYD5dfyDsXAtXjh6Ux7w9KbHgqI/7723Cd94IerLg/wA9f0gb+u + fddfwcn5r9kC1WTRrP7M4ab3Jtkccn0GU/OAB/YdLz2bmLz0Mxwp/wGP5tNEA4ZcfkovcxNmmDTL + TIjTZYflf5c+UImPsfBFgwf5iQnhel6eL7uFn5qf8dt+FCbqx5DKTdatJiIOx+2CxvNyiM2f67Hn + YRLupEd8fRQ/eTZdC/575KfzB++ueBUH6xDxhDwOrSRJPWTAzZN5WQ5FH8Aah22lywcRiNh686Qn + K4ALmKJDXFpvxZf+m9+Iv/IL3EgrA8BPlq5X33r06x/3b2b+HgCBeT2w1yuv/yz8RsS3IJALajMh + olgij4vmJS6bchZ8zBfPEqdVxJqSD4ZUjwdOWLY+F9t4AcVnO9uzaYIHpt5UNYCOfuIwoD4clH/M + JI/Vzqv0CpB8LV7RMFDP9/gjPOEDMHB68m3wDZ6qByGVZ/p1YZZ2wgnuuFMmRT9sZMfNw3niN41Z + yE0j8LuuwN0rfnaRY6LzU0wM59tVhxAQykv6zLBoXA8giA8gWIALuabTU7nhg+3MSj/Bs4xj0IWf + mADOcdod3crvbH63H9ErH/71nem21aH10+/6M0+4XvMP0O2B/ZLw5R+7YDXt/Ds09O7Dqjapi86m + Z5E1sliU2+V561X6wCfO9jTLTQhjA8UiOsjWh5+QBRRf6j3SXnZkKH3QuivEJwdNllvJwWPALD/n + Y73wmW76oX0pGG+KBpa9cAT6GvkPnA9V6AUzUdkLtqgrqSoO2sGQcruGPfmintCXfzwr/BzDXnaJ + z5G8C/6U98LfLT+in8cv/vCfcTqMxHmUf2TiT07Ojzg8+c5B5WHfRR1CDkUfxCPrdCOeVj/IxSMP + vrkOLpPi1fo0O8ieZzx6kmHVSfUlPvjolzD5x797uJouwM/C/8z0Sx+7wIgH5v7AHlhHPvGvUBW8 + YbGGKBZH/NGYMovLYvfRBrzXJT0siyfwXhzzk3joYWpH4iB86NNf2FnR9J0vQpA/zgdtZOK4iLGD + 8u9uUDzBAMPwS7QC8tjCLAds2hE/7VKkn2YvHBl8lf8Zv+3FF7zDP+0cl+MwfzqUxC4PvnAzi3+Z + l3jECrMcWzyFb/VY8qesOBOXIzgrz+TPcZOfwGMYF3HBl/FSmXVx5CmjPviY1UeZUQ8jlYd61zHl + UPRBPPbDOy8TpF/7CR4DjFK89ldxbKqH6EZGVacIVIO8Mn9S08C8+CHuJx+55Rbs2QfueuAOrGvf + +SLU44X+DIXasLioQ5ayZMxz1YXLkfXifLvMY5zXxHaJs33zMxyJRXRcI8WR/ixr0RRH6mNe+Aii + 9F5iNiefHBcxQ3aelJNfjyPPNu94iAtM+iGfE4WCflKc8yqO7MLgtdnA+TO2aIgQUcZtv4u6ApUO + xUL+CpDKaTrv0Gr6zsecO7356RdPz3vYYai7P3nZtd60Ey7XOUcreB9X49vUH1U3WLjeG+rT+DO+ + ckD+0Kc9dQM38iHzmf6GpXIgz62trRce+an3vqjqdD8/uOr3s9OJ37daH3sL/o7gBfoUtKE5skk0 + Qq/PBG0sux77kifk2kzMFoTVzCFv1scmDXwY9sE80kcQ8kd+waKJzWOEHZZ/hyMe6TfFD0Xhac6L + OA3pKOXwG/oZTha+Db60i08WFiuBOW6PQ6viGIX42ocdmf7BY8+drjhvfD78zZuOTX/zvbdP1926 + wyNxfohQxuGgeDeNDnt+D9yyL+qNAeiT9jNnttT5MYPoxnw+Y7Q/1o+HVuRRnilnWtS7jwMINCwE + qEE8oh8OhRt5NR4BfXMdGs9wbDrIjk90BuKxeImP+Fy3CEDzc15obju6Pvzk6fmf+SEY3a8XY7t/ + r5e/9ch05Jy34/Xys1zk2CyIwiWLEZsu9W3VvfobIvbmisWEnnLZhZx88uNVKb4Bj+bThAxxg0XI + 5af0UPMqfeYRDjivy3IdAm6f8h8gx00h7ApfNHhQODEhHPLG6Lw8X/LCT83P+G0fBI429DJXOHP+ + nm8YTE84f2v6oSvPm65+UP3maqnyhm/cTq/5+F3TSz9wx/Spo9ziSBN/do3cXMynjfZH5IaLm4rx + LkbZA76LH3Py2/jT34w9+HbxcL4ipwPL49BKFs7jCrh5Mi+alUJ0cic4+zhoaT8IQqLePJrIWxAU + Dx56XpUHAS3++TxUGbZwJPdEjx/Pbz566OgXT//zk+8i4v66xqfA+8vjkSP89cY6rNhkLAKvKhqe + q5livnAN7yahpS/zRDNgSnI2ccnNT61Z+gdIPd+aIeTeTRUn45be/hVPyBiwxHbguIgZ8tBHnEFR + eRK9rIvDpMJ+ySei5E3RwLIXjkBfs/gxZZl1gyCYectedIu6kqrimKZLjmxNP/D4c6c3fP5Fex5W + NME3bqdvevg509uuvnj61ivOnY5ATrfDH/1HX+SY/jjm1eqzCV95As+0juuH+uBLeufX8g7FwKkw + YQccPuSneFIPQ+aJ+TpkQg5FH8RDV0UTkadf+4n6REzGs262qzha/ew/eLkvxEu5x01ek6p/CSsc + 6Wd+n37k2JEfMPr+uyuk+83dK9/5dXizep2KFMXUZwAEkCWsselVRRaZ1VSxd0dsnmgK8i3w0nNx + yCs9bvVgWtNTb9zQh18NS33EIn/BQ/7MqOJ1ZuV/l948ipuPYVd4mvMqPzEhXM/L82W38FPzM/5l + vnST/Ok265bzq4mf7f7CI8+d/ja+V/Vgnj5383r/bdvTd+DLxP9y43ZG6THXqY3Omwu24fLCjf4I + uTYjTBidlruPjV91oV2/9uIRLhlJaOZxaCVJ8IVjx4M6Aq9lTB6OoIvB/Ze0oiqCkGCPD/LMriUP + 5J5X1UN2I/75PBiDdvB7wrjmFzzb0/pPbT//yp+fxXEfCoz6/rmuffdjpvVdv4VCX1rN51WLTRjF + RTRZyhqBU7Ha6NVdhB98s02pRQxcEA49nKkXNulzsRleGQovN4xT8Th8FTEUpWdT4cP2RJin/GPG + 7tM/xmwyokkUYw8Tiprv8Zff0M9wsvCt/DeeXfyYmOPa+sDu6Zccmv7ZVRdMV114qDGf2uOvfPLo + 9F2/c/v0/tvhNNerj5toQ7/sC2+qqCvslFcfO68KRsSGq/NDXSjOt6s2MRBcFcq5zoQNd9THZjfQ + OAECR3x6KjfmG3k1HjqIS3rwBp3jaPmVfYVnB2N+tx9RK58FLxRph1+vfOOx1Tl//P76fpY7P7O+ + r0Z+3+qcc9+A3xz6+doEKMJshF+tYR9RbC9CrkIbN8TpzRWLSZ5cLI4hJ5+WphymPhaFftl8spch + brAIufyUnuy4Si+0mrPsDMA98iY8M6Zdu+SXcszbX4nNT9gJ13iXdgs/gy/z3pQv3SS/g0u7R553 + aPrex583Pffyc1rU9/wR39KafuSjd04/8KE7p5uP4Rvz8F/rpV2IeGLz7PLW9Yw75NxU3opajayG + R+B2+enke/FwvpjwGPLZ+IbFcu3srN987H1P/ILpJasdyvfldf98D4vft8JhxUTYRNyMszHmuUWq + FYCTvAfeTUJGX+KDRTVp2CXO/uzfvLCrh+q5sM/4Il4CxdftmUfYMYTSZys7E8clgBxWfMrUfNTq + yrpASLvCk45X+tEhGrLyznQMLHvhmKivwTdw/swPvWCYV16pp91qOhdzL/7M86e3PuPie/2wogd+ + RfnX8H2tt+L7W9/4iHMjz1gHAph3v0JWnlm3HAWP/sEz06K1xk12gccwLtW59VNosq5mZFhmHodV + xukx1ML1N6y0q/UE3HE2O/mkPOpgP0POgF0Hl6ny3FQP0WVFHH/h5ceM9KOAVLnkHX6zDopna/X0 + w0947/dkLPflyFjv2+tH3/W0abXzptVq6xAXZ/kZLT9zZwlrjGJT70Vt44aIzcMmRVGhn9mFzCKX + v+FIbMMN7Y0LIui5ePRv3rle5k0vmOIoO3sQT/lXNwRvUHBQ3H7QbOEZL68WR8kRb+UtWNRh4Wfw + mdDyMt8exzR9Hd6mXvqEC2Y/piDf9+Htulu2p2+/4bbpv92Cfx6Buz4XaJPPrk8cRq038KrLpjH7 + oY0z+uDdxcP5rCsNQlZfpMh5eY6B7UM/+NAYsnjkxzRclc08MiBp6M2jibwteegv6iA7+XccPf7K + j/YCxpBy5NHjN854zuNp+9g0ffb05668wdb3zf1+eMNa/2sskg4rpsDk2HyzMea9WFFK4CTvgQfB + rCLiy2ZofhJnf/ZvXoDqgfOWicumoj55BQjZZswj7BiJ4qxBTUWA7QWQA/MD15qAWl1ZFwhpV/hM + N/3Q3oEATT8pGlj2whHoa/ANnPOFXjATEcfvT73+qRdPr/qci+7Xw4qRfvZFh6ZfedrF07990vnT + I85FmzLBfoWsPLNuOQJXeeJZZcpxk13gMYyLuOBLeyrlT6hRP3oYh0zGmXqAQeB4su+TR4rQaxCP + /cgJbuQZfWQ/Qy6U4iVv8LT4ial6iG5kNObTjxnVn4TJf/IOv1kH49hB06HD6+lf2Pq+uyuk+4z+ + R/HT7KvpWvHjgdXME382AsBAXLIYs1nCjs3j1dgccr0pcHHJt8Dv8lcOzTfgtHecQeTIBDDvXA81 + r9JnHuGA87osO07mu9QHKvExFr5o8IAElR9NhIu6SjSw7BZ+an7GP8/3ksOr6Xsef8H0Fx513oQf + WH/Ar9u319MPfeSu6Yc+fMd0J3+Ya3nF5qz+aJuV4c/6qssb+nFGvRcP57OuNAhZfZEi5+U5Bi4b + /eFDY8jikR/TOF7igpY0mYH8UBo8UudtyUN/WOc6XCiTTTz2RFPHlfOc4GyTY8K4wQfDBY5/SXr1 + F7Zf8MRXSXEf3Bj1fXPxVx1Pd34ABXvoLgfcLEx2MdZmgkFviSx6H8u+kwdf8XR+4mKNhj4dRRlm + +lxshlkKBSZa0uEh0yC986kB7qDHh+0FwC14CZdkHmpt3/Tya73qEWEKJzgmmqLHVXyJ04RvI38T + pnwILzJ/8VHnT9/9uAumy07hxxSai/vk8aN37Ezf8/47ptf9AX5WMQqvTaR1yPXCCO+z/ulyLlgf + N0Xb+cNeMM63qzYx1xXzlMfKVpiaz8MqgMbJT+Bor8j1gBsv89mP9cVjgFHg0foDXnHkBFmo14hb + PfT53X6CuIbiFU/WOfJV3vpHWT+xfdH02OnrrrpPfrXyffcl4erOl2IjjMOKxcOlzaviQs4x5iN1 + LxGb8Dj43LTk5CVeWOTmSz+Js2yceWFUD5y3bHvzUC87PwhvfcxLT++41Bw1oCdMaHsBcGvxSZ/8 + 1ONa1INTwx8lXOmH9hU/eVOk38GrOKKZct5mA8dN8oxLD09veMal0z+56sJ9eVgx9kfhr/q88skX + TK9/ykXTlfHjFKpv1i3HyF954nk2aqGjPgs8fdSlgvb1sma+nlln4PDR6wqNDOQOCq+j+Qg0jx4U + YK1ftzMD7mGHJ/sZsiCcD4Li0cPA2T9xpFOkMh3zUlhPPsZBWIunf/J1/IkTUHz4OcuHHr5leqmE + ++Dmqt7bxPxG+7Ttb7Rv4o5iqjosXitulnI2Qq/PEG1U0WnXryVPyG0VojmyuWAsR8HDQeFQ78V2 + eKUIfZoxLoevMOQv0sHE7uYyTzVJuU//GEFY+shvyPJSDtU0Lf5wL/uKh3EET1jP+B+BQ+D7rrxo + eu5n4P/MnUYXvzL88d+/c3rp+2+fPoXv9s76A3moLJvGXLA+bso79PVmkhjOt8v66BfMU+bKOwLK + Xq7CaUJA40KuaUVe5vHAPkNfSOJoGWJd0rt9jBNhTABV9hWe8xjzVpRccSQusiJv58tKt3mot4+t + VvfJN+DvozcsfKOd/1ewX23zcRW92WIErjYlnlU64HPchIdBZzcfLIqH+ly04Cfh0Jcj8Yiu9Bmf + 8YpIfN2e/JQjjNJH/NFeytMe5LD8S5/8wZF1gZh2hV/6ob0LFLwpGlj2whHoK/nOx9d/L378hdNv + ffGDT7vDipnwr/m8ED/+8LYveND0rY/CX/PhxKJ+Kg+wszEWTPVZ4F2huGs9Wz/VtOuLFdKM6wwc + PrwcSz1gULjux+urjDN5w6H8hB2mdn8SNM750E/wtPiJsH/rtS/E2+fpl36Cj3omVDiKLY4AMh7z + hZ3nDx3eWd8n34CP8OzsXrnnN9rbYbGLN4rp6mo1lXQVFQZa/ByB92eQMapIUbTiD97i6X4IYrbV + PLl4nI8yzPReHKdRirCP+GA3S1P+TOf42cRjkTOAis/hqJnwGIbhV6LjKnyEmfFy3oVKXIoBjLyE + kwPfnvvw86eXXXnhdMX5888pDXLaPfKv+fwN/jWfTx3NZd485oL1cVO2oR9vHAHifLusj3XGPOVc + Z8KGG6wrPqTn8iROgMBxWguqB9x4mW/E0XgMMAo8agfATc8HPMkPB9qFWA99frefIK6heMUTfBHf + bj/IZDXhG/BPepUI7qUbY7j3rpe/56HT4TvejQ0yvneV7FE8bT4V14snGRgVo4/Aq8hZ9D4mZxvN + E4tJngV+5ld63LRGLsGAR/NpogFDLj+lB4ZX6YM2M+K8Lmdoe+YbGZc+UCnHWPiiwYN6MSaEY7My + 7sg7Rrfg8HMVflTghz/7kukZl927P6XuyPfH/T/hr/m8+L23TR+6AxsmNynrgT+uRxuXfQh5dlFm + XZc8wiUjic2sT04piij4rA6e7GualUIBUjRrO1yKBxrhGX/0eciC8BYExYMH9UXgKg/JI/75PHnM + 6PhErAnjHH9M1Lwe4pZ8+Gs7n9i+c/XY6X+5974Bf+9+SXjkzu9FzD6sokiVSCtaNkGNAGWSsYRD + pl00TY0kXfC7uK25wi5x0mvNWzNozbw6oit9LErIvZsqTvG3MJos2tgejksB49bii64YeqpHM+T8 + 8EcOXOmH9hU/7VLMfGIEjr9N4Z981sXTG5/50DP6sGJ5vvIhR6Y3Xf2g6e8/7rzpYvxMBqvg9dhQ + nw31JocuFbSvV067rliIgNnDOKyWesCij/obltdXitBnnMlrf/bT+gJ+iychGMU36EJudshHdVC4 + epL16K/II8JnPjKoPCnO+UhgHPl8dT7MPnTrnJ179Rvww1N6PNXxJL/Rnif+bITPLGGN+ZkNo3dj + GzfE6DeMWEzydbuQVUzySsatHohPN9SDRxMNGHL5KT0wvEoftJkR53VxNK/c7tIHKvEx2p/pTcNA + Iz9OCNd4mx0f/9KjL5z+zhP5f/7u3c9NjnZ/3//grp3pe/G7t17zsTsVqOue64ORmxhF6uMsow2H + llaR87l+NAh5HFrJQhyugHszpz/7F4/8mMZdgrhoFuaNIOioN48m8rbkgZz5EWL/Hnv883kCTcj5 + eNLQ44+JmjfO984Hhu3tra177Rvw92IX7+gb7ZVkJRuphCx9NAk3W+IrScCzKXIsXMO31ZQD80Qz + YCb9JM5yX7RyFPaWHUfEhQCSVzwhj7iol7kfpM9WoGLkB0DJso+uMH9wtPxy3vFs8EN7BxK8KTqg + Z1x2RG9U/+eTLz4rDytW9PJztqYfxm+U+LXPv2S6Gj+2cR6+Md/rWn0FbM7TThcXNtbD65XTrq/X + M+3Aiw8vx1IPOyi8jsfrq1i/6gv7s58Wt/wMuVCKl35aHyz6abTLyGj0F+Mmrxn95sRnT5h3+M16 + GUc+X3M+zK2nQ4e2t++1b8APT+nxVMZXvPO5q63p512tPShRPOrzxJ+N8EkrlyzGbJawY/Mcj198 + uZjkW+B3+SuHfCA+6bEo5NFEKBhZyOWn9DJv+swjHBCny7LtAc+MSx+olGMsfNHgQeHEhHBRV1D8 + 0QsOTy97Ev6CMr6xfnDNK/CxO3emv48fg3gtfuupVmNDP84stEu9SYWHMu3iyXDioBmHVrJwHpfV + aF8easEHIm96PYhO7gQnjnoa8yqCkAaPJvIWBMWDB/VPEM0PE3kwH3F4cjx8MGHJMWF7xx+GMVSg + Jc/5WJlDz9v+xitfZ+ZTv9/zN6w1/l/A1vrvKjlsnkoyilShtaJx8y/xLkYtjfSV9AY8AEXNB/uN + ZkiZm3nm1zjzAlQPARO8NUPIvZsqTvJKT++4mizaaDLHJQBuLb7WBNTqyjwhpN3wF5j0Q3s5Yh1W + 0/l4e/h7V148ve2ahx0cVlGq5cC/k/jyz7oQP3h68XTlBWj9DfUuG9W5r5c1uS5YIU1YBg4fXg7P + Dz1g6pMT9VUsZ/FmJF7f9Gs/jisRHKWXn+Bp8ad+tIueZE47z6cfTSsfKVo8edgmn0bquc/imvMx + LiqgX2+/ODH3ZByeTpXl2nc8G78F5/UKOoq0kSoOjzzxZyMMVLQ+RjMR58OgjRsciA/FU1HJ0+1C + VjHJKxm3enDNHX40n+wbMOTyU3pgeJU+aDMjzutyhrZnvpb7YhOmuP1gK9i3MJsf8xL/9fjrNC99 + 0iXTw/G7qg6uk6sA/nri9GO/d8f0vR/E75fH97q0Lt5dgyD6uTYhNFo14WL9iA55HFpJwZXDpQUk + bBxaapfk4Qi6GADPQ8TmjSDoBk8irFjw0B/7R37Sv8fIJMzSX4sXmrSz/7QffAp4hhOd7EadBMBt + NeHne581feOTfs2oU7vf8zesnenF3HRKLkfGEkWqsFrRNuG9mC5NJpvjJvyS38V1HN0ucY4vix7h + GagQFR5kx5H5WO7dVHHSQPjIsMmijS51XMSwGVp8kpM/OFr90m74C0z6gf0fu/TI9Otf/NDpFX/8 + soPDKspzsgP/Ujf/cvfbnvGg6ZuvOA+/7jk2axKozn29rMh18Xrm+gGHD6077XR5lKg+aYdMyNVA + kiWJh+ZFo7iiHzkvP0OWK+Hhr/O0+ImZ9RH6jEzzecrk1bT8KKHCUTv8Zh0YD/dnXsNP8lMDO3wc + Wq9fkrhTHYenU2H4t++6Bv989X9Wlgw6irSRKvR54s9GGGQJawReyXfe4/CLD0U5G96wHoZvJr/s + yZdOf+6KC1S3jfU+mLxbFXjvrfz9W7dOv3Ej3gPyapu++hI6b9ac0QRu3pSajU2fhwL3NBfKmzn7 + uvHID+XcB3n4wU5XEYTUDr9AWGGC4sFD7jPq54fJiH8+T6DYIk9ZaqLHHxM1r4e47eZz/JzfXq3u + 0VvWPXvD2tp5MYM4mTcgrQYS2gtfSQLjRc9FOzl+8bJpuEjNz9xvX7RyhAfOW7a9ebLJotti6Pxh + lwRqyog/utRxCSAHFV90xdBTHX7xmPOFR3z86yd/4wkXT9d9xcOnFxwcVizqvXY9EX+Z+j889ZLp + 1fm7v1pfj6091gVP8u11wrrho/rOmtBjUF9Qf7y+6n3T+kp+Wl/Iz5DlBDfF0fuvxU9M7yO/EY34 + R9zkNaPfnGSpCdFl/MFnLQz4UhHX8JP8VICXeQB26B5+L2t4So8nO77yHU9BDG/FCe4s+5vQJo7Q + 54k/G5WSW0DFoyxaLHLnjUXYTO+inKlvWF/7iPOnf4i3qsdeuPnf/NtUk4O5U6sAf+fWD+N3b/3g + B2+f8APzY7ODTv2pXZ2dismQx6GVfr1puVdp6M3M7cK+thyKPmhz20/nkYEm7Mc8ibDCh4K2CSbs + Z+Dmh4k8hNnisM2wlRchnujxh2HZ6yFucz+YzPpgxP+iW++st54+fdNVb+02J/t86m9Y+NVESBm/ + qp1FiqLkSO+VbIQS8l74ShJwL1YrYvLmuIFfvLAsnogr47DfWMQMz44UoMKDbPvMx3Lvpjl/S1P+ + LIs2ulR+7QH3Fl9rAql5a/ml3RPwmf8/PvNh008//aEHh1UV6r59OBdvsn+Tv7/+Cy+dnodfD+31 + tM9cF6y0JixjXfHhdordXnrATqqvBBMPiWO78EkE6Xevw0p6+Qme6P+yg+z4SKcnPNBPzqcfTTsO + wloe9YYYdtbCTnyUlnyUOev6MAAcGPj1w9t/h7Oncimku23It6tp9VY6V7BRHEXXgp/xcp7Fwagi + 9RFABuKSxdj0xZt+ZsQWxAuGM+UN65JzDk0veRK+IfzYi/fFb/3cUPKzZupNNx6dvv3dt07vxve5 + 1KfahdmxKEPI49DK0mi3VmP7cMj+p1l0vPraNGbNQ6TzQCM86aLPQ06U90njgT73GzHLw4lMu+cJ + 1HTEJ4QmevwxUfN6iNvcDyYVp/NmAXiYnWQAAEAASURBVKHnW9ZV0wuf9N5udzLPp/aGtbP+Tnhl + bZ1UHC48vBisrhwtRdB74ytJ4LVoLDaexZe8OZJzwW+/0QzNLnHmMZ95AaqHoIPsOGbFNZD+Sm+c + Zah5lT4PX9ah1UNd0OJrzSJ73pAf/2/VX37cxdO7v/IR01/BuB9+RXHFd5Y+PAP/N5a/3PCf4pcb + 4gfm43KfV99h3dxO0f+1voBX3xyvr3rfRD/K07yP9jqsRn8Hj/px9J/7OnjRZ3gyO3AjbuI1DS0e + qCgcxTmftcQJKMPhJ/k5Dbvksz/8nPn2d8ngbt6Gp5M1vPa3nzBt3/Xbq62tw96koIji1LiJi0kx + WIxKqo/AMxCmWGPTF2/62cB/JrxhPfOh507/4ikPna68+MiGDA+m9kMFbsQ/oPh9H7htesVH7pi2 + 2Y+8NHpTqn9j2h1NPf6o/Xk4ZP/TrBShz/7PQ4TkvBpO0uCROm/aH+QNHjzkfiNkfpjkTlvOE2hC + xydLTdje8cdEzeshbnM/mIw8xRcFwgF2bGd96Ml39y3r7r9hbR/9bhThcCbjICKJOGQii55DBO3i + 8ESe2SmnWCQ8O6dcNFZ/N95FGC4cj3FpT7vE2Z/9R83SkUiipoormyqbTMDoAvIM/qL3Q7iTPprM + cdGFm67soysoPxp/nea1X3D59Ctf+vCDw2os6b58uhS/6/4f4XeJvRG/OPCL8Lrl9UXf4aP6QpF7 + 1598X0U7Vl9k+qNvOGM/sR8Swvnqz2zrtm9C7/ggcF+Unx43eaGSlvZ+0l3i8Ou8HY/5iHIcvQ7m + c32iQAGcDm9NO3/dwsnfFdJJw1/+2w+fDt31uwjwME9uZbdp3EQYuDzxZyPwSrKPwLMoG/1s4Bef + mobF4Zq0+EJOPusxWQ9eQ605/ZJH9jI0MOTyU3qoeZU+aDMjzutyhrZnvquJ/8PvxU+6bPq2J1wy + 8Ru9B9fpV4Ff+oM7p+96z63TR27Hv5/IvkEKuendCZzAHyjUf3hwH1oORR/24BGBCrTXoSXHAXMc + 7mP642X/za8CW84TKLjw8aShxx8TNW+c73M/mJN/590LtLPeuX197Mjj8fuyPtbtj/d8996wDh39 + W9iY+EegMukYsSlVlBzpcaya/beicXMv8ZUk0Mmf4yb8kl982Qxyj6rzsJj5dbzmLUeKL2qquO7r + NywW/QWPuWi67tlXTN9x5YMODiutwOl5+5rLz53e8kWXTd/9+POnC/j7t6rv85BAXmpDHmbZ97Fv + rNh1WLESRROnXfLudVhJH+3u/nb/lx0IR9/rSQWnvvCKT9M6NKWoQ41hjvyKl3rus7jmfJkH7IQD + KBIjbmvaOn9ra/s70/ZkxuHpROiXv/XIdPjIR2FwOZdi9gbDIPJwaMHPKEOvNwwWCbKS40g+/JmN + Ta8kw74Xp/P7zSWagnwL/C5/5dAlGHAX1/YicmQCmFdNE7IXNXBZBoqZEXG67PApl+H7VE996MTx + 4DqzKvAx/DNkf/e9t07//vfviMTY0biisb2Zs++5d0uhDaBtJHgeIrLuBJrY69DyPvGZoG5r+4yG + 9t/8KrDlfLgLPO2A8J18+HDcnB7zAsRt7idxYefAhBTPev3fd+6884rpLz/taOfY6/nk37AOHfqT + 8OXDSjG0YLEp5TxHeotkynFPLnE5Bp+3tNZOfFl0HlIn4pc+itntMg7b98WBUwMVosKDTFwtSsi9 + m6w3jvaVJh9CxoAlZn3G4n4G/nWaa59++fSGr3jkwWGlip95N/4LRD/6uRdP/+/Vl+JfzfbWOvm+ + inaswyHrM++jvQ6r0d/Bo34c/Tf6lm3pDqWHMZ9+7Ff9S1iLp/ZF2FnLvheQ4oKPMmcRB3nkVhOB + 48Tqj0znXPAniTqZ6+QPrGn1DXQlnxwjyHyTmY303JKgmPIMh2w6z5JfMnkSl2PjE7foHRn5ut3c + r+O2Hkb1EOFBtn3EFbKAiiP1HrUGdMur9EEblToP/zrN38L3qa7/E4+env9HL9KsDQ7uZ2oFnoYf + g3jjFz1k+iH8WuoH4+99uk+8ad3/0T/ZQOqz3jduJ9dn9DVlHRqwy32TNRy8wdP3De1qX0DQKeLG + HfPpx4z0w/DoUXeJw2/6N07AwLX9R2uZwy75PBHxkA//AtK0/Q0yPombozkR8F9ef9F0ztHfX22t + LmJoNJqNLEYcJjmqKBHcLnrOs2iLsYp3qvzBVzydn0FE4EOfjqIMM70Xx2GWQomLlnR4yDRI73xq + mJ73qIum7/+8B0+PufDgxxRUn7PwdhN+DOIfvv/W6Uc+fCv/GXftE/dh7CA1UGwH1Mdv5nqIahlX + b0I8rPBBuV/ed40nGzNwZR9u1bh0A726W7jYx5zXDhdAbkQXeE6UnXBg2OXH8ZXdLr6RBzA3r3cO + P/Jk/rGKk3vDOm/7z+w6rLhrcXHTMtjZaIX0dTsBvg4RGKimPAzwPONNPyQNPj7yEg4WxRNxJc48 + xpkXRvUQdJBtn/kMXvGUPuYly70JQr7qkiPTr1zziOmnv/CPHBxWUZ6zdXgQfwziqoum38Qb1xde + dlj9xb7Lfh19Fe2o48Pt5JrpOAk85nlYRZ/3mo7+zrZu+xJA93Xw8hQpP22fiTe90p7P9G+77tfx + U2s/AgnX+WxHouO9YWGrXjxtbf/p5DjeeHIH1rR+0ThRnQJlXhp5uFDO0Qrp63YC/CZ+lmrGexx+ + 4VCW4ol4MKEQzGM+82K6Hjhv2faZT/jPRdHasPgZV9jRAwgefO6h6Yfwg59v+6pHT19y+cGvKGZZ + Di5X4IkXHZ5ef/WDp9f88UumKy44pD5VA6pP3UfqK58S2bYwdmO6vyl586ec9R39nW0934/ua/vR + IVl+Wj9r/5hRb1gMqHDkjX3B2dxX1GNf5jX8MO7wRzvhakL25hMMf8dw56S+LByebLf7/mPvuHw6 + On0cwJVL5xR8okewKjoQOe5mcVLQ66RfjsAzkCW/kmcxkjfHDfz+DBKLSb5uF3LyyU855EOF5/gQ + ie1DoWIzDvPWZ5oMOAg+9DV/dHr4+fV3Nzh7Vl//7HfvnF7z8aPT33z0OdPXf8aZ+28h3t1F/oWP + 3zF909tvik2f/R1tDjIfSpbNHY3G/i/9ODyMoYINWoP4c7/JDnp2uw8bPcl0eciwz3kZp6eS+6El + fzOcYPbb5xX3eAnodo5Hvo6uj01XTN/8Of/dLJvvJ37DunPnz4FUv5UhU7STKJ6KlMG0IirI5jRk + FSHflHKM5Jb85SdxOZJ2wZ+LMCs+8ImzXy+CecFRDwETPA495WV874I5f9Hr4ZIjB7+imEvznz55 + bHrKb948vfSDd04fwL/I/Fffc8f0rLfePL395m2qz/prHCLRn2PQYcUCjfbmPhv7yofZkLOYo7+z + rXl4DdzoW9Kp8WU65tOPGeuNKE4vb4c5H5HGkc/XnC/zgB155JZ+OJ/7zHZQ4cemVs+3tPf9xAfW + 1upFSoXJgydT1RsI5TgUZiP9qSh8iCvkGY5Bt/klf/rl6s3sSLngN49x3S5xtne81oOjHoIOMnFq + CvKHHA8l2yz18/xCOiuHD+GnvZ/3jlun51932/Rh/iIpXLlu77x1PX3F226ZvvWG26eP41+vOduv + 7Ff1p/os2lE7LPpRRfKOMx7z+Kj+bEV0nW03+nPD/hKdEGZXv8c6gTm3Ff2w/+lRd4lzPmupEDBw + cU4EkQfYJV/NZx4yU+B4K3pRSHsOxz+wrr3+jyHoz2PIdXLiOWWycp5ZzkYreB9XJDXDpR1Qm/jL + T+JyJGsrkkUvQvFEXImz3+bH8OIRnWrv4na8MhZft2feZd4eGM3Zdd2Cf9HhJfgHS5/xllumX7/R + b1GqH8rgOro/WPKfwT+x9bQ33zL9U/yCvLP53HJ9ooHGoMOI3TPae/S15nmo4CPryzlerrPtQDer + e+o9Dwn7CAhOC1d48WracVBROLb78Jv+7603LATCsD5vuvadVzmCzffjH1jbR7+BJ7dKlmOkUCc+ + k5ezNtIX5/sVsj8TMLg5vvzAZuav49IPeRf8jse8ac+4Emd/FDOfckQ2wwSnPuMzXhEpjm7P+MOu + CER11txY55/GAfRUfPn3zz9y14T/g1+X1yPq19aN87fhgPtefLn49DffPP2H/3FSP+BcvGfKg+sT + DTQGHQrMcbQ391H0I+d5aDSZWF7i6zzq12YHmcsjXu4L8PCinefTj6YdBxWFYxRzPmu5DwSU4Zwv + /NGOPHbUcOSTmED8TNbquN983/vA8i/n+7M8SZVKjuBPma58siNpBs3kM/gcI55MaoZreM6Lt/GX + n8TlaMfJrNF+7b/bzf06XuvLUdhbdhyZT+SnYjO/bp+yzKmIh7NjePvNx6Yvw/el/iq+xPsEz5xF + /l6PqF9bt77OH8VfZfnG62+bvua3bpn4j0CcTVf2q+qmvop9pV3dy8m+GvtKh0aTs2bi6zxcj011 + F512gExrPbR+9GNG+mG/I5LAUWxxBNA4AQM3zgtZyxx2yZd2GM0ns0p4Z71+AYIYhKHOYe8D6xXX + fSkMH9tPTPrOVHWiU0ZRmOVsJDvn+xXyDJd2gjOpOb/kk+R3PI6j22Uc9ut4rYezeohwIRPH4na8 + gIoj9R5pH2m1h570mff8P/Bv+H3bDbdNX/62W6d33Ix/z48psgmrEM7Z65F1inpC5fpipBll/HnD + TdvTM996y/Sd77tj+sP+mgbdmXq5PlG3MaAe3jejnK5U1RP66s9WHOk7j/p1Q91Fl5Vv6yGHxJu0 + 3ohaPN1vj0frH7FwXuxB5AG85LFCSOOGv3S8tVo9fnrVO69uqc0e9z6wVseeS5J+Art00WxsUlw+ + 2Y1LfCg01O0E+PIDA9eUSZ48v+KARfHkJpr5NZ95y5FCFAwK22c+4V/FRr6lz7g4yrw9hHyGDTxH + /tXv3jE95Y2fnn7i9++KdUfazJNdWYVw4l4PTrNuUU+oan1oRjlG/Da36RW/d+f0tN/8tEb+Q6dn + 8uX6RN3GgHo48VFOV6jqCb3fTOYFcp29DKrrXnUXXVa+rYcccp1cdcVBWIun++3xaP1jsThf/mkt + PvCSxwohjRv+0nHkv+eXhcc5sFbXkKSfmMo1UqgTNppVcuAVEef7FfIM1/DlBzbywybH8174XiS6 + EQ4WxRNxJc48xpm3HNHcMChsbx7VWHFLocDm/GFXBKI6427/5VNHpy94003Td//O7dOt+T/4VF/k + z2zZlarTSF31hui6Rz1DVv1pRjnHsOc/C/id7719+qK33Dy9of8bgcCdSZfr0/sq6+F9M8rpClU9 + UTFu6pSzJq6zl0F11foMHPWehwXXyysX/R7rJF4z1htR4RjfnI9I48jna/jpecCOPBWA/ZmvDPUg + 3A7Onj2uzQfWtb91Kcg/l03YT0yXzqnWCRvNKjnw8qWiNK8hz3ANX35gIj/A55hx1EjaBb/jmcdL + fOLsl2LyliOyGSa4i9vxioj+Sm8eyzIvPyGdEcMHb9+env+OW6bn4ntM78fPU9V6MLusB59bnSny + 8npknbwuOS8eCLMx1tN1X+l7Wl/z9lunb7ru1unD+HGJM+1yfdRQvb10KLhOmbEPl6onDw18pFyo + XA+3qfWb9lcdGnzw+ox1Ja8Z9aZDhU4Z4hjm8Jv+jRNQhpwffLajZfGFA+OGv3QcuM+ZXnH9gx3J + /L75wNqersFJKbZ+YirXSIHzvDQyyGjanFcTCxG3E+Bpt+SX3HnTjx13dvvnYiZP2GUcjs/xmhfm + 9cB5y7bPfCI/AsXX7WEQ0wpEBLOQTluB/xfvH7z/9unq37hp+r8/cZcKM6srM8t68JlNuMhf9RaM + dYp6hqyy04xyjmHvdcr6r6f/8Ilj0zPe/Gn9X0XGdaZcrk/UbQyoByvSy+kKVT1j86ec9XDdalmw + HHvUXXRZeeJj36n+rLsZFYfK7QnRYcXSb42Ml+sf15wv84CdcACFA+OGv5o3cms6tP0lydnHzQfW + tHWNTlCQ9xNTudIn/kifY+DoNOfxAG27Qu68HV9+YJL8ORbuOPz2O49XRZj5ddzmLUcKUjAoHEfk + EbIiIqD0kb/kyDH8hHTaDq/9/Tunp/zGjdMP4h9a4PetnNairswu68FnrMtyvb0eWafRF7vWmeai + c7/ILtc5Rv681g9+6PbpqW+6WT/HtegsRnDaXa5P1G0MOBKyDpkS5VY/HhpNLlSuB+BVz6wjQFV3 + 0Qkh0zGffsxYb0Qtnu7X8YOXeq5/XHO+bAvELxxAsU+MY15lqIeGuyY0s2GPA2vnGp2gYOsnplNi + kIzRQXYcvec8HmaOUt4Lz/klf/lJ3hzJvOC333m8qsYsTsdtXnDUQ9BBdhyRR8gCkqf0xlmONBfx + xOxpM7wLP6bw5b954/Qt190yfRw/buB6Rl2QaK1P5pn1YIZYl83rkXUafVE8NMMfdonG4JXfXOcc + gSH/f8f/ofzWd982fRm+v8UfqzidL9c36jYGHUaRbqTnChnPennzp5w1cN1UplHPVj/qXWdYcL1U + eeJzPv2Ysd6ICsd1mq8jkcaRz9ecz/HQsvjaOpuvDPVQuGnz97F2H1j8/tVq63N1gjIZJg2qGvGc + Mj10HJtKshW8j0tF2hu/ib/8JG+OZA2+dGC/83i1iWZ+Wx65ZqUHExw6jsgj5FA0febRwljEk3Ht + 9/ETOAT+99++ZfriN944ve3TPARQmMo781vUlUkxX+L4jHXZvB6cJm70xa51prnoxDTHN7vO/45b + tqcvx6H113B4/QHiPx2v7FflFeVTHVzRli7r0uoXm9/2I3PXuZZlXkfAqu6isydaj/n0Y85602nx + 8DBJvzVSz/WPa87neBS/cADVfvMhVtsm5ws3bfw+1u4Da/vQNSDFQYkgQNJPTKeEafiVPsfAJV6x + tyS63Hk7vvwAnPw5Fi79kHDB73jm8dIucfbruM1bjshmmOBYFHx0vCJSPbo961P07UF0+/52DLH/ + 6w/fNj3lv/7h9KqP3o74ETILw4eogyTORz2odp31YJzUMuBTXYlzHaOe0FIWD55nY6znDH+c9UaH + Tj+Fn7J/Cr5M/CH8NZ/T7dxyfaJuY0D1VXD3o6rphTGeq9P6s6od69J51K8b6i66rHxbD9WfeJO2 + Nx1HgfnaF5jp8WifRSycF3sQeQAv87Ii+DKPMvR84PAvreJs2v19rN0H1rR9DaPWCRqjc6TT1mQ8 + DCjHoTAbrZC+bifA18kMg5m/k+SXf1gWT9hlMR2f4zV/OVKICg8K22f+kZ+KiHxLn3lzjAzrIeR9 + PPzXT901Xf1fPzm9+IZbp5uO8oc/mS8CZmHYVZGnJOW3qGsqiOMzu3KRv9eD0+SLegLq+tqO7miv + Mexn+Ga35E/51mM709/H32O8+jdvml6vH7kH4WlwuT5RtzGgHqpopodMXCHjKXnzp5ypum7Qs9y0 + 2qvuohNCpsQVHk+xDPIjhfwn73wdSaB4uf5xzflsx4iMo0Hml3mUoR4GDuLW+prQ1rD7wFqNn78i + eT8xXToGyR50kBoDl3ixtyS6vBe+/ACc/DkWb/oh4YLf8czjVXFmcTpu85YjspkOCsdhHgaSvAKU + PuYlyzwI4nmfDr+LH1P487910/TVb75x+p3b/KXUyBdBIx8kotF5Z1qLugoWOD6zCTeuB6eJi3qS + HTLdsHtm42ydsv7DbsmfcvLzt0O84J34EYy3nx5/zcf1jbqNAXVhZXo5XSnjWTdv/pQFFp51tp3q + ulfdRSeETMlTeDzFMjgOKlo8Oiz7OklLvwJu4Ms8wEseO2q44S8dDxxg6xMdWIufvyJJPzGVq4Ik + l4PUGLjEK6KWRJf3wpefxu/aws9J8DueebwqwixOx23ecjTCY02BV1PQLuR4KHnERbzM20PI+2i4 + HT8O8LL33To99dc/Of0ifnlcxY8YR74QqFDXe16S8lvUNRWsD5/ZxFUITnR71tH2OS//EGZj2Gsd + E5+jDXkf1x54/pDrF+HHIF78vtunG/l17z69sl9Vtyif6+GYRzkpt/rxUGlypue6eRnEQ4JWP69z + LBPm8STTMZ9+zFhvOoVjFC2OrD/14gs7zJd/epEb2AlXE/O+o+mMjzInt3Z9H2v+hrX4+SuS9BPY + KZkrT3iNgUs8XfUkurwXvvwALD8stmiQ/knwO555vCpCFNN+GVbylqMRHhxabx7VWPZSKLBhz7ha + muFHZPvo9rP4N/Ke8l8+Mf2j37kFv85lUR/EOfKFwIKz3aIOkth9mJjlnQripO6F4IR5a8z1i3m6 + od1snK1T1j9GE/E+ruPg+eNaP/JR/HjGG2+arsVf99mPP77Fekah+4C6qKKod6ZqnPGsmzd/yoWi + QSyD6ip51G+sH91m5b1Ohdc6m1FxUNHiub/fsFar9a7vY80PrGm6xic1isKkkHQ/gV06pyC9ch+4 + xCtlFUVPvoXceTu+/AAtP8DnWLiIZxO/45nHSzsvDgfG6dG85WjQlT7zDzsbLuyTT+blJ6QHfHj3 + zUen/+mNn5xe9Fs3Tr/XfvFU1bmthz9zImQWRl2feWdai7oKFvnzudWZIi+vR4xt3co/MFqHHFs8 + J7Pe83XN9Yox/N+IHyT76++5bXrmm2+a3rTP/pqP66OGi77KeqCuil8Dn/Cn5cVDpcmFqv4OHsnN + DrLqLbqsvNfH8+nHjPVGJP/EkXfOR6RxZPBV60sDXB5gR54KIP2Sz3b5MHCcl8E1gdCw68DSyQ2W + PpKzTmg8p0yGjqNTyVbwPi42Na698Jv4y0/y5mgi8eXNfu2/28GhIPbb8lAtoCo9nmHoODL/iNeK + po954SMC8dDzA3+95Q/xTfX/75PTm/7wqJosm4GRVZ0rbzfTCB+FiToYz/uirqkgTupeCE5k3bJO + oy/KPzCs1lgGMSm+6qPjrPdYN8ab67XZzw34C5DPx/e39sulTZ0F5xjlcz2yDhkt5ZYXKsZNrX5O + CEbJnUe8A1d1F11W3naSIh4N5OPKUKEVIo7inM9aKgSkqDgGn+0Uf/KFA8dDPpklcPjVPJj4PfV2 + jQOL37+aVrOfvyJbPzHJoWA4RpAaA5d48bckurwXvvwALD9swvRzEvyOZx6vqjGL03GbtxyN8KBw + HOZRjWUvhQKrODkf0zMCCQ/s7SZ878b14Hpp0SugWfyYHflCYGGE97wkLjgUSztMOH+peyE40e3t + v+KBHd2QdjaqzmF3Eust/+lnA77ibX7wuC8ubVblG3Ubg9cLUUY5+IQ/0Y+SWL8hY0qX6tt5SJB1 + AaLqIbqsfJ9PP8EXfWD/xDGK4bfWUzjyhR2AYo8EPMAu+Wo+8yhDPQwcRSU0+z7WOLCOHf5jRLCY + jK6PzpFOReGROBvswodCQ91OgNcikg9/Zv42xCPO4Et+xQvL4gk7Lhov5+PR/JisB85btn3mH3YE + iq/bwyCmyZ9+9LwPbq4HwvKiV0Sz+ihsN1PmTwvnlXWj6aKunMp6SN0LwQmq0558Uc+YV9nxPBs3 + 4ZvdrvqeAF95hh8M++pyfaJuY/B6IdJIj0/40+oHmZs665tJSe48JGj1q3qILivvdZIkh+Q14355 + w8K/h7o1rY89MfMcB9bWscdw0ic1ioJkGf04mZEM9fjjnPm0GR8KDXUjH67Om/w5v+QvPxFHx4NI + fHkTLxeTi9T8JM5+7d96gOqB85Ztn/lHvAQSoKHzhx2DWMTDqQfycj0QFlcsuxABzepTMvOFwHoI + b5wkzkOxtJMB6yF11IfPcZV/1S3qCV3x4Jnu0u1GfK47OZf1DVl2ictR8Hm/kmI/Xc436jYG1EMV + bem6QlUf6PubTubkOrhMqutedRddVr6th+oZfQDSetNp8XS/PZ7j9xcjBC95HJhCpr35JNb6DlzY + Ma6tVf2e93FgrTzpkxpkbHKSsgnoMkc8pyzKhks853sSXe68Hb+Jv/xEHB2/5BcvIiueiCtx9tvy + yDUjLsOFQ9tn/paVsfhSH/PCy5wT8bA/BtcDYbFLuOhxzeqDuZEvBMGIj/ykp+GirpzKekjdC8GJ + bm//FQ/s6IbVmo1RP+FOYr2z3nvhndfww5j20+V6RN3G4PVCoKOdXCnjmY83f8qZk+TOo/WJPhZf + 7mMI6gfy0k/Opx9NOw4uEJ50x8DDJP3WSP1x+4vWsBMOj22dzUc9rpwvHCfD32pnw4G1vdakT2ok + wSCYDJuHpjmSO2QMM1ziOQ+FhrqF3Hk7fhN/+Yk4On7JL15EVjz0xyLM/LY8GJ4dKETBBHdxHWfk + R6D4uj35PT0IKtsH/MH1QHz4UB0ioll9MGeZdYPAegjveUmch2JpV/WQuheCE93e/iseOKKbdFej + Agi7k1hvB7w3vuKFL6XFoPbR5XpE3cbg9UKcUQ4+4Y/rz/C1yZvMOV7i6zwkyDqGnnUQL/eFeG3n + +fQDlbS095PuElscuV7kER9RS77wx3iFE6DhyCcxgQ3H+fC3s+kNa2v1GEHoHCw+sT2Ss05iPKe8 + F57zPYkud97yIziTYoiDv/ws4ul8eqZdLELFGXlkHPbb8hiOBh0c2j7zH7ziKX3MS44I2qJlTA/k + 6HogTla0uiLzy7xSZr6IlgUXvus5N+8DzlQ9+Ez+Rf7lP9ahy3ST7moMe+FOYr3T3154znc/DHM/ + Xa5H1G0MXi8EOsrpChnPujGv6M+WkOtgO+W9V91Fl5UhPuokh9EH9I8PFZAjZYnDb4/n+P1Fa9gl + n/yk3+EvEx64sCN+tX4MJV7+kvC160PTzvQ4TvikRhLRhBwZco14TnkvPOd7El3uvAxSsuDhB8/J + n2PhGn7Jbx7zdbvE2W/LI9eMecq/HROnplD+UQ9G1OTB39IMHpHtg1vVlV0XTcKwnF/mlTLrBiUT + E77rObeoq2DkFdr8i/zLv+p2nHUWe/fneGV/nPWer2vwN3zlGfwMeT9drk/vK5dfbyIIdJTTC1P1 + jM2fcuYkedBpnWvfiC/3MQT1A3lddy57xpN+642I/SA945uvo+ZP2F9EwU64INKQ+4x6XOF44DhZ + /j5z4hmFywfWrW+/At/YukgQJgNjn9geVTI2gyig5qikY1zgyeOi6Mm3E+DrpAc6+XNcxiPC4EsP + jmcer4ow8+t4zVuOBh0UjiPzj/xUbNYl9TEvOSJYxJNxPVBjrQ8WPZuBsVSdW13YJBJZGOEjP+E5 + t6grp2jA/KXuheBEt7f/igd2dEO72djiOZn1dsDhZ0P/VZ7hB8O+ulyPqNsYUBdVNNNDzK5U1Q+y + 18u4TEr6zqP1iT4mS9ZddFn5Pp9+zFhvOi2e7rfHc/z+Ih/iII/cOm7HE31HSK5/4cJO86tLJp5R + uHxgbW9dlU59UsfJxiTZDADWiOeUSbAJz/nk03OT98Jv4i8/EQeTkn3jS37Pz+NVERC/4VzNlkeu + WemBKn3mH/lZ0fSZd0szeDKeB3rMOvkzFivpq+pcebuZJAo26kQLwxZ1TQXrxWc21SL/8s/5tm7l + n2b4M5ZBTF7fxOdIHwv+lOUncTkKHn2LZ6VFjn10uT5RtzGgHlmHDNYVqnpC3990CqU6u0yq6151 + F50QMq31UH25zmZ03/B5xNP99ni0/jbT+pV/WsscvOSxQkj7Hf7S8cARNvpuOjY9hjM+sNb4LjwW + m5dPahSFMrz5JPQ8fctn4nIMXOIxTQMNdTsOv+FMas4vucVxPH7FC4Yer4ow89vyGIkoRMFYUzyw + aOILWYumOFLvkQEHfXuojB/QB9cDYXHFsgtZX+XnkQGOfCGw4MJ3PecWdRWMvEKbvwpBZbe3/4on + /QNDd2MZ+BR290E/iXwf3VyPaKAxeL1UhwzWFar6AVH9mRDhWWfWL+qqh+jj0KveotOTrL3+uV7E + m7TeiLRCyTvnI/LE/UUU7MhTAdif86AeVzgeOE7aH83yRxt8YK2mx2RT6+SEcR+dI52GT46xCTqO + TnM++ehL1wnwtFvyp9/iPQ6//dp/t8s4HKfjth5R1QPnLTuOzD/zBJAADRFnyU4v/YT0gA+uB8Jm + l6BueTm/zMsjmyTzp0XmSRvNS9/yTgXrwWfyG0hJV/lXnUZflH+gGBXtNYa97HKdcyTjgj/lvfBL + P6TYT5fidqGj3lkPVTTTQ8iukPGUuA6jnpmT6+AyVT1b/aoeosvKE9/XNfpAXgEkTP6Td/jt8Ry/ + v8gBO/I4ME6E3+EvEx44ouzPYfinGOINC//bkE1HCEcm0UbnSKfhM3E5LvCYJpGGuh2H3/Dd/Ol3 + GY84F/z+DOS4u13G4Xycn/VgqYcIFzJxagryhxwPJdss9ZHhIp6YfcAG1wP5sEvaZnd+zpPBjXwh + MDHhu55zi7oKFvlLrULxqa7yrzpGP0Fb/vGsOuYY9ZPdfdBPFdg+eXB9om5j8HohxtFOqHPUn6Hr + sGoy53i5brZTXfequ+iy8rYrvHiDL/qAHs3PKObrqHnhyOCr1rfWk/OwS76az31WhnoYuLADXuwr + fNsKV7xh4fSKpvZJDRBlgtk8ANaI55RJ0HGJ5zwUGup2HH7Dww+E5M+xeCMecS74FQcsK07qWZyZ + 35YHw7ODQSe4i+u8Ij8Cxdftye/pQaCnfXFzPRAfl5t1iGtWH8xZZt0gCJZ5cSLzW9Q1Fcyfz63O + FHmV/1iHLtNNuqtxtk72V+tuQt7HdQJ85QmLkf0wf6CfXA8WEBUYg9cLwUV6fMKfqIek1p+Q8xJf + 5xFvs4PMOohX/UBeyjmffjTtOGgg/7bTYSkC21nL+AWU4ZzPdoqfPBWA7c0nswQOvwpv9N3Eb1vh + 2pr+5fX8v4OPSqebPsMpFQSlEeCUSbAJz/nk03OT98LXyQxs8ufIKs/sGl/ySw/L4slNFMW0veM1 + bzkShWBQ2D79RX6MSHypj/mYHgQZzQM/uh6Ik10STcaoZvUpmflCYGGEj/yk59yirpzKekjdC8GJ + bm//FQ/s6Cbd1agAwu4k1tsB742vPOFLaTGofXS5HlG3MXi9EGeUg0/4E/0oifUbMqZ0ia/zaH0G + ruohOlaED66fJDkkXtOOg4rCMYo5n7UwOG5/EQU78tgRJ+An85DICc8XjqL9OYzVo/ijDVvTkbse + D/ShdOqTGmQMgqRsHprmiOeURdlwied88um5yZ234zfxl5+Io+OX/OJFZMUTcSXOflseKt6Ik3DV + Snlm/saHouljHgHKruXHx/1wuR6Ij10SzcC4ZvUpmflCYMGFN04S55d1TQXzl7oXghPd3v4rHtXX + dnSXbru+1jnX3YS8jysKL7vE5QhU5YlnpTUs98WT8426jQH1UEVHX0WFqj6Q+5tOJuM6MG/nazn6 + GCDKqrcKrieZjnmvRJTVcRDW4ul+xS8tHQq4gc/xqH/IUwFkPNF3ckP/9JY4So7//2fvXYB2za6y + wPc/nU5CEjAoXoay8IIZCAm5QBDRGhFrlKmxykJriDrKCASJiKMU4zhT5eCoU17GKRwZLEdrnBKm + RDOiI1rlBS1jB+SWEGIIBJAAIUAg5Nb3TtLd55/nsp611/ue7z//6aS7z59071Pn23vt9axnrfXs + /b3f13+fPl1l3LG9/y0vwP+b4tqvoitJ/aT2k43dryczgkXhFoRTGOgK1/Pg41KjmjrFT3/nwVp5 + ePlqv3mTxwF87eF69vUybt/XyCPxEN51YS0486Z/48sx/LW/6JunC7rNiz4f3hLqUKN17r7TLwCC + EZ++09ZBV3Ixnjiuh840OTq/cKVn7TMN43bzqOdWznudG+vIeV2cB+mu1Mh9bR0tJ3SRomkPNVup + 1hN238/Rkfw6t9L1It1FF+V9TrKkP/UzaX8jGvXMvLOem98v8oGXPE6kBIw3n/Ml8cJVnHB1n+44 + /7X4Gdb1ZyukLvXuyUwwLwMAPWMdW5R1WXdxdvB1jZvwE3SKv/NUHWxKeRywuCuelTVP1YUN4Vzf + yJMzaz9g1BQ2RZv4cgy/ccZXGcVT1m2fVD+q4KXIZWBR7q/qb5v9wqDgwk8/9w66CkZeoc1/6L/z + c3+cW+cXq+P7nLGnuOAzJx/njMp3Ef6YJ2FXZVbduUC6d5LT54Uil5w+GOOp17ifoxnr4LjWc+jX + eohOCEWv/eQxqe8N19wPb70vZNc+/ciTsedzHDtrvnFufp9VZPbFxwTcP96768/GA+uaH1iV1E9q + iEIbJH4SsiYmJYW55Kc9cMFjmw5N/VL2RfhT/Mp3i/yuZ1+vTn2Xd/SxGlGJgkl7i+s6qz+JSD1m + fOzqsPJ0v7d5YT1QL09s3f51jkMXvQmih/DpmzMbOejKLTqoh9wShqsenV+4uk/wch9oxe1mJ5K/ + 71HdP5GWfySobdZR/JnJf8jTcVdkYX1KtzVBFykqeV0q7aEf/D4v49KO+CYP9Tqlh+ii/NBJ+jJP + sjKea2+IbtZRwMvvFznASx6lDV/6oB9jx0ebm+7bYdy4hgfW9XP8TaP0qTpMmBE8Z0L7yYl1bIfd + iOd++LQe9uTtPIKzKZa4+DvPoR7SHfnFC4aus/oIznlHHyvRokNCx6d/26pIfPHXvvAK50YtrsZk + PVAWFa3LwMp2+rTNfmFQcOGNk6W2DrrGwf65Jv+h/84v3UpPQDs/w2hnrnjF3cJ5J99F+GMepLlS + Q3VTAenTE/SQomkPNVsh42n5zR87TcledNL55PtLdFGeeet9J/3rHigrgISNevjQSd6e6b/p/SIH + 4oQjHQtI3pWv9xtHlPOpWsbhWXUNe35gVVI/qf1kI0l/UvESicIt+BOCtVql3axc6pYrj5vwE9B5 + sFae5LtFftezr1ci7PKOPFJBiZnemiOx60j/tlWR6oi/9oVXODdqcTUm64Gy8CuXgZW1zkMXvQmi + h/DVn/CMOujKregh9xSCGzPe+bsexOl8gdnNo57cu55NyNc1LsF3n4hgnqs2rEfptiafF4qt9rjC + 77qPsqjfsrGlIb7JQ4J6/xLQeoiOinAx95NH266DsMaxipXX9dPrPI468tGmB3HCkW7mJV9FZr9x + FYd9VUs/nlXrgTWefGTxE9szOftJzJxli5JxBzz3KdZu3ITfcDbF1hZ/57kFfj/x9/VKjV3e0cdK + pDIFQ0L3mf5tqyIC2l/7shXOjVpcjcl6oCwqysOu0ec4dPEnJwCCrT4ZYthB1zjYP9fkP/Tf+bmf + 8xPf4ZwZXvuYpH/jR9yRP7byBJe5eMSLNeerNqxP6bYmnxeKXXJS4bqP3Mcvn5eU77asg+PU90W6 + i04IxTKu8cpjSt8brp1HdLOOKvDy+0UO1E8eJ+KGzrnvnTe83ziah3unB9b29DcsSVNn6E8iHCIu + v7Tj7EXbPvP4GY0hnJdX4fXWPwF9mdKmbhXvVvXjyXp03+m37qDeXYf+Vzx1Kj0lE/Mpy37ufAM/ + 4m7Q9xI88888V+FMZg3WhwKue+R6YWNUe1zh99Cv3vzRl1gO2Yuu7BEXPUQXZRwnSwmJLz7mpUP5 + iaO557OXDgEV2LoXkSfEha/3x70TkRMvHDedb9THH7qfP/1vCSmNVOFscdcnVjt0drtPpHlW49Ao + 9e0eqh9FXP4JmH4B1p1Bv+qLfZcu2Nj1HQdxXOs2G0+To/NTF/inzTRE7+bST7jgM5uQr2tcgu96 + EcE8V21YDwpIfXryeaHYao8r/B76wX7KfsPazvDAOj97+t8S8krUu8efEH6y8674k0wL3Z3dJ0ht + I7wItLoSL66bV73eDVXVrn6VPT7p+N4QPn2nLetBd3glGPtniN50WtHSCE4z/NMWD1C7WQdQ/MFn + JmP5RT7sHf/Ac3/yd9wVWVgPCsjz6cnnhRpXu9R16AfE/KaTdqzD4BHviIseoosyxJdOSki8Gfub + jk+4ytzzEXn5/SIKceRRWidw3pUviReu4mZ95/wZVv6REIctCGeCxuwembRycr4Jvog09csl+OMn + Ytujjq6LpMUXftfjulOvVZ59uW77EdmLooPtvOnftoCqI/7aF74qONSTum7XbD1QJ29JbiGK2ena + NvuFQT2Er/7k595BV25FD7mnENyY8eM+1T7TJF3PpZ/qPtw/8rlArfxyCb77BFptjdCrsPT5lG5r + 8nmhwGqPK/yu+yjLb37HY6OGdXOc9NX5jDjY3keA7gN5ic9+8mjbdTBA+cO757MXcTe9X0Qhjjxd + QPKSj36MWiwcN52v+8Gzav3QvZL6SY0maLMZzGolM7lJdRM8U80mpj15w2945YER/syNq3omn9Z4 + cT37eiXCrk7jzNuJFh0c7jf9L1710/7al10VVJ6ybvvU58Nb0rci/aWv2OwXJVMY4aefewddBSOv + 0OY/9N/5uT/Ozfo6jumS9iR+xLlAJq5R+RQXXGZAjnkSdlVm91sXaE3Qg4qw/lRqhVof+J+y37B2 + P3THYXP4Se0nG1WbT+C+XMFlLlzw5Jlvkmmf4jccbwqG4bdmXj7arOsW+IXjYR7iUod5zGfeToRF + lQuH49N/5WclqiP+2q/tRaDVlXixHqhTTxV27LHTB1urXxiCUe/qT37GHXTlVvSQewrBjRk/zq/2 + mabPGWvZ5OO+eKN/zXbI3y+X4LtPBJD/qg31aaFL79JBylhe12yljKdufF8MXaox6+a41jPvG2Ba + D9HlBOZ+8iQrbAnH/fCuvLOe8XQdeVYcidY3p+ynD+fTfWIe/FJewZyv+9E3LPxzoULQHMepTyzG + Hj+x+onPOFyeXZyJ+LrGTfgJOsWfvLfC73pcx4wDsWpwfSOPVFDi8mNCoOtIP7bLMfy1L7zCuVGL + qzFZD5TF0683Nytzf1V/2+wXBoUTfvq5d9BVMPIKbf5D/52f+7kfZIfNNEnXc8UrLvjMycc54xL8 + MU/CrspsfeoCrcnnhSKrPa7wu+6jLL/5HY+NGtbNcdKXBEO/1kN0OQHi6zyUsO4BOPtnScof3lFH + FXj5/WKBiCOPC1PFzrvypeGFq7h9fU//W8K8mXG2PCWYFnd9YrVj+I0zXvpzoxZXY7r1T8D06/51 + q0oHduK2bviks6PuoC7bof/Oz31euvJzRtiND63hb/yIu0HfS/DHPOzlKg3rIaHrXmmCLr5H1R5K + 9v1r/WDzTR07PcledPYP/eiX7qLLCfAYs588ZuxvOqOemTf5jSNfxe34fE3YWfNVY87LPjpQi4Wj + 6T5dN4D4F4T4ofvT/5ZQ0tQZric/NqAR7Vq0raPlvvyMxhDOy6vw6rp55KyTFXu4v/TlmZckbTJi + 9U0/4+jnpVtxcrB/uacQ3Fg41cFLXPo0DzDiyzz8rPcYp3wkzrgEf8yTsKsyW4/SbU0+LxRZ7XGF + 30M/2D4vKd/tWC/HSVcSREfx5fxIF+V9To1XHlP2Nx2fsOqZeV0/q3OeFNK69/nQg/qFw7L300dF + Zr9xFYf9VZ//LaEj1AR7gZugMUsy2JqZE7/7CTtwHadcTDMGcRiTd+K5f+TvPId6ikhTXlyP655x + SFhw9uX89mO7F9y37TrSf/pU4CE+fFVB5Snrtk/WA/XzuOsysKjWeejCy5T+GRGdjOfrQdc4KIvc + pQ/XNTo/iXN+8HV+rCV/5lFP40ecCyxyTpfgj3lG5JVYWp95r6KHFE17bBS/6z7K8ps/+qYZ2YtO + OreOjINe0lt0UX7uJ48ZfW+4XvX4nsSuWfeFfB4rT3DcR/3CYTnOre8dIdlvHDdvvHf8Yw1305VL + 7Sc1muMlZ5O8NHLXjHVshy1c8NwPn9bDnrwT33mADX/mxlU94mR9Y4gXkc1T9acO52VZ6QfBTiAW + 0cG2P/3bFlB88de+8FWECEZBt3mpflGDP9nYqMfq3/qtfuEXjOeZvjkz7qArt6KH3FMIbsx48pWe + tc80pN3NpZ/qDj4zsFUIVx6X4LtPoJnnqg31aaFVYMnp80Kx1R5X+D30gz2/6aQv6+Y46SrCEQfb + +6SL8sRnP3nM6HvDNffDu+fTPv3iE+zA5zjVL1wRaUofjkvDnVdpna/7wbMKf6zh/AMKqaR+UoOM + NpvBrFYyMxl+y5+5cMFPPq35chN+uysPjPBnbt7kGXxccriefb0SYZfXOPMiqBdVHmz3m/4Xr+pv + f+3LVvoiqPUVmPp88qboMtd5cmv1C4N6CF/9yc+9g67coq7sX+4pBDdm/LpH2Wcaxu3m3TlF/5od + yNc1LsG7r5VnBV6Nlc+ndFsTdJGikteVWql5nv5mYly6kX/y6HyWfq2H6KK8z0mW9CQ+WbGgY9Qz + 88569D6rQlYeE5kPvORxIiGNW/mSeOEIc/1dH55V/BmWH1h4GAjCGVn8xPbM1P0kxjr2RXjuzyam + PXk7j+BsiiUu/s5zqGfyac041b+vVyLs+hp9rESLDgndZ/pfvOqn/bUvuyqoPKnnds/WA3VS0dxC + FOX+0lds9gsnBRd++rl30FUw8gpt/kP/nZ/7OT+yw2aapOu54hUXfObk45xxCf6YJ2FXZbY+dYHW + 5PNCkdUeV/hd91EW9Vs2tjSsm+OkLwmGfq2H6HICxNd5KGHdAzD2Nx3lD+/K6/qDI5/Hns9xqp88 + LkxA41a+NNx5WWf12f3gWbX+kRDNCcKZTYzZPYJcFCySWlyMLyJN/XIJvp/MCAh/5mM9p/hdj+ue + cTw0Dvfj2X5s9oL7tl1H+k+fABKgqXRom+wYlcfG7X+1HiiLt0SX0TW5v/Tl2Z+c8FMP4aefewdd + BSs95C59uK7R+aVT6Qlf58ea6Up27TNUcYf7x/0b9C29L8If84jjCr2obl8oCVHXyee1a9cKGU+9 + eP+WnmnJOlgm6XqR7qKL8ta78eI1Y3/T0QmFd+Wd9dz8fpEPceRxIiVgfN877uQ8G8dN51v18R8J + z5/+GZakqTNcT35sUOP65NGhlO0zj5/RGOOh4I3b++q6eeSskxV7uL/05VlvguqfEatv+hmHy8WH + CFbhlYN6yM2FVrQ0gtPMy1l+zuIBajcPP+s9xh35Y+9wiQP3MY+rujqvqpsKsO81+bxUf2qlrkM/ + IHxeJ/SePOIdcdFddFF+6CT9iXde3xuuvSG6WUcBL79f5AAveZQ2fOmDfowdH21uun6HKR4PrO3p + n2FJmjrD9eTHBjXSm12Ltqml9mub8fOhIPs2v7hulMVbkluoMnl50pdnXqa0yYjVN/1shP59nBzs + X+4pBDfM2zPydz3hgVN1ZHYi44LPbCK+rnEJvutFBPNctWE9Src1+bxQbLXHFX4P/WD7vKR8tyW+ + yUOCoV/rIToq4vi1L0fn7W9EjWMVo47oTz/yZOz50gfihGPamZd8FZn9xnHf+VSt/XxgPf0zLElT + Z+hPZj/ZfRbt0Bn3JzcPaZ7VOLQ6gts6+RPcl2zdCt4tXh7P7tuXSeXr8qQv9k0cX63HjJOD/cvN + hfE0OTo/93HZpi0eYHZzxQsXfGYT8nWNS/DdJyKY56oN61G6rUkPBda65KSuQz++iYedvqyb46Tr + RbqLTgiFtk5KyDxm7G9EPmHtz7yuH/no98PkBF/6AK9wgFQC5135er9xpHPf3Y9+6H5W/0hYSf2k + 9pONJPOJaelYJGskTc2FC74cmvrlEnznQUD4Mzdv8pB0iGTTh9A89FOcXV7Xa95OxHDDBLe46q9s + VSS+GU/+ph8L0d32lz4fHLp0qIp2+qjv9AuDwgjvPmVRVjiOcRKM/cs9heDGjHf+rgc6Mg3jdvPu + nPb3j3w5R62HLd7ci8yC7/N03BVZWI/SbU3QRYqOdq1U6wf//KaTdqyDZZKuuq+l49RDdFHe59R4 + nbMZ+xvRqGfmnfXc/H6RD3WQx4mUgPHmc740vHAVl/ui+zF/6M43N4af1CCrN7ufhN63dHXZboIv + Ik39cgm+8yDAmrLJ0/WIs/jCr3oR0TxVf8SUn5ph37ydaNG1P/1XfkaIb8ZDr9reEaSg2zi/9BPu + 3H7nr6i/5oy3RIftglb/67x5SSQnhRE+fbttNnqMaz0YQn4R0PCQ3lhq5qUrf/PAx3SsQvPwk+8Y + d+SPvcMlrvLu+LF3VYberBGcc8nnenMuqdYKtX5QzOdlXKOOPLJP6C66KOPzkVX1aAJpfyPSCRHH + Mvd8zG0cGTz6fIvIE+LI40QCGke+DvR+42g636pv/tCdl44QzmCZs3tk0soZXOYDHtsk0tQvN+E3 + /Eb+5D3WI84Dv+pFhboMAKT+1GHb++YFqBdVLmzHp//ikYjUJf7al61quFGEZd/G6Zc/69r2jz/n + E7dvfcUnbr/mOfynfjbqsdMHW6tfGIKtPhlhmQ+6xsH+uSb/yfMwf5+f+A7nzPDax6R6Gp97ZQdf + 16h8PtecV81AdZ9Yi39F3vaV3qy5L5xLPtcpRYectEdffBMPO81YBx9D6zn0az1EtxRZ+8ljxv6m + 4xNWPTOv8gFqHPk89nyuR/WTx4UJaBz76kDvN46m++5+8C8I1w/d61LrSQ6WObtHBIuCRYLqJnim + 4qXbjUvwOkSG4Xf4M7OrWY94D/yux7gZlzoc77rt70SLDg7XkXy2VRHztb/2ZSucG7W4OtN/8Sue + tb3xt37S9uf+0+duH3eH63N/6cuzPzlRN4XhCVSfshR20DUO4rjWm878NDl8HjXn/GqfaYjezZfc + jxv0vQTffVYeTFdqWB8JXXpHD+u4rpOVaj2h3Pymk6bkX3TW/5TuoovyPh9ZSshzNmN/I/IJa3/m + nfWsp86Rjzb5wEseJ1ICxve9404lXjhuHu6dHljn9UN3XjpC6vLN2T0yaeUMLjOTzzjszyZoxt7h + Eif3jfzJy2Z2cYOPSw75USHnGbfPa5z9COpFlQfb8cm3eMXT/tqXrfRFUOsrND3z2tn2tZ/6vO1N + v/WXbV/8yc/e64M6V78wqAdvVfUpS9fioGscxHHNy1b3hyaHz6PmnF/tMw3jdnPFKy74zMAe+WNf + hHdfKw8prtKwPqXbmqCLFE17KNlKGU+L97vu52jIOlgm6Xp4P7YeoovyxI/3i3hNqjoIG/XMvLOe + PGyE3vG5Hp50841zNp/zpeGF4/7h3t3BP+l+7ezp/5aQ0tQZric/NnC4/iTRom2fefwluAhqfQWn + T372Hdv//dJfsr32837p9mnPu6P6cn96E1T/vCSr79IFG9YlepRglIW91ptjtm3dCs9LXPo0D8Pw + O2mnn3yyM5P4qO/gO4U/5iHFVRrud96r6CFFR7tWqPXhm7jOY/ZjvSzTup8ndBddlPf5NF68Zu1v + Oj5h1TPzznrysGEk9xdfjg11kMcOJTCO9TlfFgvHfdfffOfXnv6T7hGb7zlpyjcJfq1PrHYMP6Rk + AM+Abo5e2Lyqr694/p3b9/6WX7Z9/Quft33infhzw90vKtblSV9uzG0dPunYHB11B32bI4Q7709g + 4UpPhVFfhOP3bnYi1UM+67vilM/Ufr0E775Wnhl6FdbWhwKWjp6gi3Ws9lCqlWo9Yff9HI1Yr6a7 + Qb/WQ3RRnvg6DyWk3iZVHYSNembeWU8eNkLv+FwPT7r5KoHzrnxJvHBk8/mrWsfhZ1jXr/8CXUnq + JzWa4KOPyccT09K5BfkVtnDBTz6t+VKP0sk78Z0HUGvKJhl2a/yuZ1+vRNjlNZ95OxEWVR4criP9 + V35Wojrir/3aXgRaXfkX/FPi9hWf8nHbm/6zT8T8HLTGflE2heFjpHSQxX1s9PmUnq2H3FMIbpQ+ + meseZZ9pkq7n3TlF/5odyNc1LsF3vYhQWyvySqxyX1tHyw5dJLjPQ5VaIeOpm9/8sdOM7DoG6Ut9 + TukuupyAz6nxOmczqg46Rj18mCRvz/T7YaJA7i8+8nMbccKRThviMZ/C1n7jKm7y4Vl1bXvO834S + hI8mqZ6cBLGImt0jkzI1tjlXkRMXPFOFT+thX4Tn/pG/8xzqEecQyfSurHmq/tThvK7bvIjqRZUL + 2/Hp37aA4ou/9oVXNdyoxUfPxG9YX//C527f/XmfuH3WJzzDevCESwd24rash+WqPqMHQTifY//S + W/HforHqAABAAElEQVTkKz3LFg/DaGcu/XxO0X/FHfljX4T3OS5+pLlSw/qUbmuCHtZ3XScrZDz7 + 8Zs/dpqSPXl0Pks/+qW36LRS6NpPHjOqDsJGPXpYznOSF3E8/xp7PkSTFpmbb8SbrwO1WLiK67qv + P7o9/My3Xdv+0Avuhetnk9RPajRXl3A+Md1SXYIqcuJYnWzlWk2okkvwnQdg5QE+c/PehN95nX/G + 7fuituHtRKs8BNqf/m2rItbf/tqXrXBu1OKjb3rh856xvfZzn7/9nRd//ParnnVH98lO3NZB1zjY + P9e8hIf+fR7cpm7rXrT+DMNvxmuu+B1+xB35Y1+EP+ZBmis1VLcvlASQTKhQ30Q493WyQsbbr4fW + Aqgv6+C41nPo13qILsoTn/dD8lim/kbkE1Y9M++sR+fvsANf+sD5k8eFdb3m60DvN47mundg+tnt + q190P/5Yg/bfnqR+UqOJuoTziemWEMoQ+jOz6YEvh6Z+uQTfeRAQ/sxU6zJ+12PcjENglYMZjs4j + 8eBqvxPbn3zpU4GH+PCJvnnK+qic/qv/5FnbD/yW529f++s/bnuGblfaOujK7nTemLjmm6d0pMnh + 86g551f7Oh+sd3OfA3WN/jWbkK9rXILvc0YE81y1YX1KtzVBTyk65KS9dNCbfNjpS3yTR+cz4mBT + B8nG8+o82U8eM6oOwhrHKvZ89vq8HOXzdp7ZB+LI0wUER76KrMXCcd/5XMb527njB9bZ2Y/q0hFS + l2/O5OwnMdaxsTyJ5374tB725GW1sgVnUwjD7/BnbtzAH/nNs57IyROc7dHHSsTqDENC4nQppINt + VTTsVVfFNYGoPqpfnoM/r/VnP/U52/f95udvX/hJzyz5DrqmX+rFNS8b9RnD51H6jXOzvo6TjgzH + 75P4EXfkj6244DIX3+QfpV2Jpfst3dYEPa3jktMXtfWBv+/n6MQ6+Bhaz1N6iC7KWPfGgzl5+xvR + qGfmnfWsp86Rz/XwhJuvEjDefNVE9pmvy1v3DoX9KJH1Dev8R5PUT2qQoVlWz9k91oyg2CSYuOC5 + Hz6th30RvvMAG/7MzVv1iJP1jSFeRDZP1Z86nNf1mhfBvahyYTs+/dsWUHzx177wVcShnlHaR+Xy + 1z/nju01L3ve9m0v//jt1+CPROx0ZUfRg2tetkP/0luwdY8IbR6G0c5c8YrLOWcG5sgf+yL8MQ8p + rtJQ3VQgOlImFMg3MceS0woZb//8piOw8CYoOsTTrntcfvGLLsozz3p/s4LkVR2EjXpm3lmPzp9Q + jD0fbe6ClzxdQHArXxIvXMVVffCPB9bZtae/YdUZric/NnTm7WjbZx4/hcXgm/ZjcHz+L71ze/3n + /ZLtz/+G5+hPy+eT1Q8p30FdtkP/wWnmpSs/Z+kHrXbz8JPvGHeDvpfgj3mu2tFYD12wulfRw/eo + 2kPZvn/G0/KbP3b6kr3obtCPfuktuijPa5v95DFjfyNSfuJY3/4ciTSOfB57PscxsvmqMePI14Fa + LBxN5xM7n1EY/oZ1x/Wnv2HVGfoTAofIjwYeEmcv2tbRtp8yYgjn5cfa6zNxS/7Er3n29sbf9Anb + F//KO91e+qel26xr1a1bt9IP/mlLP4bhd6m785PP+q+4G/QtvXe4xJEX/snfhV2RhfVAhdHR1w16 + WMdqrxUynnr5zR877chedDfoR7/0kOBRZuikhNTbjP1NZ9TDh0ny9kw/z7/GyjP7QJxwAFUC41a+ + 3m8cCZ1P7HxGYfiB9dyX/Qx8jwrC5GxuzO6RScclqyInruNMxNc1LsH3kxkRu3yjjpvxqw5ENk/F + RUzXSW2rj5xZ1+XE9qd/41WR+GY8daJdLfZitfyxtvpV+I+q//ZnPHf715/9vO1F+NPy6p9N8hIe + +pfecFn30rPs3T1ieHCZD/cP2zfwJ9+OP3HFM/OI4wq9WJ/SbU14G/pCLTl9UVtPvonxK3Zasg6W + qfU8pYfooozPp/HiNWN/0xn1zLzJbxwZKg6FL74cG+oljx0CMt58Hej9xtF0n2jj/o3PKAw/sF55 + hj+Htb1NEHh5GfQErdk9Mmnl5Exc5gO+HJr65RJ8P5kRsMt3op5T/K7HdSdel3qX13Xb34kWHRyu + I/2nTzlUWNepuuhX+FiU/TE8vQJ/Zut1r/j47a992sdtz78T2uD8lxBu3OdR+uV+wNX6YY0o3+Ha + xyT/8f5x/8gfW3nCn7l4Jr84rtCL9Snd1qQ3Mcvse1UKtZ6w+eaPnZasg+PUt+7nwtHvfUTwvMRL + fPb1NOm8/Y2ocTyvPR9zG0c+jz2f62Fk81VjxpGvA7VYOJqV7+z8JzY+ozD8wOJq80/h/aRGE2yK + zfAS0JsZ69iKGrjguW9RtPILcRiTd+JP8XeeqmPij/ziRWXNU3UF57yjj5xZ14XikNDx6b/qtWP4 + 0wdntTUWZX+MT/zT8l/6yc/c3vi5H7+9CvM1bozh84hOpSf8fT5Y63wz9zmA5xbOO8L7XHNeF+cZ + pV2JpfXRhat7FT2sY98rvGnpaT3rzR87zVgHX8Mb3jcA0e990kX5uZ88ZuxvRMof3lFHzot+8VVc + 55l9IE44FSKg6yGf47JYOO5XvvOztxdqPrD8U3g++Rg8Z7XCSyQKuDlXkRPXcWQfTdCMfRGe+0f+ + znOoZ/JpLXofQvMwP9XY1em6zYugXhRMcNaR/o0XUHwznvwVxyIqD5dPpfH8Z5xtf/UFz96+87Of + u/3GT8A/JtbQOWO9O++yJTvWu3l3TtG/ZnIe9b0E3/eg8pDiKg3rUxdoTXiL8h7Pdte91j7fxLmf + QvrFOjtOulKfvG/EV+8v0UV5n0/jxVt8rIOOUc/M6/rpdR5HHflo04NzFI502tC9MF9FZr9xFcf9 + a342cWd9w6qfwvtJjebqzb6ezEw6Lhn9tAeOxci2Q/5+uQTfeRCgPMBnbt6b8Duv8884HhqH6/Rs + fycqv23Xkf4rTiKCB4Fdp/qmrfCxKPspNn36c+/Y/uXLn7v9nRd+3PYrn4mruNN93YvWz3LrPdHn + hT3F5ZwzU8sWmgbGKf6BP+Zx0NV5tT66UHWv6t77KZH22Ch+D/3qze/41Y91syyt5yk9RCeEglsn + 6ck85uTDRG/AUQ8fOsnbs3DkqzgQdH5smQ9x4asEzrvyJfHCka/ynfvfEHJnPbCuP/p2bvhJjaRo + liSc3WPNwMS+CM99PmR2o+zJG37Db+TvPFXHxB/5xYvKZr0SYZfX/ZkXWXtR5cJ2fPq3LSB52l/7 + sqvLY7+75p86xu/5FXfiHxOft33tpzxrw8/opec8tz6fyJ95d07Rv2bKd9T3EvwxDymu0mB9+3tV + 19FPidGuccZDBr6J657PfuSv+4jpYt1FJ4TCW6eqJzL3N6JRz8w769H7rIrZ8+XYcI7kcWEjL8+3 + A73fOJp1/tdPfcN6xjP1rw315ATLnN0jk0ZUzONJyawTr8zl15ovl+AZf+RP3lvhdz2uY8bt87pu + +1FTL6o82K4j/dgWkPW3v/ZlV4fw11/qWRtP3enj8POsP/PrnrV962fyb4KgbqUnJLnhnLHnY+Bt + jq7Rf8XlHAUyUMsd/yV5OvY2L/Smpi6+UHPSw4jlya06jVOf3OebGL9iCyI8dXZc63lKD9EJYXYk + arx4zdjfdJCPQ8c48ia/cWTw4P7iSx+olzx2CGgc++hA7zeOZvX5DP+RBu6sb1hf9iL8NTPnb9aT + k0nRLNnmE1O9Ikgz/RgTF3w5NPXLJfjOg4DwZ27e1EXS4gu/6kBk81T9wblO12veTiQK0cHh+PRf + /UlE6hF/7cuuCkDwqCVJSU/Z+QEI8b/81Ae3L37Lg9Kzzw+K9PlgrXPIXOfpc4r+NVPJ8nOpcQn+ + VJ6E3u5Zb1bVXxdoTXiL5n2VKmkvHfSwGnajyDd5ZI842NJbdFoptHWqejTB09+IRj16WBZA59Q4 + 8nns+XJsqIM8XUDuAevrQC0Wjib919+8fdnL/ZeMYmc9sATf7tKTEyxzdo9MWjk516Nx4pg9+1iY + Ma+X4Bl35E/e5r0Jv/M6/4xLHa7TdduPwnrBfduuI/2nTwAJ0FR1tl0NiiDNPjVnyvkP3/Xw9jmv + v3/76+/44PbB66XfOLcbzhkxPgbfF59T9H/87tNVOxH16QtV96p00Lu67qOK9sU0Hvt8E+NX7PRl + 3fqa2n9Kd9FJcbPj3i79yWvG/kY06pl5k984MlTcji99gJc8TiQg483Xgd5vHE0EnJ3dJUe93PDA + 0pOTZGiW1c8npqXDNqnoz3zAl0NTv1yC7zwICH/m1NEzSYsv/K5nXy/xwckvk+LVdi+W7TrSP/el + sgG7eOpTcSfqSV1PlfmH7n90+50/8MD2VT/60PauD+Eqnjpv6V76Yy35M5/C515RxMN5x/a55rxq + FnyfhxRXadx4r6JH3lep1vev9cSbWA+tgx7Woa+p9R/60d/Xne8LPT2Iz37yOC8fJgpoHM29vkQa + Rz6PPZ/rYWTzVd3Gka8DtVg4muC9frMH1p3P/k6ArpPFT2zPaoXNm8Kzmgaa8wGvzOXXmi+X4P3E + VYnNn7y3wq86ENk8Vdc+r+s1L2rqRZUH2/Hp37aA4ou/9oWvDo/91vbH+vQePJz+5I89tP22Nz6w + /cB9+rN9atnnEZ1KT3j6fLCW/JkvuR85R5Hz5RL8MU/HXZGF9akLtCY9FEZ7bBS/h3715o++aUf2 + 5NF9HXGwpbfotFJo6yQ9iTdjfyNSfsvNh0ny9kx/P3XG+RaRJ8QJB+7e90Ms+Xq/cewaxHc8epcr + 8uv+G9YffuF78Uh7C4P5BMzsHpl0XLIqcuKCF/VoYtoX4f3E3fMnb/OmLhIe+MWLCpun6g/OeRlW + feTMikeTtLe4Ew8hnK/95sn27E/rp8DLw9Dvb/7MB7fP+r77tr/3Cw9L19m29MOGdaz7VLbOFevd + 3Oew7l2fO4nLz6XGJfg+Z4CZ56oN66MLNa8X36QqdbXri9p68qGBX7HTl+xFd7HuoovyPh9ZSkhe + M/Y3nVHPzJv8xpGh4kCw+HJs4CWPHQIy3nwd6P3GwTw/+8H58ysC9g8shVy7i1X7ie3ZPTLpuGTj + SXnEi6YfnbIQ6KYmb8eRlw+jA3/yNi51kfLA7yf+vl7GBee8I4/EWzyiE9ziTjyUM0/7zZNtlpM8 + Wn+Mv7zufQ9vn/f6+7b/6W0f2PgD9qlzWvd5RKe6T5LpcM7YA4POn7HWfX//uH+Dvo/xPonjCr1Y + n3mvSge9q2e7vqitJ/x888dOS9bNca3neL/Q731E8H3RebKfPGbsb0SNY30rb/IbR76K6zx5v3Mf + ceTpAlhn+uhALRYO5tn5XeXt6cQD6/pdvBx+YntWK2weYWm1n7D1UJh4sUuUzoNAN7XDJQ95T/An + 77GeU/yuZ18v4/Z5R57VyKIT3OK6TuN1uOpzxoNg0Xee0fHH3PInH3p0+wM/eP/2e978wPaTD+If + /9g/uxw6p2mfR+l3s3NmOH6fxI+4nGP4Y/uc9veVGO6LF2vOV224Xwq47pHrlaJpD2XTrv5kjfs5 + mrIOTaf++33DuOghuigz95PHpP2NyCdcZY46WDd56ef511h5yq8JccIxIPvpowO1WDiYh59fEXDj + A4s/x7p+ft1PbJDy0gDYM9axSTBxLEa2HXxdo5q6CH+Kv/OENzNZh0g2fQjNQz/F2eUdfeTM2g8W + wS2u66z+7Bj+2l/0nYe1fKwNfov68z/x0Pabvvfe7dvf+4jbk75om9bQOb1LPxjWcd2LPh+G0Z+5 + z8HndozLOQLucQn+mCdhV2VWf7t7FT2k6GjXChlPvcb9HM1YL+pdPFqc0F10Ud7nI0t6El/y8mTo + 0AmFd89nL3ECKrB1LyJPiAtf76cPhTmBsjnvOd2Hn18ReeMDiz/Hura9xU9sRPEhAWDPIrVNgolj + t7Lt4Osa1dRF+FP8ydu8N+F33n29Pj2L6byjD50Syuu6sJb2FnfiyzH86bvDx2K1/NG+okSv+fkP + bi//7nu3v/72h7ZHsGGd4aBuvluYJdyu3eCs47oXtHWuQO/mPgfyFj4zmcvfSS7BH/N03BVZWJ/S + bU14l1L12S7toR/8emgd9LDOfSw+p6Ff6yG6KE/8en87j9K7DsJGPTOv66fX5+WoIx9telC/cKTT + RuVlXxWZ/eCubzf8/IrIGx9Y3D0/v8tPbDTDprHVM91lY9I+s04897GhqV/K3uESJ3jlwTr8mW+F + X7yI7DqZjyLs8o4+cmbtd2LHp5/qjxWJb8aT39vqsXi0/hh4+Q/3PrJ9wffds33VWx/Y3v2hR/e6 + sr/owfXQmSaHz6Pmm50zsH3OiQs+swn5ukafm89Z+Qa+7wEiyH/VhvWZ96p00GN83CvZdR/RxFPl + Gxb+g+e7Tp3Z6QfWdsddfmLzCQixENmzRLNNwonjJZZtB1/XAA/HRfhT/MnbvDfhd959vX5Tzbyj + D247QdVl23VUH3ovFJD1l73qIp/Cx6Lsj9LpFz90fftqPKT4sHoz/piC+zvoyt6iB9d6eEUIbljn + nse53XDOALWeiQs+s4n4ukYJr3MPLjNQxzwr8GqsVLcvVN2r0qEer32v/DhXP6yc31TmN510Yx36 + WIw/pYeusxRXaOukhDxnM/Y3olHPzOv6XY/OvwrZ87kedtZ849zM14Guh/lY3qP7P39VqAu+Yd15 + 5+7PY7lHJo2omCEGx+6TDcVkfzZRwJviGXfkT16quMvjxOLLi/MaN+NSh+Ndr/2I7AX3bbuO5Euf + ABKgqepsuyoQQar56Jv5xxS+8e0Pbi/793dv3/JzHyhBMh10ZXvpn2tewkP/Pg9uU7fSU2GHc2Z4 + 7WPa40fckT/2jn/gfY7gK35yX6VhfUq3NelhxDqXnO7AePZD/Zae6ck6OK71PKWH6IRQaOukhOQ1 + I/PoYDhj6BhH3llPP+WEG++PiiNR81UC5135krgeYid/fsU6Tn/DOvx5LPfIpHW5OEMMDs0oYjfb + IX+/XILvJzMCdvkYdwv8rsd1JN4qzzpdr/2dSCWqPDhcR/pJn3KosK5TddFfHfai7I+i6XXv/dD2 + G7/rfdvX/fiD24OP8r+nSd/p76Are0v/XOs2RwhuzPhxfrUv/bHezaWfzvEWzjvCX4Tvc6o8mK7U + UN0WuvSOHrmvKZd23UesnhLfsLbTP7+iIqcfWPRs689jSTJeIuz2JeMlpV2XdTfbIX+/XIL3E3fP + n7y8nJfxy1+fADNOb6au0/Xaj81ecN+260i+6o9A9TnjEVDb6rH60/qj5OUn8EcTft+b7t1+9xvv + 2X7qITyoqIfeROnbbdPR55M+owdD9PDyfaDJ4fOoOedX+5Id691cvIoLPrMJ+brGJfiuFxFqa0Ve + iZX1QWXR0dcJ6lvHag+10q77KIvvw2WnGevWdNZ/6Nd6iC7K+3xkKSF5zdjfiEY9M6/rZ3UIQJ6M + lcdE5gOvcEBVAuNWvt4nbjv98yvmuPiBdX7920jSn1RsnlT4rbmKvOgTbjbBRLEvwneewd95qo7U + M/m0Fr0rax7WR3F2ddKsPlYji05wi+s6jVfH4pvx5G/6sUhFV3e+H/+67+t+7IHtc//9+7Z/9Ysf + RBs8Z9RLwXnCpYMs7ssf3bThfomTewrBDbqNs46+R9lnGnp38yl8zt2BfF3jEnyfMyKY56oN61O6 + rQm6RLdUbKVaT/h9XsY1inpMHtkndBcdFXF86yQ96x7Ia76FI/2ej7lVL99nNfZ88CsN4phPaWfe + la+Ahbv2beE7zhc/sL7iZa/bzq+/bT4xmSqt9hO2HgqyUV329bCY2aqpHW7guX/kl32L/M7r/DMu + dTgv6k+e1YiqVHkItL/6KFuHpjri96wzyFmJYDZ89dbU5e//3EPbS7/jvdv/gZ9X8edWq184CeAJ + d9/0c++gq2CFk1sBXPWQ3rA03+ycgWHak/gRV4U0f+wd/8BzX7zFvwKvxsr9lm5rgvq+UOs60V7v + Kz00hp1urIPPq/U8pYfooox1b7x4zag66Bj1PBnfsJDwp7YvffHrXMWNrxc/sM7OzrdrZ/94PjEt + nVvoJz5E4eWRnZl5uD9H2TvcwHcexFhTHBLWF+GP/MIhonmqruDMYz7zdiJVqfLgcHz6qfysRHzx + 135tLwKtruTLm+55ePv873rv9kd/8N7tPfg3gdYr/bBflE1h+DYvHWTpGA+6xkEc13x3iYCGx+Tv + +wGX9XUc0yXtSfy4H0f+2IoLLvOJPK7q6ry639JtTdBDiqa9Vqj1gX9+00lH1sHHIF11X+sekwW2 + 92HwvDpP9n0SOcb+RtQ4Xos9H3MbRz6PlWf2gTjydAGph3wdqAVw/w/qy24513TxA4uYR85f059U + vAzYSqv+hGDv2KEYc2asROGiRtk7XOIA4f6RX/bkHfgjv3jB0DwVF5zzjjyrERWo8pDQ8enHtg5X + fPHXvvD7/sq6MtO78JdS/dEfvGf7bd/9vo1/tmqnD6pc/cKg4Dzh0kEWdTrqGgdxck8huFH6ZB7n + 1vnhY7p1DGJSPU/EfUKqKzWoQwk9J+gRHVKuccZTL7/5Yzeq72fpKrvuMUCtu+ii/NxPHjOqDsJG + PXzoJG/P9ON8M1Ye8pGfr4gTrjeqHvLRj7EWr/HG6debP7C+8uX/gX/jHzn7yUnusknpJzuflCga + STXbwdc1qqkdbuC5L15EhD9z8w48Ei1urJzX+WdccM47+mC4geIRHWzXkX4Wr3jaX/uyq4xDPbV7 + 26YPXT/f/vpP3L+95K534x8DP6CHA4tpnave1S+c1IPI6lOWZD7oGgdxXPOyHfqX3nBpHufW+RlG + f+ZRz62cd/Lt+G+SB2mu1LA+pduaoIcUTXuo2QoZT8tv/thpyjpQ79JVi/V+pF96iy7KE5/95DGj + 6iBs1KOHJXm5m5l+nn+NPZ/rYUXNlzjlZX0dyDX+dtGX6K9qD99xvvkDi+iza9+kVngZaOJ3bLlZ + LJPP2Q6+rlFN7XCJA6qfzFiHP/Ot8IsXkc1T9URM5x15ViOqUeUhoePTj21VJL74a1/4ajEEZd6u + 6UH8d39/9x0Pbq/4jvfgB+v3b/jvlXU+qWenDzZXv8QRxfOs/uTn3kFXbkUPuacQ3Jjx437UPtO0 + /FjLln4Vl3uRGZicI5cal+C7T4DJfy/+RcOrfviB7R0fwL8Nvc1Db+rcl+ho2aELlZntWin2o334 + 9dAqW5vCm6DodK79vim/dYYBXX0C1nvpz3OGS17zLRx1rPcF/aMe8ylM+4uPOO4jjn11Acm78hF4 + fnb2TWa5+PXyB9b1a/+AGfvJCYM10ObQzGS0M9shf79cgj/F33nCm/kEv+txHTMOhakE1+d67cd2 + L7hv23Wkn/QJoPqb8ey34roeEt6ewfK/5Wce3F702ndtf+It+GMKD/g/UvYn26qrdR66+JMTBIKl + LzKmv4OucbB/rnEu0Zkmh8+j5nFunR8YpmO85lFP36MRd+SPrTzBZa78O37s/ZN34c+bfc8921/4 + yfprcbB3O4YepuvCSYC6XtBj6s7qrFDrCXt+00n91oF6Dz1P6SG6KOPzWfrznM3oe5P84a33Basq + 4OX3ixyIYx9OxA3F972jff06/lK1Z32LnDd5ufyB9Uc+812I/47jJ1Y/YSEKu5SdmQm5P0fZO9zA + n+K3trfG73pcx4xLHc7LsiheldeLZduffoxHgwZoSjzrqjj2eex39v4Er99094e23/qd795e/ea7 + 8QN1NpW6MfOW1OXKPhHWyzMvk8pXaPrCLBxfD7rGwf7l5sJ4mhyTv+9H7Ss/1ru54hWXe5HZhHxd + 4xK8z9H1zTyU53/Hf8j9Wd9z7/YP8B9203c7hvVBdvaxJp8XClpyUte6j9zHL5/XCb0nj3hHHGz2 + Kl7dB8e3TnZ0Xt8bBPiEq8w9n72sn8wee770gTjydAHcTx8JvPba7ctf8O7wXDRf/sBS5Nk38YnK + FpWTcxXpJzuS02YRKT5zMl+CZ9yRXzbjwpuZnAd+53X+GRec63Pd9oOjF0UH23WkH9sCqo74a1/4 + avBQT+0+odMvfODR7Svf9H48rN6zvenuh6sdnlDqxswTg24ZrXPVu/oFQjDiR7zoDrqSLHpwTf5D + /9JbMOefNtOQdjePem7lvJNPvLkXmSvvjj/5Ks8vfvDR7Y/9yIPbF7z+nu3776m/MgeYJ2tYj9Jt + TT4vFLHktFKtHxB6aC2ASrYOfSyIv0B30UUZ4ut9Jz6esxXob0Q6qfDW+0L1GXj5/SIf4sijtBWn + vDPf9Zv+sN1V3ewPjgah+Zn/GJfoPqZKq/6EgF2XdTczhvtzlL3Dseixf+SXfYv85jHfjEsdzut6 + 7UdxvahyYRNHcSdeQNURv2edQdqsPmbLT9SaP1D/az9+3/bSf/uu7e//7ENVb9pxQdYDdfLEcgtR + kPur+ttmvzCoh/DTz72DroKRV2jzH/rv/NKt9CQ7bMkuVsfLrnjF5V5kTj7OGZfgbzXPD95/ffsd + 33/v9pX4D77fyf/Nz5M0rA8FLB09+bxQQ7XHFX4P/WD3/Ry1Wremk848d+fhfukuupzA3E8ek/Y3 + IuUP756PyMvvF1GII48PmhtVD/m4ff7Adsf5t8pxycutfcN61affh7/U71+4JRaJJPUm2D3JKUrt + zzeJargE3096gMOfmV3t8pAweUSeeoybccE53jj7O5EYRAeH60i+xSue9te+7C6gFk/s9C9+Af84 + g59Tfd2P3LM9gAeX+2O9SzdWoH4585bwVtRwf8OPQF6m9M8IHkDHw+TGMU4BxMk9hXCiFe/802Y1 + jNvNKqDy3sJ5u+CL8V3vreQB5lvxj4evwM+3/upPPbR9gLo+wcN6lG5rgi7OXXKgCivV+sH2ee1r + lH/ykCA6kgW29BadVupw7SePG/e94dp5RFf3QLtV4OX3i2jcH/J0Aamn7u352T/a/puXPkDkZePW + HlhkObv2GrdUEtabwE92PimtVp7o802iIi7B9ycAwMpDsZl28lL04jnye9/+GRececxnfyda5cFB + HMWdeFWkOuJPXZwVPhZlP87T2+5/ePtd3/WL2xe//r36gfr+E4v1Lt2Y2npg5i2py5X91qdw7hcG + HcKPePV30FUwJ7RbwnG3R+dXYaUnvNZXWXy+2DvWw3qt/4pTg80uIlk7XOLkrvuDtfgz14Ht4gr/ + EP4N61/+yYe2z/7ue7b/Dz+gfyKH8rOyOrg+P72rve38VHjpoIfVsFOj+2m6G/Sj3zqTLooQn/3k + SVafLxDacH2jDm5gXH6/iEIcebqA5CUf/ibk7eyW/nGQTLf+wHr0Q/8cCd+VVvuJz+bZ9JxVI5Fj + SCRqdRrfT3qEUIq2L8Bb9MUvXkQe44KTn5qBz/ydSCQqr/3pp+qV2Kx7xseuGkSw6nm8Vvc+fH37 + H95y9/ZZ//YXtrve80EW4Dp4+qOR2lZ/zG09MAtHoMfqH/EYttkvDMGKv/rxdNDVgVUH4yQMd3t0 + fhVWesLb+bFW+Zk7H/NH/xXnApueRDKU5wT+MeUBU+olKf/R8FVvuW/7z99wz/aW+56Yn285X+m2 + Jp+X6mElHD6Y1KcPFygX25iqf/JcpLvoorzjZElP8prR9yb5ieN5rbzJf/n9IgfidA+xrASMJ9/1 + 6+fv3j700L8l6lbGrT+wXv2Kh0H4mrTqTwjeLau0m5mZ+3OUvcOx6LHvo0FPCmeTt85vHvPNuNTh + vOazH+S9qHJhE6dLwbrKrkXbDou/mqw+yvqIJ/5Tyd99+/3bi//Nz2/f+BP3bf1//EtdOOxVP/uO + iQVG6yocK/Zwf8OPQPcLv2DpKzyMO+jKra6DbgTSHqPzC3eTc2Y4fp/Ej/tx5I+tuOAyF594w5+5 + 6tzFjfxYegD3Rvww/vNff+/2x/HzLf7Fho/ncL+l25r0JmaeJSd1HfoB0fdzFOR++lis5yk9RBdl + rLssJfQ9Uv7cL84YdM+8rh/7wpHBg/uLz3GqP3zVmHHId+3a39/8bAnFTedbf2CR5vr538ST8RH3 + 7CL1pGWREIddaSaW9hxl73ADz33xIib8mZt34I/8zuv8My4452VZlUeqIlnX5cT2px/jVRFxIF7x + savJ4inrI5re8L4Pbr/pte/cvvpN79veg3+b5X5GHtbBa9EO1hsTCwz1y1k4Aj129WNr9QtDsPQV + HsYddOVW9JCbBRlPk6PzC1d61r7Kxno3V7zics6ZTcjXNS7Bd5+IuDQPMKm3E1Td1zF/yzs/sL3s + u+7evgF/HOLxem45X+m2Jp+X6kkl1HXoB8T8ptMo1dvH4n6Gfq2H6KKI+5YlPX2PyNn3i/eHNqaZ + N3pdfr8YDV7yOBE3ovcj16+d/a/auMWXx/bA+sqX/Udk/Rb3zOywIAq72c12yN8vxGHscImrffFi + Hf7Mt8IvXkRynnHYYNrK69n+TlR+245PPxUnsdnnjI+tcDpq8eFPP/fQI9uXvuE92+ff9QvbD+G/ + ++NY/RQv87AOnv5opLbdZ8UpXjgCPRaf6139wi9Y8bdujDvoyq2ug24WZD66OMjb883OGSC3cQI/ + 4o78sZUnuMyVX7zhz3yTugBZQ/2tvvnzrf/5bQ9un/s9d2//4hc/8p9vWZ/SbU0+V1Sx5KQudR+5 + j19880ffFGwdHNd6ntJDdFHG59R48Zqx7xfvD4bkGHmT3zgyeHB/8TlO9ZPHDgEr/lu2L/mMn0/s + rcyP7YFFxuvnfwmveMC6SM0sknZm4srPpcYleMb7aECDgLYn7034lR+Rx7jUIT81Sx6Jp0SrvPan + H+NVkeqY8eyXtsLHouzHMH2AP+z9kbu3l/zrd27/7zvur3pMvKuXntTB07dQ2GS9MVec4MIR6LH4 + Fk5vApqCmUh6Ycv9HXQlVdfBuCkEnXQv/nkvOj8wKj/zKfxNzrsKc57gMlf+Hf/N8hQe0xrqb/Rd + np968JHtv37zvdvvesPd23+s/5pgBd36yvqUbmvCaUW3cPlgWk/4fV7GNSrngW31PeonpnUXXZSZ + +8ljxv5GNOqZeWc9Ov8qZOVxfT5W6EgeFyYkvrniD7df+4sVdsvTY39g8VvW+fbP+glbl1U2qsv+ + bELVEIexww089y0ZRCxcZl7OXZyJ+NrDeY2bcanD8c5vP0J7wX3briP5ql4C1eeMR0Btq4jqrwu6 + xcW3/dwD20v/9c9tf+Gtd28PPoJ/X1I8cx5lrjp4+u1gvTHZSOrGLByBHu5v+BHoT2z4BUtf4WHc + QVduVUKheCurbro4Zv19frWvsrHezRWvuFs47+S7CN993kqeUS+WHupv9N3b1uW77n50+zx82/pT + P/rA9j78i5HHOqxP6bYmnxfIlpzMV/eR+/jl83IdyWsdHCddR/3EtB6ii/JzP3nM6HujSG2IbtZR + BV5+vxiO+vELkwvEdO3s7J9tX/YZP07vYxmP/YFF9vPtr/QTti6rbDSRfV7S3Sh7hxt47lsy9MQU + sRkXXGYSF19yOK/zJ16nXjjnnbyINFAUgsF23uqjbAFVR/yedQZp81BP6rpo/rF7P7T9jrt+fvv9 + 3/OL2zvwqa0x+nM/M18xpQ6eftfPemO6oI4XjkAP91f1Y2v1C0MwE3W86A66kqrrYJyE4m6PFU++ + 0hPezo+1ys9c+iku+MxkLT+XGpfgH1MeEKbeYq/+Rt/lWLjzDV+Mt/8L/6H5y77z/dvf+mn/fxs7 + /pKFeUq3NeG0JPhol/bQD349tA56iG/y0D/0o196i04rVbj2k8eF9zeiUc/M6/ohE/3Ik7Hny7Gh + fuGAqrofffTsryTmscwf3gPrj7z0e1Hk65jIT3aIwaIpSorPnGrK3uEGnvuWjCKYN3PzDjwShVmz + 8zr/jAvOeScvwgyseNuuI/1UfxKb/c342AqnoxY3n96Pn9p+zX947/bZ/+ad279/zweWXgwb/bmf + ma94mYd18Pp1/aw3puvoeOEI9HB/5uXO6heGYMVf/Xg66OrAqoNxLMh56eLo/Cqs9Kx9lY31bu58 + zB/9V9yRP7bynMB3n7eSZ9SLpceoO3XSkb6wEo72vfhm/D/ir5z+HPyPPPg/9LiVYZ7SbU0+V+UJ + C/MsHfTQGHajVK+PQfWO+olpPUS3Olr7yWPGvl/dJ6sYdZCfvPTz/Gvs+VyP6heOAYp7Hb5dfW9i + Hsv84T2wmOH6uZ6QetKiiN1M/2iCZuwdLnFyQwzO+K2Zl7D22eQuDvvh45JDfkRwnnHBOd44+xHU + i6KD7fjkW7ziaX/ty1b6Iqj1iYmfxn/rbfduL/qX79j+NuZHSh/XXQHpE2b2Xc+gZxzzUinM7o/1 + xsRixgtHoMfiWzi9CWgKZqKVn3EHXbnVddDNgsxHF8eKJ1/pWfsqG+vdXPGKCz6zCfm6xiX47hMR + l+YBJvV2AvU3+i7Hwrlf28Dh10/hf+rxu7//7u2VP3DP9rYH+g+iNOVcJK51tOziIW7JyTxDPyB8 + Xs4fTvHVMajfUb/58r4gXRRx341XHjP2/UI+x7OKUUf0p198FYf9xZc+rE858L8cvPbnjH7srx/+ + A+tVL/l2lPr9+qRgkSw6M+sYTaissne4ge8nM8CUqO3JO/BHfvHyMIFPvC8D5TOftY0fmwaW37bj + 00/FOVD4Pf9o89ivWP3yHe9+aHv5t//s9jVves/2fvxF6kqr0wt/gUd/7sf+UaYTYsOfbIhTXtab + dka/dDMPeDN29Svcl6loFBGdGOO2DrrGoTpgkN9AejRm/fNedH6gWBWr1VzxiosOmcl44I99Ef4x + 5RG9dWMqDear/KmT+8pnQMHcgd7M5f/2d39o+1x82/oz+PvI+PdwnRrmgU95evJ5iSdRxR99gOAN + WnUYJ3vR2T/0o199iE4rBa795Ck+3RuuuZ/6Vt7kv/x+MRpxi+/7ti954V3c/XDGh//A4t+7fL59 + vZ/sEAPiUHx/cqAU2nOUvcMNPPctmSVqe/IO/JHfeZ3fZ+J6gnNellV5cmZdF4pFoP3px3Y5hr/2 + ha8mi2e2/NP4t0i//7vftf1O/KzqP97Hf1QY9en6hL+iRn/uJ/VwHnmYl/FutHhjGtjxwhHosfpf + OL3ZaApm3o4XbNSdQjirDsZxIWBlcd00xHPoi2mI3s0Vv8OPuCN/7Ivw3N/xJ9+pPKkTcw/iKn94 + 6FM+gdyvbeiDX8IVP79B/w38jz74P/z4pp95CP9A0syOFg4RytOTeAgoGq7wu+6jLOZZNrY0rMPg + Ee/CtR6iWx2t/eQpPuYlTPnDu+ezlzgBFbjnc5zqL75Hz7YP62dXIsfLh//AIsOnvuQf4RX/1hAi + sujM9I0maMbe4Qa+n/SAWlOIo7DBO/DhIzWHn/iuY8YF57zG2Y+gXlR5sF1H+lm84ml/7ctW+iLw + mn/r51/4ofdvL8E//vHfAnrwUEd9vg1Vd0FGf+4n9Qx66Qyb8V0/eWMyj+M0C0egh/sbfgT6Ext+ + wUy08jNu1J1z7TronkIQv+ef96LzA6PyMxev8kaHzCbk6xqX4B9THrCm306g/kbf5Vi4qTNw+KV+ + qi4wKoL/BvFP/vB92+fhfwDyve9fP98yT+m2JvEwsGnE4zq0rzzLVhLhc26l66hfcbBdHyyeV9XX + OikheYmm13wLR96VNzoYRz6PPV/6QBx+nV/ffnz7Qy/8p8F+OPNH9sD6gjP8662zv8wu/YSvmZVI + lFFS2Ttc4gRnUyX2tBkXXGbSHvjFS1GwL56KC855GRY/OAwkm2E8I/nTj20BxRd/7QuvcG6QZfuH + +HNUn/HP37H9xbe+b+PfVqK8gviSdH5dn+kHaPSXuMbnTqQOxnf9rDemgR0vHIEei2/hdJloCmai + jhfsoCupug7GTSHoXH2J59CXygZmN0u/igs+swn5usYl+O4TEZfmASb9dgL1N/oux8It/Zjh+A0L + jIpQmSjgR+5/ZPvC77t7+5I33bP9NP6tsHngUJ6exMPAao8r/HYd2oft8zI/9zjEt+jKHnEglA6i + 06rjvJ882nYddCi/65l5owP71vk7THkXn+Nan7OzvwTsvvCKu9VJJd0q+CTu350/Y/vJN//02dm1 + T5bKvLynBvcpWl3C3Qy8mpzzwDWvDvc0v5/sdZjkqXwR84Z8ndB8C16XTxsiwksdiqajH24O4D/n + lz57+/73+X/4wMPtOANku072e/QL5Lq5ZH5NzNemF1VHATCVroXn/kV5en/H7/iZiDiNTLBdR28c + 2qt9R+1fc24187ITLb5TM/3MN2bpwfhTg/us9zB/WHkmf/HdwKM60gECyl4PrZBUvdWoedIXw9qB + +qt80uGX2CucOw1ov3mSSbPqHTywoyP93Ufyive4T6DYqj5FamPWXxu9r0W97PNgE/lQyc8/+qs/ + /VM2fcmZ6Me2/si+YTGXv2X9bxK/LplKkCijmLJ3uIHvJhHiw8qhQb3gMpP2wC9eRDYP/cAH57w0 + w9uJyGaY4PTXZSi7L0v7zcNCuwws3sCHFdOST6c+LxVvwaivboXrZgUYo7/s7+olphKIX4nCW3mr + oI5nHupQY/Exzn24XxiCEe99+/k66i7+VQfdCiCwR+dXvUuHzg8k07n6mc/1Kn7osYSuFLPP4DKT + F/4df/Kdiit8MXsadYeHjvTlymOjP/wSrviXH0FwuJ7SoexyzEk8zsNXDiuUvM6z9DSm6hBv6Trq + Fwts1weL5yXe1FXxQKT8vl+NI+/KO+u5+f1idsX91Y/0YUWmj/yBRZYHnvN/oqgfY7frk5mijCGR + qJVV3c2A6ZOAM37riHj5ar95b8LvvM4/4yKm8408KxGycB8vCHQd1UfZ5Rh+44xXuAmEr/qrE9el + DEpgfsDlL56i6D5hJ67xrI9D+lW8G8Um6628amTEM09uocKHrm07XoILP+KV96Ar4pLQ7mqc+zVm + /ce+VDZwu3nWnXPOTM7yF33byhNcZsGrT6wvzVP45uZCgo6+y5m+ANCObeDwS3m6zvgBg4M4PWzE + a7sccxIPiZtGeVyH9pVn2dzjsA6O6zpO6cGysA8kw6quikeFyav7SVjjWObKGx2ME5DgAx9t7p7/ + 2PVPuhPPiI98PD4PrD/xAvxt/udfw+ry5LUoo0CJRK3QXHCZAeO+tMTampZ9Af7I77zOn3iptcs7 + 8uTM2u/ErqP6AFF4la/sxU9/9ag6bcuPw2Unjidm2csf/uI46KEo8DpfYZKHfO1gnpjMs3j9SUmg + h/sbfvE7XnzFm7rJmz6czvxJaDc8BlaWPX+fN7ydX6ywM1e88kaHzGQ98Me+CP+Y8oi++mIuDuar + /Orbu6rfS+OVn+eMXzt91FmVLXnor/tQtk5MeSod0844JbJCzmN/81RNnKzD4Bn1x+/6YKEvMu33 + k0fbroOwxrHaqp+7dR6X3y9yXPua7b/EM+JxGI/PA4uFfMVL/9X59fN/mievRRkVSiRqpdPaz4Bx + 35JZorYvwB/5nReXIjwVF5z8SLD8SKqEOhXD2m8e+sMrQPvTB+fqUflsixaHywSOJ2bZy188RcE3 + SfBzFv6Yh3ztYFxMAzteOAI9uO+whdObjaZgJup4wVzXjEtCu+FpIVYersRz6Es8lS5pVz7mLx0y + m4iva1S+Hf/Ac/+W84A1+TsB+YsvPPQtnDovG/Xil3CtQ/wIgsP1pK/wyFF+TeJxHr5ykKfiZDHP + songUF2LruyFc37iAEZf5nWcLDvslxdAOpTfcTNvdGDf5iP2yMeN83/6yJd8+r+S83F4efweWCzm + 2rX/Fk/e+1SXRBkVlu1PAohOGyL1kxq2jwbbCON+5sYN/BSJWcxjvhkXnPNNXgQZyHDDYDtv6lu8 + ArS/9mUr3ARlY9KlYgLXpQx4GfX5Ngw/3QufONdjerJ0HYxXIl23bDdfxwtHoMfi020U3p+c8Atm + 3o4XbNRd55iEdiMw+yMPl+I59KWyKx3jZVf8Dj/ijvyxL8Jz/5bzpE7MPVhP5Q8PfconkDovG/rg + 1+wDSKOqQddT5wugebSQAEpH/hnXeca9UJ5lC8K4ImieUT8xzs8ZBvpiJo6170Lllxc2YY2jufK6 + fnqJE5DgHR989z+63fnH5XicXh7fB9aXf+bPoK4/q9pGE9P2JwEOl36o009q2JbMEnFf9sQN/BSJ + /OYx34wLzvmMsx9BvSjNYTtv6lu84ml/7ctWdyYoG5MOlwlcFzG+JOaPn2HrsI96KKp1oIUhPSpe + icKbdswX3ss+AV0P6wQ3+Vin+ggP9w66ClY4uRXAVY/Or3qXDs6nLEqXtCfxNzlvF0w5WEfxZ0YV + jylP4bt4LgavZC5n6nQHlZ/64JdwElIEipBZeuqhJt4V5zyVjmmpP2dP2on+2leepSf3OKzD4FGe + hWs9JPjqaO3L0Xl9b8Rc/Kxiz2cv4qB7xuTD7tdtf+gFPxvf4zE/vg8sVvTxL/lG1K8fwO8KrKb8 + SVBPaojaT2peNgREyt2TP7jMJB4i2XTkMS4452VY5VmJGG6YtKc/9RnPayRA+2u/tptgwXS4jFNe + A/BqG7Dyh18A3bbg5yw86+Vg38xDpdpB3pgGdrxwBHpw32EL537hF8xEHS/YqJuJOLoOrOvNof16 + WfHkWzp0fobhN9mO9TR+xClfcWuqOpQnuMzkTZ/hz3wqrvCY1lB/o+/ypC9X7jzsQG9mYG70YxMN + up7SoexyzEk8TFVlciWC8DrP0pNYDvnFK7qyF875ixc6mddxspSQeNG5DjqU33H9vuBuAX0PBVTg + ynP+xuvPfOE3avNxfHn8H1ivPMN/9Xntq/pRnWIlEsTkjGZ3MzD9ZMZaR8TLV/un8Ed+8SGieSpP + cM438uiUlAAv3McLEjo+9dkux/DXvvAKN0HZqrvejq5LGZSg65M//MURXWAmrvG5E+oLfr0LBCze + pRvZOl44VuSx+Exom/3CLxgW6iN+xh105VbXQbcCuNuj8wtXesLb+bFmuqQ9iR96uMCmd/7iO3U/ + HlOe8Ax69zf6Ll/qdOXuR/qgE/UjIQke+pWeethEN+FKtzUhasQppxVKXj006jyqJKOat3SVfUJ3 + 0anSjpNV9WiCp+/XqKfrp7+AxpHBo3U/21696VkQz+MzP/4PLNb1qhf/O/wA/pt3JfJSY/iToJ78 + aLqf1Lyc9OO3NS273gy7OBPxtYd5zJf4XDqCHO/Zfmz2gvu2idOlUN6KI3DYi7/inEB8BUMfJnRd + AiiB+REnf/jpxzjowa3GWz4nZDmM7/rZd0wDk9c4Aj0W38K5X/gFM1HHC3bQlVTRg2teXtpjrHjy + 3eScGY7fJ/Ej7sgfW3HBZS4+8YY/c9W5ixv5sfRQf6Pv3k6fnl03cPg1++AJcSid5LnsXpUOM84M + eB36Kc+yBVEe6ux8XccpPVgW9ld9s27ymrHv16in3xfJJxbmJZ8H9Tg7P//mh//gC9+YvcdzfmIe + WKzw0Wf/KRT/ni62mtKTmU3RzgxQP5mxtqa8BN5v3MBPkZhDfDxM8M644Jxv5MmZdV1O7PjUt3jF + o7OZ/PQzO4b66Ql3xwlclwB4GfXJH376MUZ/iXM9J/LodiJGBZB36UaqjheOingsPhdu2/ESrng7 + XrBRdxpOv6RF3a6jkmBa8SzM8fR2fqx1TpmLV3HBZ3YgX9e4BP+Y8oA19XYC9Tf6LsfCLf3Yid7M + O574sdn3pnQouxxzEg9TVXtciSB5nWfpSSyH/HUM0nXUH7/3YfG8xOu4xrMPl+066Ggcy1x5Zz3m + I1bjvQ8/uv13MR7v+Yl7YL360/CwuvZ1XbBEolZQAarsZoD0ZOaM3zoiXtbaP4U/iGQ+RDRP5QnO + +UaelQhZuI8XJHR86rNdjuGvfeEVboKyVXd1oryCOEHXJ3/4iyO6wExc4xnOob4wMV6JwhvTwI4X + jkCPxbdw/uSEXzDzdrxgB11J1XUwDoESkA6PFU++0hOuzo+1ys9c8YoLPjMpD/yxL8I/pjyitx5M + paH+Rt+9HZxn5UcnT3/DKtm28z+1/eEXvrfketyndZMfd2oQ/sPzO7b73vK9uHevyKXmk1mXac6A + 7i4v7eHX5YTd84lahce12X3S1aUjfPLxqpGO72UvPBlel0/54gew7M7TfmA42l+06Yj7Gk7oeMBv + 8Bcq+Job3zRYqJzaEM6ffG7H+x13yNP7O/765NzpEf5V1+Sf/bq9whu+f805nHgIMEq8cwZOdY7Z + +Yg8MQ78uScXPrRAobyDP/l27MV7Aw/3oysDyl4PrbBUvQU3T+4/w9ohOqUjHX65vsmDHeHjN08Q + mougebCYfXUf4onyriN6mMesro9r9zHrN27tl/3Gh//gp79C6yfo5Yn7hsWC9UO3s6/CATzaYkvN + EhtvmojSYiIs4mXmZRUuM7klOhce5jFuxgXn+Hk4iDNQBDlD15F8xs/bZH/tI77LUF+2RVtvQ9fF + FL6cHT8ugQrgy+gvcY333egEvNSrftYb08COF44VeSy+hXv6GxZltR7RyYKO+zT0KyU1OQ44/PJ1 + Ck/0BUz3hP6b3as6P54XxiqHdsVxX3mWTSyH6lCecQ8O98n1AYx9MnHs7wN5ta08rHvhyLvyRi/f + Q+zjh0Db9oxXO/qJe31iH1is+1Wf+f14/RsWiVpZ1d0MgD4JOOO3johi1z5VPOLDB4iG/IhonsoT + nONHnpWo4jEhoeOTz3Y5hr/2hXd+5SlbdVcnrosYJ+z6ZIe/ONIn0bpUnsXHcA71hYnx7WC9MQ3s + eOEI9Oj8O37Hi694O150B11J1XVgzVtefM5Cc9Rx6EtlM4w0mU/hR9yRP7byBJeZvODb8d8sT+Ex + rcF6ii88dKYvVx4b+uCXcNXH8iMIDtdjHWOXY07icR6+clih5HWe4jHAKNXLPKJznaf0EN3qqHVS + 3eQ1ad8v5Q/vyjvr8VPu/G88/Ad/wxPyg/bR5uP0Hz9PxlPr5z3jT+N/QvZ6uvxJgMOFmFSnn9QU + l378tqZlT9zAWyQyepjHfIn36ZExeT3bj81ecN82eXQplLfiCBz24q84JxBfwdCHCdMfkEpgflqj + LsZzjP4S13jDVx2M7/rZd8w9bz4BnYDhQ1ds2na8+Ip35VdhN8QlobLxlrOAMVY8C7vJOSOm9cRa + ccFnJu+BP/ZFeO6LN/yZi2cXJ/p9/eKv/OEBzPVxsTs/9Idfs4/lBxQO11M6lF2OOYlH7F0OF0M/ + 5Vk2sRzuh7Poyl641kN0q6O1nzzFhzxqiLP4ae75tM+Kz7Y3PHLfp/33Aj7BL0/8Nyw28MoXfWi7 + 9szfh9bu1pOZlwmXger2k5qXA9BIyX3ZEzfwfBPMYR7zzbjgnA/8zYtoA0UjOtj2pz7bAqqO+Gtf + +Kqi/UVbnbguYtxZ55cd/uIY/SWu8Wk3eRjf9bPemAZ2vHAEeiy+hdObjaZgJup4wQ66kqrrYNwU + gs7Vl3gOfalsYHYz+RIXfGY75O+XS/DdJwIuzQNM+t3xV/7E07dwo15k8DefU34EgcD13Oxepc7w + phLaFcf8+OXzMq5ROQ9sq17ZIw6290m3OmqdpCfxZmQeBXDGEB020n9mPK3ufmS7/srt1WcPO/KJ + fX1yHljs4cte+Pbt/NqX+5MA4lE0iijxqCHEACxStj1xA2/RlzjmMZ94Ki44+XkGybMSiYRwnZH8 + qc/4cgx/7YuvalA+7guGPkzouohZtvukHX4tEbjXI373U5jkIV87GBdzz6s6eNs6vHRmAIb1cLzv + polSt2Guy+kcl4SyyF98lUa84T/2JR44d/Oop/FDjyN/bNUZXObuK6qP+VSewrPeHsQVX+qkL7qY + MTb0wS/hWgfrJBOOpTN5V5z6kB09RpyKoW39lV95li0IXqwD5+IZ9SsOtuuDhb6A5HbVVfHKo231 + o4DGkXfljQ7Xr13/0u0P4L39JI0n74HFhl71mf8ET+ZvoKp6QmeGnLO7BAAAKHpJREFUi7aPxlK2 + TXGDy0wuic6Fh/jAcIwLTn4eZvLkzIpHU/tTn/GqiID2177sLmDCdLjccF3EOGHnlx3+4hj9Ja7x + DOdIHYxHfvfHPDEN7HjhCPRYfAvnT2z4BTNRxwt20JVUXQfjphB0rr7Ec+hLZQOzm8mXuOAz2yF/ + v1yC7z4RcGkeYNLvjr/yJ56+hRv1IsNT8RsW5PmGR3//p39Ef0d7632Liyf3gYWirj/32p8+Pzt7 + vT8R9k9sXoFcDn8i8b2AHVzO3czmuD+G/Lg2x7jgHG8+5VmJxCI6OByffJU/b/r2177sKkJ1ch/l + Yuvpb1g+H+t+k3OGVtKrznOHz7lT4vJzqXEJ3ufIc8h53CQPMMprZr/qIMd9Kt/Cjf547/Br9uHM + VbbuST3UxJt8cqiw2q57U3HK6Q6S13mWnilZ/kXnfoZ+9Ls+ROhhv+r3fvKYUfeXDinoeuY3rOvn + 19/w8H0veFJ+buWK/KqS5saTsv67P/Jrz7aH3wQtnn+zT0KJTHHr8vR8okjzHD7pEge8/Dw0HqJs + vPTCZ2h4XT7lHcCyO0/7geFof9H6etTlEAAvI/8NfmJcZy3aHmWOPHV0zDt5ZZvHV9B+X9Kxv8O5 + rr0e4VcZQ7fsY0aCohkL43evOYea+80DUFW3n3NOY7a+7OjEOPDnnnxYeSb9RfVyvyvGsuz10ApJ + 1Vtw1+OHDXWjLR7lMY314D0uWlENHPd53vjleAH8cuSBnXtPQOuRvDrw4z6BoavFDneG/2XZ+d0P + nz/j5dsf+HVvN/LJe33Sv2GpNfw86/xR/wfSEb3FBMCHlUODaDhd4TKTRKKLTS/mqcsgt+OCc/w8 + HIDGrcgZuo7kM35/qWZdowwSgK8mHDEPe16qZSvtuARqgC+jP/dzqJeY5GF81888MZnHcZqFI9DD + /Q0/An354RfMRCs/4w66cqvroBuBtMdY8eRbOnR+huE3ozRXvOKCz0zeA3/si/CPKY/o9/W7v9E3 + a8BQPq+GDRx+zT7cWZUteaIz9QiPFmVHD9ex2rVCyXvRw8o6OF/XMfRrPUQnRNffeBSWvLq/dOiE + wms98D+Dws+tnvyHFau5PQ8sZv6KF78GIn4zPwE4/M0lhzZs+qGicJkdoLi8mMc4n4nj9GYqfmrf + eZi2D4/7tu1PPuMFVB0znvwVxyLaX7S+vspHNwD4PeqTbT65+TL6cz/2jzJXHsa3g7wxmWfx8mHU + t7D2HbZwehPQpKN4V37ujbolFGHkFdr82SccY8U7/7SVH5jdXPHCRYfMJuTrGpfgybPjR6TsU3Hw + pb5OQFzlDw99C0fBYkMf/Jr88JQfExyuxzrGLsecxGNeheOFPBUni3mW3SjVyzyic51Dv9ZDdKuj + tZ88ZvS94Zr74VVm/NzqBU/qz61UQL3cvgcWC3j4k/4Y/ln4h7nsTwCsrSkvgfd5Cv4EqdkBfO0h + PyKaB4fl0+PhhGfkyZkRJz9ekNDxyVdxdgz/5FM4N8qvSZeKK9elDErQ9QGh3covI33CSFzjDV95 + GG+BgGaemHtef1IS6LH4Fu7pb1hL7+hkQcd9GvqVkpp8TsDhV99Xe8qP6ZbuVZ1f3wuF44XntO6R + 8yy7Ucf7J3vh1rmTTpUqdO0njxl9b7jOPdH+Wx++51Of9J9buSK/3t4H1qs/+cHt2jN+H0R7sJ/0 + JVHb+sioTxQcAvc1MtuqffslfcX5cHhGvDWe7e9Ei679yVdxDjzEh68LKL+PWJ9QiOt6fZ1lK7/s + 8BfH6C9xnLtewtKH3gVlK0/lpV/bNQtHBo/Ft3B6E9AUDAvMKz/jDrpyq+ugWwHc7bHiybd06PxA + Ml3SnsSPOOVrdgYysuoMLnPt7/ixJ/tUXHgw91B/o+9ypE5XXvmpD35N/uVHIBzuu3QouxxzEg9T + VZlciSB5nWfpSSyH/OIdfZ7SQ3SqtONkKSF5te066FB+TQ9i+cVP1p+3chU3vt7eBxbr+dIX/TCE + +Op+0mPLmvISQHyIThV3M+O4P4b8iGieigvO8ebbnVnxaILD8clX+VmJ+OKv/dpWGe2v+n19xecy + dS2KH/HyF0/6SJ+w3Y/nrpe45NG7oGz1vXQzzPr4k5IMHjt9sLX6hSEY9Z75GXfQlVtdB90K4G6P + WX+fH7ydH2umsyozH/NH/5rJynxzlK08J/CPKY/oT/AXb+pk+vTlymOjTvwSrus0n0zJQ3/6WnGt + I9smvxShn9k4uFg6OM+yBSGKAcpTPLIXrvUQnTMxdu0njxl9b4TQBrxf/aFXvuCt9t6+19v/wGLv + X/aSb8InxDdYsjoiXha4/MnBJ79PQzNjaI/hfeNmXHCON5/9CO5F0cEmTpdC+WwLOOzFX3Gso/1F + W9ev6x224mWHvxrBJQt+zqPMlUe3s/Ki4kp/Y7xwZPBwfyuvbcdLj+Jd+Rl30JVb6Vdu8NMeY8Xv + z63zA2sdaq54xUWHzOQ98Me+CP+Y8oh+X7/7G32zBgzl82rYwOGXzyk8nlW25LnsXkWPEdd5XAfN + 2/ENCz9k/4aHX/kbvknl3OaXq/HAggjXf/rFX3t2vv29vsS8rNj3JwcOGzYvkWaKRnsM79s/44Jz + vPnsR3Avig42cTf/JJx1jTJUn23R1tux6x328jtftzH6S5zrOZGHfF0/+45pXTpeOAI9Ft/CuV/4 + BTNRxwt20JVU6Zdrvitpj7HiybfOrfMzDL8ZpbniFRd8ZvIe+GNfhH9MeUS/r9/9jb5ZA4byeTVs + 4PBr9uHOqmzJc9m9ih6uY7VrhZL3yf6GdX5+/e89/NZP/Vo1ewVeqPHVGf/u/Blnb3/Lt+Pofrs+ + IXFq+QTtNwXfHCeGP1HrUsC/iytbl5hvAtl46QXxvlzNo40BLPtGPzAc7S/avB25r8HZ/fgK2lZc + ITipbi+063ymrw0l2OMGb+XruEMdvb/DRWcV0Hm90KvqslzVz+i3Agw89cp3n/CeL3yYIFaq1Ln3 + ec34W+A/9bAptaPGxXkmf+W9oV7uNxOWZa+HVkiIwyi4eeohiIJoi0d5TOM6eY+LlvGLoCz6zaON + vBx5YOu8lcf5zLuvv/srnNIx6/n5az/4y3/9Fz4e/4v5lPiRzlfmG5Ya+YKzR86vP/OLsNafhG9x + cdl1uJkJjrgK9GHw8Hfi500iOA4JhMuPzXErRNf+5Fu8ytf+2pfdBcy7hzP3pVDdgixbaetWLD/r + WZcw+7t6ycNCmZfxIgpvTNqpOzgCPRbfwj39DWvpFZ2s87hPQ79SUpPPCTj88nFYVyrPcev3qs5v + xpkBr+NeKM+yBVEe5OO98ISZi4Vb5046VarQtc96idc2/3Do6z/4rE/4oqv0sGJl6ya7zqvx+s0/ + 8svOzh/9Pqj3qfomUeL7NE6X7G8O9cnDxvKw0uHY1uHwEOXHSy/or8OW359Myw+gAMXDS1N2K9j+ + opWj4qSqL4nrBM8NfoHMyyX5NI16vaEEyh+76nE7h7hDns6/45/9rHjSK5wT8JNf9e3ac5xiji+H + 8+s3SdGLF+uegVe+MTsfESfGgT/35MPKM+mL9wYe7kdX4steD62QVL0FNw8fCuyPYe0QHU2qeJpH + ASK23zzJZIcJmkd5Fq77SF4p7jqUV/uwr5//xAef8azP3X7vr37C/qrjXd2Pwbha37BSOP5OaBzK + F+JE36lD5ZtFp7DEx0bQmnP4u0PxrVj+uiQ+HGz3godm2/HJ58Oct2nPX3HMoPp60qVjnOsSQAk6 + flwWejXSJ4zENT7tJg/ju37miWlgxwtHoMfiW7inv2EtvaOTBfX5SeahXympyToDh18+DusKxvJj + gsO6130ouxxzEg8DdR/NIIJ5nqf+cVB+8Y57cLhPro90WpkdibzPelHf9e2d2513fuFVfFixYNZ6 + dcffeeuLz+64/t2Q8eP7E1Zi31iyvwFQfF4engla07vYLe4+seXHi4EiW3DG8xOQ8QNYdudpv8J9 + CYBf28xbG84g2/EU/ug3j/JySSJN6UdmJ9jjXK/LPcQd8nT+Hf+xX6Yxj8J3dWQf86692q8yd1PO + oWa/eUteAFX3nPkmQn7harawRJ4YB/6c+4eVZ9JfVC/3oyvxZa+HVkiq3oK7nvTFsHaITulIh19U + U25RDVz7zSN3XoqgebCIjoS0HiJWBkVmH3+I+77za3f+5g/93k/5oVBetflqfsOKSl/xGT90fn72 + RfgbDR/R4eby0i/RA/Rh8NQjfvDB2Z6HhthxK3KGjq/LAL/iCCSgbIfFrhraX7T1NnQ8Mb505qdF + O/xaInBdwsQ13vBVB+NdCIIZF3PPqzzgzVh8C/f0N6zDOVAsCTruUwmYcwFAO7aBw6++F/aUHxMc + 1r3Ot+xyzEk8DGR6Dy4qDivnWXaBxO88oit74da5k06VKjT716+ff+Ds2h1fdJUfViz4aj+wWOGX + v+i1+Jr6ZVD5Ok9xffJT9DW8X58o/397VxtraVWdzzn3zgxQgQEKSP1ATSWhlaaJ/dNUrYxNTEYL + CRoxVRtpy4/GtJo2TfnRxFT7o/5ppLRNjKlaCtoMatKG4h/LxJg2/eE/EezwVQlWKylcYGBm7syc + t896nrX2Xvs97713mLngPTPvvtyz9seznrX22muv+56Zwx1Mc+xJZyiNJZkCcWZ+meMMDceksAkA + gxedMpZ+jN2Hsk4YckcGpE8P8JL843rwO0faX+jJH8P17DA7MccF43W7Diz6xJnHapVPhBpL3/YL + wrJPjghLfocjblDLDJQM+GuxT5z0xWfxpZVWZr8jDiGl6MwutsCXfQLe2BvSAyb8LUaS36FvaxVX + 48dzRdyIc37t0PBQwoL88Tj42BeyAEvwhiciCLv84QLCGBcU/ZW94keKn+yHP0RQ1ebn3WSOSnDz + 0ZvecH/w7VS58wuWRe53rrPPZ/2xnb5+omEOh5Gb5rXOI7Z1HqJwXMdC/EShuoCkibHW3Y7jQSR7 + jb7xJzdor8CQUmY3+ZvGUPN1w8s/OpH2F/ONvwYKO8ZHorATQ/EVfeIMqFb5Km58wuqdg4WKcU75 + lOLnkaRQnIHDl45DcQWBr0NgQXHfLK/8/LKeGEiQz1PnFXYIUh7RTsqDXj7JP6NjT+zMv+kfHnvf + m/5FTDv7dTkKlsXwlrd8Fn8i+Jnyk4VBr8HVvH7y2FFy7ElnKI0ltY7J0qlnaLjxCcvjxbhZ9Hpx + tSmLP+JnqR+X27rRdB7Og3PIY4YdwEb6eeqcHJ/0aC/ITW6B1znKvy3tkI47qRa4v7RvX4l9+M59 + X8Dhi3bcr7oORSzIn9iXxr6QBXnMVKFhhF3P5mmnjg1rTXGTXvEjxa/Eg7UqIkLVPz/2/jf8LXtL + 8NI7pSXw+AsP/NlsNvn00CXhKfsh5UMrp8/D0uFqHfstHfSbdSUF6JQMyro2uZBVWve4MclT0iwk + lwyU5Cnm/Rio73bNHc/agndY7IfryX83X/QanLtoovKJsI4dVPaRL+GiXw1/EwiEKc4hS9DT3SEZ + +kPS3WqE47bFTkPsg8yPKfObzeZTK09OQPB0uc5egetYbd3yBfoCYh0dH5fpsFTMWEd6Yk08fT8q + ndvxCeDkp8yFfTB98uhNV38q0ez4rsVg+doXH7gVSfA5OF/816Xzw8SOdJktOQThOrKiuZzMhVj3 + 3AE+P2FJ33ko+usePmalzJFWWVHsowNgsr+wLh76bd3sdx0WAy0u8fb1enbq/mPf/f3UeXqkYYpb + mbDthJupQ632hbcRei7L5bF94VvxSjLOKUkaMv2h5rzBH/K07GR+513goR/hORR8XN8WBon76xsU + jxcfhqMsMBA0Z3SWJ0FLqoQr6+Lhcrw4QeFBh+dtE2ixD3woFBOz3zt60+u/EKrLIi0uy9m++MAH + 4fyXcD/3lA2waPQuL0/Pt8kssLtll9TvWOnYBL4x1noctsa+kIUng/OYE7RfBOhSMbB1N1DsY0bm + wz/ISDJDG5/L7CbtxHpacPNFr8GRSS/F/mb8JQ7hRy+uRpX0o0iEGV4OrDcSi3R3SMY5DckgzdJx + DX/YG+LHHI/3dPidj+ZNPzXaj3PGvI3dElHVHOJnOE4QKJyPyzQjZOtU947HkaPEExCbB4EdR+Fh + xydiHRLF6hj8+MiRm66+J6kvTVc3Ymnc7Tn6Dw9cP+26r+MA/HfD+2ECxsseh+ZjHaoVEb9rpaOx + 4LZuh2+HnYA+1mXP68BYK+uupuvhhgjAi/RIu7BuGIP7kbiUvUwjv1pc4u3r9exUPtnROO+nzssh + vtIvhSP5h4lwt3aEb17jHFzyHABQHAYkcPQrSfKb/lBzXt1Wi48cOy07mX8jHvphcXB/fDz8ZOQw + uoV94asWF48A7bjbhBtOYwjr4dv3xVHlsdXS+jwYRxwNg3/M+NlusnLj0Zte962is2Qdi8tyty8+ + dN10euKbSIQrIqmbS8lD9G2aYC4PFy1liYpGFC2pF0XX9xTCbc30sq9LrBRLxY9RFk/xT+4wqbSM + 9UgyTBjOWsFrKAOxTkOBq365ootQ5DDxhV6Kh0HMbhMnw3nyc9n5kn9tIGwb9bLQfxuTBfRDMgI5 + JIFfaI7bFjsL5JjI/DYMjM2npqLo54x5G9cdFhrFAzvnOgPgONpxnOmHpWJGONnRehS95Ib4wet0 + bscnjHc+f+rk6vT69Ruu5m/4zbrL1F+evyXcKKq3XPvdbmX2a7hkj9hx1ssdp2fZYXfQxpJMAZsu + HZvXWPriCbwr9vSDz9jRCr/T+rWkXQHwmvzjuvvFdVt2u+iGnvxx/wwXdky/+G96MbSNJH3iDKhW + +SpOxRnrhImo2je95DcDhanihy1DMeYNjlb1ja/dF90GppGuT73AhxShvda2Bb7sExpb2gEm/C0G + uL+0b1+ouBo/s8Ai0vDEOiYZnvTDy8e+kAV5zFQNp/Gk+AERP0wNF01xkx73m/3vJo/Op7NfXfZi + ZXtVVGPXyyzvfOSK6fwYnrQ6PHHZoXoS5suks2dyah2g0jEFjaXvyYp1JSk7bXKZHZ+GpgHb9YXk + koHin8w5f+inS2J8nI79cCg7Pp/9d/MtX+Bc1USxvxm/7xvC+Rb9wgJZGZ8mEAiDF5tGAk2+IRn6 + Q5JWei+Oa/jhT30Sgd+naqdHzWHmd54yn/CyZ+eD+BjO9IplG1v8PB6WD5wgUDgfl2lGyNbxzSa+ + ui+z4zwBgeQ67bh1N4z57xzZvWv/ZP9VTyX40naX/wkrQv/bP/+TrrvgHUiOgzxiZYmyBRhdKkmt + Y7J0HIaxLrMup61TTx3ite7zXHcHaE88pPWklb5hdH2KPsfB7xxIssBnST5TtxZ2TL8smF4MBSz6 + xBlQrdg3BTSNpU8+5y36hMkvmZNeGNQyVpxPVuq+yNPbF3kAbGTyx255X6/PH+MGF3plX/DjVOw4 + HqI2BjTt21ciLmKOfQKHryY+tGzrUGR4vKiRt+oRwPWIh+Lr4YCyCMKu7Hh8qregAS7zYIzfuvCt + Iyurv362FCvbrqKTNr703fse3jP7ybEv4QQ/GMmsrPHdluSxpMEcxsPrnqyOz9nguQE1+0lX1cnj + Y9IyiXNy1eTTepj3YyCx26Vbmqed5Gb4G0laxwN8iQddtsq3CT/3kS/hol/VLniaQNgw4pckrDf7 + zuPQH5Jyu3113LbYaZk1yvyYMb/ZbD412o9zxryNkRHWI8ppFA/DcYJA4XxcpsNSMSM+2TFWxDN4 + aEEvXI9jwBT+R+Z7jux+/Ucm+6fHEmzpu8rYpd9GbwP4P6Zn//jQ5/A3iLfmS6VDjUsIHeaCQuC1 + AnBb1yWr6wASYKK/7rbLutNG0to8m0m/vOhZ0hFZ1gkiP3s+X4uL1rMfFZd4+3o9O5Uv9t3fT50X + v+z29aofyS/vLoh6axnHevksDhGvJIGnvSRlr9zi1kSPH1Xh9O1k5o14bL54bhvQuL4tDBL31+Ha + N+INvB2TSfKY5DhYsR60pEo4m7c8wpf0CdCL8+CjC589csPr/ghGTPGsahaXs7at3Png7+JQ70Dy + n89NMguQFMgWpoCPmT0GaNbj0gjP68QsszFH4rFcMz1r/XVLKkuuCgDIeQ3Okak5AfXTus83/kIn + DFKvbKSYb/kIDwdNWfakpvlBfu4r4iRPKy7563y8tLEPzOlS6lJRzy6pzRt+SPKyYWVIAr/QHLct + dhbIMZH5bRgYm0+N9uOcMW/jusNCo3gYjrwECufjMh2WihnxyY75YXF0HjBEk93JEfyDER8/euPr + Ph/zZ5v0zDvbtpX2c9dD181Odl+bzaZv5qHjUjEFyq1RCLxW8LKXIpOBBMRlV1ERkdsq60ruxSco + GSyXHklHpOmlZutsLgs+YMVOxm1S5Hp2Kp/0Nc77qfPyI9yJuCW7iE+4WzvCN6+8jdBzWS9fvdrG + qqtpMN9PkuQ3/aHmvMEf8rTsZH7nXeChH+ExFHysYuJD8ri/vjHxqNjwGIOHdqQnVuw/aAsPZoiH + wFe/aIH70RPT1RvXb7hqqT+2wO1u8nL2/KH7Rpv88LXfnZ+YvRWfQ/mqZUFJvno7lBIDSRV4XquS + VJFMUKg5VLLNYUwqA5g9NRko9nk95U9xHVkc+Cyb5A0/TL8smF4MZa/oE2dAtWLf/dJY+uRz3qJP + Ovklc+IPg1rGStlntWM98vT2RR6sNTL5Y8Wpr9fnj3GDCz232/CHvSE74SdkaYZzvuCxtYgLeoRq + DH/xRVyJQ6wDhgXDlSLjY1/IgjyyQ3q8GI/Hg6PEE5DJ5K4Xj+355bO9WNl2LcbnTLO3iEjCO7Dh + 85GLngsKgY2Vo558nAAmgD5efCLx8JX1SDE3QH3DaFyecHwsfucwVOBdFrzclD+8Sz5BHJIYkqnd + 1+vZGcZJ3wnoTPVDvvX16Df9cN/dro9a4ZffA+yXN+I0IO1y236SlD3b4UDr8Z+RnUzvvCo2yU+b + j7ga3se1aAWJ++vwXLQsXDYmD+2IRlliRclpSZVwNo8vK374g/Wj+F1Wt+It4F2EnQMvnvXnwE5j + i3iLuNp1X8ORv5l3jLngYVC2+OWPSxNFBEBmWRF+qTQmfX8dScXkoiFDyEC5/JiR+bAPieQt665X + x7RSDNq8E3ChDhOfWS32pV/5hKvjxM/t5iK46BfjEfy8dG4Xc7nYkN/2ZfOGH5KhPySBX2iO2xY7 + C+SYyPw2DIzNp1aKEBDcF9djh4VG8QCCRUpAsKBDO0WAxfmLGet48UavFKv5/Jx4C5hCze7Z/5aw + v2O8RTyxPnsrcuBLzI1SFJgNXix0OW1dl50dyxZf93mfpgmvFqVoWDJCQfqGqGOoYWRj52EPL0je + wGdJvOCmID9MvyyYXgwFLPrEGVDN5qVWcSqqWCdMREWfMPmV9cKglrFiDqRW9Y2v3Rd53JxpZV7q + BT6k8fb4Y7wR3uZP2Q7pW/+1v7Rv8wEt9oVeGgOHr7yPug4YFuSPx8HHvpAFeWSH9HhRhMKuv628 + 68WLzo23gBGFkL1TiulzQ67c/dBH8cth/w7JMPC3iJ6smyWXXQque7zsUvkYwpPYk5SQmnxaJ7xe + Auq7XdO3sUvi47TSvN8Sxw3wuT4B/qLLswU/95Ev4aJfcNDtQjaBsGHEL0mgm33ncegPSfe7EY7b + FjsNsQ8yP6bMbzabT+2VfMKCF0dPTrtbj7733HkLmELNblyB/vy5M7a3iJPJ13CJ8beIusTlbRxv + l4WICy7sEusSMosjgiw2pq/k1hOU65VQp0vvPznj0kfAo0jFfC0ujih23DCLRuLlOPYB2bNT+aSv + cd5PnS9uo9PXi426OW08NtGXfvmjqOmSR5wGJPC0l6Ts2YEMtB7/GdnJ9M674K/NR1wN72O9XfMh + edxfh5fiBjyPMXhMWt5IKL+C1nnw0cLvTbvpzYfP8r8F5HY3efGs3wRxLiz9/fcvXNkz/Wt8zu6j + VoTyJWYSeVFQluluMgcxr3UPErMwrSMLS3EjxMKdigtHsqdlZW2/ONQxUTKArs2bv/TLRfhFJP1O + /JzUOOMG+QGo8z2/nbfald8xNjO8nMA10ubxLbaejEAOSSPsN8c1/GEP2Jdkp89t48zvfITZfGql + CNm5Yt7GdWeFRnGwfCAvgcL5uEzTc1vHNxsz6K4XfuaqW/GvMB+N2XNVWozH5hFYvfP7+6az6R3d + tPuFthgge+ySUqQiFLlp+mVdudZ/sokkLkUgrq3ppUa7Nvb5gg9YseMTxKUi2Nfr2al80tdY+rwk + SZ9uFTO2b3OrTHg8iCr++qgVvI3Qc6lLHnEakMDRryQVX/NgoPX4z8hOpt/IX5uPuBrexy/DE9aD + k9nsDw7vf/WO/9dscthezr5n38tpYsm4D3aru/7nvz6Gvy7+FD5sepFy1sKEJLXLWoVfqnRX++v2 + ExVf5ZJ7kqtI1JQv69TfpPjEaXnRoB7vjhbcfLWXcekYiv20nmjSPnORWvSL8QCv/PDAuB0WJfA3 + UlEsV92jqrEXhyg2jUy+l67jG/6w91LtFNLUyfzOx1WbT03F188Z8zauJ6taZmEuOPISKJyPy7Ql + 2GTyHMafxFPV3+Cp6oRNjE0RiCswxqMfgQOPv3rX+vrtk9n0A7lK6bJ7EYrcNF1mZRG4hH4dbZ5N + 41IsFtYdFXiXBV9o0IFdFglTIW6xmBS9np0y3/Dn/chQ5a9+cbvZP/qhdfnh/b7gbTS/paDLy22E + d60Ejn4mSX7TH2rOG/whT8tO5nfeBR76YXFyf3y8HU9YID2A3wr68Rfec8WPsytjXxFQdo7R2DAC + q1/G28RudscEbxN1aVJRwuWNu0ICu8x+iS2VrWiNT1ip+CAmisuAjEAOyaHTcRyLCc/hDOxsxe9+ + E2Z2U1Mx25YnrAeRX+PbvxTboe5YsIai0p+zt4n/+/DHupP+NhHXjk8gcfsMP1CsSvUin34ilyec + eKYwvdTqk43mCz5gxY5PUN8vK3hCv+j17JR5t6tx3k+1S7eKGbuUlT/vV7hwkKP2pVeEFp5YjBff + Ec6hIiR7bbEoRnr88VPktOwUUnMI9hCnBR4WrfDYcfHDKYbkcX99Y+JxPqjbGAaem8/n49s/xmvr + l02ybGvlcw5hbxNPrN+OS/4BphqTmTmtUFgRwIILpLD/5LUJNpOpuHBkeF+nYlr3+VpkxEID6FJP + jnCB6jFvM0mfAH+pfLJbxwXg+8hFatGvhj+KhlPwcsJ+I7FGd4dk6A9J52yE4xr+sDfEjzlG/3T4 + nY/2TT812o9zxryN3RJR1RziZzhOENfBnzvn3cpt49u/FNAtun5TtkCNy00EVr/8yL7ZrLsDtw9v + E33Jq4ULJqff+gBApktPRSS3KaTGImRjn9+smFCNuMTb1+vZqXyyq7H0VU3qvPj5Cndy8XL/svtu + V+jea7213JcuOaMT3rUSeNpLkvEwnqHW40dVOH07mX8jHvphcXJ/fHyqf4aF/wfwwenK7Nbn333l + f2RzY3/rCLS3ZWv8iIgIHOhWdp98+Gbk6p/ioxC/xGm7tH6JLZXHJ6zxCctrnp6suul38fuqPvPC + 4Sv+afKB6clIpVGeegTGgnXqsdoQufKVR9+DJ67b8Psd3+Y1i8WqVC9q6idyeVKJZ4rek4mtCy5Z + 8HFSpShm3PiEVZ7AGJ+d9YTVzSf/PlmZ/eXz77783g2TaFw4pQjENTgl8AjaPAKrBx59G35Z4G3T + 2eQ94xPW4ts6SzbFZUDWRxG9Hc7jobD7Ot9eokg18qXa2Yrf+Qgzu6np7a29XcZ+MW9je7a2HaJ/ + 34lu8ukj+1/9n0ll7J5BBMaCdQbB20h194FHr8P/VH0bPgpxM56QVoRTEpcnpvEJa9PixTjh8ocs + T1BDQffihQrRFDsVk4HiCI4oLsEfsqF3vgUem4/zMwUfW9HCf3ir1x2Yz1f/4vD+yx5s+MbBGUdg + LFhnHMJNCA789xt3zY//yXQ6uwUPAedFkvNyQE0lDNLexljztzNl3efrWDDiCIce7470qR7zBk36 + NoxW+UIv/kDdEU7U4hbfdjb8USycgpccPI3EWr7qZr2MQ39IOmcjHNfwh73Mi/6WdhpiH2R+5+OK + zaemYmbxm78AS59fX9312aPvuuQHCTJ2tzECythtJBypBiLw5ceu3L3afQIrvz+bTi/WJfVr5EUl + tKxIsLmsRcMRNg+CFrdYTIpeXNcFPtkRTvqqHnVefoQ7UdSSf/Qj+eXdBdErQrrkqVhBwVgVF0jg + 6VeSLI69YlHs9PjjSeu07BRSc0gbXOChH/IY3afxj2rdMb1gz+3Pvn3vM1l97G9/BDz7tp94ZByI + wIFHL95tRWsy/QQ+hHNluaS5SOEGlGKzUGScM823RcYvfeaDivHldkr8XhRZRKi/6BeLSPD75Q47 + Q0Wn7BegUpxM38ahPySDNEvHbYudzBv9zI8585fN5q11kyfR+6vn1n/2c5PfnL6oyfH15Y5Am8kv + t7WRv0Rg99efeMvsxPEP4dL/Fr5fHwuluKSixMsdJ2XzmGhxi8WkFKV4dlngE6Fw0lcVqfP0ye0W + PudhsaIf7nnMx0ay7BWhhScWYM0M92kSeNpLUvYMMdB6/Bs+GUF1SzuZ3nmTv0+A4CsnT87uOvzu + Sx/I0LH/ykTA0/GVMTZaGYgAfjPbeV99Yt98Mv8Q7vz7cCAX9YtDHbt+Kj5tkfFLH8Uj45Lpyqfj + r+PE70WRRYQ8i0WRRQQqph9FIswMFR2zVoqS6eXxBkWHvEGaZS4msN/Yy7zob2kn80Y/+OfdM/jL + k3tOzlbvPrxv77ex2Q2qZiiO8uWMgJ3l2HZKBA4+ft7up2fvXem6D+Et434Ugt2bFRO6vUExKXpx + XYlTcalFKMbjExbjZUUKDWIdJfAb+Fly97OvufSfJ784XefC+PJTj8BYsH7qR7CBA/f+4JLz1ifv + xyPJh/FJ+rcDZXdKLRWf8Qlr8e1j80RlgcN3ebLzJ6fyRBhFChDgvg22uyd7ZveMf4DuubbDRFyB + HebW6E4Tgft+dPn568fxa266fd1sdj3elOCfKPOjo1x8uzY+YfnbRASyFCv0c/HCb0l4ZDad3Y8P + Th2c7971b4ffceFTTdzHwY6LwFiwdtyRbO3Q+fc9+dru+ORd+Dfa9uE22q91fq1pWe3Kb/c4wfn2 + mEsxS09q0nPbTtTiFotiwx9PLk7R/JkS+NIfXDdFoxST0B+SztkIx700O92T3by7fzJbOXhsNv/m + kesve7LhHAc7PgJtJu94d0cHhyKw51+fvGblJJ68Jp0Vr+uBubwUmygPC8VJRy+cipFXO5qweXVc + WNFBt85jHRMBqx3hm9deETqV4kW/oBeS/MYz1Hr88Xavm89/BP2D80l3cNKt3P/sb+x9bEh9nFue + CHhWLo/Do6dbRAB/Uvyqb/z4WvwPItdPZngCm3TvxCftL81aLAKYMGmtjjlUFWIxykVq5z9hwfGn + 8Ynzb+Fzbvcfn3YHD++77Hu+o1GcJREYC9ZZcpAbbgMF7Px7/+/nVqbHr5lMZ9fgf8y+Bk8g7OPh + 60345P0q/8ddK156hCJVFDM+oGGmFjVPGce/0k9YcPEE/Hwcny44hBJ6CNs7BPcOrc92HzryzvN/ + CEc3eAzbMELjwhJFYCxYS3RY2+4qfvXzRcefeuNkPkUBQyHDNxLiGpSna3DrX4Nixtuv4vXKPWF1 + MDvt5j/sJrNDkIcwftgK1ImTq4eeX7nwsfFfktn2TFgawrFgLc1RvcKOfqfbdeHzz188Pd5dMpkd + 3zudrO6dTE/unUxWICeXwJu9+EN/jPGNMYqa+jEnd9cg1vBEt4aiszaxD2HOpmvzebcG3TX8Mru1 + ldnkGfwt3RrK4Rp41ubT2drhC161NvmV6XFRjK9jBGoE/h//xb5CiJqhJQAAAABJRU5ErkJggg== + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: true + type: MultiNamespace + - supported: true + type: AllNamespaces + install: + strategy: deployment + spec: + deployments: + - name: clickhouse-operator + spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse-operator + template: + metadata: + labels: + app: clickhouse-operator + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + divisor: "1m" + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + divisor: "1m" + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + divisor: "1Mi" + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + divisor: "1Mi" + # Honor the OperatorGroup's target namespaces so every advertised + # installMode works (AllNamespaces sends an empty string = watch all). + - name: WATCH_NAMESPACES + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + image: docker.io/altinity/clickhouse-operator:0.27.2 + imagePullPolicy: Always + name: clickhouse-operator + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 512Mi + - image: docker.io/altinity/metrics-exporter:0.27.2 + imagePullPolicy: Always + name: metrics-exporter + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + serviceAccountName: clickhouse-operator + permissions: + - serviceAccountName: clickhouse-operator + rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - clickhouse-operator + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # discovery.* resources + # + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operator's specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete + skips: + - clickhouse-operator.v0.27.1 + - clickhouse-operator.v0.27.0 + - clickhouse-operator.v0.26.3 + - clickhouse-operator.v0.26.2 + - clickhouse-operator.v0.26.1 diff --git a/deploy/operatorhub/0.27.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.27.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..7ad285b82 --- /dev/null +++ b/deploy/operatorhub/0.27.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1847 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# OPERATOR_VERSION=0.27.2 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.27.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + nullable: true + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + nullable: true + x-kubernetes-preserve-unknown-fields: true + actionPlan: + type: object + description: "Action Plan" + nullable: true + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + # StringBool is polymorphic — accepts native YAML bool (true/false), + # integer (0/1), or string from the recognized vocabulary + # (true/True/TRUE, yes/Yes, on/On, 1, enable/enabled, and their + # false/no/off/0/disable/disabled counterparts). Validation moves + # into the operator: pkg/apis/common/types StringBool.UnmarshalJSON + # normalizes input, IsValid() rejects garbage at normalize time. + # Structural-schema rules don't natively support bool|int|string + # union, so we use x-kubernetes-preserve-unknown-fields: true as + # the documented escape hatch (k8s apiextensions.k8s.io/v1). + x-kubernetes-preserve-unknown-fields: true + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + # both humped and all-lowercase accepted + - "" + - "RollingUpdate" + - "rollingupdate" + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Installation. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "Auto" + - "auto" + - "Manual" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: &TypeReconcile + type: object + description: "[OBSOLETED] Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) + enum: + - "" + - "Wait" + - "wait" + - "NoWait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" + enum: + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + macros: + type: object + description: "macros parameters" + properties: + sections: + type: object + description: "sections behaviour for macros" + properties: + users: + type: object + description: "sections behaviour for macros on users" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + profiles: + type: object + description: "sections behaviour for macros on profiles" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + quotas: + type: object + description: "sections behaviour for macros on quotas" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + settings: + type: object + description: "sections behaviour for macros on settings" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + files: + type: object + description: "sections behaviour for macros on files" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + runtime: &TypeReconcileRuntime + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "The maximum number of cluster shards that may be reconciled in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: &TypeReconcileStatefulSet + type: object + description: "Optional, StatefulSet reconcile behavior tuning" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is. + 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards. + 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Delete" + - "delete" + - "Ignore" + - "ignore" + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for StatefulSet to be 'Ready' during update" + minimum: 0 + maximum: 3600 + pollInterval: + type: integer + description: "How many seconds to wait between checks for StatefulSet status during update" + minimum: 1 + maximum: 600 + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is. + 2. rollback - delete Pod and rollback StatefulSet to previous Generation. Follow 'abort' path afterwards. + 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Rollback" + - "rollback" + - "Ignore" + - "ignore" + recreate: + type: object + description: "Behavior during recreate StatefulSet" + properties: + onDataLoss: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to PVC data loss or missing volumes. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate - proceed and recreate StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Recreate" + - "recreate" + onUpdateFailure: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to update failure or StatefulSet not ready. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate - proceed and recreate StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Recreate" + - "recreate" + host: &TypeReconcileHost + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: + !!merge <<: *TypeStringBool + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + replicas: + type: object + description: "Whether the operator during reconcile procedure should wait for replicas to catch-up" + properties: + all: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for all replicas to catch-up" + new: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for new replicas to catch-up" + delay: + type: integer + description: "replication max absolute delay to consider replica is not delayed" + probes: + type: object + description: "What probes the operator should wait during host launch procedure" + properties: + startup: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for startup probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to do not wait. + readiness: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for ready probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to wait. + drop: + type: object + properties: + replicas: + type: object + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted or recreated + properties: + onDelete: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted + onLostVolume: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica volume is lost + active: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated + hooks: &TypeReconcileHooks + type: object + description: "hooks to execute before and after host reconcile" + properties: + pre: + type: array + description: "actions to execute before reconcile" + nullable: true + items: &TypeHookAction + type: object + required: + - events + properties: + sql: + type: object + properties: + queries: + type: array + nullable: true + items: + type: string + shell: + type: object + properties: + command: + type: array + nullable: true + items: + type: string + container: + type: string + http: + type: object + properties: + url: + type: string + method: + type: string + target: + type: string + description: "where to execute hook for cluster-level hooks: FirstHost (default), AllHosts, AllShards" + # Both camelCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (firstHost == firsthost). + enum: + - "" + - "FirstHost" + - "firsthost" + - "AllHosts" + - "allhosts" + - "AllShards" + - "allshards" + events: + type: array + minItems: 1 + description: | + Reconcile lifecycle events that trigger this hook. Required, must be non-empty. + The hook is skipped on any reconcile whose classifier does not emit at least one + of the listed events. Supported values: + Any - wildcard match: fires on every hook-evaluation point, + including the pre-delete sweep on the dying host + HostCreate - first reconcile that creates a host (no ancestor); best + paired with POST hooks because PRE hooks on first creation + are skipped (no live pod yet) + HostUpdate - reconcile that has prior state for the host; catch-all + for ongoing reconciles + HostStart - host transitions from stopped to running + HostStop - host is being stopped (current spec marks it stopped) + HostConfigRestart - in-place software restart for a configuration change + HostRollout - pod-template change forces a StatefulSet rollout + HostShutdown - aggregate: fires whenever the pod is going down for any + reason (Stop, ConfigRestart, Rollout, or Delete) + HostDelete - host is being removed from the cluster (downsize); fires + on the dying host before tear-down. Always emitted + alongside HostShutdown. + items: + type: string + # Both PascalCase and all-lowercase forms are accepted; the + # runtime comparison is case-insensitive (strings.EqualFold). + enum: + - "Any" + - "any" + - "HostCreate" + - "hostcreate" + - "HostDelete" + - "hostdelete" + - "HostUpdate" + - "hostupdate" + - "HostStart" + - "hoststart" + - "HostStop" + - "hoststop" + - "HostConfigRestart" + - "hostconfigrestart" + - "HostRollout" + - "hostrollout" + - "HostShutdown" + - "hostshutdown" + failurePolicy: + type: string + description: | + Controls what happens when this hook returns an error. + Fail (default): error propagates — pre-hook aborts reconcile / host deletion. + Ignore: error is logged and the reconcile continues. + # Both PascalCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (Fail == fail). + enum: + - "Fail" + - "fail" + - "Ignore" + - "ignore" + post: + type: array + description: "actions to execute after reconcile" + nullable: true + items: + !!merge <<: *TypeHookAction + cluster: + type: object + description: | + CHI-level cluster reconcile defaults inherited by every cluster's + spec.configuration.clusters[N].reconcile section. Use this as a single + place to define cluster-level hooks that should apply to all clusters + in this CHI; per-cluster hooks (under clusters[N].reconcile.hooks) + are appended to (and dedup'd against) the inherited set. + properties: + hooks: + type: object + description: "cluster-level hooks inherited by every cluster" + properties: + pre: + type: array + description: "actions to execute before each cluster reconcile" + nullable: true + items: &TypeClusterHookAction + type: object + required: + - events + properties: + sql: + type: object + properties: + queries: + type: array + nullable: true + items: + type: string + shell: + type: object + properties: + command: + type: array + nullable: true + items: + type: string + container: + type: string + http: + type: object + properties: + url: + type: string + method: + type: string + target: + type: string + description: "where to execute hook for cluster-level hooks: FirstHost (default), AllHosts, AllShards" + # Both camelCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (firstHost == firsthost). + enum: + - "" + - "FirstHost" + - "firsthost" + - "AllHosts" + - "allhosts" + - "AllShards" + - "allshards" + events: + type: array + minItems: 1 + description: | + Cluster-scope reconcile lifecycle events. Required, non-empty. + Any - wildcard match: fires on every cluster reconcile pass + and the cluster delete sweep + ClusterCreate - all hosts in the cluster are new (no ancestor); fires + only on the first reconcile of a brand-new cluster + ClusterReconcile - ongoing reconcile pass over an existing cluster (at + least one host has prior state). Fires on every + operator reconcile cycle the upstream gates allow, + including taskID-only force-reconciles. NOT a "spec + changed" signal. + ClusterDelete - cluster is being removed (delete sweep — wiring follow-up) + items: + type: string + # Both PascalCase and all-lowercase forms are accepted; the + # runtime comparison is case-insensitive (strings.EqualFold). + enum: + - "Any" + - "any" + - "ClusterCreate" + - "clustercreate" + - "ClusterDelete" + - "clusterdelete" + - "ClusterReconcile" + - "clusterreconcile" + failurePolicy: + type: string + description: | + Controls what happens when this hook returns an error. + Fail (default) propagates the error; Ignore logs a warning and continues. + # Both PascalCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (Fail == fail). + enum: + - "Fail" + - "fail" + - "Ignore" + - "ignore" + post: + type: array + description: "actions to execute after each cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + reconcile: + !!merge <<: *TypeReconcile + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "no" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" + enum: + - "" + - "StatefulSet" + - "statefulset" + - "Operator" + - "operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion (case-insensitive). + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + availabilityZone: + type: string + description: "availability zone for Zookeeper node" + keeper: + type: object + description: | + reference to a ClickHouseKeeperInstallation (CHK) resource. + The operator resolves this to ZooKeeper node addresses automatically. + properties: + name: + type: string + description: "name of the ClickHouseKeeperInstallation custom resource" + namespace: + type: string + description: "namespace of the CHK resource, defaults to the CHI namespace if omitted" + serviceType: + type: string + description: | + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry + enum: + - "" + - "Replicas" + - "replicas" + - "Service" + - "service" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + use_compression: + !!merge <<: *TypeStringBool + description: "Enables compression in Keeper protocol if set to true" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica (case-insensitive)" + enum: + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) + - "" + - "None" + - "none" + - "All" + - "all" + shard: + type: string + description: "how schema is propagated between shards (case-insensitive)" + enum: + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) + - "" + - "None" + - "none" + - "All" + - "all" + - "DistributedTablesOnly" + - "distributedtablesonly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + security: + type: object + description: | + Per-cluster security toggles for outbound TLS connections the operator makes + to this cluster's ClickHouse and ZooKeeper / Keeper hosts. Nil fields fall + through to the operator-wide defaults in ClickHouseOperatorConfiguration. + See docs/security_hardening.md for details. + x-kubernetes-preserve-unknown-fields: true + pdbManaged: + !!merge <<: *TypeStringBool + description: | + Specifies whether the Pod Disruption Budget (PDB) should be managed. + During the next installation, if PDB management is enabled, the operator will + attempt to retrieve any existing PDB. If none is found, it will create a new one + and initiate a reconciliation loop. If PDB management is disabled, the existing PDB + will remain intact, and the reconciliation loop will not be executed. By default, + PDB management is enabled. + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + !!merge <<: *TypeReconcileRuntime + host: + !!merge <<: *TypeReconcileHost + hooks: + # Per-cluster cluster-scope hooks. The schema (TypeClusterHookAction) is + # defined once at spec.reconcile.cluster.hooks above; this block just + # references it. Hooks defined here are merged with any inherited from + # CHI-level spec.reconcile.cluster.hooks via dedup'd MergeFrom. + type: object + description: "cluster-level hooks to execute before and after cluster reconcile" + properties: + pre: + type: array + description: "actions to execute before cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + post: + type: array + description: "actions to execute after cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + shardsCount: + type: integer + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + shards: + type: array + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClusterScopeIndex" + - "clusterscopeindex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhouse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + # both humped and all-lowercase accepted + - "" + - "Unspecified" + - "unspecified" + - "OnePerHost" + - "oneperhost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" + - "ShardAntiAffinity" + - "shardantiaffinity" + - "ReplicaAntiAffinity" + - "replicaantiaffinity" + - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" + - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" + - "MaxNumberPerNode" + - "maxnumberpernode" + - "NamespaceAffinity" + - "namespaceaffinity" + - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" + - "ClusterAffinity" + - "clusteraffinity" + - "ShardAffinity" + - "shardaffinity" + - "ReplicaAffinity" + - "replicaaffinity" + - "PreviousTailAffinity" + - "previoustailaffinity" + - "CircularReplication" + - "circularreplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "Shard" + - "shard" + - "Replica" + - "replica" + - "Cluster" + - "cluster" + - "ClickHouseInstallation" + - "clickhouseinstallation" + - "Namespace" + - "namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + security: + type: object + description: | + CHI-level security defaults, applied to every cluster that does not override + them. Each cluster can shadow these via spec.configuration.clusters[].security. + See docs/security_hardening.md for details. + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" + enum: + # List useTypeXXX constants from model (both humped and all-lowercase accepted) + - "" + - "Merge" + - "merge" diff --git a/deploy/operatorhub/0.27.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.27.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..d33ca14f7 --- /dev/null +++ b/deploy/operatorhub/0.27.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1847 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# OPERATOR_VERSION=0.27.2 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.27.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + nullable: true + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + nullable: true + x-kubernetes-preserve-unknown-fields: true + actionPlan: + type: object + description: "Action Plan" + nullable: true + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + # StringBool is polymorphic — accepts native YAML bool (true/false), + # integer (0/1), or string from the recognized vocabulary + # (true/True/TRUE, yes/Yes, on/On, 1, enable/enabled, and their + # false/no/off/0/disable/disabled counterparts). Validation moves + # into the operator: pkg/apis/common/types StringBool.UnmarshalJSON + # normalizes input, IsValid() rejects garbage at normalize time. + # Structural-schema rules don't natively support bool|int|string + # union, so we use x-kubernetes-preserve-unknown-fields: true as + # the documented escape hatch (k8s apiextensions.k8s.io/v1). + x-kubernetes-preserve-unknown-fields: true + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + # both humped and all-lowercase accepted + - "" + - "RollingUpdate" + - "rollingupdate" + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Installation. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "Auto" + - "auto" + - "Manual" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: &TypeReconcile + type: object + description: "[OBSOLETED] Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) + enum: + - "" + - "Wait" + - "wait" + - "NoWait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" + enum: + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + macros: + type: object + description: "macros parameters" + properties: + sections: + type: object + description: "sections behaviour for macros" + properties: + users: + type: object + description: "sections behaviour for macros on users" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + profiles: + type: object + description: "sections behaviour for macros on profiles" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + quotas: + type: object + description: "sections behaviour for macros on quotas" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + settings: + type: object + description: "sections behaviour for macros on settings" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + files: + type: object + description: "sections behaviour for macros on files" + properties: + enabled: + !!merge <<: *TypeStringBool + description: "enabled or not" + runtime: &TypeReconcileRuntime + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "The maximum number of cluster shards that may be reconciled in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: &TypeReconcileStatefulSet + type: object + description: "Optional, StatefulSet reconcile behavior tuning" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is. + 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards. + 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Delete" + - "delete" + - "Ignore" + - "ignore" + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for StatefulSet to be 'Ready' during update" + minimum: 0 + maximum: 3600 + pollInterval: + type: integer + description: "How many seconds to wait between checks for StatefulSet status during update" + minimum: 1 + maximum: 600 + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is. + 2. rollback - delete Pod and rollback StatefulSet to previous Generation. Follow 'abort' path afterwards. + 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Rollback" + - "rollback" + - "Ignore" + - "ignore" + recreate: + type: object + description: "Behavior during recreate StatefulSet" + properties: + onDataLoss: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to PVC data loss or missing volumes. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate - proceed and recreate StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Recreate" + - "recreate" + onUpdateFailure: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to update failure or StatefulSet not ready. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate - proceed and recreate StatefulSet. + enum: + - "" + - "Abort" + - "abort" + - "Recreate" + - "recreate" + host: &TypeReconcileHost + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: + !!merge <<: *TypeStringBool + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + replicas: + type: object + description: "Whether the operator during reconcile procedure should wait for replicas to catch-up" + properties: + all: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for all replicas to catch-up" + new: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for new replicas to catch-up" + delay: + type: integer + description: "replication max absolute delay to consider replica is not delayed" + probes: + type: object + description: "What probes the operator should wait during host launch procedure" + properties: + startup: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for startup probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to do not wait. + readiness: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for ready probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to wait. + drop: + type: object + properties: + replicas: + type: object + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted or recreated + properties: + onDelete: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted + onLostVolume: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica volume is lost + active: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated + hooks: &TypeReconcileHooks + type: object + description: "hooks to execute before and after host reconcile" + properties: + pre: + type: array + description: "actions to execute before reconcile" + nullable: true + items: &TypeHookAction + type: object + required: + - events + properties: + sql: + type: object + properties: + queries: + type: array + nullable: true + items: + type: string + shell: + type: object + properties: + command: + type: array + nullable: true + items: + type: string + container: + type: string + http: + type: object + properties: + url: + type: string + method: + type: string + target: + type: string + description: "where to execute hook for cluster-level hooks: FirstHost (default), AllHosts, AllShards" + # Both camelCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (firstHost == firsthost). + enum: + - "" + - "FirstHost" + - "firsthost" + - "AllHosts" + - "allhosts" + - "AllShards" + - "allshards" + events: + type: array + minItems: 1 + description: | + Reconcile lifecycle events that trigger this hook. Required, must be non-empty. + The hook is skipped on any reconcile whose classifier does not emit at least one + of the listed events. Supported values: + Any - wildcard match: fires on every hook-evaluation point, + including the pre-delete sweep on the dying host + HostCreate - first reconcile that creates a host (no ancestor); best + paired with POST hooks because PRE hooks on first creation + are skipped (no live pod yet) + HostUpdate - reconcile that has prior state for the host; catch-all + for ongoing reconciles + HostStart - host transitions from stopped to running + HostStop - host is being stopped (current spec marks it stopped) + HostConfigRestart - in-place software restart for a configuration change + HostRollout - pod-template change forces a StatefulSet rollout + HostShutdown - aggregate: fires whenever the pod is going down for any + reason (Stop, ConfigRestart, Rollout, or Delete) + HostDelete - host is being removed from the cluster (downsize); fires + on the dying host before tear-down. Always emitted + alongside HostShutdown. + items: + type: string + # Both PascalCase and all-lowercase forms are accepted; the + # runtime comparison is case-insensitive (strings.EqualFold). + enum: + - "Any" + - "any" + - "HostCreate" + - "hostcreate" + - "HostDelete" + - "hostdelete" + - "HostUpdate" + - "hostupdate" + - "HostStart" + - "hoststart" + - "HostStop" + - "hoststop" + - "HostConfigRestart" + - "hostconfigrestart" + - "HostRollout" + - "hostrollout" + - "HostShutdown" + - "hostshutdown" + failurePolicy: + type: string + description: | + Controls what happens when this hook returns an error. + Fail (default): error propagates — pre-hook aborts reconcile / host deletion. + Ignore: error is logged and the reconcile continues. + # Both PascalCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (Fail == fail). + enum: + - "Fail" + - "fail" + - "Ignore" + - "ignore" + post: + type: array + description: "actions to execute after reconcile" + nullable: true + items: + !!merge <<: *TypeHookAction + cluster: + type: object + description: | + CHI-level cluster reconcile defaults inherited by every cluster's + spec.configuration.clusters[N].reconcile section. Use this as a single + place to define cluster-level hooks that should apply to all clusters + in this CHI; per-cluster hooks (under clusters[N].reconcile.hooks) + are appended to (and dedup'd against) the inherited set. + properties: + hooks: + type: object + description: "cluster-level hooks inherited by every cluster" + properties: + pre: + type: array + description: "actions to execute before each cluster reconcile" + nullable: true + items: &TypeClusterHookAction + type: object + required: + - events + properties: + sql: + type: object + properties: + queries: + type: array + nullable: true + items: + type: string + shell: + type: object + properties: + command: + type: array + nullable: true + items: + type: string + container: + type: string + http: + type: object + properties: + url: + type: string + method: + type: string + target: + type: string + description: "where to execute hook for cluster-level hooks: FirstHost (default), AllHosts, AllShards" + # Both camelCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (firstHost == firsthost). + enum: + - "" + - "FirstHost" + - "firsthost" + - "AllHosts" + - "allhosts" + - "AllShards" + - "allshards" + events: + type: array + minItems: 1 + description: | + Cluster-scope reconcile lifecycle events. Required, non-empty. + Any - wildcard match: fires on every cluster reconcile pass + and the cluster delete sweep + ClusterCreate - all hosts in the cluster are new (no ancestor); fires + only on the first reconcile of a brand-new cluster + ClusterReconcile - ongoing reconcile pass over an existing cluster (at + least one host has prior state). Fires on every + operator reconcile cycle the upstream gates allow, + including taskID-only force-reconciles. NOT a "spec + changed" signal. + ClusterDelete - cluster is being removed (delete sweep — wiring follow-up) + items: + type: string + # Both PascalCase and all-lowercase forms are accepted; the + # runtime comparison is case-insensitive (strings.EqualFold). + enum: + - "Any" + - "any" + - "ClusterCreate" + - "clustercreate" + - "ClusterDelete" + - "clusterdelete" + - "ClusterReconcile" + - "clusterreconcile" + failurePolicy: + type: string + description: | + Controls what happens when this hook returns an error. + Fail (default) propagates the error; Ignore logs a warning and continues. + # Both PascalCase and all-lowercase forms are accepted; the + # runtime normalizes via strings.EqualFold (Fail == fail). + enum: + - "Fail" + - "fail" + - "Ignore" + - "ignore" + post: + type: array + description: "actions to execute after each cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + reconcile: + !!merge <<: *TypeReconcile + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "no" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" + enum: + - "" + - "StatefulSet" + - "statefulset" + - "Operator" + - "operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion (case-insensitive). + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + availabilityZone: + type: string + description: "availability zone for Zookeeper node" + keeper: + type: object + description: | + reference to a ClickHouseKeeperInstallation (CHK) resource. + The operator resolves this to ZooKeeper node addresses automatically. + properties: + name: + type: string + description: "name of the ClickHouseKeeperInstallation custom resource" + namespace: + type: string + description: "namespace of the CHK resource, defaults to the CHI namespace if omitted" + serviceType: + type: string + description: | + how to discover keeper endpoints (case-insensitive): + Replicas (default) — enumerate per-host services, one ZK node per keeper replica + Service — use the CR-level headless service as a single ZK node entry + enum: + - "" + - "Replicas" + - "replicas" + - "Service" + - "service" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + use_compression: + !!merge <<: *TypeStringBool + description: "Enables compression in Keeper protocol if set to true" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.env`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like {common}, {users}, {hosts} or config.d, users.d, conf.d, wrong prefixes will be ignored, subfolders also will be ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica (case-insensitive)" + enum: + # List SchemaPolicyReplicaXXX constants from model (both humped and all-lowercase accepted) + - "" + - "None" + - "none" + - "All" + - "all" + shard: + type: string + description: "how schema is propagated between shards (case-insensitive)" + enum: + # List SchemaPolicyShardXXX constants from model (both humped and all-lowercase accepted) + - "" + - "None" + - "none" + - "All" + - "all" + - "DistributedTablesOnly" + - "distributedtablesonly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + security: + type: object + description: | + Per-cluster security toggles for outbound TLS connections the operator makes + to this cluster's ClickHouse and ZooKeeper / Keeper hosts. Nil fields fall + through to the operator-wide defaults in ClickHouseOperatorConfiguration. + See docs/security_hardening.md for details. + x-kubernetes-preserve-unknown-fields: true + pdbManaged: + !!merge <<: *TypeStringBool + description: | + Specifies whether the Pod Disruption Budget (PDB) should be managed. + During the next installation, if PDB management is enabled, the operator will + attempt to retrieve any existing PDB. If none is found, it will create a new one + and initiate a reconciliation loop. If PDB management is disabled, the existing PDB + will remain intact, and the reconciliation loop will not be executed. By default, + PDB management is enabled. + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + !!merge <<: *TypeReconcileRuntime + host: + !!merge <<: *TypeReconcileHost + hooks: + # Per-cluster cluster-scope hooks. The schema (TypeClusterHookAction) is + # defined once at spec.reconcile.cluster.hooks above; this block just + # references it. Hooks defined here are merged with any inherited from + # CHI-level spec.reconcile.cluster.hooks via dedup'd MergeFrom. + type: object + description: "cluster-level hooks to execute before and after cluster reconcile" + properties: + pre: + type: array + description: "actions to execute before cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + post: + type: array + description: "actions to execute after cluster reconcile" + nullable: true + items: + !!merge <<: *TypeClusterHookAction + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + shardsCount: + type: integer + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + shards: + type: array + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClusterScopeIndex" + - "clusterscopeindex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhouse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + # both humped and all-lowercase accepted + - "" + - "Unspecified" + - "unspecified" + - "OnePerHost" + - "oneperhost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" + - "ShardAntiAffinity" + - "shardantiaffinity" + - "ReplicaAntiAffinity" + - "replicaantiaffinity" + - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" + - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" + - "MaxNumberPerNode" + - "maxnumberpernode" + - "NamespaceAffinity" + - "namespaceaffinity" + - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" + - "ClusterAffinity" + - "clusteraffinity" + - "ShardAffinity" + - "shardaffinity" + - "ReplicaAffinity" + - "replicaaffinity" + - "PreviousTailAffinity" + - "previoustailaffinity" + - "CircularReplication" + - "circularreplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "Shard" + - "shard" + - "Replica" + - "replica" + - "Cluster" + - "cluster" + - "ClickHouseInstallation" + - "clickhouseinstallation" + - "Namespace" + - "namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + security: + type: object + description: | + CHI-level security defaults, applied to every cluster that does not override + them. Each cluster can shadow these via spec.configuration.clusters[].security. + See docs/security_hardening.md for details. + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clickhouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit` (case-insensitive)" + enum: + # List useTypeXXX constants from model (both humped and all-lowercase accepted) + - "" + - "Merge" + - "merge" diff --git a/deploy/operatorhub/0.27.2/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml b/deploy/operatorhub/0.27.2/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml new file mode 100644 index 000000000..26c3ee738 --- /dev/null +++ b/deploy/operatorhub/0.27.2/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml @@ -0,0 +1,937 @@ +# Template Parameters: +# +# OPERATOR_VERSION=0.27.2 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + labels: + clickhouse-keeper.altinity.com/chop: 0.27.2 +spec: + group: clickhouse-keeper.altinity.com + scope: Namespaced + names: + kind: ClickHouseKeeperInstallation + singular: clickhousekeeperinstallation + plural: clickhousekeeperinstallations + shortNames: + - chk + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + - name: suspend + type: string + description: Suspend reconciliation + # Displayed in all priorities + jsonPath: .spec.suspend + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + endpoints: + type: array + description: "All endpoints" + nullable: true + items: + type: string + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + nullable: true + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + nullable: true + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + hostsWithReplicaCaughtUp: + type: array + description: "List of hosts with replica caught up" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + description: | + Allows to stop all ClickHouse Keeper clusters defined in a CHK. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + # StringBool is polymorphic — accepts native YAML bool (true/false), + # integer (0/1), or string from the recognized vocabulary. + # See pkg/apis/common/types/string_bool.go (UnmarshalJSON + IsValid). + # Structural-schema rules don't natively support bool|int|string + # union, so we use x-kubernetes-preserve-unknown-fields: true. + x-kubernetes-preserve-unknown-fields: true + suspend: + !!merge <<: *TypeStringBool + description: | + Suspend reconciliation of resources managed by a ClickHouse Keeper. + Works as the following: + - When `suspend` is `true` operator stops reconciling all resources. + - When `suspend` is `false` or not set, operator reconciles all resources. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + (case-insensitive) + enum: + - "" + - "Wait" + - "wait" + - "NoWait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default (case-insensitive)" + enum: + # List ObjectsCleanupXXX constants from model (both humped and all-lowercase accepted) + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "no" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator (case-insensitive)" + enum: + - "" + - "StatefulSet" + - "statefulset" + - "Operator" + - "operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion (case-insensitive). + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "retain" + - "Delete" + - "delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates. used for customization of the `Service` resource, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + serviceTemplates: + type: array + description: "optional, template names from chi.spec.templates.serviceTemplates. used for customization of the `Service` resources, created by `clickhouse-operator` to cover all clusters in whole `chi` resource" + nullable: true + items: + type: string + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + settings: &TypeSettings + type: object + description: | + allows configure multiple aspects and behavior for `clickhouse-keeper` instance + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level and replica-level + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + pdbManaged: + !!merge <<: *TypeStringBool + description: | + Specifies whether the Pod Disruption Budget (PDB) should be managed. + During the next installation, if PDB management is enabled, the operator will + attempt to retrieve any existing PDB. If none is found, it will create a new one + and initiate a reconciliation loop. If PDB management is disabled, the existing PDB + will remain intact, and the reconciliation loop will not be executed. By default, + PDB management is enabled. + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + security: + type: object + description: | + Per-cluster security toggles. Nil fields fall through to CHK spec security and + then to operator-wide defaults in ClickHouseOperatorConfiguration. Symmetric + with chi.spec.configuration.clusters[].security. See docs/security_hardening.md. + x-kubernetes-preserve-unknown-fields: true + insecure: + !!merge <<: *TypeStringBool + description: | + optional, controls exposure of the plaintext Keeper + client port (zk:2181). Defaults to "yes" (legacy + behavior, byte-stable). Set to "no" to: + - suppress zk:2181 from the per-cluster Service and + from per-host StatefulSet container ports; + - emit in the per-host overlay + so the Keeper process binds no plaintext listener at + all (the liveness probe automatically falls back to + `pgrep clickhouse-keeper` because 4LW does not work + over TLS upstream). + Must be paired with `secure: yes` — the normalizer + aborts reconcile with reason NoKeeperListener when + `insecure: no` is set without `secure: yes` (no + client port would be emitted). + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure (TLS) Keeper client ports for this cluster. + Translates into 1 in Raft server entries so peer + Keeper instances dial each other over TLS. Mirrors the cluster.secure + flag on CHI. + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + zkPortSecure: + type: integer + description: "optional, secure (TLS) Keeper client port; emitted alongside zkPort when the cluster opts into secure mode" + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + security: + type: object + description: | + CHK-level security defaults, applied to every cluster that does not override + them. Symmetric with chi.spec.security. See docs/security_hardening.md. + x-kubernetes-preserve-unknown-fields: true + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClusterScopeIndex" + - "clusterscopeindex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhouse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + zkPortSecure: + type: integer + description: "optional, secure (TLS) Keeper client port; emitted alongside zkPort when the cluster opts into secure mode" + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + # both humped and all-lowercase accepted + - "" + - "Unspecified" + - "unspecified" + - "OnePerHost" + - "oneperhost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "ClickHouseAntiAffinity" + - "clickhouseantiaffinity" + - "ShardAntiAffinity" + - "shardantiaffinity" + - "ReplicaAntiAffinity" + - "replicaantiaffinity" + - "AnotherNamespaceAntiAffinity" + - "anothernamespaceantiaffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "anotherclickhouseinstallationantiaffinity" + - "AnotherClusterAntiAffinity" + - "anotherclusterantiaffinity" + - "MaxNumberPerNode" + - "maxnumberpernode" + - "NamespaceAffinity" + - "namespaceaffinity" + - "ClickHouseInstallationAffinity" + - "clickhouseinstallationaffinity" + - "ClusterAffinity" + - "clusteraffinity" + - "ShardAffinity" + - "shardaffinity" + - "ReplicaAffinity" + - "replicaaffinity" + - "PreviousTailAffinity" + - "previoustailaffinity" + - "CircularReplication" + - "circularreplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants (both humped and all-lowercase accepted) + - "" + - "Unspecified" + - "unspecified" + - "Shard" + - "shard" + - "Replica" + - "replica" + - "Cluster" + - "cluster" + - "ClickHouseInstallation" + - "clickhouseinstallation" + - "Namespace" + - "namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operatorhub/0.27.2/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.27.2/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..ccd472390 --- /dev/null +++ b/deploy/operatorhub/0.27.2/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,734 @@ +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.27.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + jsonPath: .status + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" + properties: + namespaces: + type: object + description: "List of namespaces where clickhouse-operator watches for events." + x-kubernetes-preserve-unknown-fields: true + configuration: + type: object + description: "Behavior when ClickHouseOperatorConfiguration resources change" + properties: + onChange: + type: string + enum: + - none + - None + - ignore + - Ignore + - restart + - Restart + description: "none/ignore — do nothing; restart — exit process so the operator pod restarts" + clickhouse: + type: object + description: "Clickhouse related parameters used by clickhouse-operator" + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + description: | + Each 'path' can be either absolute or relative. + In case path is absolute - it is used as is. + In case path is relative - it is relative to the folder where configuration file you are reading right now is located. + properties: + common: + type: string + description: | + Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + Default value - config.d + host: + type: string + description: | + Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + Default value - conf.d + user: + type: string + description: | + Path to the folder where ClickHouse configuration files with users settings are located. + Files are common for all instances within a CHI. + Default value - users.d + user: + type: object + description: "Default parameters for any user which will create" + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + description: "Default network parameters for any user which will create" + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + configurationRestartPolicy: + type: object + description: "Configuration restart policy describes what configuration changes require ClickHouse restart" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + rules: + type: array + description: "Set of configuration rules for specified ClickHouse version" + items: + type: object + description: "setting: value pairs for configuration restart policy" + x-kubernetes-preserve-unknown-fields: true + access: + type: object + description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" + properties: + scheme: + type: string + description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + rootCA: + type: string + description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + rootCASecretRef: + type: object + description: "Reference to a k8s Secret (in the operator namespace) holding the PEM root certificate authority used when verifying ClickHouse server certificates over https. Alternate source for rootCA; mutually exclusive with the inline rootCA above (inline wins)." + properties: + name: + type: string + description: "Name of the k8s Secret holding the PEM CA bundle. Empty name = not used" + key: + type: string + description: "Key within the Secret whose value is the PEM CA bundle. When empty, the operator tries 'ca.crt' then 'tls.crt'" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "Port to be used by operator to connect to ClickHouse instances" + timeouts: + type: object + description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" + properties: + connect: + type: integer + minimum: 1 + maximum: 10 + description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." + query: + type: integer + minimum: 1 + maximum: 600 + description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + addons: + type: object + description: "Configuration addons specifies additional settings" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + spec: + type: object + description: "spec" + properties: + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + properties: + users: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + settings: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + files: + type: object + description: "see same section from CR spec" + x-kubernetes-preserve-unknown-fields: true + metrics: + type: object + description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" + properties: + timeouts: + type: object + description: | + Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + Specified in seconds. + properties: + collect: + type: integer + minimum: 1 + maximum: 600 + description: | + Timeout used to limit metrics collection request. In seconds. + Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + All collected metrics are returned. + tablesRegexp: + type: string + description: | + Regexp to match tables in system database to fetch metrics from. + Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. + Default is "^(metrics|custom_metrics)$". + excludeRegexp: + type: array + items: + type: string + description: | + List of regexps to match ClickHouse metrics to exclude from collection/export. + Regexps match internal metric names before Prometheus normalization and prefixing. + security: + type: object + description: | + Per-component security toggles for outbound connections the operator establishes: + ClickHouse-client TLS (clickhouse.tls.verify, clickhouse.tls.minVersion, clickhouse.tls.serverName, clickhouse.tls.rootCA, clickhouse.tls.rootCASecretRef), + ZooKeeper/Keeper-client TLS (zookeeper.tls.verify, zookeeper.tls.minVersion), + Kubernetes-client TLS (kubernetes.tls.verify=Strict refuses an insecure kubeconfig at startup; kubernetes.tls.minVersion floors the K8s API client transport, coerced to 1.3 under FIPS/Enforced), + operator↔metrics-exporter IPC channel hardening (ipc.mode, ipc.bindHost, ipc.tokenPath). + Operator-wide master switch (security.policy): Permissive (default) preserves 0.27.0 + behavior; Enforced coerces all per-component knobs above to their Strict positions + at startup (clickhouse.tls.verify=Strict, clickhouse.tls.minVersion=1.3, + zookeeper.tls.verify=Strict, zookeeper.tls.minVersion=1.3, + kubernetes.tls.verify=Strict, kubernetes.tls.minVersion=1.3, ipc.mode=Secure) and + rejects CHIs that cannot be served in a FIPS-compatible posture (e.g. CHIs + referencing plain-text external ZooKeeper). Independent of the Go runtime FIPS + toolchain — when this operator binary is built with GOFIPS140=v1.0.0 (the Go FIPS + module currently in CMVP review, not yet a completed validation) the startup gate + ORs runtime detection with this knob. + FIPS image policy (security.images.policy): orthogonal to security.policy. + Permissive (default) accepts any image; Required refuses CRs whose CH/Keeper + images lack the "fips" tag substring (admission) AND aborts running CRs whose + `SELECT version()` lacks "fips" (post-Ready confirmation). + All fields default to nil — current behavior preserved on upgrade. + See docs/security_hardening.md for details. + x-kubernetes-preserve-unknown-fields: true + template: + type: object + description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" + properties: + chi: + type: object + properties: + policy: + type: string + description: | + CHI template updates handling policy + Possible policy values: + - ReadOnStart. Accept CHIT updates on the operators start only. + - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + enum: + # both humped and all-lowercase accepted + - "" + - "ReadOnStart" + - "readonstart" + - "ApplyOnNextReconcile" + - "applyonnextreconcile" + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileCHIsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileCHKsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHKs in parallel, 1 by default" + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: + type: object + description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + recreate: + type: object + description: "Behavior during recreate StatefulSet" + properties: + onDataLoss: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to PVC data loss or missing volumes. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate (default) - proceed and recreate StatefulSet. + onUpdateFailure: + type: string + description: | + What to do in case operator needs to recreate StatefulSet due to update failure or StatefulSet not ready. + Possible options: + 1. abort - abort the process, do nothing with the problematic StatefulSet. + 2. recreate (default) - proceed and recreate StatefulSet. + host: + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: &TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" + # StringBool is polymorphic — accepts native YAML + # bool (true/false), integer (0/1), or string from + # the recognized vocabulary. Normalized by + # pkg/apis/common/types StringBool.UnmarshalJSON. + # Structural-schema rules don't natively support + # bool|int|string union, so we use the documented + # escape hatch x-kubernetes-preserve-unknown-fields. + x-kubernetes-preserve-unknown-fields: true + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + replicas: + type: object + description: "Whether the operator during reconcile procedure should wait for replicas to catch-up" + properties: + all: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for all replicas to catch-up" + new: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for new replicas to catch-up" + delay: + type: integer + description: "replication max absolute delay to consider replica is not delayed" + probes: + type: object + description: "What probes the operator should wait during host launch procedure" + properties: + startup: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for startup probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to do not wait. + readiness: + !!merge <<: *TypeStringBool + description: | + Whether the operator during host launch procedure should wait for readiness probe to succeed. + In case probe is unspecified wait is assumed to be completed successfully. + Default option value is to wait. + drop: + type: object + properties: + replicas: + type: object + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted or recreated + properties: + onDelete: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica is deleted + onLostVolume: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop replicas when replica volume is lost + active: + !!merge <<: *TypeStringBool + description: | + Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated + hooks: + type: object + description: "default host-level hooks to execute before and after host reconcile" + properties: + pre: + type: array + description: "actions to execute before host reconcile" + nullable: true + items: &TypeHookActionChop + type: object + properties: + sql: + type: object + properties: + queries: + type: array + nullable: true + items: + type: string + shell: + type: object + properties: + command: + type: array + nullable: true + items: + type: string + container: + type: string + http: + type: object + properties: + url: + type: string + method: + type: string + target: + type: string + description: "where to execute hook: FirstHost (default), AllHosts, AllShards" + enum: + - "" + - "FirstHost" + - "firsthost" + - "AllHosts" + - "allhosts" + - "AllShards" + - "allshards" + post: + type: array + description: "actions to execute after host reconcile" + nullable: true + items: + !!merge <<: *TypeHookActionChop + coordination: + type: object + description: "Coordination with external systems during reconcile" + properties: + keeper: + type: object + description: "Keeper-related coordination settings" + properties: + readyTimeout: + type: integer + minimum: 1 + description: | + How long the operator waits for a referenced ClickHouseKeeper to become ready + before aborting CHI reconcile. In seconds. Default is 120. + onKeeperResourceUpdate: + type: string + description: | + Reaction when a referenced CHK resource changes. + none (default) — do nothing + reconcile — trigger CHI reconcile + enum: + - "" + - "None" + - "none" + - "Reconcile" + - "reconcile" + recovery: + type: object + description: "Auto-recovery from reconcile failures, scoped by CHI status" + properties: + onStatus: + type: object + description: "Recovery scopes keyed by the CHI .status.status they apply to" + properties: + aborted: + type: object + description: "Recovery while Status=Aborted" + properties: + onPodReady: + type: string + description: | + Reaction when a pod belonging to an Aborted CHI transitions to Ready. + retry (default) — re-enqueue the CHI for reconcile + none — do nothing, CHI stays Aborted + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + completed: + type: object + description: "Recovery while Status=Completed when a child pod regresses to Ready=False and stays NotReady" + properties: + onPodNotReady: + type: string + description: | + Reaction when a pod belonging to a Completed CHI flips Ready=True -> Ready=False + and stays NotReady for at least onPodNotReadyThreshold. OFF by default — opt-in only, + because force-recreating the pod is destructive (can interrupt replica recovery; + hard downtime for a single-replica shard). + none (default) — do nothing + retry — re-enqueue the CHI for reconcile (force-restart the stuck host) + enum: + - "" + - "None" + - "none" + - "Retry" + - "retry" + onPodNotReadyThreshold: + type: string + description: "Minimum duration a pod must stay Ready=False before recovery fires (Go duration string, e.g. '5m'; default 5m)" + annotation: + type: object + description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + include annotations with names from the following list + items: + type: string + exclude: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + exclude annotations with names from the following list + items: + type: string + label: + type: object + description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + include labels from the following list + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + !!merge <<: *TypeStringBool + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" + statefulSet: + type: object + description: "define StatefulSet-specific parameters" + properties: + revisionHistoryLimit: + type: integer + description: "revisionHistoryLimit is the maximum number of revisions that will be\nmaintained in the StatefulSet's revision history. \nLook details in `statefulset.spec.revisionHistoryLimit`\n" + pod: + type: object + description: "define pod specific parameters" + properties: + terminationGracePeriod: + type: integer + description: "Optional duration in seconds the pod needs to terminate gracefully. \nLook details in `pod.spec.terminationGracePeriodSeconds`\n" + logger: + type: object + description: "allow setup clickhouse-operator logger behavior" + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. diff --git a/dev/generate_helm_chart.sh b/dev/generate_helm_chart.sh index 8a0664efa..4acc1517b 100755 --- a/dev/generate_helm_chart.sh +++ b/dev/generate_helm_chart.sh @@ -271,6 +271,17 @@ function update_configmap_resource() { local data data=$(yq e '.data' "${file}") + local name_suffix="${name/etc-clickhouse-operator-/}" + local name_suffix="${name_suffix/etc-keeper-operator-/keeper-}" + local camel_cased_name + camel_cased_name=$(to_camel_case "${name_suffix}") + + # The operator config ConfigMap gets special handling: fullname substitution, + # config.yaml block-scalar → map conversion (so the configmap-files helper can + # patch the nested watch.namespaces.include), and the configmap-files helper + # (which wires in the top-level watchNamespaces value). Every other ConfigMap + # uses the generic configmap-data helper. + local data_helper if [ "${name}" = "etc-clickhouse-operator-files" ]; then local search='name: "clickhouse-operator"' local replace="name: '{{ include \"altinity-clickhouse-operator.fullname\" . }}'" @@ -279,18 +290,17 @@ function update_configmap_resource() { search='config.yaml: |' replace='config.yaml:' data=${data/"${search}"/"${replace}"} - fi - local name_suffix="${name/etc-clickhouse-operator-/}" - local name_suffix="${name_suffix/etc-keeper-operator-/keeper-}" - local camel_cased_name - camel_cased_name=$(to_camel_case "${name_suffix}") + data_helper='{{ include "altinity-clickhouse-operator.configmap-files" (list . .Values.configs.files .Values.watchNamespaces) | nindent 2 }}' + else + data_helper='{{ include "altinity-clickhouse-operator.configmap-data" (list . .Values.configs.'"${camel_cased_name}"') | nindent 2 }}' + fi yq e -i '.metadata.name |= "{{ printf \"%s-'"${name_suffix}"'\" (include \"altinity-clickhouse-operator.fullname\" .) }}"' "${file}" yq e -i '.metadata.namespace |= "{{ include \"altinity-clickhouse-operator.namespace\" . }}"' "${file}" yq e -i '.metadata.labels |= "{{ include \"altinity-clickhouse-operator.labels\" . | nindent 4 }}"' "${file}" yq e -i '.metadata.annotations |= "{{ include \"altinity-clickhouse-operator.annotations\" . | nindent 4 }}"' "${file}" - yq e -i '.data |= "{{ include \"altinity-clickhouse-operator.configmap-data\" (list . .Values.configs.'"${camel_cased_name}"') | nindent 2 }}"' "${file}" + data_helper="${data_helper}" yq e -i '.data |= strenv(data_helper)' "${file}" if [ -z "${data}" ]; then yq e -i '.configs.'"${camel_cased_name}"' |= null' "${values_yaml}" diff --git a/dev/run_vet.sh b/dev/run_vet.sh index 14faa6706..459f6fa6b 100755 --- a/dev/run_vet.sh +++ b/dev/run_vet.sh @@ -1,9 +1,39 @@ #!/bin/bash +set -o errexit +set -o nounset +set -o pipefail -# The WithCancel, WithDeadline, and WithTimeout functions take a Context (the parent) -# and return a derived Context (the child) and a CancelFunc. -# Calling the CancelFunc cancels the child and its children, removes the parent's reference to the child, -# and stops any associated timers. +# Run `go vet` over all packages as a local static-analysis gate. # -# Failing to call the CancelFunc leaks the child and its children until the parent is canceled or the timer fires. -# The go vet tool checks that CancelFuncs are used on all control-flow paths. +# Two analyzers are disabled because in this codebase they only fire on +# deliberate, accepted patterns — not real defects — and would otherwise bury +# the actionable findings under ~50 lines of known noise: +# +# printf — the announcer logging API (a.V(n).M(x).F().Info(msg, args...)) +# legitimately forwards a runtime-built, non-constant format +# string. Every printf hit is that logging pattern, not a bug. +# copylocks — generated deepcopy/fake code (zz_generated.deepcopy.go, +# pkg/client/**/fake) copies lock-bearing structs by value, which +# is unavoidable in generated code, plus one mergo.Merge call. +# +# Everything else stays enabled (unreachable, struct tag, unusedresult, …), so +# real problems still fail the run. This mirrors the govet settings planned for +# .golangci.yml so the local gate and the linter agree. +# +# NOTE: the suite is run with `go test -vet=off` (see run_go_tests.sh) because +# the disabled analyzers above also trip the test-binary compile; this script is +# the dedicated place vet actually runs. + +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source "${CUR_DIR}/go_build_config.sh" + +cd "${SRC_ROOT}" + +echo "Running go vet over ${SRC_ROOT} (printf, copylocks disabled — see header)..." +if go vet -copylocks=false -printf=false ./...; then + echo "go vet: no findings" +else + rc=$? + echo "go vet: found issues (exit ${rc})" >&2 + exit "${rc}" +fi diff --git a/dev/start_new_release_branch.sh b/dev/start_new_release_branch.sh index c7decfddd..aa8d2cf30 100755 --- a/dev/start_new_release_branch.sh +++ b/dev/start_new_release_branch.sh @@ -69,6 +69,11 @@ esac read -p "Press enter to start new release" echo "Starting new release: ${NEW_RELEASE}" +# Pull latest master +git checkout master +git pull +git pull altinity master + # Create release branch git branch "${NEW_RELEASE}" git checkout "${NEW_RELEASE}" @@ -96,5 +101,9 @@ git -C "${SRC_ROOT}" commit -m "env: manifests" git -C "${SRC_ROOT}" add deploy/helm/ git -C "${SRC_ROOT}" commit -m "env: helm chart" +# Push new branch to altinity +git push altinity + +# Repository status echo "git status:" git -C "${SRC_ROOT}" status diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index b9f573ec1..33b4838fa 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -82,6 +82,22 @@ spec: name: "" # Port where to connect to ClickHouse instances to port: 8123 + # `rootCA`: inline PEM CA bundle the operator uses to verify the ClickHouse + # server certificate when connecting over https (scheme: https, or auto when + # only TLS ports are open). Verification is enforced when TLS hardening is + # opted in — security.clickhouse.tls.verify: Strict, or a non-empty + # minVersion/serverName; otherwise the CA is loaded but verification stays + # relaxed for backward compatibility. + rootCA: "" + # `rootCASecretRef`: alternate source — read the PEM CA from a Kubernetes + # Secret in the operator's own namespace instead of inlining it above. The + # operator resolves it into `rootCA` once at config load (rotate the Secret + + # restart the operator to pick up a new CA). Mutually exclusive with the + # inline `rootCA` above (inline wins). Empty `name` = not used. When `key` is + # empty, the operator tries "ca.crt" then "tls.crt". + rootCASecretRef: + name: "" + key: "" ################################################ ## @@ -98,7 +114,7 @@ spec: # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function. # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics. tablesRegexp: "^(metrics|custom_metrics)$" - # List of regexps to match ClickHouse metrics to exclude from export. + # List of regexps to match ClickHouse metrics to exclude from collection/export. # Regexps match internal metric names before Prometheus normalization and prefixing. # Default is the per-CPU OS metrics filter shown below; set to [] to disable. excludeRegexp: @@ -380,11 +396,11 @@ spec: ################################################ ## - ## Auto-recovery from aborted reconcile + ## Auto-recovery from aborted/completed reconcile ## ################################################ recovery: - from: + onStatus: aborted: # When a reconcile lands in status=Aborted (due to FIPS coercion # conflict, plain-text ZK under FIPS, etc.), the operator can @@ -394,6 +410,20 @@ spec: # none — disable auto-recovery; operator user must # edit the CR spec to retrigger normalize. onPodReady: retry + completed: + # When a Completed CHI's pod regresses Ready=True→Ready=False and + # stays NotReady for at least onPodNotReadyThreshold, the operator + # can force-restart (recreate) the stuck host. + # OFF by default (opt-in): recreating a Completed CHI's pod is + # destructive — it can interrupt a replica's in-progress recovery + # and means hard downtime for a single-replica shard. + # Values: + # none (default) — do nothing. + # retry — re-enqueue the CHI so the stuck host is recreated. + onPodNotReady: none + # Minimum time a pod must stay Ready=False before recovery fires, once + # enabled (Go duration string; default 5m). Raise it for slow replicas. + onPodNotReadyThreshold: 5m ################################################ ## diff --git a/docs/operator_configuration.md b/docs/operator_configuration.md index 8fa03e877..7e4d58af1 100644 --- a/docs/operator_configuration.md +++ b/docs/operator_configuration.md @@ -115,6 +115,14 @@ chPassword: clickhouse_operator_password chPort: 8123 ``` +When the operator connects over HTTPS, it verifies the ClickHouse server certificate +with the CA from `clickhouse.access.rootCA` (inline PEM) or `clickhouse.access.rootCASecretRef` +(a Secret in the operator's own namespace; key defaults to `ca.crt` then `tls.crt`, inline +`rootCA` wins). Verification is enforced when TLS hardening is opted in — +`security.clickhouse.tls.verify: Strict`, or a non-empty `minVersion`/`serverName`; otherwise +the CA is loaded but verification stays relaxed for backward compatibility. +See the [operator config example](chi-examples/70-chop-config.yaml). + ## ClickHouse Installation settings Operator deploys ClickHouse clusters with different defaults, that can be configured in a flexible way. diff --git a/docs/operator_upgrade.md b/docs/operator_upgrade.md index 8b2929ab2..ffc096d04 100644 --- a/docs/operator_upgrade.md +++ b/docs/operator_upgrade.md @@ -18,7 +18,9 @@ **ACVP responder** (optional, build-tagged): a NIST ACVP test responder can be embedded into operator and metrics-exporter binaries via the `acvp_wrapper` build tag. Default builds do NOT include the responder. See `docs/security_hardening_fips.md` § ACVP for the test-evidence pipeline. -**0.27.0 → 0.27.1 auto-recovery from Aborted:** the new `reconcile.recovery.from.aborted.onPodReady` knob re-enqueues a CHI when a pod transitions NotReady → Ready while the CR sits in `Aborted` status. Default is `retry` (auto-recover on pod-Ready transition); set to `none` to opt out and preserve the pre-0.27.1 manual-intervention behaviour. +**0.27.0 → 0.27.1 auto-recovery from Aborted:** the new `reconcile.recovery.from.aborted.onPodReady` knob re-enqueues a CHI when a pod transitions NotReady → Ready while the CR sits in `Aborted` status. Default is `retry` (auto-recover on pod-Ready transition); set to `none` to opt out and preserve the pre-0.27.1 manual-intervention behaviour. **(Renamed in 0.27.2 — see the 0.27.1 → 0.27.2 note below.)** + +**0.27.1 → 0.27.2 recovery config key renamed (backward-incompatible):** the auto-recovery config key `reconcile.recovery.from.{aborted,completed}` is renamed to `reconcile.recovery.onStatus.{aborted,completed}` in `ClickHouseOperatorConfiguration`. The `from` grouping level is removed; the per-status scopes (`aborted`, `completed`) and their action keys (`onPodReady`, `onPodNotReady`, `onPodNotReadyThreshold`) are otherwise unchanged. The operator **silently ignores** the obsolete `from` key (unknown config keys are dropped at load). **Action required only if you explicitly set `reconcile.recovery.from.aborted.onPodReady: none`** on 0.27.0/0.27.1 to *disable* auto-recovery of Aborted CHIs: after upgrade that setting is ignored, the accessor falls back to its default (`retry`), and Aborted auto-recovery is **silently re-enabled**. Re-apply it under the new path: `reconcile.recovery.onStatus.aborted.onPodReady: none`. (The `completed` scope — sustained-NotReady host recovery — is new in 0.27.2 and off by default, so no migration is needed for it.) **0.27.0 → 0.27.1 reconcile hooks (preview):** new `spec.reconcile.host.hooks` and `spec.reconcile.cluster.hooks` blocks accept `events:` + `sql:` / `http:` / `shell:` actions. Only `sql:` is wired end-to-end in 0.27.1; `http:` and `shell:` currently emit a "not yet implemented" Fatal at validation time, so defer adopting those action types until the corresponding runners ship. diff --git a/docs/security_hardening_fips.md b/docs/security_hardening_fips.md index 41c8a73cd..93b99f731 100644 --- a/docs/security_hardening_fips.md +++ b/docs/security_hardening_fips.md @@ -137,7 +137,7 @@ kubectl get chi -o json | jq -r '.items[].status.errors[]? | select(startswith(" Recovery is via spec edit: `kubectl apply` a corrected CHI (set `secure: true` on every ZK node, or remove the `zookeeper:` block and use a CHK reference). The informer's `UpdateFunc` re-enqueues the CR and normalize re-runs cleanly. -Note: this recovery path does NOT depend on `recovery.from.aborted.onPodReady` +Note: this recovery path does NOT depend on `recovery.onStatus.aborted.onPodReady` — that path requires pod-readiness transitions, which never fire for CHIs rejected at the normalizer (pods are never created). diff --git a/go.mod b/go.mod index 0737dd5b0..6b349ada7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/altinity/clickhouse-operator -go 1.26.3 +go 1.26.4 replace ( github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.0 diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go index d80a5ceba..498cef9d7 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go @@ -417,6 +417,11 @@ func (cluster *Cluster) HostsCount() int { return count } +// IsSingleNode reports whether the cluster has fewer than two hosts (i.e. no Raft quorum peers). +func (cluster *Cluster) IsSingleNode() bool { + return cluster.HostsCount() < 2 +} + func (cluster *Cluster) IsZero() bool { return cluster == nil } diff --git a/pkg/apis/clickhouse.altinity.com/v1/interface.go b/pkg/apis/clickhouse.altinity.com/v1/interface.go index 141154928..ddd52b5e7 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/interface.go +++ b/pkg/apis/clickhouse.altinity.com/v1/interface.go @@ -135,6 +135,7 @@ type ICluster interface { WalkHosts(func(host *Host) error) []error HostsCount() int + IsSingleNode() bool FindShard(needle interface{}) IShard FindHost(needleShard interface{}, needleHost interface{}) *Host diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go index 1244d4a1c..b5b4eb6b6 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "fmt" + "strings" + "github.com/altinity/clickhouse-operator/pkg/apis/swversion" "github.com/imdario/mergo" @@ -421,13 +423,15 @@ func (cr *ClickHouseInstallation) FoundIn(haystack []*ClickHouseInstallation) bo return false } -// Possible templating policies +// Possible templating policies (canonical humped form; CRD also accepts all-lowercase) const ( - TemplatingPolicyAuto = "auto" - TemplatingPolicyManual = "manual" + TemplatingPolicyAuto = "Auto" + TemplatingPolicyManual = "Manual" ) -// IsAuto checks whether templating policy is auto +// IsAuto checks whether templating policy is auto. +// Uses EqualFold: this is read from template CRs that are NOT run through the normalizer +// (readCHITemplates → GetAutoTemplates), so the raw letter-casing must be tolerated here. func (cr *ClickHouseInstallation) IsAuto() bool { if cr == nil { return false @@ -435,7 +439,7 @@ func (cr *ClickHouseInstallation) IsAuto() bool { if (cr.Namespace == "") && (cr.Name == "") { return false } - return cr.GetSpecT().GetTemplating().GetPolicy() == TemplatingPolicyAuto + return strings.EqualFold(cr.GetSpecT().GetTemplating().GetPolicy(), TemplatingPolicyAuto) } // IsStopped checks whether CR is stopped @@ -454,12 +458,13 @@ const ( RestartRollingUpdate = "RollingUpdate" ) -// IsRollingUpdate checks whether CHI should perform rolling update +// IsRollingUpdate checks whether CHI should perform rolling update. +// The restart field is read un-normalized, so fold casing here (RollingUpdate/rollingupdate). func (cr *ClickHouseInstallation) IsRollingUpdate() bool { if cr == nil { return false } - return cr.GetSpecT().GetRestart().Value() == RestartRollingUpdate + return cr.GetSpecT().GetRestart().EqualFoldString(RestartRollingUpdate) } // IsTroubleshoot checks whether CHI is in troubleshoot mode diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go index e5d3dc148..220ad9a51 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go @@ -463,6 +463,12 @@ func (cluster *Cluster) HostsCount() int { return count } +// IsSingleNode reports whether the cluster has fewer than two hosts (i.e. no replication/sharding +// peers). remote_servers on such a cluster references only localhost, which always resolves. +func (cluster *Cluster) IsSingleNode() bool { + return cluster.HostsCount() < 2 +} + func (cluster *Cluster) IsZero() bool { return cluster == nil } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index d1ffbfb6f..a11de3b1b 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -46,22 +46,29 @@ const ( // a referenced ClickHouseKeeper to become ready during CHI reconcile. defaultKeeperReadyTimeout = 120 + // Consts below use the canonical humped form; the CRD also accepts all-lowercase and + // the accessors compare case-insensitively (EqualFold). + // KeeperOnResourceUpdateNone means do nothing when referenced CHK changes (default). - KeeperOnResourceUpdateNone = "none" + KeeperOnResourceUpdateNone = "None" // KeeperOnResourceUpdateReconcile means trigger CHI reconcile when referenced CHK changes. - KeeperOnResourceUpdateReconcile = "reconcile" + KeeperOnResourceUpdateReconcile = "Reconcile" // OnConfigurationChangeNone means ignore ClickHouseOperatorConfiguration changes (default). - OnConfigurationChangeNone = "none" + OnConfigurationChangeNone = "None" // OnConfigurationChangeIgnore is an alias for OnConfigurationChangeNone. - OnConfigurationChangeIgnore = "ignore" + OnConfigurationChangeIgnore = "Ignore" // OnConfigurationChangeRestart means exit the process so the pod restarts with the new config. - OnConfigurationChangeRestart = "restart" + OnConfigurationChangeRestart = "Restart" - // RecoveryActionNone means do nothing, CHI stays Aborted. - RecoveryActionNone = "none" + // RecoveryActionNone means do nothing, CHI stays in its current state. + RecoveryActionNone = "None" // RecoveryActionRetry means re-enqueue CHI for reconcile (default). - RecoveryActionRetry = "retry" + RecoveryActionRetry = "Retry" + + // defaultCompletedOnPodNotReadyThreshold is the minimum time a pod must remain in + // Ready=False before the operator considers the host stuck and re-enqueues a reconcile + defaultCompletedOnPodNotReadyThreshold = 5 * time.Minute // Default values for ClickHouse user configuration // 1. user/profile @@ -79,8 +86,9 @@ const ( ChSchemeHTTP = "http" // ChSchemeHTTPS specifies HTTPS access scheme ChSchemeHTTPS = "https" - // ChSchemeAuto specifies that operator has to decide itself should https or http be used - ChSchemeAuto = "auto" + // ChSchemeAuto specifies that operator has to decide itself should https or http be used. + // Humped canonical form; the normalizer folds any casing (auto/Auto) to this. + ChSchemeAuto = "Auto" // Username and Password to be used by operator to connect to ClickHouse instances for // 1. Metrics requests @@ -107,6 +115,10 @@ const ( // Used in case no other specified in config defaultReconcileCHIsThreadsNumber = 1 + // defaultReconcileCHKsThreadsNumber specifies default number of CHK controller threads running concurrently. + // Used in case no other specified in config + defaultReconcileCHKsThreadsNumber = 1 + // defaultReconcileShardsThreadsNumber specifies the default number of threads usable for concurrent shard reconciliation // within a single cluster reconciliation. Defaults to 1, which means strictly sequential shard reconciliation. defaultReconcileShardsThreadsNumber = 1 @@ -138,45 +150,78 @@ const ( PasswordReplacer = "***" ) +// StatefulSet failure-action consts use the canonical humped form; the CRD also accepts +// all-lowercase and the normalizer folds any casing to these via util.FoldEnum. const ( // What to do in case StatefulSet can't reach new Generation - abort CHI reconcile - OnStatefulSetCreateFailureActionAbort = "abort" + OnStatefulSetCreateFailureActionAbort = "Abort" // What to do in case StatefulSet can't reach new Generation - delete newly created problematic StatefulSet - OnStatefulSetCreateFailureActionDelete = "delete" + OnStatefulSetCreateFailureActionDelete = "Delete" // What to do in case StatefulSet can't reach new Generation - do nothing, keep StatefulSet broken and move to the next - OnStatefulSetCreateFailureActionIgnore = "ignore" + OnStatefulSetCreateFailureActionIgnore = "Ignore" ) const ( // What to do in case StatefulSet can't reach new Generation - abort CHI reconcile - OnStatefulSetUpdateFailureActionAbort = "abort" + OnStatefulSetUpdateFailureActionAbort = "Abort" // What to do in case StatefulSet can't reach new Generation - delete Pod and rollback StatefulSet to previous Generation // Pod would be recreated by StatefulSet based on rollback-ed configuration - OnStatefulSetUpdateFailureActionRollback = "rollback" + OnStatefulSetUpdateFailureActionRollback = "Rollback" // What to do in case StatefulSet can't reach new Generation - do nothing, keep StatefulSet broken and move to the next - OnStatefulSetUpdateFailureActionIgnore = "ignore" + OnStatefulSetUpdateFailureActionIgnore = "Ignore" ) const ( // What to do in case StatefulSet needs to be recreated due to PVC data loss or missing volumes // Abort - Loss: abort CHI reconcile - OnStatefulSetRecreateOnDataLossActionAbort = "abort" + OnStatefulSetRecreateOnDataLossActionAbort = "Abort" // Recreate - Loss: proceed and recreate StatefulSet - OnStatefulSetRecreateOnDataLossActionRecreate = "recreate" + OnStatefulSetRecreateOnDataLossActionRecreate = "Recreate" // What to do in case StatefulSet needs to be recreated due to update failure or StatefulSet not ready // Abort - Failure: abort CHI reconcile - OnStatefulSetRecreateOnUpdateFailureActionAbort = "abort" + OnStatefulSetRecreateOnUpdateFailureActionAbort = "Abort" // Recreate - Failure: proceed and recreate StatefulSet - OnStatefulSetRecreateOnUpdateFailureActionRecreate = "recreate" + OnStatefulSetRecreateOnUpdateFailureActionRecreate = "Recreate" ) +// Canonical (humped) candidate lists for the StatefulSet failure-action enums. OnDataLoss and +// OnUpdateFailure share the Abort/Recreate set. The Normalize* helpers fold any accepted casing +// to the canonical const; an unrecognized value passes through so the caller's default applies. +var ( + onStatefulSetCreateFailureActions = []string{ + OnStatefulSetCreateFailureActionAbort, OnStatefulSetCreateFailureActionDelete, OnStatefulSetCreateFailureActionIgnore, + } + onStatefulSetUpdateFailureActions = []string{ + OnStatefulSetUpdateFailureActionAbort, OnStatefulSetUpdateFailureActionRollback, OnStatefulSetUpdateFailureActionIgnore, + } + onStatefulSetRecreateActions = []string{ + OnStatefulSetRecreateOnDataLossActionAbort, OnStatefulSetRecreateOnDataLossActionRecreate, + } +) + +// NormalizeOnStatefulSetCreateFailureAction folds any accepted casing to its canonical const. +func NormalizeOnStatefulSetCreateFailureAction(value string) string { + return util.FoldEnum(value, onStatefulSetCreateFailureActions...) +} + +// NormalizeOnStatefulSetUpdateFailureAction folds any accepted casing to its canonical const. +func NormalizeOnStatefulSetUpdateFailureAction(value string) string { + return util.FoldEnum(value, onStatefulSetUpdateFailureActions...) +} + +// NormalizeOnStatefulSetRecreateAction folds any accepted casing to its canonical const +// (used for both onDataLoss and onUpdateFailure, which share the Abort/Recreate set). +func NormalizeOnStatefulSetRecreateAction(value string) string { + return util.FoldEnum(value, onStatefulSetRecreateActions...) +} + const ( defaultMaxReplicationDelay = 10 ) @@ -379,6 +424,15 @@ type OperatorConfigClickHouse struct { Username string `json:"username,omitempty" yaml:"username,omitempty"` Password string `json:"password,omitempty" yaml:"password,omitempty"` RootCA string `json:"rootCA,omitempty" yaml:"rootCA,omitempty"` + // RootCASecretRef sources the RootCA PEM from a Secret in the operator's pod + // namespace (inline RootCA wins; empty Name = unused; Key defaults ca.crt then + // tls.crt). Mirrors security.clickhouse.tls.rootCASecretRef. Value struct, not + // *core.SecretKeySelector: Access is anonymous, where a heap-bearing pointer + // field would break deepcopy generation. + RootCASecretRef struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Key string `json:"key,omitempty" yaml:"key,omitempty"` + } `json:"rootCASecretRef,omitempty" yaml:"rootCASecretRef,omitempty"` // Location of k8s Secret with username and password to be used by the operator to connect to ClickHouse instances // Can be used instead of explicitly specified (above) username and password @@ -563,21 +617,23 @@ type OperatorConfigReconcile struct { } // OperatorConfigReconcileRecovery specifies auto-recovery behavior for reconcile. -// Event→action mappings are scoped by the CHI state we recover FROM (under .From). +// Event→action mappings are scoped by the CHI status they apply to (under .OnStatus). // Global policy knobs (future: retries, backoff, cooldown, enabled) sit as flat peers -// of .From at this level. Multi-scope design anticipates future states beyond Aborted +// of .OnStatus at this level. Multi-scope design anticipates future states beyond Aborted // (e.g. Failed, Broken). type OperatorConfigReconcileRecovery struct { - // From maps the CHI state we recover from (e.g. aborted) to event→action mappings. - From OperatorConfigReconcileRecoveryFrom `json:"from,omitempty" yaml:"from,omitempty"` + // OnStatus maps a CHI status (e.g. aborted, completed) to event→action mappings + // that apply while the CHI is in that status. + OnStatus OperatorConfigReconcileRecoveryOnStatus `json:"onStatus,omitempty" yaml:"onStatus,omitempty"` } -// OperatorConfigReconcileRecoveryFrom groups recovery-event mappings by the CHI state -// being recovered from. Each sub-field is a scope whose keys are on recovery -// triggers. -type OperatorConfigReconcileRecoveryFrom struct { - // Aborted scope — recovery from Status=Aborted. +// OperatorConfigReconcileRecoveryOnStatus groups recovery-event mappings by the CHI status +// they apply to. Each sub-field is a scope whose keys are on recovery triggers. +type OperatorConfigReconcileRecoveryOnStatus struct { + // Aborted scope — recovery while Status=Aborted. Aborted OperatorConfigReconcileRecoveryScope `json:"aborted,omitempty" yaml:"aborted,omitempty"` + // Completed scope — recovery while Status=Completed when a child pod regresses to Ready=False + Completed OperatorConfigReconcileRecoveryCompletedScope `json:"completed,omitempty" yaml:"completed,omitempty"` // Future: Failed, Broken, etc. } @@ -592,8 +648,26 @@ type OperatorConfigReconcileRecoveryScope struct { // Future: OnKeeperReady, OnOperatorRestart. } +// OperatorConfigReconcileRecoveryCompletedScope holds the event→action mappings for the +// Completed scope. +type OperatorConfigReconcileRecoveryCompletedScope struct { + // OnPodNotReady controls reaction when a pod belonging to a Completed CHI flips + // Ready=True → Ready=False and stays NotReady for at least OnPodNotReadyThreshold. + // OFF by default (opt-in), unlike the Aborted scope's OnPodReady — force-recreating a + // Completed CHI's pod is destructive (can interrupt replica recovery; single-replica downtime): + // nil / "none" (default) — do nothing, host stays Ready=False until external action + // "retry" — re-enqueue CHI for reconcile so shouldForceRestartHost + // can decide whether to restart the host + OnPodNotReady *types.String `json:"onPodNotReady,omitempty" yaml:"onPodNotReady,omitempty"` + // OnPodNotReadyThreshold is the minimum duration a pod must remain in Ready=False + // before this scope fires. Accepts any time.ParseDuration string (default "5m" + // when unset, empty, or unparseable). + OnPodNotReadyThreshold *types.String `json:"onPodNotReadyThreshold,omitempty" yaml:"onPodNotReadyThreshold,omitempty"` +} + type OperatorConfigReconcileRuntime struct { ReconcileCHIsThreadsNumber int `json:"reconcileCHIsThreadsNumber" yaml:"reconcileCHIsThreadsNumber"` + ReconcileCHKsThreadsNumber int `json:"reconcileCHKsThreadsNumber" yaml:"reconcileCHKsThreadsNumber"` ReconcileShardsThreadsNumber int `json:"reconcileShardsThreadsNumber" yaml:"reconcileShardsThreadsNumber"` ReconcileShardsMaxConcurrencyPercent int `json:"reconcileShardsMaxConcurrencyPercent" yaml:"reconcileShardsMaxConcurrencyPercent"` @@ -1189,22 +1263,27 @@ func (c *OperatorConfig) normalizeSectionReconcileStatefulSet() { c.Reconcile.StatefulSet.Update.PollInterval = defaultStatefulSetUpdatePollInterval } - // Default action on Create/Update failure - to keep system in previous state + // Default action on Create/Update failure - to keep system in previous state. + // Fold any accepted casing to the canonical const first, then default if empty. // Default Create Failure action - delete + c.Reconcile.StatefulSet.Create.OnFailure = NormalizeOnStatefulSetCreateFailureAction(c.Reconcile.StatefulSet.Create.OnFailure) if c.Reconcile.StatefulSet.Create.OnFailure == "" { c.Reconcile.StatefulSet.Create.OnFailure = OnStatefulSetCreateFailureActionDelete } // Default Updated Failure action - revert + c.Reconcile.StatefulSet.Update.OnFailure = NormalizeOnStatefulSetUpdateFailureAction(c.Reconcile.StatefulSet.Update.OnFailure) if c.Reconcile.StatefulSet.Update.OnFailure == "" { c.Reconcile.StatefulSet.Update.OnFailure = OnStatefulSetUpdateFailureActionRollback } // Default Recreate actions - recreate + c.Reconcile.StatefulSet.Recreate.OnDataLoss = NormalizeOnStatefulSetRecreateAction(c.Reconcile.StatefulSet.Recreate.OnDataLoss) if c.Reconcile.StatefulSet.Recreate.OnDataLoss == "" { c.Reconcile.StatefulSet.Recreate.OnDataLoss = OnStatefulSetRecreateOnDataLossActionRecreate } + c.Reconcile.StatefulSet.Recreate.OnUpdateFailure = NormalizeOnStatefulSetRecreateAction(c.Reconcile.StatefulSet.Recreate.OnUpdateFailure) if c.Reconcile.StatefulSet.Recreate.OnUpdateFailure == "" { c.Reconcile.StatefulSet.Recreate.OnUpdateFailure = OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -1241,7 +1320,9 @@ func (c *OperatorConfig) normalizeSectionClickHouseAccess() { // 1. Metrics requests // 2. Schema maintenance // User credentials can be specified in additional ClickHouse config files located in `chUsersConfigsPath` folder - switch strings.ToLower(c.ClickHouse.Access.Scheme) { + // Fold any accepted casing (http/HTTP, https/HTTPS, auto/Auto) to the canonical const, + // falling back to the default scheme for unrecognized values. + switch util.FoldEnum(c.ClickHouse.Access.Scheme, ChSchemeHTTP, ChSchemeHTTPS, ChSchemeAuto) { case ChSchemeHTTP: c.ClickHouse.Access.Scheme = ChSchemeHTTP case ChSchemeHTTPS: @@ -1323,6 +1404,9 @@ func (c *OperatorConfig) normalizeSectionReconcileRuntime() { if c.Reconcile.Runtime.ReconcileCHIsThreadsNumber == 0 { c.Reconcile.Runtime.ReconcileCHIsThreadsNumber = defaultReconcileCHIsThreadsNumber } + if c.Reconcile.Runtime.ReconcileCHKsThreadsNumber == 0 { + c.Reconcile.Runtime.ReconcileCHKsThreadsNumber = defaultReconcileCHKsThreadsNumber + } if c.Reconcile.Runtime.ReconcileShardsThreadsNumber == 0 { c.Reconcile.Runtime.ReconcileShardsThreadsNumber = defaultReconcileShardsThreadsNumber } @@ -1454,6 +1538,20 @@ func (c *OperatorConfig) RequiresStrictK8sTLS() bool { c.Security.GetKubernetes().GetTLS().GetVerify() == TLSVerifyStrict } +// ResolveK8sTLSMinVersion returns the K8s-API TLS floor for GetClientset. Under +// hardened posture (policy=Enforced or fips.enforced) returns TLSMinVersion13; +// otherwise security.kubernetes.tls.minVersion. Callable on raw file-based config +// before applyEnforcedHardening runs. Empty = Go stdlib default. +func (c *OperatorConfig) ResolveK8sTLSMinVersion() TLSMinVersion { + if c == nil { + return TLSMinVersion("") + } + if c.Security.RequiresHardening() { + return TLSMinVersion13 + } + return c.Security.GetKubernetes().GetTLS().GetMinVersion() +} + // coerceTypedString one-way coerces a *types.String-valued config field (TLSVerify, // TLSMinVersion, IPCMode — all type aliases of types.String) to the FIPS-strict // target value and logs the change. Caller passes the address of the struct field @@ -1489,11 +1587,17 @@ func (c *OperatorConfig) applyEnvVarParams() { c.Watch.Namespaces.Include = types.NewStrings([]string{ns}) } - if nss := os.Getenv(deployment.WATCH_NAMESPACES); len(nss) > 0 { - // We have WATCH_NAMESPACES explicitly specified - if namespaces := c.splitNamespaces(nss); len(namespaces) > 0 { - c.Watch.Namespaces.Include = types.NewStrings(namespaces) + // LookupEnv, not Getenv+len: OLM's AllNamespaces mode sets WATCH_NAMESPACES to an empty + // string (present-but-empty), which must mean watch-all - distinct from a non-OLM deploy + // that leaves it unset. Present-and-empty is coerced to the watch-all pattern so it does + // not fall through to applyDefaultWatchNamespace()'s own-namespace inference. Supersedes + // the singular WATCH_NAMESPACE above. + if nss, ok := os.LookupEnv(deployment.WATCH_NAMESPACES); ok { + namespaces := c.splitNamespaces(nss) + if len(namespaces) == 0 { + namespaces = []string{".*"} } + c.Watch.Namespaces.Include = types.NewStrings(namespaces) } if nss := os.Getenv(deployment.WATCH_NAMESPACES_EXCLUDE); len(nss) > 0 { @@ -1619,19 +1723,46 @@ func (c *OperatorConfig) copyWithHiddenCredentials() *OperatorConfig { // RestartOnOperatorConfigurationChange reports whether the operator process should exit when // ClickHouseOperatorConfiguration changes (so the pod restarts). func (c *OperatorConfig) RestartOnOperatorConfigurationChange() bool { - return strings.ToLower(c.Watch.Configuration.OnChange.String()) == OnConfigurationChangeRestart + return c.Watch.Configuration.OnChange.EqualFoldString(OnConfigurationChangeRestart) } // ShouldRecoverAbortedOnPodReady reports whether the operator should re-enqueue a CHI // reconcile when a pod belonging to an Aborted CHI transitions to Ready. Default is to retry. -// Backed by reconcile.recovery.from.aborted.onPodReady config key. +// Backed by reconcile.recovery.onStatus.aborted.onPodReady config key. func (c *OperatorConfig) ShouldRecoverAbortedOnPodReady() bool { - value := strings.ToLower(c.Reconcile.Recovery.From.Aborted.OnPodReady.String()) - if value == "" { + onPodReady := c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady + if onPodReady.Value() == "" { // Default behavior — retry return true } - return value == RecoveryActionRetry + return onPodReady.EqualFoldString(RecoveryActionRetry) +} + +// ShouldRecoverCompletedOnPodNotReady reports whether the operator should re-enqueue a +// CHI reconcile when a pod belonging to a Completed CHI flips to Ready=False and stays +// there for longer than CompletedOnPodNotReadyThreshold. Default is OFF: force-recreating +// a Completed CHI's pod is destructive (it can interrupt a replica's in-progress recovery +// and means hard downtime for a single-replica shard), so this is opt-in — only an explicit +// onPodNotReady: retry enables it. Unlike the Aborted scope, which retries by default. +// Backed by reconcile.recovery.onStatus.completed.onPodNotReady config key. +func (c *OperatorConfig) ShouldRecoverCompletedOnPodNotReady() bool { + return c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady.EqualFoldString(RecoveryActionRetry) +} + +// CompletedOnPodNotReadyThreshold returns the minimum duration a pod must remain in +// Ready=False before the Completed recovery scope fires. Falls back to the package +// default (5m) if the config value is unset, empty, or unparseable. +// Backed by reconcile.recovery.onStatus.completed.onPodNotReadyThreshold config key. +func (c *OperatorConfig) CompletedOnPodNotReadyThreshold() time.Duration { + raw := strings.TrimSpace(c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReadyThreshold.String()) + if raw == "" { + return defaultCompletedOnPodNotReadyThreshold + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return defaultCompletedOnPodNotReadyThreshold + } + return d } // IsNamespaceWatched returns whether specified namespace is in a list of watched diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go index 1efc717f2..4b855ba85 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_recovery_test.go @@ -16,6 +16,7 @@ package v1 import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -23,7 +24,7 @@ import ( ) // TestShouldRecoverAbortedOnPodReady verifies the accessor's behavior across the -// full matrix of possible values for reconcile.recovery.from.aborted.onPodReady. +// full matrix of possible values for reconcile.recovery.onStatus.aborted.onPodReady. func TestShouldRecoverAbortedOnPodReady(t *testing.T) { tests := []struct { name string @@ -45,15 +46,79 @@ func TestShouldRecoverAbortedOnPodReady(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { c := &OperatorConfig{} - c.Reconcile.Recovery.From.Aborted.OnPodReady = tc.onReady + c.Reconcile.Recovery.OnStatus.Aborted.OnPodReady = tc.onReady require.Equal(t, tc.expected, c.ShouldRecoverAbortedOnPodReady()) }) } } -// TestRecoveryActionConstants documents the stable enum values published in the CRD. -// Changes here would break users' CHOPCONF CRs. +// TestRecoveryActionConstants documents the canonical (humped) enum values. The CRD also +// accepts the all-lowercase forms, and the accessors compare case-insensitively (EqualFold), +// so existing lowercase CHOPCONF CRs keep working. func TestRecoveryActionConstants(t *testing.T) { - require.Equal(t, "none", RecoveryActionNone) - require.Equal(t, "retry", RecoveryActionRetry) + require.Equal(t, "None", RecoveryActionNone) + require.Equal(t, "Retry", RecoveryActionRetry) +} + +// TestShouldRecoverCompletedOnPodNotReady verifies the accessor's behavior across the +// full matrix of possible values for reconcile.recovery.onStatus.completed.onPodNotReady. +// Mirrors TestShouldRecoverAbortedOnPodReady so symmetric config keys behave identically. +func TestShouldRecoverCompletedOnPodNotReady(t *testing.T) { + tests := []struct { + name string + onPodNotRdy *types.String + expected bool + }{ + {"nil defaults to off (opt-in only — destructive recreate)", nil, false}, + {"empty string defaults to off", types.NewString(""), false}, + {"retry lowercase", types.NewString("retry"), true}, + {"Retry mixed case", types.NewString("Retry"), true}, + {"RETRY upper case", types.NewString("RETRY"), true}, + {"none lowercase — opt-out", types.NewString("none"), false}, + {"None mixed case", types.NewString("None"), false}, + {"NONE upper case", types.NewString("NONE"), false}, + {"unknown value treated as no-retry (fail safe)", types.NewString("bogus"), false}, + {"whitespace-only treated as no-retry", types.NewString(" "), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &OperatorConfig{} + c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReady = tc.onPodNotRdy + require.Equal(t, tc.expected, c.ShouldRecoverCompletedOnPodNotReady()) + }) + } +} + +// TestCompletedOnPodNotReadyThreshold verifies the threshold parser. Unparseable, +// empty, and non-positive values must fall back to the package default — operators +// who *really* want to disable the safety net should use onPodNotReady=none, not +// pass a malformed duration. +func TestCompletedOnPodNotReadyThreshold(t *testing.T) { + tests := []struct { + name string + raw *types.String + expected time.Duration + }{ + {"nil falls back to default", nil, defaultCompletedOnPodNotReadyThreshold}, + {"empty falls back to default", types.NewString(""), defaultCompletedOnPodNotReadyThreshold}, + {"whitespace falls back to default", types.NewString(" "), defaultCompletedOnPodNotReadyThreshold}, + {"unparseable falls back to default", types.NewString("five minutes"), defaultCompletedOnPodNotReadyThreshold}, + {"zero falls back to default (don't accidentally disable)", + types.NewString("0s"), defaultCompletedOnPodNotReadyThreshold}, + {"negative falls back to default", types.NewString("-30s"), defaultCompletedOnPodNotReadyThreshold}, + {"30 seconds — aggressive", types.NewString("30s"), 30 * time.Second}, + {"5 minutes — the documented default in string form", + types.NewString("5m"), 5 * time.Minute}, + {"1 hour — conservative", types.NewString("1h"), time.Hour}, + {"complex duration: 1h30m", types.NewString("1h30m"), 90 * time.Minute}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := &OperatorConfig{} + c.Reconcile.Recovery.OnStatus.Completed.OnPodNotReadyThreshold = tc.raw + require.Equal(t, tc.expected, c.CompletedOnPodNotReadyThreshold()) + }) + } } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go new file mode 100644 index 000000000..99ded8924 --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop_watch_test.go @@ -0,0 +1,55 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/altinity/clickhouse-operator/pkg/apis/deployment" +) + +// TestApplyEnvVarParamsWatchNamespaces verifies that WATCH_NAMESPACES (wired by the OLM CSV +// to the OperatorGroup's olm.targetNamespaces annotation) maps every advertised OLM install +// mode to the right watch set. The decisive case is AllNamespaces: OLM sets the var to an +// empty string, which must mean "watch all namespaces", not "watch own namespace". +func TestApplyEnvVarParamsWatchNamespaces(t *testing.T) { + tests := []struct { + name string // OLM install mode under test + value string // WATCH_NAMESPACES as OLM sets it from olm.targetNamespaces + expected []string + }{ + {"OwnNamespace", "openshift-operators", []string{"openshift-operators"}}, + {"SingleNamespace", "team-a", []string{"team-a"}}, + {"MultiNamespace comma", "team-a,team-b", []string{"team-a", "team-b"}}, + {"MultiNamespace colon", "team-a:team-b", []string{"team-a", "team-b"}}, + {"AllNamespaces empty -> watch all", "", []string{".*"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(deployment.WATCH_NAMESPACES, tt.value) + + c := &OperatorConfig{} + c.applyEnvVarParams() + + // ElementsMatch, not Equal: the include set is order-insensitive (NewStrings + // dedups via a map), and watch.namespaces is consumed as a set downstream. + require.ElementsMatch(t, tt.expected, c.Watch.Namespaces.Include.Value(), + "WATCH_NAMESPACES=%q", tt.value) + }) + } +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_host.go b/pkg/apis/clickhouse.altinity.com/v1/type_host.go index 362ac428e..d3aeb1c1e 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_host.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_host.go @@ -748,7 +748,7 @@ func (host *Host) ShouldIncludeIntoCluster() bool { switch { case host.IsStopped(): return false - case host.GetCluster().HostsCount() < 2: + case host.GetCluster().IsSingleNode(): return false default: return true diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go index 514990d6f..68dc626ef 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref.go @@ -14,14 +14,16 @@ package v1 +import "github.com/altinity/clickhouse-operator/pkg/util" + // KeeperServiceType describes how keeper endpoints are discovered for a KeeperRef. type KeeperServiceType string const ( // KeeperServiceTypeReplicas discovers per-host services, one ZK node per keeper replica. - KeeperServiceTypeReplicas KeeperServiceType = "replicas" + KeeperServiceTypeReplicas KeeperServiceType = "Replicas" // KeeperServiceTypeService uses the CR-level headless service as a single ZK node entry. - KeeperServiceTypeService KeeperServiceType = "service" + KeeperServiceTypeService KeeperServiceType = "Service" ) // IsEmpty returns true if no service type is set. @@ -36,9 +38,9 @@ type KeeperRef struct { // Namespace is the namespace of the CHK resource. Defaults to the CHI namespace if omitted. // +optional Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` - // ServiceType controls how keeper endpoints are discovered: - // "replicas" (default) — enumerate per-host services, one ZK node per keeper replica - // "service" — use the CR-level headless service as a single ZK node entry + // ServiceType controls how keeper endpoints are discovered (case-insensitive): + // "Replicas" (default) — enumerate per-host services, one ZK node per keeper replica + // "Service" — use the CR-level headless service as a single ZK node entry // +optional ServiceType KeeperServiceType `json:"serviceType,omitempty" yaml:"serviceType,omitempty"` } @@ -61,10 +63,18 @@ func (r *KeeperRef) GetNamespace(defaultNamespace string) string { return r.Namespace } -// GetServiceType returns the service type, defaulting to replicas if empty. +// GetServiceType returns the service type, defaulting to Replicas if empty. The raw value +// is read un-normalized from the spec, so fold any accepted casing to the canonical const; +// an unrecognized value passes through unchanged so the resolver can report it as invalid. func (r *KeeperRef) GetServiceType() KeeperServiceType { if r == nil || r.ServiceType.IsEmpty() { return KeeperServiceTypeReplicas } - return r.ServiceType + return KeeperServiceType( + util.FoldEnum( + string(r.ServiceType), + string(KeeperServiceTypeReplicas), + string(KeeperServiceTypeService), + ), + ) } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go new file mode 100644 index 000000000..7091ca58a --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_keeper_ref_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestKeeperRefGetServiceType verifies the accessor defaults to Replicas, folds any +// accepted casing to the canonical const, and passes unrecognized values through so the +// resolver can flag them as invalid. +func TestKeeperRefGetServiceType(t *testing.T) { + tests := []struct { + name string + ref *KeeperRef + want KeeperServiceType + }{ + {"nil defaults to Replicas", nil, KeeperServiceTypeReplicas}, + {"empty defaults to Replicas", &KeeperRef{}, KeeperServiceTypeReplicas}, + {"canonical Replicas", &KeeperRef{ServiceType: "Replicas"}, KeeperServiceTypeReplicas}, + {"lowercase replicas folds", &KeeperRef{ServiceType: "replicas"}, KeeperServiceTypeReplicas}, + {"canonical Service", &KeeperRef{ServiceType: "Service"}, KeeperServiceTypeService}, + {"uppercase SERVICE folds", &KeeperRef{ServiceType: "SERVICE"}, KeeperServiceTypeService}, + {"unrecognized passes through", &KeeperRef{ServiceType: "bogus"}, KeeperServiceType("bogus")}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.ref.GetServiceType()) + }) + } +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go b/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go index b0da8faf2..035e1d36b 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_reconcile.go @@ -235,21 +235,23 @@ func (r *ChiReconcile) GetConfigMapPropagationTimeoutDuration() time.Duration { return time.Duration(r.GetConfigMapPropagationTimeout()) * time.Second } -// Possible reconcile policy values +// Possible reconcile policy values (canonical humped form; CRD also accepts all-lowercase) const ( - ReconcilingPolicyUnspecified = "unspecified" - ReconcilingPolicyWait = "wait" - ReconcilingPolicyNoWait = "nowait" + ReconcilingPolicyUnspecified = "Unspecified" + ReconcilingPolicyWait = "Wait" + ReconcilingPolicyNoWait = "NoWait" ) -// IsReconcilingPolicyWait checks whether reconcile policy is "wait" +// IsReconcilingPolicyWait checks whether reconcile policy is "Wait". +// EqualFold so both humped and all-lowercase inputs match regardless of normalization order. func (r *ChiReconcile) IsReconcilingPolicyWait() bool { - return strings.ToLower(r.GetPolicy()) == ReconcilingPolicyWait + return strings.EqualFold(r.GetPolicy(), ReconcilingPolicyWait) } -// IsReconcilingPolicyNoWait checks whether reconcile policy is "no wait" +// IsReconcilingPolicyNoWait checks whether reconcile policy is "NoWait". +// EqualFold so both humped and all-lowercase inputs match regardless of normalization order. func (r *ChiReconcile) IsReconcilingPolicyNoWait() bool { - return strings.ToLower(r.GetPolicy()) == ReconcilingPolicyNoWait + return strings.EqualFold(r.GetPolicy(), ReconcilingPolicyNoWait) } // GetCleanup gets cleanup diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_security.go b/pkg/apis/clickhouse.altinity.com/v1/type_security.go index d501eda92..45b24ad29 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_security.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_security.go @@ -63,10 +63,8 @@ type ClusterSecurityKubernetes struct { } // ClusterSecurityKubernetesTLS holds knobs for the operator's outbound -// Kubernetes API client. The k8s client-go respects whatever's in the -// kubeconfig; the operator never builds the kubeconfig's tls.Config itself, -// so these knobs are evaluated as a LOAD-TIME GATE — the operator refuses -// to start with a kubeconfig that doesn't meet the requested posture. +// Kubernetes API client. Verify is a load-time gate against the kubeconfig's +// Insecure flag; MinVersion is applied to the client transport by GetClientset. type ClusterSecurityKubernetesTLS struct { // Verify gates startup against the kubeconfig's TLSClientConfig.Insecure // field. Valid values are TLSVerifyStrict and TLSVerifyNone. @@ -77,12 +75,8 @@ type ClusterSecurityKubernetesTLS struct { // override the kubeconfig; it only refuses to load an insecure one. Verify *types.String `json:"verify,omitempty" yaml:"verify,omitempty"` // MinVersion floors TLS at the named protocol version. Valid values are - // TLSMinVersion12 and TLSMinVersion13. Nil = Go stdlib default. - // - // Declared for shape symmetry with ClickHouse/Zookeeper and coerced under - // FIPS, but not yet enforced on the operator's K8s API transport — a future - // enhancement will wire it into rest.Config when the operator wraps the - // kubeconfig with stricter TLS settings. + // TLSMinVersion12 and TLSMinVersion13. Nil = Go stdlib default. Coerced to + // 1.3 under FIPS/Enforced; applied on the K8s API transport by GetClientset. MinVersion *types.String `json:"minVersion,omitempty" yaml:"minVersion,omitempty"` } @@ -420,8 +414,7 @@ func (t *ClusterSecurityKubernetesTLS) GetVerify() TLSVerify { } // GetMinVersion returns the resolved TLSMinVersion for the operator's K8s client. -// Nil-safe; returns empty value when unset. Declared for shape consistency and -// FIPS coercion uniformity — not yet wired into the K8s API transport. +// Nil-safe; returns empty value when unset. func (t *ClusterSecurityKubernetesTLS) GetMinVersion() TLSMinVersion { if (t == nil) || (t.MinVersion == nil) || !t.MinVersion.HasValue() { return TLSMinVersion("") diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go index 71f013252..c294a4903 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_security_fips_test.go @@ -51,6 +51,36 @@ func TestSecurity_GetFIPS_IsEnforced_NilSafe(t *testing.T) { require.True(t, (&OperatorConfigSecurity{FIPS: &OperatorConfigSecurityFIPS{Enforced: types.NewStringBool(true)}}).GetFIPS().IsEnforced()) } +// TestResolveK8sTLSMinVersion verifies hardened posture forces TLS 1.3 and that +// explicit security.kubernetes.tls.minVersion is honored when not hardened. +func TestResolveK8sTLSMinVersion(t *testing.T) { + require.Equal(t, TLSMinVersion(""), (*OperatorConfig)(nil).ResolveK8sTLSMinVersion()) + require.Equal(t, TLSMinVersion(""), (&OperatorConfig{}).ResolveK8sTLSMinVersion()) + + explicit12 := &OperatorConfig{} + explicit12.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion12))}, + } + require.Equal(t, TLSMinVersion12, explicit12.ResolveK8sTLSMinVersion()) + + explicit13 := &OperatorConfig{} + explicit13.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion13))}, + } + require.Equal(t, TLSMinVersion13, explicit13.ResolveK8sTLSMinVersion()) + + enforcedOver12 := &OperatorConfig{} + enforcedOver12.Security.Policy = types.NewString(string(SecurityPolicyEnforced)) + enforcedOver12.Security.Kubernetes = &ClusterSecurityKubernetes{ + TLS: &ClusterSecurityKubernetesTLS{MinVersion: types.NewString(string(TLSMinVersion12))}, + } + require.Equal(t, TLSMinVersion13, enforcedOver12.ResolveK8sTLSMinVersion()) + + fipsForced := &OperatorConfig{} + fipsForced.Security.FIPS = &OperatorConfigSecurityFIPS{Enforced: types.NewStringBool(true)} + require.Equal(t, TLSMinVersion13, fipsForced.ResolveK8sTLSMinVersion()) +} + // TestSecurity_RequiresHardening_NilSafe verifies the union accessor used to // gate per-CR security checks (plain-text ZK rejection, FIPS-bypass rejection, // ZK digest-auth rejection). Fires when EITHER security.policy=Enforced OR diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go b/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go index 3f69d82a3..cf59e18d7 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go @@ -17,7 +17,7 @@ package v1 // HostTemplatesIndex describes index of host templates type HostTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*HostTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*HostTemplate `testdiff:"ignore"` } // NewHostTemplatesIndex creates new HostTemplatesIndex object @@ -71,7 +71,7 @@ func (i *HostTemplatesIndex) Walk(f func(template *HostTemplate)) { // PodTemplatesIndex describes index of pod templates type PodTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*PodTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*PodTemplate `testdiff:"ignore"` } // NewPodTemplatesIndex creates new PodTemplatesIndex object @@ -125,7 +125,7 @@ func (i *PodTemplatesIndex) Walk(f func(template *PodTemplate)) { // VolumeClaimTemplatesIndex describes index of volume claim templates type VolumeClaimTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*VolumeClaimTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*VolumeClaimTemplate `testdiff:"ignore"` } // NewVolumeClaimTemplatesIndex creates new VolumeClaimTemplatesIndex object @@ -179,7 +179,7 @@ func (i *VolumeClaimTemplatesIndex) Walk(f func(template *VolumeClaimTemplate)) // ServiceTemplatesIndex describes index of service templates type ServiceTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*ServiceTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*ServiceTemplate `testdiff:"ignore"` } // NewServiceTemplatesIndex creates new ServiceTemplatesIndex object diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go index e898a6794..011e1c69c 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go @@ -15,6 +15,8 @@ package v1 import ( + "strings" + core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -42,6 +44,19 @@ func NewPVCProvisionerFromString(s string) PVCProvisioner { return PVCProvisioner(s) } +// Normalize folds any letter-casing of a recognized value to its canonical +// PVCProvisioner const, so the CRD can accept both humped and all-lowercase forms +// (e.g. "operator" -> "Operator"). Unrecognized values are returned unchanged for +// the caller's IsValid()/reset to handle. +func (v PVCProvisioner) Normalize() PVCProvisioner { + for _, known := range []PVCProvisioner{PVCProvisionerStatefulSet, PVCProvisionerOperator} { + if strings.EqualFold(string(v), string(known)) { + return known + } + } + return v +} + // IsValid checks whether PVCProvisioner is valid func (v PVCProvisioner) IsValid() bool { switch v { @@ -84,6 +99,19 @@ func NewPVCReclaimPolicyFromString(s string) PVCReclaimPolicy { return PVCReclaimPolicy(s) } +// Normalize folds any letter-casing of a recognized value to its canonical +// PVCReclaimPolicy const, so the CRD can accept both humped and all-lowercase forms +// (e.g. "delete" -> "Delete"). Unrecognized values are returned unchanged for the +// caller's IsValid()/reset to handle. +func (v PVCReclaimPolicy) Normalize() PVCReclaimPolicy { + for _, known := range []PVCReclaimPolicy{PVCReclaimPolicyRetain, PVCReclaimPolicyDelete} { + if strings.EqualFold(string(v), string(known)) { + return known + } + } + return v +} + // IsValid checks whether PVCReclaimPolicy is valid func (v PVCReclaimPolicy) IsValid() bool { switch v { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go new file mode 100644 index 000000000..38437de8e --- /dev/null +++ b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template_test.go @@ -0,0 +1,65 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestPVCReclaimPolicyNormalize verifies casing-folding to the canonical humped const, +// so the CRD can accept both humped and all-lowercase forms. +func TestPVCReclaimPolicyNormalize(t *testing.T) { + tests := []struct { + in string + expected PVCReclaimPolicy + }{ + {"Retain", PVCReclaimPolicyRetain}, + {"retain", PVCReclaimPolicyRetain}, + {"RETAIN", PVCReclaimPolicyRetain}, + {"Delete", PVCReclaimPolicyDelete}, + {"delete", PVCReclaimPolicyDelete}, + {"DELETE", PVCReclaimPolicyDelete}, + {"", PVCReclaimPolicyUnspecified}, + {"bogus", "bogus"}, // unrecognized: returned unchanged (caller's IsValid resets) + } + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + require.Equal(t, tc.expected, PVCReclaimPolicy(tc.in).Normalize()) + }) + } +} + +// TestPVCProvisionerNormalize verifies casing-folding to the canonical humped const. +func TestPVCProvisionerNormalize(t *testing.T) { + tests := []struct { + in string + expected PVCProvisioner + }{ + {"StatefulSet", PVCProvisionerStatefulSet}, + {"statefulset", PVCProvisionerStatefulSet}, + {"STATEFULSET", PVCProvisionerStatefulSet}, + {"Operator", PVCProvisionerOperator}, + {"operator", PVCProvisionerOperator}, + {"", PVCProvisionerUnspecified}, + {"bogus", "bogus"}, + } + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + require.Equal(t, tc.expected, PVCProvisioner(tc.in).Normalize()) + }) + } +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go index a3cf2ef4f..dd8160f73 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go +++ b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go @@ -2127,7 +2127,7 @@ func (in *OperatorConfigReconcile) DeepCopy() *OperatorConfigReconcile { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OperatorConfigReconcileRecovery) DeepCopyInto(out *OperatorConfigReconcileRecovery) { *out = *in - in.From.DeepCopyInto(&out.From) + in.OnStatus.DeepCopyInto(&out.OnStatus) return } @@ -2142,18 +2142,45 @@ func (in *OperatorConfigReconcileRecovery) DeepCopy() *OperatorConfigReconcileRe } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OperatorConfigReconcileRecoveryFrom) DeepCopyInto(out *OperatorConfigReconcileRecoveryFrom) { +func (in *OperatorConfigReconcileRecoveryCompletedScope) DeepCopyInto(out *OperatorConfigReconcileRecoveryCompletedScope) { + *out = *in + if in.OnPodNotReady != nil { + in, out := &in.OnPodNotReady, &out.OnPodNotReady + *out = new(types.String) + **out = **in + } + if in.OnPodNotReadyThreshold != nil { + in, out := &in.OnPodNotReadyThreshold, &out.OnPodNotReadyThreshold + *out = new(types.String) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryCompletedScope. +func (in *OperatorConfigReconcileRecoveryCompletedScope) DeepCopy() *OperatorConfigReconcileRecoveryCompletedScope { + if in == nil { + return nil + } + out := new(OperatorConfigReconcileRecoveryCompletedScope) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OperatorConfigReconcileRecoveryOnStatus) DeepCopyInto(out *OperatorConfigReconcileRecoveryOnStatus) { *out = *in in.Aborted.DeepCopyInto(&out.Aborted) + in.Completed.DeepCopyInto(&out.Completed) return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryFrom. -func (in *OperatorConfigReconcileRecoveryFrom) DeepCopy() *OperatorConfigReconcileRecoveryFrom { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorConfigReconcileRecoveryOnStatus. +func (in *OperatorConfigReconcileRecoveryOnStatus) DeepCopy() *OperatorConfigReconcileRecoveryOnStatus { if in == nil { return nil } - out := new(OperatorConfigReconcileRecoveryFrom) + out := new(OperatorConfigReconcileRecoveryOnStatus) in.DeepCopyInto(out) return out } diff --git a/pkg/apis/common/types/string.go b/pkg/apis/common/types/string.go index 4ec9dd589..a563fdc3b 100644 --- a/pkg/apis/common/types/string.go +++ b/pkg/apis/common/types/string.go @@ -95,6 +95,14 @@ func (s *String) EqualFold(other *String) bool { return strings.EqualFold(s.Value(), other.Value()) } +// EqualFoldString reports whether the String case-insensitively equals a plain string +// value (typically an enum const). Nil-safe: a nil String compares as "" (so it never +// equals a non-empty value). Lets callers fold enum casing without wrapping the const +// in a *String, e.g. field.EqualFoldString(api.RecoveryActionRetry). +func (s *String) EqualFoldString(value string) bool { + return strings.EqualFold(s.Value(), value) +} + // MergeFrom merges value from another variable func (s *String) MergeFrom(from *String) *String { if from == nil { diff --git a/pkg/apis/deployment/affinity.go b/pkg/apis/deployment/affinity.go index f5039c414..cc47f84db 100644 --- a/pkg/apis/deployment/affinity.go +++ b/pkg/apis/deployment/affinity.go @@ -14,6 +14,8 @@ package deployment +import "github.com/altinity/clickhouse-operator/pkg/util" + // Possible pod distributions const ( PodDistributionUnspecified = "Unspecified" @@ -60,3 +62,55 @@ const ( PortDistributionUnspecified = "Unspecified" PortDistributionClusterScopeIndex = "ClusterScopeIndex" ) + +// podDistributionTypes enumerates every recognized PodDistribution.Type value in canonical (humped) form. +var podDistributionTypes = []string{ + PodDistributionUnspecified, + PodDistributionClickHouseAntiAffinity, + PodDistributionShardAntiAffinity, + PodDistributionReplicaAntiAffinity, + PodDistributionAnotherNamespaceAntiAffinity, + PodDistributionAnotherClickHouseInstallationAntiAffinity, + PodDistributionAnotherClusterAntiAffinity, + PodDistributionNamespaceAffinity, + PodDistributionClickHouseInstallationAffinity, + PodDistributionClusterAffinity, + PodDistributionShardAffinity, + PodDistributionReplicaAffinity, + PodDistributionPreviousTailAffinity, + PodDistributionMaxNumberPerNode, + PodDistributionCircularReplication, + PodDistributionOnePerHost, +} + +// podDistributionScopes enumerates every recognized PodDistribution.Scope value in canonical (humped) form. +var podDistributionScopes = []string{ + PodDistributionScopeUnspecified, + PodDistributionScopeShard, + PodDistributionScopeReplica, + PodDistributionScopeCluster, + PodDistributionScopeClickHouseInstallation, + PodDistributionScopeNamespace, + PodDistributionScopeGlobal, +} + +// portDistributionTypes enumerates every recognized PortDistribution.Type value in canonical (humped) form. +var portDistributionTypes = []string{ + PortDistributionUnspecified, + PortDistributionClusterScopeIndex, +} + +// NormalizePodDistributionType folds any accepted casing of a PodDistribution type to its canonical const. +func NormalizePodDistributionType(value string) string { + return util.FoldEnum(value, podDistributionTypes...) +} + +// NormalizePodDistributionScope folds any accepted casing of a PodDistribution scope to its canonical const. +func NormalizePodDistributionScope(value string) string { + return util.FoldEnum(value, podDistributionScopes...) +} + +// NormalizePortDistributionType folds any accepted casing of a PortDistribution type to its canonical const. +func NormalizePortDistributionType(value string) string { + return util.FoldEnum(value, portDistributionTypes...) +} diff --git a/pkg/apis/deployment/affinity_test.go b/pkg/apis/deployment/affinity_test.go new file mode 100644 index 000000000..1235c18e3 --- /dev/null +++ b/pkg/apis/deployment/affinity_test.go @@ -0,0 +1,47 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deployment + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNormalizePodDistributionType verifies casing folds to the canonical const +// while unrecognized values pass through (the normalizer maps those to Unspecified). +func TestNormalizePodDistributionType(t *testing.T) { + require.Equal(t, PodDistributionClickHouseAntiAffinity, NormalizePodDistributionType("clickhouseantiaffinity")) + require.Equal(t, PodDistributionClickHouseAntiAffinity, NormalizePodDistributionType("ClickHouseAntiAffinity")) + require.Equal(t, PodDistributionCircularReplication, NormalizePodDistributionType("CIRCULARREPLICATION")) + require.Equal(t, PodDistributionMaxNumberPerNode, NormalizePodDistributionType("maxnumberpernode")) + require.Equal(t, "bogus", NormalizePodDistributionType("bogus")) + require.Equal(t, "", NormalizePodDistributionType("")) +} + +// TestNormalizePodDistributionScope verifies scope casing folds to the canonical const. +func TestNormalizePodDistributionScope(t *testing.T) { + require.Equal(t, PodDistributionScopeShard, NormalizePodDistributionScope("shard")) + require.Equal(t, PodDistributionScopeCluster, NormalizePodDistributionScope("Cluster")) + require.Equal(t, PodDistributionScopeClickHouseInstallation, NormalizePodDistributionScope("clickhouseinstallation")) + require.Equal(t, "bogus", NormalizePodDistributionScope("bogus")) +} + +// TestNormalizePortDistributionType verifies port-distribution casing folds to the canonical const. +func TestNormalizePortDistributionType(t *testing.T) { + require.Equal(t, PortDistributionClusterScopeIndex, NormalizePortDistributionType("clusterscopeindex")) + require.Equal(t, PortDistributionUnspecified, NormalizePortDistributionType("UNSPECIFIED")) + require.Equal(t, "bogus", NormalizePortDistributionType("bogus")) +} diff --git a/pkg/chop/config_access_rootca_test.go b/pkg/chop/config_access_rootca_test.go new file mode 100644 index 000000000..8ea79cb38 --- /dev/null +++ b/pkg/chop/config_access_rootca_test.go @@ -0,0 +1,135 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chop + +import ( + "testing" + + sigsyaml "github.com/kubernetes-sigs/yaml" + "github.com/stretchr/testify/require" + + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" +) + +// TestAccessRootCASecretRefUnmarshalAndResolve guards the clickhouse.access +// rootCASecretRef wiring end to end with the SAME yaml package the config loader +// uses (kubernetes-sigs/yaml -> YAML->JSON->json.Unmarshal), so it exercises the +// JSON struct tags exactly as getFileBasedConfig does. It then runs the shared +// resolver to confirm the Secret PEM lands in Access.RootCA, which is what +// NewClusterConnectionParamsFromCHOpConfig reads into the operator's TLS config. +func TestAccessRootCASecretRefUnmarshalAndResolve(t *testing.T) { + // Explicit key. + const cfg = ` +clickhouse: + access: + rootCA: "" + rootCASecretRef: + name: my-ca-secret + key: my.crt +` + var oc api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(cfg), &oc)) + require.Equal(t, "my-ca-secret", oc.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "my.crt", oc.ClickHouse.Access.RootCASecretRef.Key) + require.Equal(t, "", oc.ClickHouse.Access.RootCA) + + fakeGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"my.crt": []byte("PEM-EXPLICIT")}, nil + } + resolveRootCAFromSecret(&oc.ClickHouse.Access.RootCA, oc.ClickHouse.Access.RootCASecretRef.Name, + oc.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test", fakeGet) + require.Equal(t, "PEM-EXPLICIT", oc.ClickHouse.Access.RootCA) + + // Empty key -> ca.crt default. + const cfgDefault = ` +clickhouse: + access: + rootCASecretRef: + name: only-name +` + var oc2 api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(cfgDefault), &oc2)) + require.Equal(t, "only-name", oc2.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "", oc2.ClickHouse.Access.RootCASecretRef.Key) + + defGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"ca.crt": []byte("PEM-DEFAULT")}, nil + } + resolveRootCAFromSecret(&oc2.ClickHouse.Access.RootCA, oc2.ClickHouse.Access.RootCASecretRef.Name, + oc2.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test", defGet) + require.Equal(t, "PEM-DEFAULT", oc2.ClickHouse.Access.RootCA) +} + +// TestAccessRootCASecretRefMergesFromCR proves the ClickHouseOperatorConfiguration +// (CRD) path reaches the same resolver as the file config: getAllCRBasedConfigs -> +// buildUnifiedConfig -> OperatorConfig.MergeFrom (mergo deep-merge) must carry the +// nested anonymous-struct access.rootCASecretRef from a CR spec into the unified +// config, where fetchAccessRootCA then resolves it. Guards CRD parity for the field. +func TestAccessRootCASecretRefMergesFromCR(t *testing.T) { + // base = file config with no access ref; cr = a chopconf CR spec carrying the ref. + base := &api.OperatorConfig{} + const crSpec = ` +clickhouse: + access: + rootCASecretRef: + name: cr-ca-secret + key: cr.crt +` + var cr api.OperatorConfig + require.NoError(t, sigsyaml.Unmarshal([]byte(crSpec), &cr)) + require.NoError(t, base.MergeFrom(&cr)) + + // The ref survived the mergo deep-merge of the anonymous Access struct. + require.Equal(t, "cr-ca-secret", base.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "cr.crt", base.ClickHouse.Access.RootCASecretRef.Key) + + // ...and resolves through the shared resolver into Access.RootCA. + fakeGet := func(ns, name string) (map[string][]byte, error) { + return map[string][]byte{"cr.crt": []byte("PEM-FROM-CR-SECRET")}, nil + } + resolveRootCAFromSecret(&base.ClickHouse.Access.RootCA, base.ClickHouse.Access.RootCASecretRef.Name, + base.ClickHouse.Access.RootCASecretRef.Key, "op-ns", "test cr-merge", fakeGet) + require.Equal(t, "PEM-FROM-CR-SECRET", base.ClickHouse.Access.RootCA) +} + +// TestAccessRootCASecretRefMergePrecedence locks how access CA settings merge across +// layered config sources (file + ClickHouseOperatorConfiguration CRs). RootCASecretRef +// is a value struct, so OperatorConfig.MergeFrom (mergo WithOverride) merges it FIELD +// BY FIELD: a higher-priority source's empty field does NOT clear a lower-priority +// non-empty one. This matches every other clickhouse.access.* value field (username, +// password, secret.*); only the security.clickhouse.tls pointer ref replaces wholesale. +// Consequence: layered configs should set the FULL ref, not a partial override. +func TestAccessRootCASecretRefMergePrecedence(t *testing.T) { + // A higher-priority CR ref does NOT override a lower-priority inline rootCA + // (mergo keeps the non-empty inline); the resolver then applies inline-wins. + base := &api.OperatorConfig{} + base.ClickHouse.Access.RootCA = "FILE-INLINE" + cr := &api.OperatorConfig{} + cr.ClickHouse.Access.RootCASecretRef.Name = "cr-secret" + require.NoError(t, base.MergeFrom(cr)) + require.Equal(t, "FILE-INLINE", base.ClickHouse.Access.RootCA) + require.Equal(t, "cr-secret", base.ClickHouse.Access.RootCASecretRef.Name) + + // A higher-priority CR overriding only the name RETAINS the lower-priority key + // (field-by-field merge) — hence "set the full ref in layered configs". + b2 := &api.OperatorConfig{} + b2.ClickHouse.Access.RootCASecretRef.Name = "file-secret" + b2.ClickHouse.Access.RootCASecretRef.Key = "file.crt" + c2 := &api.OperatorConfig{} + c2.ClickHouse.Access.RootCASecretRef.Name = "cr-secret" + require.NoError(t, b2.MergeFrom(c2)) + require.Equal(t, "cr-secret", b2.ClickHouse.Access.RootCASecretRef.Name) + require.Equal(t, "file.crt", b2.ClickHouse.Access.RootCASecretRef.Key) +} diff --git a/pkg/chop/config_manager.go b/pkg/chop/config_manager.go index dafe98ba8..25952d682 100644 --- a/pkg/chop/config_manager.go +++ b/pkg/chop/config_manager.go @@ -120,6 +120,7 @@ func (cm *ConfigManager) Init() error { cm.fetchSecretCredentials() cm.fetchSecurityRootCA() + cm.fetchAccessRootCA() // From now on we have one unified CHOP config log.V(1).Info("Unified CHOP config - with secret data fetched (but not post-processed yet):") @@ -488,6 +489,23 @@ func (cm *ConfigManager) fetchSecurityRootCA() { fetchSecurityRootCAResolve(cm.config.Security.GetClickHouse().GetTLS(), ns, cm.getSecretData) } +// fetchAccessRootCA resolves clickhouse.access.rootCASecretRef into the inline +// clickhouse.access.rootCA via the shared resolver (see resolveRootCAFromSecret). +// Unlike the security path it does not clear the ref afterwards: Access.RootCASecretRef +// is read only here and never flows into per-CHI merges, so there is nothing to clear. +func (cm *ConfigManager) fetchAccessRootCA() { + ns, _ := cm.GetRuntimeParam(deployment.OPERATOR_POD_NAMESPACE) + access := &cm.config.ClickHouse.Access + resolveRootCAFromSecret( + &access.RootCA, + access.RootCASecretRef.Name, + access.RootCASecretRef.Key, + ns, + "chopconf clickhouse.access", + cm.getSecretData, + ) +} + // getSecretData adapts the operator's kubeClient to the secretDataGetter seam. func (cm *ConfigManager) getSecretData(namespace, name string) (map[string][]byte, error) { secret, err := cm.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, controller.NewGetOptions()) @@ -497,64 +515,65 @@ func (cm *ConfigManager) getSecretData(namespace, name string) (map[string][]byt return secret.Data, nil } -// fetchSecurityRootCAResolve carries the pure decision logic of fetchSecurityRootCA. -// Mutates tls in place: inlines RootCA on success, clears RootCASecretRef on any -// terminal outcome (success or failure). -// -// Nil-safe on every input: nil tls, nil RootCASecretRef, or nil getSecret all -// return early. Empty `ref.Name` is the documented "not used" sentinel — the -// ref is cleared silently with no warning. -func fetchSecurityRootCAResolve(tls *api.ClusterSecurityClickHouseTLS, operatorNs string, getSecret secretDataGetter) { - if (tls == nil) || (tls.RootCASecretRef == nil) { - return - } - if getSecret == nil { - // Defensive: a test seam or future refactor passing nil would otherwise - // panic at the call site below. Treat as fetch-error: clear the ref so - // downstream merges don't propagate a stub, log so it's diagnosable. - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef=%q: secret getter is nil — clearing ref", tls.RootCASecretRef.Name) - tls.RootCASecretRef = nil +// resolveRootCAFromSecret is the shared decision logic for inlining a CA bundle +// from a Kubernetes Secret into an inline RootCA PEM string at config-load time. +// Used by BOTH chopconf security.clickhouse.tls (fetchSecurityRootCA) and +// clickhouse.access (fetchAccessRootCA). It sets *inline on success and is +// nil-safe / fail-open: empty secretName is the "not used" sentinel; an inline +// value already set wins; any fetch/key failure leaves *inline untouched and logs +// at Warning so operators see it at default verbosity (fix the Secret, restart). +// `what` labels the config path in log messages. Callers own clearing their own +// ref form (e.g. the *core.SecretKeySelector on the security TLS struct). +func resolveRootCAFromSecret(inline *string, secretName, secretKey, operatorNs, what string, getSecret secretDataGetter) { + if (inline == nil) || (secretName == "") { + // Empty secretName is the documented "not used" sentinel — silent no-op. return } - ref := tls.RootCASecretRef - if ref.Name == "" { - // Empty Name is the "not used" sentinel — let users keep the ref block - // in their chopconf with empty values without forcing them to comment - // it out. Clear the ref so downstream merges don't propagate the stub. - tls.RootCASecretRef = nil + if *inline != "" { + // Inline rootCA wins; operators see the warning and pick one. + log.Warning("%s: both rootCA and rootCASecretRef=%q set — using inline rootCA, ignoring ref", what, secretName) return } - if tls.RootCA != "" { - // Inline RootCA wins; clear the ref so downstream merges don't propagate - // a conflict per CHI. Operators see the warning and pick one. - log.Warning("chopconf security.clickhouse.tls: both rootCA and rootCASecretRef=%q set — using inline rootCA, ignoring ref", ref.Name) - tls.RootCASecretRef = nil + if getSecret == nil { + // Defensive: a nil getter would panic below. Treat as a fetch failure. + log.Warning("%s rootCASecretRef=%q: secret getter is nil — ignoring ref", what, secretName) return } if operatorNs == "" { - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef=%q: operator namespace unknown; clearing ref (chopconf-level CA disabled)", ref.Name) - tls.RootCASecretRef = nil + log.Warning("%s rootCASecretRef=%q: operator namespace unknown; secret-sourced CA disabled", what, secretName) return } - keys := []string{ref.Key} - if ref.Key == "" { + keys := []string{secretKey} + if secretKey == "" { keys = []string{"ca.crt", "tls.crt"} } - data, err := getSecret(operatorNs, ref.Name) + data, err := getSecret(operatorNs, secretName) if err != nil { - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef: unable to fetch %s/%s: %v — clearing ref (fix the Secret and restart the operator)", operatorNs, ref.Name, err) - tls.RootCASecretRef = nil + log.Warning("%s rootCASecretRef: unable to fetch %s/%s: %v — ignoring ref (fix the Secret and restart the operator)", what, operatorNs, secretName, err) return } for _, k := range keys { if v, ok := data[k]; ok { - tls.RootCA = string(v) - tls.RootCASecretRef = nil - log.V(1).Info("chopconf security.clickhouse.tls: inlined RootCA from %s/%s key=%s", operatorNs, ref.Name, k) + *inline = string(v) + log.V(1).Info("%s: inlined RootCA from %s/%s key=%s", what, operatorNs, secretName, k) return } } - log.Warning("chopconf security.clickhouse.tls.rootCASecretRef: secret %s/%s exists but none of keys %v found — clearing ref (fix the Secret and restart the operator)", operatorNs, ref.Name, keys) + log.Warning("%s rootCASecretRef: secret %s/%s exists but none of keys %v found — ignoring ref (fix the Secret and restart the operator)", what, operatorNs, secretName, keys) +} + +// fetchSecurityRootCAResolve resolves the chopconf security.clickhouse.tls +// rootCASecretRef via the shared resolveRootCAFromSecret, then ALWAYS clears +// RootCASecretRef: this ref is operator-scoped and terminal once processed here +// (success or failure), so it must not propagate into the per-CHI MergeFrom +// inheritance where the normalizer would re-resolve it against every CHI's +// namespace. Nil-safe: nil tls or nil RootCASecretRef return early. +func fetchSecurityRootCAResolve(tls *api.ClusterSecurityClickHouseTLS, operatorNs string, getSecret secretDataGetter) { + if (tls == nil) || (tls.RootCASecretRef == nil) { + return + } + ref := tls.RootCASecretRef + resolveRootCAFromSecret(&tls.RootCA, ref.Name, ref.Key, operatorNs, "chopconf security.clickhouse.tls", getSecret) tls.RootCASecretRef = nil } diff --git a/pkg/chop/config_manager_test.go b/pkg/chop/config_manager_test.go index ed260d52a..76f2fa9a0 100644 --- a/pkg/chop/config_manager_test.go +++ b/pkg/chop/config_manager_test.go @@ -208,3 +208,39 @@ func TestFetchSecurityRootCAResolve_ClearOnFailure(t *testing.T) { }) } } + +// TestResolveRootCAFromSecret covers the shared resolver used by both the +// chopconf security.clickhouse.tls path and the clickhouse.access path: +// inline-wins precedence, fail-open on every error (empty result, never panic), +// and ca.crt -> tls.crt key defaulting. +func TestResolveRootCAFromSecret(t *testing.T) { + fakeGet := func(want map[string][]byte, err error) secretDataGetter { + return func(ns, name string) (map[string][]byte, error) { return want, err } + } + tests := []struct { + name string + secretName string + secretKey string + operatorNs string + inline string + get secretDataGetter + wantRootCA string + }{ + {name: "empty name sentinel — no-op", secretName: "", operatorNs: "op", get: fakeGet(nil, nil), wantRootCA: ""}, + {name: "inline wins over secret", secretName: "ca", operatorNs: "op", inline: "INLINE", get: fakeGet(map[string][]byte{"ca.crt": []byte("FROM-SECRET")}, nil), wantRootCA: "INLINE"}, + {name: "nil getter — fail open, empty", secretName: "ca", operatorNs: "op", get: nil, wantRootCA: ""}, + {name: "empty namespace — fail open, empty", secretName: "ca", operatorNs: "", get: fakeGet(map[string][]byte{"ca.crt": []byte("X")}, nil), wantRootCA: ""}, + {name: "default key ca.crt", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"ca.crt": []byte("CA-PEM")}, nil), wantRootCA: "CA-PEM"}, + {name: "default key falls back to tls.crt", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"tls.crt": []byte("TLS-PEM")}, nil), wantRootCA: "TLS-PEM"}, + {name: "explicit key wins over ca.crt", secretName: "ca", secretKey: "custom", operatorNs: "op", get: fakeGet(map[string][]byte{"custom": []byte("CUSTOM"), "ca.crt": []byte("WRONG")}, nil), wantRootCA: "CUSTOM"}, + {name: "key missing — fail open, empty", secretName: "ca", operatorNs: "op", get: fakeGet(map[string][]byte{"other": []byte("...")}, nil), wantRootCA: ""}, + {name: "fetch error — fail open, empty", secretName: "ca", operatorNs: "op", get: fakeGet(nil, errors.New("boom")), wantRootCA: ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + inline := tc.inline + resolveRootCAFromSecret(&inline, tc.secretName, tc.secretKey, tc.operatorNs, "test clickhouse.access", tc.get) + require.Equal(t, tc.wantRootCA, inline, "rootCA") + }) + } +} diff --git a/pkg/chop/kube_machinery.go b/pkg/chop/kube_machinery.go index 282a27a92..6c1438f32 100644 --- a/pkg/chop/kube_machinery.go +++ b/pkg/chop/kube_machinery.go @@ -15,7 +15,9 @@ package chop import ( + "crypto/tls" "fmt" + "net/http" "os" "os/user" "path/filepath" @@ -30,6 +32,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/apis/deployment" chopclientset "github.com/altinity/clickhouse-operator/pkg/client/clientset/versioned" + "github.com/altinity/clickhouse-operator/pkg/util/tlsutil" ) // lastKubeConfigInsecure records whether the most recently loaded kubeconfig @@ -80,8 +83,11 @@ func getKubeConfig(kubeConfigFile, masterURL string) (*kuberest.Config, error) { return captureInsecure(conf, nil) } -// GetClientset gets k8s API clients - both kube native client and our custom client -func GetClientset(kubeConfigFile, masterURL string) ( +// GetClientset gets k8s API clients - both kube native client and our custom client. +// chopConfigFile supplies the file-based chopconf path used to resolve the K8s-API +// TLS minVersion floor before the first network call (same timing as the +// insecure-kubeconfig gate in ConfigManager.Init). +func GetClientset(kubeConfigFile, masterURL, chopConfigFile string) ( *kube.Clientset, *apiextensions.Clientset, *chopclientset.Clientset, @@ -93,6 +99,11 @@ func GetClientset(kubeConfigFile, masterURL string) ( os.Exit(1) } + minVerStr, hardened := resolveK8sTLSMinVersion(chopConfigFile) + if minVer := tlsutil.VersionUint16(minVerStr); minVer != 0 { + applyK8sClientTLSMinVersion(kubeConfig, minVer, hardened) + } + // Layer on k8s client rate limiting overrides if specified in CHOP config. if maybeQps := os.Getenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT); maybeQps != "" { parsedQps, err := strconv.ParseFloat(maybeQps, 32) @@ -139,3 +150,66 @@ func GetClientset(kubeConfigFile, masterURL string) ( return kubeClientset, apiextensionsClientset, chopClientset, dynamicClientset } + +// resolveK8sTLSMinVersion reads the file-based chopconf and returns the effective +// K8s-API TLS floor ("1.2"|"1.3"|"") plus whether a hardened (FIPS/Enforced) posture +// requires it. Uses a nil-client ConfigManager because file loading never touches the +// API. Errors yield ("", false) - no floor. +// +// File-only by design, mirroring the insecure-kubeconfig gate (RequiresStrictK8sTLS): +// this runs before the first secure API call, so hardening declared ONLY in a CR-based +// ClickHouseOperatorConfiguration (merged later, after the API client exists) is not +// visible here and will not floor the K8s transport. Declare fips.enforced / policy in +// the file-based chopconf for the K8s-API floor to apply. +func resolveK8sTLSMinVersion(chopConfigFile string) (minVersion string, hardened bool) { + cm := newConfigManager(nil, nil, chopConfigFile) + fileConfig, err := cm.getFileBasedConfig(chopConfigFile) + if err != nil || fileConfig == nil { + return "", false + } + return string(fileConfig.ResolveK8sTLSMinVersion()), fileConfig.Security.RequiresHardening() +} + +// applyK8sClientTLSMinVersion stamps MinVersion onto the rest.Config transport via +// rest.Config.Wrap, preserving client-go's TLS/proxy/HTTP2 setup. client-go invokes the +// wrapper on the freshly-built *http.Transport before any request, and crypto/tls reads +// MinVersion at handshake, so the floor takes effect on the actual ClientHello (h1 and h2). +// +// If the built RoundTripper is not *http.Transport the floor cannot be enforced. Under a +// hardened (FIPS/Enforced) posture that is fatal - the operator must not silently negotiate +// below the required floor - mirroring the fail-closed insecure-kubeconfig gate. For a +// user-chosen floor without hardening, it degrades to a warning (best-effort). +func applyK8sClientTLSMinVersion(cfg *kuberest.Config, minVer uint16, hardened bool) { + cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper { + out, err := floorTransportTLSMinVersion(rt, minVer) + if err != nil { + if hardened { + log.F().Fatal("k8s client TLS minVersion floor (0x%04x) unenforceable under hardened posture: %v", minVer, err) + os.Exit(1) + } + log.F().Warning("k8s client TLS minVersion floor (0x%04x) not applied: %v", minVer, err) + return rt + } + log.F().Info("k8s client TLS minVersion floor applied: 0x%04x - K8s API servers below this version will be refused", minVer) + return out + }) +} + +// floorTransportTLSMinVersion sets MinVersion on the transport's TLS config. It clones the +// TLS config before mutating because client-go caches *http.Transport keyed on TLS options +// (transport/cache.go), so the config may be shared across clientsets built from the same +// rest.Config - all want the same floor, but cloning avoids mutating shared state in place. +// Returns an error (floor cannot be enforced) if rt is not an *http.Transport. +func floorTransportTLSMinVersion(rt http.RoundTripper, minVer uint16) (http.RoundTripper, error) { + t, ok := rt.(*http.Transport) + if !ok { + return rt, fmt.Errorf("transport is %T, not *http.Transport", rt) + } + tlsConfig := &tls.Config{} + if t.TLSClientConfig != nil { + tlsConfig = t.TLSClientConfig.Clone() + } + tlsConfig.MinVersion = minVer + t.TLSClientConfig = tlsConfig + return t, nil +} diff --git a/pkg/chop/kube_machinery_test.go b/pkg/chop/kube_machinery_test.go new file mode 100644 index 000000000..a0dd2b6c3 --- /dev/null +++ b/pkg/chop/kube_machinery_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chop + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +// notATransport is a RoundTripper that is NOT *http.Transport, used to exercise the +// "floor cannot be enforced" path (the case that must fail-closed under hardening). +type notATransport struct{} + +func (notATransport) RoundTrip(*http.Request) (*http.Response, error) { return nil, nil } + +// TestFloorTransportTLSMinVersion covers the pure floor-stamping helper that backs the +// K8s-API TLS minVersion enforcement (PR #2020). This is the transport-enforcement path +// that the PR's config-resolution tests did not reach. +func TestFloorTransportTLSMinVersion(t *testing.T) { + t.Run("stamps MinVersion when TLS config is nil", func(t *testing.T) { + tr := &http.Transport{} + out, err := floorTransportTLSMinVersion(tr, tls.VersionTLS13) + require.NoError(t, err) + require.Same(t, tr, out) + require.NotNil(t, tr.TLSClientConfig) + require.Equal(t, uint16(tls.VersionTLS13), tr.TLSClientConfig.MinVersion) + }) + + t.Run("clones existing TLS config instead of mutating it in place", func(t *testing.T) { + // client-go caches and shares *http.Transport across clientsets; the helper must not + // mutate the shared *tls.Config in place. + orig := &tls.Config{ServerName: "api.example"} + tr := &http.Transport{TLSClientConfig: orig} + _, err := floorTransportTLSMinVersion(tr, tls.VersionTLS13) + require.NoError(t, err) + require.Equal(t, uint16(0), orig.MinVersion, "original shared config must be untouched") + require.NotSame(t, orig, tr.TLSClientConfig, "transport must hold a clone") + require.Equal(t, uint16(tls.VersionTLS13), tr.TLSClientConfig.MinVersion) + require.Equal(t, "api.example", tr.TLSClientConfig.ServerName, "clone preserves other fields") + }) + + t.Run("errors (floor unenforceable) when not *http.Transport", func(t *testing.T) { + _, err := floorTransportTLSMinVersion(notATransport{}, tls.VersionTLS13) + require.Error(t, err) + }) +} + +// TestFloorTransportTLSMinVersion_EnforcesHandshake proves the floor actually governs the +// TLS handshake: a client floored to 1.3 must refuse a server that caps at 1.2, while a +// client floored to 1.2 connects. This is the end-to-end behavior the operator relies on +// under FIPS/Enforced and which minikube e2e cannot deterministically exercise. +func TestFloorTransportTLSMinVersion_EnforcesHandshake(t *testing.T) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + srv.TLS = &tls.Config{MaxVersion: tls.VersionTLS12} // server refuses anything above 1.2 + srv.StartTLS() + defer srv.Close() + + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + newTransport := func() *http.Transport { + return &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}} + } + + t.Run("floor 1.3 rejects a TLS-1.2-max server", func(t *testing.T) { + rt, err := floorTransportTLSMinVersion(newTransport(), tls.VersionTLS13) + require.NoError(t, err) + _, err = (&http.Client{Transport: rt}).Get(srv.URL) + require.Error(t, err, "handshake must fail: server max 1.2 is below client floor 1.3") + }) + + t.Run("floor 1.2 accepts a TLS-1.2 server", func(t *testing.T) { + rt, err := floorTransportTLSMinVersion(newTransport(), tls.VersionTLS12) + require.NoError(t, err) + resp, err := (&http.Client{Transport: rt}).Get(srv.URL) + require.NoError(t, err) + _ = resp.Body.Close() + }) +} diff --git a/pkg/controller/chi/controller-chk-watcher.go b/pkg/controller/chi/controller-chk-watcher.go index bb45c2f7a..5c9100dfe 100644 --- a/pkg/controller/chi/controller-chk-watcher.go +++ b/pkg/controller/chi/controller-chk-watcher.go @@ -83,7 +83,9 @@ func (c *Controller) StartCHKWatcher(ctx context.Context) { // isKeeperWatchEnabled checks if the CHK watch is configured. func (c *Controller) isKeeperWatchEnabled() bool { policy := chop.Config().Reconcile.Coordination.Keeper.OnKeeperResourceUpdate - return policy.HasValue() && policy.Value() == api.KeeperOnResourceUpdateReconcile + // EqualFoldString: the CRD accepts both "Reconcile" and "reconcile"; the chopconf value + // is not re-folded after load, so compare case-insensitively here. + return policy.HasValue() && policy.EqualFoldString(api.KeeperOnResourceUpdateReconcile) } // onCHKUpdate handles CHK update events. Triggers CHI reconcile only when CHK transitions to Completed. diff --git a/pkg/controller/chi/controller-keeper-resolver.go b/pkg/controller/chi/controller-keeper-resolver.go index 73dbd1f3d..26b4126f2 100644 --- a/pkg/controller/chi/controller-keeper-resolver.go +++ b/pkg/controller/chi/controller-keeper-resolver.go @@ -115,11 +115,19 @@ func (c *Controller) resolveKeeperByService(ctx context.Context, namespace, name } // resolveKeeperByReplicas resolves using per-host services (one node per replica). -// CHK host services are discovered by CHK labeler's LabelCRName and LabelService/LabelServiceValueHost labels. +// +// A CHK host exposes a ready-only client Service (LabelServiceValueHostClient, +// publishNotReadyAddresses=false) alongside the peer/Raft Service (LabelServiceValueHost, +// publishNotReadyAddresses=true). ClickHouse clients must resolve only Ready Keeper nodes, so +// the client tier is selected first. The peer tier is the fallback for CHK clusters that predate +// the split (single host Service) or use a user-supplied replicaServiceTemplate. func (c *Controller) resolveKeeperByReplicas(ctx context.Context, namespace, name string, domainPattern *types.String) (api.ZookeeperNodes, error) { - // Discover CHK host services by label selector - opts := chkHostServiceListOptions(name, namespace) - services, err := c.kubeClient.CoreV1().Services(namespace).List(ctx, opts) + // Prefer the ready-only client services + services, err := c.kubeClient.CoreV1().Services(namespace).List(ctx, chkClientServiceListOptions(name, namespace)) + if (err == nil) && (len(services.Items) == 0) { + // No client-tier services - fall back to the peer/host services (pre-split / templated CRs) + services, err = c.kubeClient.CoreV1().Services(namespace).List(ctx, chkHostServiceListOptions(name, namespace)) + } // Handle errors and empty list switch { @@ -156,7 +164,7 @@ func chkListOptions(name, namespace string) meta.ListOptions { }) } -// chkHostServiceListOptions builds meta.ListOptions to select CHK per-host services. +// chkHostServiceListOptions builds meta.ListOptions to select CHK per-host peer/Raft services. func chkHostServiceListOptions(name, namespace string) meta.ListOptions { chk := chkApi.NewClickHouseKeeperInstallation(name, namespace) l := chkLabeler.New(chk) @@ -165,3 +173,14 @@ func chkHostServiceListOptions(name, namespace string) meta.ListOptions { l.Get(commonLabeler.LabelService): l.Get(commonLabeler.LabelServiceValueHost), }) } + +// chkClientServiceListOptions builds meta.ListOptions to select CHK per-host client-facing +// services (publishNotReadyAddresses=false), the tier ClickHouse clients should resolve. +func chkClientServiceListOptions(name, namespace string) meta.ListOptions { + chk := chkApi.NewClickHouseKeeperInstallation(name, namespace) + l := chkLabeler.New(chk) + return controller.NewListOptions(map[string]string{ + l.Get(commonLabeler.LabelCRName): name, + l.Get(commonLabeler.LabelService): l.Get(commonLabeler.LabelServiceValueHostClient), + }) +} diff --git a/pkg/controller/chi/labeler/labeler.go b/pkg/controller/chi/labeler/labeler.go index 87d0e9fe2..db9683d10 100644 --- a/pkg/controller/chi/labeler/labeler.go +++ b/pkg/controller/chi/labeler/labeler.go @@ -111,8 +111,7 @@ func (l *Labeler) LabelMyObjectsTree(ctx context.Context) error { // Put labels on the Deployment err = l.labelDeployment(ctx, replicaSet) if err != nil { - fmt.Errorf("%w %s err: %v", ErrUnableToLabelDeployment, util.NamespacedName(replicaSet), err) - return err + return fmt.Errorf("%w %s err: %v", ErrUnableToLabelDeployment, util.NamespacedName(replicaSet), err) } return nil diff --git a/pkg/controller/chi/worker-boilerplate.go b/pkg/controller/chi/worker-boilerplate.go index 97b157602..aab67967b 100644 --- a/pkg/controller/chi/worker-boilerplate.go +++ b/pkg/controller/chi/worker-boilerplate.go @@ -149,8 +149,10 @@ func (w *worker) processReconcilePod(ctx context.Context, cmd *cmd_queue.Reconci case cmd_queue.ReconcileUpdate: // Detect NotReady → Ready transition for pods belonging to Aborted CHIs // and re-enqueue the CHI for reconcile. Controlled by config option - // reconcile.recovery.from.aborted.onPodReady (default: retry). + // reconcile.recovery.onStatus.aborted.onPodReady (default: retry). w.recoverAbortedReconcileOnPodReady(ctx, cmd.Old, cmd.New) + // Symmetric path for Ready→NotReady on Completed CHIs. + w.recoverCompletedReconcileOnPodNotReady(ctx, cmd.Old, cmd.New) return nil case cmd_queue.ReconcileDelete: w.a.V(1).M(cmd.Old).F().Info("Delete Pod. %s/%s", cmd.Old.Namespace, cmd.Old.Name) diff --git a/pkg/controller/chi/worker-pod-retry.go b/pkg/controller/chi/worker-pod-retry.go index 86b901b46..bffe6d5cc 100644 --- a/pkg/controller/chi/worker-pod-retry.go +++ b/pkg/controller/chi/worker-pod-retry.go @@ -17,6 +17,7 @@ package chi import ( "context" "strings" + "time" core "k8s.io/api/core/v1" @@ -28,6 +29,14 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/k8s" ) +// stuckHostMinDelay floors the deferred-re-enqueue delay so a 1-second flap +// can't produce an immediate reconcile. +const stuckHostMinDelay = 5 * time.Second + +// stuckHostExtraDelay buffers threshold against apiserver/informer latency +// so the eventual reconcile observes an up-to-date LastTransitionTime. +const stuckHostExtraDelay = 2 * time.Second + // normalizeTimeAbortReasons enumerates Aborted reasons that cannot recover via // pod transitions — the spec itself must be edited. Auto-recovery skips these // to avoid metrics churn on pod-Ready flips that would just re-trigger the same @@ -41,7 +50,7 @@ var normalizeTimeAbortReasons = []string{ // recoverAbortedReconcileOnPodReady inspects a pod update event and re-enqueues the parent // CHI for reconcile when the pod transitioned NotReady → Ready and the CHI is Aborted. -// Controlled by reconcile.recovery.from.aborted.onPodReady config option (default: retry). +// Controlled by reconcile.recovery.onStatus.aborted.onPodReady config option (default: retry). // The decision to re-enqueue is based on the CHI's Status alone, not on ActionPlan — // see shouldTriggerAutoRecovery for the rationale. func (w *worker) recoverAbortedReconcileOnPodReady(ctx context.Context, oldPod, newPod *core.Pod) { @@ -132,3 +141,128 @@ func isPodNotReadyToReadyTransition(oldPod, newPod *core.Pod) bool { isReadyNow := !k8s.PodHasNotReadyContainers(newPod) return wasNotReady && isReadyNow } + +// recoverCompletedReconcileOnPodNotReady is the symmetric counterpart of +// recoverAbortedReconcileOnPodReady. It inspects a pod update event and schedules a +// delayed CHI reconcile when a child pod of a Completed CHI transitions Ready → NotReady. +// The delay equals the configured threshold (default 5m), so by the time the reconcile +// fires, shouldForceRestartHost can observe a sustained Ready=False and decide whether +// to restart the host. +func (w *worker) recoverCompletedReconcileOnPodNotReady(ctx context.Context, oldPod, newPod *core.Pod) { + if !chop.Config().ShouldRecoverCompletedOnPodNotReady() { + return + } + + if !isPodReadyToNotReadyTransition(oldPod, newPod) { + return + } + + // Skip pods that are terminating — the Ready→NotReady flip is normal + // shutdown bookkeeping, not a host regression. + if newPod.GetDeletionTimestamp() != nil && !newPod.GetDeletionTimestamp().IsZero() { + return + } + + // Skip pods already in a kubelet-driven failure mode (ImagePullBackOff, + // CrashLoopBackOff, Pending, etc.) — kubelet is handling those and an + // operator-driven StatefulSet rollout would just race it. + if podIsInKubeletFailureMode(newPod) { + return + } + + cr, err := w.c.GetCR(&newPod.ObjectMeta) + if err != nil || cr == nil { + return + } + + if !shouldTriggerStuckHostRecovery(cr) { + return + } + + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + delay := stuckHostScheduleDelay(newPod, threshold, time.Now()) + + w.a.V(1).M(cr).F(). + WithEvent(cr, a.EventActionReconcile, a.EventReasonStuckHostRecoveryTriggered). + Info( + "Stuck-host recovery scheduled: pod %s became NotReady while CHI %s/%s is Completed; "+ + "re-enqueue in %s (threshold %s)", + newPod.Name, cr.Namespace, cr.Name, delay.Truncate(time.Second), threshold, + ) + + scheduled := cr + time.AfterFunc(delay, func() { + w.c.enqueueObject(cmd_queue.NewReconcileCHI(cmd_queue.ReconcileAdd, nil, scheduled)) + }) +} + +// shouldTriggerStuckHostRecovery reports whether the given CHI is a valid stuck-host +// recovery target: status is Completed and the CHI is not being deleted. +func shouldTriggerStuckHostRecovery(cr *api.ClickHouseInstallation) bool { + if cr == nil { + return false + } + status := cr.EnsureStatus() + if status.GetStatus() != api.StatusCompleted { + return false + } + if !cr.GetDeletionTimestamp().IsZero() { + return false + } + return true +} + +// crHasHostNeedingStuckRecovery reports whether the CR is a stuck-host recovery target +// (Completed, not deleting, feature enabled) with at least one host whose pod has been +// NotReady past the configured threshold. reconcileCR uses this to proceed past the +// "no reconcile work" gate: a sustained-NotReady pod is live runtime state that never +// surfaces as an ActionPlan diff or object drift, so without this the delayed recovery +// reconcile self-aborts before shouldForceRestartHost is ever consulted. +func (w *worker) crHasHostNeedingStuckRecovery(ctx context.Context, cr *api.ClickHouseInstallation) bool { + if !chop.Config().ShouldRecoverCompletedOnPodNotReady() { + return false + } + if !shouldTriggerStuckHostRecovery(cr) { + return false + } + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + found := false + cr.WalkHosts(func(host *api.Host) error { + if !found && w.isPodSustainedNotReady(ctx, host, threshold) { + found = true + } + return nil + }) + return found +} + +// isPodReadyToNotReadyTransition reports whether the pod transitioned from "all containers +// ready" to "some container not ready". The dual of isPodNotReadyToReadyTransition. +func isPodReadyToNotReadyTransition(oldPod, newPod *core.Pod) bool { + if oldPod == nil || newPod == nil { + return false + } + wasReady := !k8s.PodHasNotReadyContainers(oldPod) + isNotReadyNow := k8s.PodHasNotReadyContainers(newPod) + return wasReady && isNotReadyNow +} + +// stuckHostScheduleDelay computes how long to wait before firing the stuck-host +// re-enqueue. It returns max(threshold − elapsed + extra, minDelay), clamped to +// non-negative. +func stuckHostScheduleDelay(newPod *core.Pod, threshold time.Duration, now time.Time) time.Duration { + elapsed := time.Duration(0) + if newPod != nil { + for _, cond := range newPod.Status.Conditions { + if cond.Type == core.PodReady && !cond.LastTransitionTime.IsZero() { + elapsed = now.Sub(cond.LastTransitionTime.Time) + break + } + } + } + delay := threshold - elapsed + stuckHostExtraDelay + if delay < stuckHostMinDelay { + delay = stuckHostMinDelay + } + return delay +} diff --git a/pkg/controller/chi/worker-pod-retry_test.go b/pkg/controller/chi/worker-pod-retry_test.go index 39893bb18..829375919 100644 --- a/pkg/controller/chi/worker-pod-retry_test.go +++ b/pkg/controller/chi/worker-pod-retry_test.go @@ -145,3 +145,186 @@ func TestShouldTriggerAutoRecovery(t *testing.T) { }) } } + +// TestIsPodReadyToNotReadyTransition verifies the dual of isPodNotReadyToReadyTransition: +// fires only on Ready→NotReady, mirrors the same nil/edge-case handling. +func TestIsPodReadyToNotReadyTransition(t *testing.T) { + tests := []struct { + name string + old, new *core.Pod + expected bool + }{ + {"nil old", nil, pod(false), false}, + {"nil new", pod(true), nil, false}, + {"both nil", nil, nil, false}, + {"ready → not ready (the target case)", pod(true), pod(false), true}, + {"ready → ready (no transition)", pod(true), pod(true), false}, + {"not ready → ready (wrong direction, handled by sibling)", pod(false), pod(true), false}, + {"not ready → not ready", pod(false), pod(false), false}, + {"multi-container: all ready → one not ready", multiContainerPod(true, true), multiContainerPod(false, true), true}, + {"multi-container: one not ready → all ready", multiContainerPod(true, false), multiContainerPod(true, true), false}, + {"multi-container: all ready → all ready", multiContainerPod(true, true), multiContainerPod(true, true), false}, + {"empty statuses → not ready (fires; empty counts as ready)", + &core.Pod{}, pod(false), true}, + {"12-container pod: last flips to not ready", + multiContainerPod(true, true, true, true, true, true, true, true, true, true, true, true), + multiContainerPod(true, true, true, true, true, true, true, true, true, true, true, false), + true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := isPodReadyToNotReadyTransition(tc.old, tc.new) + require.Equal(t, tc.expected, got) + }) + } +} + +// TestShouldTriggerStuckHostRecovery verifies the CR-state gate used by +// recoverCompletedReconcileOnPodNotReady. +func TestShouldTriggerStuckHostRecovery(t *testing.T) { + makeCR := func(status string, deleting bool) *api.ClickHouseInstallation { + cr := &api.ClickHouseInstallation{ + ObjectMeta: meta.ObjectMeta{Name: "chi", Namespace: "ns"}, + } + cr.EnsureStatus().Status = status + if deleting { + now := meta.NewTime(time.Now()) + cr.ObjectMeta.DeletionTimestamp = &now + } + return cr + } + + tests := []struct { + name string + cr *api.ClickHouseInstallation + expected bool + }{ + {"nil CR — reject", nil, false}, + // The target case: Completed CHI whose host has just regressed. + {"Completed, not deleting — accept (the target case)", makeCR(api.StatusCompleted, false), true}, + // Aborted is the sibling path's responsibility; firing stuck-host recovery on it + // would double-enqueue with recoverAbortedReconcileOnPodReady once the pod + // eventually becomes Ready again. + {"Aborted — reject (handled by sibling recoverAbortedReconcileOnPodReady path)", + makeCR(api.StatusAborted, false), false}, + // InProgress means a reconcile is already in flight; let it observe the pod state + // on its own rather than racing another enqueue. + {"InProgress — reject (reconcile already running)", makeCR(api.StatusInProgress, false), false}, + {"Terminating — reject", makeCR(api.StatusTerminating, false), false}, + {"Completed but being deleted — reject", makeCR(api.StatusCompleted, true), false}, + // Fresh CR with no status field set yet — happens between Create and the first + // status update by the operator. + {"empty status (fresh CR) — reject", makeCR("", false), false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, shouldTriggerStuckHostRecovery(tc.cr)) + }) + } +} + +// TestStuckHostScheduleDelay verifies the delay computation for the deferred re-enqueue. +// The helper is pure (clock + threshold injected as args), so we can exercise the +// boundary conditions without time-mocking the rest of the controller. +func TestStuckHostScheduleDelay(t *testing.T) { + now := time.Date(2026, 5, 28, 12, 0, 0, 0, time.UTC) + + // podWithReadyTransition builds a pod whose PodReady condition transitioned at the + // given offset from "now". Negative offset means the transition is in the past. + podWithReadyTransition := func(offset time.Duration) *core.Pod { + return &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(offset))}, + }, + }, + } + } + + tests := []struct { + name string + pod *core.Pod + threshold time.Duration + // expected delay must satisfy lo <= got <= hi (small tolerance for arithmetic). + expectMin time.Duration + expectMax time.Duration + }{ + { + // Fresh transition: full threshold + small extra padding for apiserver + // catch-up. With threshold=5m and elapsed=0, delay should be ~5m02s. + name: "fresh transition: schedule full threshold + extra", + pod: podWithReadyTransition(0), + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay - time.Second, + expectMax: 5*time.Minute + stuckHostExtraDelay + time.Second, + }, + { + // Already half-elapsed: remaining ~2.5m + extra. + name: "half-elapsed: schedule the remainder", + pod: podWithReadyTransition(-150 * time.Second), + threshold: 5 * time.Minute, + expectMin: 150*time.Second + stuckHostExtraDelay - time.Second, + expectMax: 150*time.Second + stuckHostExtraDelay + time.Second, + }, + { + // Threshold already past at schedule time (e.g. operator restart after + // long outage): clamp to stuckHostMinDelay rather than firing instantly, + // so a single quick flap doesn't produce an immediate restart. + name: "threshold already past: clamp to minDelay", + pod: podWithReadyTransition(-10 * time.Minute), + threshold: 5 * time.Minute, + expectMin: stuckHostMinDelay, + expectMax: stuckHostMinDelay, + }, + { + // Nil pod: no LastTransitionTime info → treat as elapsed=0 → full threshold. + name: "nil pod: full threshold", + pod: nil, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Pod has no PodReady condition (very early in lifecycle): elapsed=0. + name: "pod missing PodReady condition: full threshold", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{}}}, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Zero LastTransitionTime (apiserver hasn't stamped it yet): treat as + // elapsed=0, schedule full threshold. + name: "zero LastTransitionTime: full threshold", + pod: &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse}, + }, + }, + }, + threshold: 5 * time.Minute, + expectMin: 5*time.Minute + stuckHostExtraDelay, + expectMax: 5*time.Minute + stuckHostExtraDelay, + }, + { + // Threshold smaller than minDelay: minDelay still floors the result. + name: "tiny threshold: clamp to minDelay", + pod: podWithReadyTransition(0), + threshold: 1 * time.Second, + expectMin: stuckHostMinDelay, + expectMax: stuckHostMinDelay, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := stuckHostScheduleDelay(tc.pod, tc.threshold, now) + require.GreaterOrEqual(t, got, tc.expectMin, "delay below expected minimum") + require.LessOrEqual(t, got, tc.expectMax, "delay above expected maximum") + }) + } +} diff --git a/pkg/controller/chi/worker-reconciler-chi.go b/pkg/controller/chi/worker-reconciler-chi.go index 86ed40796..debfdf6ee 100644 --- a/pkg/controller/chi/worker-reconciler-chi.go +++ b/pkg/controller/chi/worker-reconciler-chi.go @@ -111,6 +111,8 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal w.a.M(new).F().Info("CR has reconcile work - continue reconcile") case w.isAfterFinalizerInstalled(new.GetAncestorT(), new): w.a.M(new).F().Info("isAfterFinalizerInstalled - continue reconcile-2") + case w.crHasHostNeedingStuckRecovery(ctx, new): + w.a.M(new).F().Info("CR has a sustained-NotReady host - continue reconcile for stuck-host recovery") default: w.a.M(new).F().Info("No reconcile work - abort reconcile") metrics.CRReconcilesCompleted(ctx, new) @@ -358,6 +360,7 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *api.ClickHo cr.GetRuntime().UnlockCommonConfig() w.includeAllHostsIntoCluster(ctx, cr) + w.restartNewlyAddedHosts(ctx, cr) return err } @@ -371,6 +374,114 @@ func (w *worker) includeAllHostsIntoCluster(ctx context.Context, cr *api.ClickHo }) } +// restartNewlyAddedHosts works around issue #2013: a replica added to an already-existing +// multi-host cluster starts ClickHouse before the operator publishes the full remote_servers, +// so any cluster-dependent object (Distributed / DICTIONARY / refreshable MV) fails its async +// startup table load with CLUSTER_DOESNT_EXIST. ClickHouse never re-runs terminal FAILED/CANCELED +// loader jobs, so the operator's later SYSTEM RELOAD CONFIG (which only adds the cluster to +// system.clusters) does not recover them - the new replica stays broken until a manual restart. +// The complete remote_servers is published just above (reconcileConfigMapCommon + +// includeAllHostsIntoCluster), so restarting the newly-added hosts here re-runs their loader jobs +// against the real cluster. +// +// Why a restart and not "seed the full remote_servers before the pod boots": remote_servers is in +// the shared common ConfigMap, and getRemoteServersGeneratorOptions() deliberately excludes a host +// that has no StatefulSet yet (advertising it would break existing pods' cluster-wide operations - +// see the comment there). So the new host cannot be handed the complete topology before it exists +// without harming existing pods; restarting only the new host after publish is the resolution. +// +// This ONLY ever restarts newly-added hosts. Existing hosts (ObjectStatusFound/Same, HasAncestor) +// are never in the set - they pick up config changes via ConfigMap-propagation-wait + ClickHouse's +// own config auto-reload, and must never be restarted. Gated to scale-up only, AND only to hosts +// that actually recorded a terminal CLUSTER_DOESNT_EXIST load failure (the positive #2013 signal) - +// so a fresh install, a single-host cluster, or a plain-replica scale-up with no cluster-dependent +// objects (including a legitimately-lagging replica) is never restarted. +func (w *worker) restartNewlyAddedHosts(ctx context.Context, cr *api.ClickHouseInstallation) { + if util.IsContextDone(ctx) { + log.V(1).Info("Reconcile is aborted. Restart newly added hosts: %s ", cr.GetName()) + return + } + + cr.WalkHosts(func(host *api.Host) error { + // Only hosts created during THIS reconcile. reconcileHost flips a Requested host to + // ObjectStatusCreated; on any later reconcile it HasAncestor() and is Found/Same, so this + // restart is one-shot and never loops. + if !host.GetReconcileAttributes().GetStatus().Is(types.ObjectStatusCreated) { + return nil + } + // Single-node clusters reference only localhost in remote_servers, which always resolves - + // there is no boot-before-cluster window to close. + if host.GetCluster().IsSingleNode() { + return nil + } + // Scale-up only. On an INITIAL cluster creation every host is ObjectStatusCreated too, but + // none can hit CLUSTER_DOESNT_EXIST (no pre-existing cluster-dependent objects at first + // boot) - restarting them would just add a pointless restart to every fresh install. + // IsInNewCluster() (no ancestor AND all hosts added this reconcile) is true exactly for a + // brand-new cluster; a scale-up added only some hosts, so it is false there. Same predicate + // shouldMigrateTables uses to tell a scale-up host from a fresh-cluster host. + if host.IsInNewCluster() { + return nil + } + if host.IsStopped() || host.IsTroubleshoot() { + return nil + } + // Restart ONLY if this host actually hit the #2013 condition: a terminal CLUSTER_DOESNT_EXIST + // async-load failure (a Distributed / DICTIONARY / refreshable-MV object that failed to load + // because the host booted before its remote_servers was complete). ClickHouse never re-runs + // such terminal loaders after a config reload, so a restart is the only recovery. Gating on + // this positive per-host failure signal - rather than on host-newness or pod-readiness - is + // what makes the restart precise: a plain-ReplicatedMergeTree replica with no cluster-dependent + // objects (e.g. a legitimately-lagging replica) records zero such errors and is left untouched, + // so its normal sync/delay behavior is never disrupted. + if !w.hostHitClusterDoesNotExist(ctx, host) { + w.a.V(1).M(host).F().Info("Skip restart of newly-added host - no CLUSTER_DOESNT_EXIST load failure. Host: %s", host.GetName()) + return nil + } + + w.a.V(1).M(host).F().Info("Restart newly-added host to re-load cluster-dependent objects against full remote_servers. Host: %s", host.GetName()) + w.task.WaitForConfigMapPropagation(ctx, host) + if err := w.hostSoftwareRestart(ctx, host); err != nil { + // Best-effort, matching includeAllHostsIntoCluster: the reconcile already succeeded and + // the host serves queries; if the restart fails the cluster-dependent objects stay + // broken and need a manual restart, so surface it but do not fail the reconcile. + w.a.V(1).M(host).F().Warning("Failed to restart newly-added host; it may need a manual restart. Host: %s err: %v", host.GetName(), err) + return nil + } + // The restart also wipes non-persistent schema the operator migrated to this host BEFORE the + // restart - notably the contents of an Engine=Memory database, which live in RAM only and are + // gone after the reboot with nothing to restore them (they are not ZK-replicated). Re-run table + // migration so those objects are recreated. HostCreateTables issues CREATE ... IF NOT EXISTS, + // so persistent objects are untouched; best-effort, so it never fails the reconcile. + if err := w.migrateTables(ctx, host, NewMigrateTableOptions()); err != nil { + w.a.V(1).M(host).F().Warning("Post-restart table re-migration failed on newly-added host. Host: %s err: %v", host.GetName(), err) + } + return nil + }) +} + +// hostHitClusterDoesNotExist reports whether the host recorded a terminal CLUSTER_DOESNT_EXIST +// async-load failure (the #2013 signal). Best-effort: on a query error it returns false - safer to +// skip the restart than to disrupt a healthy/syncing host on an inconclusive read. +func (w *worker) hostHitClusterDoesNotExist(ctx context.Context, host *api.Host) bool { + n, err := w.ensureClusterSchemer(host).HostClusterDoesNotExistErrorCount(ctx, host) + if err != nil { + w.a.V(1).M(host).F().Warning("Cannot read CLUSTER_DOESNT_EXIST error count on newly-added host; skipping restart. Host: %s err: %v", host.GetName(), err) + } + return clusterDoesNotExistErrorIndicatesRestart(n, err) +} + +// clusterDoesNotExistErrorIndicatesRestart maps a CLUSTER_DOESNT_EXIST error-count read to the +// #2013 restart decision. A read error is inconclusive and yields false (skip) - safer to leave a +// healthy/syncing host alone than to restart on an unreliable read; only a positive count (the +// terminal load-failure signal) yields true. +func clusterDoesNotExistErrorIndicatesRestart(n int, err error) bool { + if err != nil { + return false + } + return n > 0 +} + // reconcileConfigMapCommon reconciles common ConfigMap func (w *worker) reconcileConfigMapCommon( ctx context.Context, @@ -534,7 +645,14 @@ func hostRequiresStatefulSetRollout(host *api.Host) bool { func (w *worker) hostForceRestart(ctx context.Context, host *api.Host, opts *statefulset.ReconcileOptions) error { w.a.V(1).M(host).F().Info("Reconcile host. Force restart: %s", host.GetName()) - if host.IsStopped() || (w.hostSoftwareRestart(ctx, host) != nil) { + // A sustained-NotReady pod won't be healed by an in-place software restart: the + // unreadiness may originate outside ClickHouse, and hostSoftwareRestart's readiness + // wait would just time out before falling back to scale-down. Recreate the pod + // directly (scale-down here, scale-up by the caller's StatefulSet reconcile). + stuckNotReady := chop.Config().ShouldRecoverCompletedOnPodNotReady() && + w.isPodSustainedNotReady(ctx, host, chop.Config().CompletedOnPodNotReadyThreshold()) + + if host.IsStopped() || stuckNotReady || (w.hostSoftwareRestart(ctx, host) != nil) { _ = w.hostScaleDown(ctx, host, opts) } diff --git a/pkg/controller/chi/worker-reconciler-chi_test.go b/pkg/controller/chi/worker-reconciler-chi_test.go index 2a9ed7e9f..28381362c 100644 --- a/pkg/controller/chi/worker-reconciler-chi_test.go +++ b/pkg/controller/chi/worker-reconciler-chi_test.go @@ -15,6 +15,7 @@ package chi import ( + "errors" "testing" "github.com/stretchr/testify/require" @@ -171,3 +172,33 @@ func TestHostRequiresStatefulSetRollout(t *testing.T) { require.False(t, hostRequiresStatefulSetRollout(hostWith(cur, desired))) }) } + +// TestClusterDoesNotExistErrorIndicatesRestart exercises the pure decision that gates the #2013 +// scale-up recovery restart (restartNewlyAddedHosts): a newly-added host is rebooted only if it +// recorded a terminal CLUSTER_DOESNT_EXIST async-load failure. +// +// This is the piece a live minikube run cannot deterministically reach — the boot-before- +// remote_servers race is timing-dependent and observed restartCount=0 in e2e, so the firing path +// is otherwise correct-by-construction only. The decision must fire iff count > 0, and a read +// error must NEVER trigger a restart (inconclusive read → leave a healthy/syncing host alone, +// which is what keeps a legitimately-lagging plain replica — test_010056 — untouched). +func TestClusterDoesNotExistErrorIndicatesRestart(t *testing.T) { + readErr := errors.New("dial tcp: connection refused") + + for _, tc := range []struct { + name string + n int + err error + want bool + }{ + {"positive count → restart", 5, nil, true}, + {"single occurrence → restart", 1, nil, true}, + {"zero count → skip (no #2013 failure, e.g. lagging plain replica)", 0, nil, false}, + {"read error → skip (inconclusive, never restart on a bad read)", 3, readErr, false}, + {"read error with zero count → skip", 0, readErr, false}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, clusterDoesNotExistErrorIndicatesRestart(tc.n, tc.err)) + }) + } +} diff --git a/pkg/controller/chi/worker-statefulset-rollback.go b/pkg/controller/chi/worker-statefulset-rollback.go index 42cb8600e..b647886fa 100644 --- a/pkg/controller/chi/worker-statefulset-rollback.go +++ b/pkg/controller/chi/worker-statefulset-rollback.go @@ -58,7 +58,9 @@ func (c *Controller) OnStatefulSetCreateFailed(ctx context.Context, host *api.Ho return common.ErrCRUDIgnore } - return common.ErrCRUDUnexpectedFlow + // This is unexpected flow + // Keep it commented out for not to have linter complain + // return common.ErrCRUDUnexpectedFlow } // OnStatefulSetUpdateFailed handles situation when StatefulSet update failed in k8s level @@ -102,7 +104,9 @@ func (c *Controller) OnStatefulSetUpdateFailed(ctx context.Context, rollbackStat return common.ErrCRUDIgnore } - return common.ErrCRUDUnexpectedFlow + // This is unexpected flow + // Keep it commented out for not to have linter complain + // return common.ErrCRUDUnexpectedFlow } // shouldContinueOnCreateFailed return nil in case 'continue' or error in case 'do not continue' diff --git a/pkg/controller/chi/worker-status-helpers.go b/pkg/controller/chi/worker-status-helpers.go index 8da4e493d..40c6c5486 100644 --- a/pkg/controller/chi/worker-status-helpers.go +++ b/pkg/controller/chi/worker-status-helpers.go @@ -18,6 +18,8 @@ import ( "context" "time" + core "k8s.io/api/core/v1" + log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" @@ -52,6 +54,94 @@ func (w *worker) isPodReady(ctx context.Context, host *api.Host) bool { return false } +// isPodSustainedNotReady reports whether the host's pod is currently Ready=False AND +// has been so for at least `threshold`. Returns false for pods whose failure mode is +// already being handled by kubelet (ImagePullBackOff, CrashLoopBackOff, Pending, etc.) +// so the operator does not race kubelet on its own recovery path. +func (w *worker) isPodSustainedNotReady(ctx context.Context, host *api.Host, threshold time.Duration) bool { + if threshold <= 0 { + // Threshold of 0/negative means "feature disabled" + return false + } + pod, err := w.c.kube.Pod().Get(ctx, host) + if err != nil || pod == nil { + return false + } + if podIsInKubeletFailureMode(pod) { + return false + } + return podIsSustainedNotReady(pod, threshold, time.Now()) +} + +// podIsSustainedNotReady is the pure inner predicate of isPodSustainedNotReady, +// extracted so it can be exercised without a kube client. Returns true iff the pod +// has a PodReady condition that is currently not True and whose LastTransitionTime +// is at least `threshold` in the past relative to `now`. +func podIsSustainedNotReady(pod *core.Pod, threshold time.Duration, now time.Time) bool { + if pod == nil || threshold <= 0 { + return false + } + for _, cond := range pod.Status.Conditions { + if cond.Type != core.PodReady { + continue + } + if cond.Status == core.ConditionTrue { + return false + } + // Status is False or Unknown. Treat both as "not ready" + if cond.LastTransitionTime.IsZero() { + return false + } + return now.Sub(cond.LastTransitionTime.Time) >= threshold + } + // No PodReady condition at all. + return false +} + +// kubeletDrivenWaitingReasons is the set of container Waiting.Reason values that +// indicate kubelet is already actively recovering the pod and a parallel +// operator-driven StatefulSet rollout would just race kubelet. +var kubeletDrivenWaitingReasons = map[string]struct{}{ + "CrashLoopBackOff": {}, + "ImagePullBackOff": {}, + "ErrImagePull": {}, + "InvalidImageName": {}, + "CreateContainerError": {}, + "RunContainerError": {}, + "ContainerCannotRun": {}, + "CreateContainerConfigError": {}, +} + +// podIsInKubeletFailureMode reports whether the pod is in a state where kubelet +// (or the kube-scheduler) is already handling the failure: not yet scheduled, +// in Pending phase, or any container in a kubelet-driven waiting reason. +// In those states an operator-driven reconcile would race kubelet without value. +func podIsInKubeletFailureMode(pod *core.Pod) bool { + if pod == nil { + return false + } + if pod.Status.Phase == core.PodPending { + return true + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting == nil { + continue + } + if _, hit := kubeletDrivenWaitingReasons[cs.State.Waiting.Reason]; hit { + return true + } + } + for _, cs := range pod.Status.InitContainerStatuses { + if cs.State.Waiting == nil { + continue + } + if _, hit := kubeletDrivenWaitingReasons[cs.State.Waiting.Reason]; hit { + return true + } + } + return false +} + func (w *worker) isPodStarted(ctx context.Context, host *api.Host) bool { if pod, err := w.c.kube.Pod().Get(ctx, host); err == nil { return k8s.PodHasAllContainersStarted(pod) @@ -185,6 +275,18 @@ func (w *worker) getRemoteServersGeneratorOptions() *commonConfig.HostSelector { // Base model specifies to exclude: // 1. all newly added hosts // 2. all explicitly excluded hosts + // + // Excluding newly-added (ObjectStatusRequested) hosts is DELIBERATE, not incidental: a host + // whose StatefulSet does not exist yet is an unreachable cluster member, and remote_servers + // lives in the single COMMON ConfigMap mounted by every pod. Advertising a not-yet-created + // host would hand every existing pod a cluster definition pointing at a host that cannot be + // reached, breaking cluster-wide operations during the reconcile window (existing replicas' + // Distributed queries, ON CLUSTER DDL, and the operator's own clusterAllReplicas/remote() + // schema-migration and health queries). The new host is added to remote_servers only in the + // final phase, once its StatefulSet exists. Do NOT drop this to "seed" the full topology into + // the preliminary ConfigMap: because the ConfigMap is shared, that necessarily re-advertises + // the not-yet-created host to existing pods. Newly-added hosts recover their cluster-dependent + // objects via the post-publish restart in restartNewlyAddedHosts, never by seeding. return commonConfig.NewHostSelector().ExcludeReconcileAttributes( types.NewReconcileAttributes(). SetStatus(types.ObjectStatusRequested). diff --git a/pkg/controller/chi/worker-status-helpers_test.go b/pkg/controller/chi/worker-status-helpers_test.go new file mode 100644 index 000000000..cb4d7391b --- /dev/null +++ b/pkg/controller/chi/worker-status-helpers_test.go @@ -0,0 +1,201 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chi + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestPodIsSustainedNotReady covers the pure post-fetch decision used by +// isPodSustainedNotReady. +func TestPodIsSustainedNotReady(t *testing.T) { + now := time.Date(2026, 5, 28, 12, 0, 0, 0, time.UTC) + + withReady := func(status core.ConditionStatus, transitionOffset time.Duration) *core.Pod { + return &core.Pod{ + Status: core.PodStatus{ + Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: status, + LastTransitionTime: meta.NewTime(now.Add(transitionOffset))}, + }, + }, + } + } + + tests := []struct { + name string + pod *core.Pod + threshold time.Duration + expected bool + }{ + { + name: "nil pod — never sustained", + pod: nil, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "zero threshold — feature disabled, never fires", + pod: withReady(core.ConditionFalse, -30*time.Minute), + threshold: 0, + expected: false, + }, + { + name: "negative threshold — feature disabled, never fires", + pod: withReady(core.ConditionFalse, -30*time.Minute), + threshold: -1 * time.Second, + expected: false, + }, + { + name: "no PodReady condition — early lifecycle, never sustained", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodInitialized, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + }}}, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=True — not sustained even with old LastTransitionTime", + pod: withReady(core.ConditionTrue, -30*time.Minute), + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=False but only 1m ago — under threshold (transient)", + pod: withReady(core.ConditionFalse, -1*time.Minute), + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "PodReady=False for exactly the threshold — fires (>= semantics)", + pod: withReady(core.ConditionFalse, -5*time.Minute), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=False for 26h — the production incident, fires", + pod: withReady(core.ConditionFalse, -26*time.Hour), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=Unknown for 10m — treated as not-ready, fires", + pod: withReady(core.ConditionUnknown, -10*time.Minute), + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodReady=False but LastTransitionTime is zero — conservative, don't fire", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{{Type: core.PodReady, Status: core.ConditionFalse}}}}, + threshold: 5 * time.Minute, + expected: false, + }, + { + name: "multiple PodReady entries — use first match", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + {Type: core.PodReady, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now)}, + }}}, + threshold: 5 * time.Minute, + expected: true, + }, + { + name: "PodScheduled present alongside PodReady=False — still fires on Ready", + pod: &core.Pod{Status: core.PodStatus{Conditions: []core.PodCondition{ + {Type: core.PodScheduled, Status: core.ConditionTrue, + LastTransitionTime: meta.NewTime(now.Add(-1 * time.Hour))}, + {Type: core.PodReady, Status: core.ConditionFalse, + LastTransitionTime: meta.NewTime(now.Add(-10 * time.Minute))}, + }}}, + threshold: 5 * time.Minute, + expected: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, podIsSustainedNotReady(tc.pod, tc.threshold, now)) + }) + } +} + +// TestPodIsInKubeletFailureMode locks in the kubelet-recovery filter: any pod whose +// failure mode is already being handled by kubelet (image pull errors, crash loops, +// pending, etc.) must NOT trigger the stuck-host recovery path. +func TestPodIsInKubeletFailureMode(t *testing.T) { + waitingContainer := func(reason string) *core.Pod { + return &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + ContainerStatuses: []core.ContainerStatus{ + {Name: "clickhouse", State: core.ContainerState{ + Waiting: &core.ContainerStateWaiting{Reason: reason}, + }}, + }, + }} + } + waitingInit := func(reason string) *core.Pod { + return &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + InitContainerStatuses: []core.ContainerStatus{ + {Name: "init", State: core.ContainerState{ + Waiting: &core.ContainerStateWaiting{Reason: reason}, + }}, + }, + }} + } + + tests := []struct { + name string + pod *core.Pod + expected bool + }{ + {"nil pod", nil, false}, + {"no statuses, running phase", &core.Pod{Status: core.PodStatus{Phase: core.PodRunning}}, false}, + {"Pending phase — scheduler/kubelet handling", &core.Pod{Status: core.PodStatus{Phase: core.PodPending}}, true}, + {"ImagePullBackOff — kubelet handling", waitingContainer("ImagePullBackOff"), true}, + {"ErrImagePull — kubelet handling", waitingContainer("ErrImagePull"), true}, + {"InvalidImageName — kubelet handling", waitingContainer("InvalidImageName"), true}, + {"CrashLoopBackOff — kubelet handling", waitingContainer("CrashLoopBackOff"), true}, + {"CreateContainerError — kubelet handling", waitingContainer("CreateContainerError"), true}, + {"RunContainerError — kubelet handling", waitingContainer("RunContainerError"), true}, + {"ContainerCannotRun — kubelet handling", waitingContainer("ContainerCannotRun"), true}, + {"CreateContainerConfigError — kubelet handling", waitingContainer("CreateContainerConfigError"), true}, + {"init container in ImagePullBackOff — kubelet handling", waitingInit("ImagePullBackOff"), true}, + {"ContainerCreating — transient, not kubelet failure", waitingContainer("ContainerCreating"), false}, + {"PodInitializing — transient, not kubelet failure", waitingContainer("PodInitializing"), false}, + {"running container, no waiting state", &core.Pod{Status: core.PodStatus{ + Phase: core.PodRunning, + ContainerStatuses: []core.ContainerStatus{ + {Name: "clickhouse", Ready: true, State: core.ContainerState{ + Running: &core.ContainerStateRunning{}, + }}, + }, + }}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, podIsInKubeletFailureMode(tc.pod)) + }) + } +} diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index caf251699..77e1eed29 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -27,6 +27,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" + "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/controller/chi/metrics" "github.com/altinity/clickhouse-operator/pkg/controller/common" a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" @@ -193,6 +194,17 @@ func (w *worker) shouldForceRestartHost(ctx context.Context, host *api.Host) boo w.a.V(1).M(host).F().Info("Host with unknown version and in CrashLoopBackOff should be restarted. It most likely is unable to start due to bad config. Host: %s", host.GetName()) return true + case chop.Config().ShouldRecoverCompletedOnPodNotReady() && + w.isPodSustainedNotReady(ctx, host, chop.Config().CompletedOnPodNotReadyThreshold()): + // Closes the gap where Completed CHIs with a sustained-NotReady host + // were left stuck indefinitely. + threshold := chop.Config().CompletedOnPodNotReadyThreshold() + w.a.V(1).M(host).F(). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonHostStuckNotReady). + Info("Host pod has been Ready=False past threshold %s — force restart. Host: %s", + threshold, host.GetName()) + return true + default: w.a.V(1).M(host).F().Info("Host force restart is not required. Host: %s", host.GetName()) return false diff --git a/pkg/controller/chk/controller.go b/pkg/controller/chk/controller.go index d5a9ea6ae..bef2cd13f 100644 --- a/pkg/controller/chk/controller.go +++ b/pkg/controller/chk/controller.go @@ -50,11 +50,21 @@ type Controller struct { //pvcDeleter *volume.PVCDeleter } -func (c *Controller) new() { - c.namer = managers.NewNameManager(managers.NameManagerTypeKeeper) - c.kube = kube.NewAdapter(c.Client, c.APIReader, c.namer) - //labeler: NewLabeler(kube), - //pvcDeleter := volume.NewPVCDeleter(managers.NewNameManager(managers.NameManagerTypeKeeper)) +// NewController creates a CHK Controller with its request-independent collaborators +// (name manager, kube adapter) initialized once, at construction. controller-runtime +// reuses a single Controller across all MaxConcurrentReconciles goroutines, so these +// fields must NOT be (re)assigned per-Reconcile — doing so is a data race once +// reconcileCHKsThreadsNumber > 1. The CHI controller initializes the same way. +func NewController(c client.Client, apiReader client.Reader, scheme *apiMachinery.Scheme, extClient apiExtensions.Interface) *Controller { + namer := managers.NewNameManager(managers.NameManagerTypeKeeper) + return &Controller{ + Client: c, + APIReader: apiReader, + Scheme: scheme, + ExtClient: extClient, + namer: namer, + kube: kube.NewAdapter(c, apiReader, namer), + } } func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -85,7 +95,6 @@ func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, err } - c.new() w := c.newWorker() if w.ensureFinalizer(ctx, new) { diff --git a/pkg/controller/chk/worker-reconciler-chk.go b/pkg/controller/chk/worker-reconciler-chk.go index e696979d0..1bd95a4e7 100644 --- a/pkg/controller/chk/worker-reconciler-chk.go +++ b/pkg/controller/chk/worker-reconciler-chk.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" log "github.com/altinity/clickhouse-operator/pkg/announcer" @@ -493,23 +494,37 @@ func (w *worker) hostScaleDown(ctx context.Context, host *api.Host, opts *statef return nil } -// reconcileHostService reconciles host's Service +// reconcileHostService reconciles host's Service(s). A host may expose more than one Service +// (the peer/Raft Service and the client-facing Service — see creator.createServiceHost), so all +// of them are reconciled and registered; failing to register every reconciled Service would let +// the cleanup pass purge it as an unknown object. func (w *worker) reconcileHostService(ctx context.Context, host *api.Host) error { - service := w.task.Creator().CreateService(interfaces.ServiceHost, host).First() - if service == nil { + services := w.task.Creator().CreateService(interfaces.ServiceHost, host) + if len(services) == 0 { // This is not a problem, service may be omitted return nil } - prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()).First() - err := w.reconcileService(ctx, host.GetCR(), service, prevService) - if err == nil { - w.a.V(1).M(host).F().Info("DONE Reconcile service of the host: %s", host.GetName()) - w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) - } else { - w.a.V(1).M(host).F().Warning("FAILED Reconcile service of the host: %s", host.GetName()) - w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) + prevServices := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()) + for i, service := range services { + if service == nil { + continue + } + // Pair with the previous-generation Service at the same index (creators emit a stable order). + var prevService *core.Service + if i < len(prevServices) { + prevService = prevServices[i] + } + err := w.reconcileService(ctx, host.GetCR(), service, prevService) + if err == nil { + w.a.V(1).M(host).F().Info("DONE Reconcile service %s of the host: %s", service.GetName(), host.GetName()) + w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) + } else { + w.a.V(1).M(host).F().Warning("FAILED Reconcile service %s of the host: %s", service.GetName(), host.GetName()) + w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) + return err + } } - return err + return nil } // reconcileCluster reconciles cluster, excluding nested shards diff --git a/pkg/controller/common/announcer/event-emitter.go b/pkg/controller/common/announcer/event-emitter.go index f7247b6c5..e71547ad1 100644 --- a/pkg/controller/common/announcer/event-emitter.go +++ b/pkg/controller/common/announcer/event-emitter.go @@ -65,6 +65,15 @@ const ( // reconcile was aborted, on observing a recovery signal (e.g. a pod became Ready). EventReasonAutoRecoveryTriggered = "AutoRecoveryTriggered" + // EventReasonStuckHostRecoveryTriggered fires when the operator re-enqueues a + // Completed CHI for reconcile because one of its hosts has been Ready=False for + // longer than the configured threshold. + EventReasonStuckHostRecoveryTriggered = "StuckHostRecoveryTriggered" + + // EventReasonHostStuckNotReady fires when shouldForceRestartHost decides to force + // a host restart because the pod has been Ready=False past the configured threshold. + EventReasonHostStuckNotReady = "HostStuckNotReady" + // EventReasonKeeperUpdateNoEndpointChange fires when the operator observes a referenced // CHK reconcile completing but decides not to trigger a CHI reconcile because the resolved // zookeeper endpoints have not changed. diff --git a/pkg/interfaces/label_type.go b/pkg/interfaces/label_type.go index f152328e3..2596cc55c 100644 --- a/pkg/interfaces/label_type.go +++ b/pkg/interfaces/label_type.go @@ -22,10 +22,11 @@ const ( LabelConfigMapHost LabelType = "Label cm host" LabelConfigMapStorage LabelType = "Label cm storage" - LabelServiceCR LabelType = "Label svc chi" - LabelServiceCluster LabelType = "Label svc cluster" - LabelServiceShard LabelType = "Label svc shard" - LabelServiceHost LabelType = "Label svc host" + LabelServiceCR LabelType = "Label svc chi" + LabelServiceCluster LabelType = "Label svc cluster" + LabelServiceShard LabelType = "Label svc shard" + LabelServiceHost LabelType = "Label svc host" + LabelServiceHostClient LabelType = "Label svc host client" LabelExistingPV LabelType = "Label existing pv" LabelNewPVC LabelType = "Label new pvc" diff --git a/pkg/interfaces/name_type.go b/pkg/interfaces/name_type.go index b98acba1d..b277b00f0 100644 --- a/pkg/interfaces/name_type.go +++ b/pkg/interfaces/name_type.go @@ -33,6 +33,7 @@ const ( NameInstanceHostname NameType = "NameInstanceHostname" NameStatefulSet NameType = "NameStatefulSet" NameStatefulSetService NameType = "NameStatefulSetService" + NameStatefulSetServiceClient NameType = "NameStatefulSetServiceClient" NamePodHostname NameType = "NamePodHostname" NameFQDN NameType = "NameFQDN" NameFQDNs NameType = "NameFQDNs" diff --git a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go index 9f0bc0d80..78095b4b5 100644 --- a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go +++ b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go @@ -142,16 +142,20 @@ const ( type MetricsFetcher struct { connectionParams *clickhouse.EndpointConnectionParams tablesRegexp string + // Used to filter system-metric names while fetching metrics. Nil means keep all. + metricsFilter MetricsFilter } // NewMetricsFetcher creates new clickhouse fetcher object func NewMetricsFetcher( endpointConnectionParams *clickhouse.EndpointConnectionParams, tablesRegexp string, + metricsFilter MetricsFilter, ) *MetricsFetcher { return &MetricsFetcher{ connectionParams: endpointConnectionParams, tablesRegexp: tablesRegexp, + metricsFilter: metricsFilter, } } @@ -179,11 +183,9 @@ func (f *MetricsFetcher) buildMetricsSQL() string { } // getClickHouseQueryMetrics requests metrics data from ClickHouse. -// Exclusion of "noisy" metrics is enforced solely by the writer-side filter -// (see CHIPrometheusWriter.metricsFilter). A SQL-side filter was tried and -// dropped: wrapping the UNION-ALL chain in `FROM (...) WHERE NOT (...)` left -// the metrics query returning zero rows across restart-then-scrape windows; -// the writer-side filter is sufficient and avoids that fragility. +// Excluded names are dropped during row scan so they never enter the in-memory buffer. +// SQL-side filtering was tried and abandoned: wrapping the UNION-ALL in +// `FROM (...) WHERE NOT (...)` caused zero rows on restart-then-scrape windows. func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( ctx, @@ -191,13 +193,27 @@ func (f *MetricsFetcher) getClickHouseQueryMetrics(ctx context.Context) (Table, func(rows *sql.Rows, data *Table) error { var metric, value, description, _type string if err := rows.Scan(&metric, &value, &description, &_type); err == nil { - *data = append(*data, []string{metric, value, description, _type}) + f.appendMetricRow(data, metric, value, description, _type) } return nil }, ) } +// appendMetricRow adds a scanned system-metrics row to the buffer, dropping excluded +// names first. This is a memory pre-filter on the highest-cardinality fetch path (the +// system.metrics/asynchronous_metrics UNION, whose per-CPU OS series scale with core +// count) so excluded rows never enter the in-memory Table. Names synthesized by the +// other query paths (parts, mutations, disks, replicas) never reach this scan, so the +// writer-side filter stays authoritative for those; where both paths see a name the +// filter is identical, so re-checking writer-side is idempotent. +func (f *MetricsFetcher) appendMetricRow(data *Table, metric, value, description, _type string) { + if IsExcluded(f.metricsFilter, metric) { + return + } + *data = append(*data, []string{metric, value, description, _type}) +} + // getClickHouseSystemParts requests data sizes from ClickHouse func (f *MetricsFetcher) getClickHouseSystemParts(ctx context.Context) (Table, error) { return f.clickHouseQueryScanRows( diff --git a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher_test.go b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher_test.go new file mode 100644 index 000000000..0f89c6414 --- /dev/null +++ b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher_test.go @@ -0,0 +1,72 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +import "testing" + +// TestAppendMetricRowFiltersExcluded verifies the fetch-side memory pre-filter: +// excluded metric names are dropped during row scan so they never enter the +// in-memory Table (the OOM-reduction goal of the fetch-side filter). stubMetricsFilter +// is shared with prometheus_writer_test.go. +func TestAppendMetricRowFiltersExcluded(t *testing.T) { + fetcher := NewMetricsFetcher( + nil, + "", + &stubMetricsFilter{excluded: map[string]bool{ + "metric.OSUserTimeCPU0": true, + "metric.OSUserTimeCPU1": true, + }}, + ) + + var data Table + rows := [][]string{ + {"metric.OSUserTimeCPU0", "1", "", "gauge"}, // excluded + {"metric.Query", "2", "", "gauge"}, // kept + {"metric.OSUserTimeCPU1", "3", "", "gauge"}, // excluded + {"metric.TCPConnection", "4", "", "gauge"}, // kept + } + for _, r := range rows { + fetcher.appendMetricRow(&data, r[0], r[1], r[2], r[3]) + } + + want := Table{ + {"metric.Query", "2", "", "gauge"}, + {"metric.TCPConnection", "4", "", "gauge"}, + } + if len(data) != len(want) { + t.Fatalf("expected %d rows after filtering, got %d: %v", len(want), len(data), data) + } + for i := range want { + for j := range want[i] { + if data[i][j] != want[i][j] { + t.Fatalf("row %d col %d: expected %q, got %q", i, j, want[i][j], data[i][j]) + } + } + } +} + +// TestAppendMetricRowNilFilterKeepsAll verifies a nil filter is a no-op (keeps every +// row), guarding the nil-safe IsExcluded contract on the fetch path. +func TestAppendMetricRowNilFilterKeepsAll(t *testing.T) { + fetcher := NewMetricsFetcher(nil, "", nil) + + var data Table + fetcher.appendMetricRow(&data, "metric.OSUserTimeCPU0", "1", "", "gauge") + fetcher.appendMetricRow(&data, "metric.Query", "2", "", "gauge") + + if len(data) != 2 { + t.Fatalf("nil filter should keep all rows, got %d: %v", len(data), data) + } +} diff --git a/pkg/metrics/clickhouse/exporter.go b/pkg/metrics/clickhouse/exporter.go index ac494ab02..7d0fd3e23 100644 --- a/pkg/metrics/clickhouse/exporter.go +++ b/pkg/metrics/clickhouse/exporter.go @@ -120,10 +120,10 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher { case api.ChSchemeAuto: switch { case types.IsPortAssigned(host.HTTPPort): - clusterConnectionParams.Scheme = "http" + clusterConnectionParams.Scheme = api.ChSchemeHTTP clusterConnectionParams.Port = int(host.HTTPPort) case types.IsPortAssigned(host.HTTPSPort): - clusterConnectionParams.Scheme = "https" + clusterConnectionParams.Scheme = api.ChSchemeHTTPS clusterConnectionParams.Port = int(host.HTTPSPort) } case api.ChSchemeHTTP: @@ -135,6 +135,7 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *MetricsFetcher { return NewMetricsFetcher( clusterConnectionParams.NewEndpointConnectionParams(host.Hostname), chop.Config().ClickHouse.Metrics.TablesRegexp, + e.metricsFilter, ) } diff --git a/pkg/metrics/clickhouse/metrics_filter.go b/pkg/metrics/clickhouse/metrics_filter.go index 95fc74d66..02980c43f 100644 --- a/pkg/metrics/clickhouse/metrics_filter.go +++ b/pkg/metrics/clickhouse/metrics_filter.go @@ -24,10 +24,17 @@ package clickhouse // be added there without touching this file. // // Nil-safe by contract: implementations must return false from IsExcluded when -// the receiver is nil so callers can pass a typed-nil filter as a no-op. The -// consumer (CHIPrometheusWriter) additionally guards a raw-nil interface value -// at the call site, so a writer constructed via struct literal without setting -// metricsFilter is also safe (no panic, no exclusions). +// the receiver is nil so callers can pass a typed-nil filter as a no-op. A raw-nil +// interface value is handled by the package-level IsExcluded helper below — see it +// for the shared call-site guard. type MetricsFilter interface { IsExcluded(name string) bool } + +// IsExcluded is a nil-safe wrapper over MetricsFilter: a raw-nil interface value +// is treated as a no-op filter (excludes nothing). Both filter call sites — the +// fetch-side row scan and the writer-side emission — funnel through this so the +// nil-guard contract lives in one place instead of being duplicated per call site. +func IsExcluded(filter MetricsFilter, name string) bool { + return (filter != nil) && filter.IsExcluded(name) +} diff --git a/pkg/metrics/clickhouse/prometheus_writer.go b/pkg/metrics/clickhouse/prometheus_writer.go index 2b69a86e1..40893844b 100644 --- a/pkg/metrics/clickhouse/prometheus_writer.go +++ b/pkg/metrics/clickhouse/prometheus_writer.go @@ -263,9 +263,11 @@ func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( value string, metricLabels map[string]string, ) { - // Nil-guard: a writer constructed via struct literal without setting metricsFilter - // would panic on dispatch; treat nil interface as "no exclusions" (no-op filter). - if (w.metricsFilter != nil) && w.metricsFilter.IsExcluded(name) { + // Writer-side filter is authoritative for metric names synthesized outside the + // metrics scan (table parts, mutations, disks, replicas) — these never reach + // appendMetricRow. Names that do flow through the scan are already filtered + // fetch-side; re-checking here is idempotent. See appendMetricRow. + if IsExcluded(w.metricsFilter, name) { return } diff --git a/pkg/metrics/operator/machinery.go b/pkg/metrics/operator/machinery.go index 4e6b619b5..b0d6637cc 100644 --- a/pkg/metrics/operator/machinery.go +++ b/pkg/metrics/operator/machinery.go @@ -106,8 +106,14 @@ func serveMetrics(addr, path string) { handler := promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{ ErrorHandling: promhttp.ContinueOnError, }) - http.Handle(path, handler) - err := http.ListenAndServe(addr, nil) + // Serve a private mux, NOT http.DefaultServeMux. controller-runtime (pulled in by + // the CHK controller) transitively imports net/http/pprof, whose init() registers + // /debug/pprof/* on DefaultServeMux. Binding DefaultServeMux here — ListenAndServe( + // addr, nil) — would expose those pprof endpoints (CPU profile, heap, goroutine) on + // this public metrics port. A dedicated mux serves only the metrics handler. + mux := http.NewServeMux() + mux.Handle(path, handler) + err := http.ListenAndServe(addr, mux) if err != nil { fmt.Printf("error serving http: %v", err) } diff --git a/pkg/model/chi/namer/namer.go b/pkg/model/chi/namer/namer.go index 4b9cda877..b93316028 100644 --- a/pkg/model/chi/namer/namer.go +++ b/pkg/model/chi/namer/namer.go @@ -98,10 +98,12 @@ func (n *Namer) Name(what interfaces.NameType, params ...any) string { return n.createClusterPDBName(cluster) default: + // Delegate any type not handled above to the common namer, whose Name() + // has no default case and panics on an unspecified type. That is the + // single fail-loud guard, so an unknown type cannot sneak through to a + // silent zero-value return here. return n.commonNamer.Name(what, params...) } - - panic("unknown name type") } func (n *Namer) Names(what interfaces.NameType, params ...any) []string { @@ -112,7 +114,9 @@ func (n *Namer) Names(what interfaces.NameType, params ...any) []string { excludeSelf := params[2].(bool) return n.createFQDNs(obj, scope, excludeSelf) default: + // Delegate any type not handled above to the common namer, whose Names() + // panics on an unspecified type. That is the single fail-loud guard, so + // an unknown type cannot sneak through to a silent nil return here. return n.commonNamer.Names(what, params...) } - panic("unknown names type") } diff --git a/pkg/model/chi/normalizer/normalizer.go b/pkg/model/chi/normalizer/normalizer.go index 4c975e282..a7b75c8d5 100644 --- a/pkg/model/chi/normalizer/normalizer.go +++ b/pkg/model/chi/normalizer/normalizer.go @@ -238,14 +238,15 @@ func (n *Normalizer) normalizeStop(stop *types.StringBool) *types.StringBool { // normalizeRestart normalizes .spec.restart func (n *Normalizer) normalizeRestart(restart *types.String) *types.String { - switch strings.ToLower(restart.Value()) { - case strings.ToLower(chi.RestartRollingUpdate): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; anything else becomes empty. + // Kept as a switch so future restart policies are just new cases. + switch util.FoldEnum(restart.Value(), chi.RestartRollingUpdate) { + case chi.RestartRollingUpdate: return types.NewString(chi.RestartRollingUpdate) + default: + // Unknown value - just use empty + return nil } - - // In case it is unknown value - just use empty - return nil } // normalizeTroubleshoot normalizes .spec.stop @@ -302,6 +303,10 @@ func (n *Normalizer) normalizeDefaults(defaults *chi.Defaults) *chi.Defaults { if defaults.StorageManagement == nil { defaults.StorageManagement = chi.NewStorageManagement() } + // Fold casing + validate the default StorageManagement (provisioner/reclaimPolicy). + // This path was previously left un-normalized, so a lowercase reclaimPolicy at the + // defaults level was written verbatim into the PVC label. + templates.NormalizeStorageManagement(defaults.StorageManagement) // Ensure field if defaults.Templates == nil { //defaults.Templates = api.NewChiTemplateNames() @@ -359,15 +364,11 @@ func (n *Normalizer) normalizeTemplating(templating *chi.ChiTemplating) *chi.Chi if templating == nil { templating = chi.NewChiTemplating() } - switch strings.ToLower(templating.GetPolicy()) { - case strings.ToLower(chi.TemplatingPolicyAuto): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the default. + switch util.FoldEnum(templating.GetPolicy(), chi.TemplatingPolicyAuto, chi.TemplatingPolicyManual) { + case chi.TemplatingPolicyAuto: templating.SetPolicy(chi.TemplatingPolicyAuto) - case strings.ToLower(chi.TemplatingPolicyManual): - // Known value, overwrite it to ensure case-ness - templating.SetPolicy(chi.TemplatingPolicyManual) default: - // Unknown value, fallback to default templating.SetPolicy(chi.TemplatingPolicyManual) } return templating @@ -399,16 +400,13 @@ func (n *Normalizer) normalizeReconcile(reconcile *chi.ChiReconcile) *chi.ChiRec reconcile = chi.NewChiReconcile().SetDefaults() } - // Policy - switch strings.ToLower(reconcile.GetPolicy()) { - case strings.ToLower(chi.ReconcilingPolicyWait): - // Known value, overwrite it to ensure case-ness + // Policy — fold any accepted casing to the canonical const; unknown values fall back to default. + switch util.FoldEnum(reconcile.GetPolicy(), chi.ReconcilingPolicyWait, chi.ReconcilingPolicyNoWait) { + case chi.ReconcilingPolicyWait: reconcile.SetPolicy(chi.ReconcilingPolicyWait) - case strings.ToLower(chi.ReconcilingPolicyNoWait): - // Known value, overwrite it to ensure case-ness + case chi.ReconcilingPolicyNoWait: reconcile.SetPolicy(chi.ReconcilingPolicyNoWait) default: - // Unknown value, fallback to default reconcile.SetPolicy(chi.ReconcilingPolicyUnspecified) } @@ -450,7 +448,8 @@ func (n *Normalizer) normalizeReconcileRuntime(runtime chi.ReconcileRuntime) chi } func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) chi.ReconcileStatefulSet { - // Create + // Create — fold casing to canonical const, then default if empty. + sts.Create.OnFailure = chi.NormalizeOnStatefulSetCreateFailureAction(sts.Create.OnFailure) if sts.Create.OnFailure == "" { sts.Create.OnFailure = chi.OnStatefulSetCreateFailureActionDelete } @@ -461,13 +460,16 @@ func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) if sts.Update.PollInterval == 0 { sts.Update.PollInterval = defaultStatefulSetUpdatePollInterval } + sts.Update.OnFailure = chi.NormalizeOnStatefulSetUpdateFailureAction(sts.Update.OnFailure) if sts.Update.OnFailure == "" { sts.Update.OnFailure = chi.OnStatefulSetUpdateFailureActionRollback } // Recreate + sts.Recreate.OnDataLoss = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnDataLoss) if sts.Recreate.OnDataLoss == "" { sts.Recreate.OnDataLoss = chi.OnStatefulSetRecreateOnDataLossActionRecreate } + sts.Recreate.OnUpdateFailure = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnUpdateFailure) if sts.Recreate.OnUpdateFailure == "" { sts.Recreate.OnUpdateFailure = chi.OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -505,15 +507,13 @@ func (n *Normalizer) normalizeCleanup(str *string, value string) { if str == nil { return } - switch strings.ToLower(*str) { - case strings.ToLower(chi.ObjectsCleanupRetain): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the supplied default. + switch util.FoldEnum(*str, chi.ObjectsCleanupRetain, chi.ObjectsCleanupDelete) { + case chi.ObjectsCleanupRetain: *str = chi.ObjectsCleanupRetain - case strings.ToLower(chi.ObjectsCleanupDelete): - // Known value, overwrite it to ensure case-ness + case chi.ObjectsCleanupDelete: *str = chi.ObjectsCleanupDelete default: - // Unknown value, fallback to default *str = value } } @@ -973,30 +973,20 @@ func (n *Normalizer) normalizeClusterSchemaPolicy(policy *chi.SchemaPolicy) *chi policy = chi.NewClusterSchemaPolicy() } - switch strings.ToLower(policy.Replica) { - case strings.ToLower(schemer.SchemaPolicyReplicaNone): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the default. + switch util.FoldEnum(policy.Replica, schemer.SchemaPolicyReplicaNone, schemer.SchemaPolicyReplicaAll) { + case schemer.SchemaPolicyReplicaNone: policy.Replica = schemer.SchemaPolicyReplicaNone - case strings.ToLower(schemer.SchemaPolicyReplicaAll): - // Known value, overwrite it to ensure case-ness - policy.Replica = schemer.SchemaPolicyReplicaAll default: - // Unknown value, fallback to default policy.Replica = schemer.SchemaPolicyReplicaAll } - switch strings.ToLower(policy.Shard) { - case strings.ToLower(schemer.SchemaPolicyShardNone): - // Known value, overwrite it to ensure case-ness + switch util.FoldEnum(policy.Shard, schemer.SchemaPolicyShardNone, schemer.SchemaPolicyShardAll, schemer.SchemaPolicyShardDistributedTablesOnly) { + case schemer.SchemaPolicyShardNone: policy.Shard = schemer.SchemaPolicyShardNone - case strings.ToLower(schemer.SchemaPolicyShardAll): - // Known value, overwrite it to ensure case-ness - policy.Shard = schemer.SchemaPolicyShardAll - case strings.ToLower(schemer.SchemaPolicyShardDistributedTablesOnly): - // Known value, overwrite it to ensure case-ness + case schemer.SchemaPolicyShardDistributedTablesOnly: policy.Shard = schemer.SchemaPolicyShardDistributedTablesOnly default: - // unknown value, fallback to default policy.Shard = schemer.SchemaPolicyShardAll } diff --git a/pkg/model/chi/normalizer/templates_cr/const.go b/pkg/model/chi/normalizer/templates_cr/const.go index 9d2b2c6d0..5c617747a 100644 --- a/pkg/model/chi/normalizer/templates_cr/const.go +++ b/pkg/model/chi/normalizer/templates_cr/const.go @@ -15,6 +15,6 @@ package templates_cr const ( - // .spec.useTemplate.useType - UseTypeMerge = "merge" + // .spec.useTemplate.useType (canonical humped form; CRD also accepts all-lowercase) + UseTypeMerge = "Merge" ) diff --git a/pkg/model/chi/normalizer/templates_cr/normalizer.go b/pkg/model/chi/normalizer/templates_cr/normalizer.go index 2968cbbb7..de6167f8d 100644 --- a/pkg/model/chi/normalizer/templates_cr/normalizer.go +++ b/pkg/model/chi/normalizer/templates_cr/normalizer.go @@ -16,6 +16,7 @@ package templates_cr import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/util" ) // NormalizeTemplateRefList normalizes list of templates use specifications @@ -38,10 +39,11 @@ func normalizeTemplateRef(templateRef *api.TemplateRef) *api.TemplateRef { // So far do nothing with empty namespace } - // Ensure UseType - switch templateRef.UseType { + // Ensure UseType — fold any accepted casing to the canonical const; unknown values + // fall back to the default. Kept as a switch so future use types are just new cases. + switch util.FoldEnum(templateRef.UseType, UseTypeMerge) { case UseTypeMerge: - // Known use type, all is fine, do nothing + templateRef.UseType = UseTypeMerge default: // Unknown use type - overwrite with default value templateRef.UseType = UseTypeMerge diff --git a/pkg/model/chi/schemer/schemer.go b/pkg/model/chi/schemer/schemer.go index 6b503bedd..8e5273714 100644 --- a/pkg/model/chi/schemer/schemer.go +++ b/pkg/model/chi/schemer/schemer.go @@ -167,6 +167,12 @@ func (s *ClusterSchemer) HostActiveQueriesNum(ctx context.Context, host *api.Hos return s.QueryHostInt(ctx, host, s.sqlActiveQueriesNum()) } +// HostClusterDoesNotExistErrorCount returns how many CLUSTER_DOESNT_EXIST errors the host has +// recorded (issue #2013 signal - see sqlClusterDoesNotExistErrorCount). +func (s *ClusterSchemer) HostClusterDoesNotExistErrorCount(ctx context.Context, host *api.Host) (int, error) { + return s.QueryHostInt(ctx, host, s.sqlClusterDoesNotExistErrorCount()) +} + // HostClickHouseVersion returns ClickHouse version on the host func (s *ClusterSchemer) HostClickHouseVersion(ctx context.Context, host *api.Host) (string, error) { return s.QueryHostString(ctx, host, s.sqlVersion()) diff --git a/pkg/model/chi/schemer/sql.go b/pkg/model/chi/schemer/sql.go index c6f6fb126..b8182c46c 100644 --- a/pkg/model/chi/schemer/sql.go +++ b/pkg/model/chi/schemer/sql.go @@ -256,6 +256,15 @@ func (s *ClusterSchemer) sqlActiveQueriesNum() string { return `SELECT count() FROM system.processes` } +// sqlClusterDoesNotExistErrorCount counts CLUSTER_DOESNT_EXIST occurrences on a host. +// Non-zero means a cluster-dependent object (Distributed / DICTIONARY / refreshable MV) hit a +// terminal async startup-load failure because the host booted before its remote_servers was +// complete - i.e. the issue #2013 condition. A host with no cluster-dependent objects (or one that +// loaded them cleanly) reports 0. +func (s *ClusterSchemer) sqlClusterDoesNotExistErrorCount() string { + return `SELECT sum(value) FROM system.errors WHERE name = 'CLUSTER_DOESNT_EXIST'` +} + func (s *ClusterSchemer) sqlVersion() string { return `SELECT version()` } diff --git a/pkg/model/chi/tags/labeler/labeler.go b/pkg/model/chi/tags/labeler/labeler.go index 09ada58d2..0707b31be 100644 --- a/pkg/model/chi/tags/labeler/labeler.go +++ b/pkg/model/chi/tags/labeler/labeler.go @@ -46,9 +46,12 @@ func (l *Labeler) Label(what interfaces.LabelType, params ...any) map[string]str return l.labelConfigMapHost(params...) default: + // Delegate any type not handled above to the base labeler, whose Label() + // has no default case and panics on an unspecified type. That is the + // single fail-loud guard, so an unknown type cannot sneak through to a + // silent zero-value return here. return l.Labeler.Label(what, params...) } - panic("unknown label type") } // Selector diff --git a/pkg/model/chi/tags/labeler/list.go b/pkg/model/chi/tags/labeler/list.go index 2bf27b5e7..b138cd542 100644 --- a/pkg/model/chi/tags/labeler/list.go +++ b/pkg/model/chi/tags/labeler/list.go @@ -47,6 +47,7 @@ var list = types.List{ labeler.LabelServiceValueCluster: "cluster", labeler.LabelServiceValueShard: "shard", labeler.LabelServiceValueHost: "host", + labeler.LabelServiceValueHostClient: "host-client", labeler.LabelPVCReclaimPolicyName: clickhouse_altinity_com.APIGroupName + "/" + "reclaimPolicy", // Supplementary service labels - used to cooperate with k8s diff --git a/pkg/model/chk/creator/service.go b/pkg/model/chk/creator/service.go index 46d50e8e0..e2fb0856e 100644 --- a/pkg/model/chk/creator/service.go +++ b/pkg/model/chk/creator/service.go @@ -72,7 +72,7 @@ func (m *ServiceManager) CreateService(what interfaces.ServiceType, params ...an var host *chi.Host if len(params) > 0 { host = params[0].(*chi.Host) - return []*core.Service{m.createServiceHost(host)} + return m.createServiceHost(host) } } panic("unknown service type") @@ -214,7 +214,7 @@ func crExposesSecureZK(cr chi.ICustomResource) bool { } exposed := false cr.WalkHosts(func(h *chi.Host) error { - if h != nil && h.ZKPortSecure.HasValue() { + if (h != nil) && h.ZKPortSecure.HasValue() { exposed = true } return nil @@ -272,33 +272,59 @@ func (m *ServiceManager) createServiceShard(shard chi.IShard) *core.Service { return nil } -// createServiceHost creates new core.Service for specified host -func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { +// createServiceHost builds the per-host (replica-level) Services for a Keeper host. +// +// When the user supplies a replicaServiceTemplate it is honored as-is (single Service) — the +// user is in full control and back-compat is preserved. +// +// Otherwise two operator-managed headless Services are emitted (issue #1982): +// - peer : intra-keeper Raft + StatefulSet pod DNS. publishNotReadyAddresses=true so Raft +// peers can reach each other BEFORE pods are Ready and bootstrap quorum. Keeps the existing +// NameStatefulSetService name + LabelServiceHost label → byte-identical to the pre-split +// layout, no pod re-roll, and the Raft /STS serviceName binding is unchanged. +// - client : ClickHouse-facing. publishNotReadyAddresses=false so DNS only resolves Ready +// Keeper nodes; raft port omitted. The keeper-ref resolver selects this tier via +// LabelServiceHostClient so clients never connect to a not-yet-Ready Keeper. +func (m *ServiceManager) createServiceHost(host *chi.Host) []*core.Service { if host.IsZero() { return nil } if template, ok := host.GetServiceTemplate(); ok { // .templates.ServiceTemplate specified - return creator.CreateServiceFromTemplate( - template, - host.GetRuntime().GetAddress().GetNamespace(), - m.namer.Name(interfaces.NameStatefulSetService, host), - m.tagger.Label(interfaces.LabelServiceHost, host), - m.tagger.Annotate(interfaces.AnnotateServiceHost, host), - m.tagger.Selector(interfaces.SelectorHostScope, host), - m.or.CreateOwnerReferences(m.cr), - m.macro.Scope(host), - m.labeler, - ) + return []*core.Service{ + creator.CreateServiceFromTemplate( + template, + host.GetRuntime().GetAddress().GetNamespace(), + m.namer.Name(interfaces.NameStatefulSetService, host), + m.tagger.Label(interfaces.LabelServiceHost, host), + m.tagger.Annotate(interfaces.AnnotateServiceHost, host), + m.tagger.Selector(interfaces.SelectorHostScope, host), + m.or.CreateOwnerReferences(m.cr), + m.macro.Scope(host), + m.labeler, + ), + } } - // Create default Service - // We do not have .templates.ServiceTemplate specified or it is incorrect + // No user template - emit the two default per-host Services. + peer := m.buildDefaultHostService(host, + m.namer.Name(interfaces.NameStatefulSetService, host), interfaces.LabelServiceHost, + true /* publishNotReady */, true /* includeRaftPort */) + client := m.buildDefaultHostService(host, + m.namer.Name(interfaces.NameStatefulSetServiceClient, host), interfaces.LabelServiceHostClient, + false /* publishNotReady */, false /* includeRaftPort */) + return []*core.Service{peer, client} +} + +// buildDefaultHostService builds one operator-managed headless per-host Service. publishNotReady +// toggles ServiceSpec.PublishNotReadyAddresses; includeRaftPort keeps the Raft port (peer) or +// drops it (client, which only needs the ZK client ports). +func (m *ServiceManager) buildDefaultHostService(host *chi.Host, name string, label interfaces.LabelType, publishNotReady, includeRaftPort bool) *core.Service { svc := &core.Service{ ObjectMeta: meta.ObjectMeta{ - Name: m.namer.Name(interfaces.NameStatefulSetService, host), + Name: name, Namespace: host.GetRuntime().GetAddress().GetNamespace(), - Labels: m.macro.Scope(host).Map(m.tagger.Label(interfaces.LabelServiceHost, host)), + Labels: m.macro.Scope(host).Map(m.tagger.Label(label, host)), Annotations: m.macro.Scope(host).Map(m.tagger.Annotate(interfaces.AnnotateServiceHost, host)), OwnerReferences: m.or.CreateOwnerReferences(m.cr), }, @@ -306,10 +332,10 @@ func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { Selector: m.tagger.Selector(interfaces.SelectorHostScope, host), ClusterIP: TemplateDefaultsServiceClusterIP, Type: "ClusterIP", - PublishNotReadyAddresses: true, + PublishNotReadyAddresses: publishNotReady, }, } - appendHostExposedPorts(svc, host) + appendHostExposedPorts(svc, host, includeRaftPort) m.labeler.MakeObjectVersion(svc.GetObjectMeta(), svc) return svc } @@ -320,11 +346,15 @@ func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { // matching per-host XML overlay emits so the Keeper // process binds no plaintext listener at all; liveness probe falls back to // pgrep. Other ports (zk-secure, raft) flow through unchanged. -func appendHostExposedPorts(svc *core.Service, host *chi.Host) { +func appendHostExposedPorts(svc *core.Service, host *chi.Host, includeRaftPort bool) { host.WalkSpecifiedPorts(func(name string, port *types.Int32, protocol core.Protocol) bool { if (name == chi.KpDefaultZKPortName) && !host.IsInsecure() { return false } + // The client-facing Service omits the Raft port — clients only use the ZK client ports. + if (name == chi.KpDefaultRaftPortName) && !includeRaftPort { + return false + } svc.Spec.Ports = append(svc.Spec.Ports, core.ServicePort{ Name: name, Protocol: protocol, diff --git a/pkg/model/chk/creator/service_host_ports_test.go b/pkg/model/chk/creator/service_host_ports_test.go new file mode 100644 index 000000000..a17b2dd1c --- /dev/null +++ b/pkg/model/chk/creator/service_host_ports_test.go @@ -0,0 +1,65 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package creator + +import ( + "testing" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/apis/common/types" +) + +// portNames extracts the port names emitted onto a Service for assertion. +func portNames(svc *core.Service) []string { + names := make([]string, 0, len(svc.Spec.Ports)) + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + return names +} + +// TestAppendHostExposedPortsRaftToggle pins the per-host Service port partition (issue #1982): +// the peer/Raft Service keeps the raft port (includeRaftPort=true) while the client-facing +// Service drops it (includeRaftPort=false) and retains only the ZK client ports. +func TestAppendHostExposedPortsRaftToggle(t *testing.T) { + newHost := func() *chi.Host { + return &chi.Host{ + HostSecure: chi.HostSecure{Insecure: types.NewStringBool(true)}, + HostPorts: chi.HostPorts{ + ZKPort: types.NewInt32(2181), + RaftPort: types.NewInt32(9234), + }, + } + } + + t.Run("peer keeps raft port", func(t *testing.T) { + svc := &core.Service{} + appendHostExposedPorts(svc, newHost(), true) + names := portNames(svc) + require.Contains(t, names, chi.KpDefaultZKPortName) + require.Contains(t, names, chi.KpDefaultRaftPortName) + }) + + t.Run("client drops raft port, keeps zk", func(t *testing.T) { + svc := &core.Service{} + appendHostExposedPorts(svc, newHost(), false) + names := portNames(svc) + require.Contains(t, names, chi.KpDefaultZKPortName) + require.NotContains(t, names, chi.KpDefaultRaftPortName) + }) +} diff --git a/pkg/model/chk/creator/service_host_split_test.go b/pkg/model/chk/creator/service_host_split_test.go new file mode 100644 index 000000000..b014dcb00 --- /dev/null +++ b/pkg/model/chk/creator/service_host_split_test.go @@ -0,0 +1,150 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is an external test package (creator_test) so it can import the managers +// package to build a real tagger; managers imports creator, so an in-package test +// would form an import cycle. +package creator_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + + chk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/chop" + "github.com/altinity/clickhouse-operator/pkg/interfaces" + chkNormalizer "github.com/altinity/clickhouse-operator/pkg/model/chk/normalizer" + chkLabeler "github.com/altinity/clickhouse-operator/pkg/model/chk/tags/labeler" + commonNormalizer "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer" + commonLabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler" + "github.com/altinity/clickhouse-operator/pkg/model/managers" +) + +// The normalizer reads the global operator config (chop.Config()) via the labeler; +// initialize a default instance once for the package. +func init() { chop.New(nil, nil, "") } + +// serviceLabelKey is the fully-qualified label key that carries the Service tier value +// (host vs host-client), resolved through the CHK labeler so the test does not hard-code it. +func serviceLabelKey(t *testing.T) string { + t.Helper() + l := chkLabeler.New(chk.NewClickHouseKeeperInstallation("x", "ns")) + return l.Get(commonLabeler.LabelService) +} + +func portNames(svc *core.Service) []string { + names := make([]string, 0, len(svc.Spec.Ports)) + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + return names +} + +// normalizeSingleHostCHK builds a minimal one-cluster CHK and normalizes it, returning the +// ready-to-use CR plus a ServiceManager wired exactly as the controller wires it. +func normalizeSingleHostCHK(t *testing.T) (*chk.ClickHouseKeeperInstallation, interfaces.IServiceManager) { + t.Helper() + src := chk.NewClickHouseKeeperInstallation("kpr", "ns") + src.Spec.Configuration = &chk.Configuration{Clusters: []*chk.Cluster{{Name: "keeper"}}} + cr, err := chkNormalizer.New().CreateTemplated(src, commonNormalizer.NewOptions[chk.ClickHouseKeeperInstallation]()) + require.NoError(t, err) + require.NotNil(t, cr) + + sm := managers.NewServiceManager(managers.ServiceManagerTypeKeeper) + sm.SetCR(cr) + sm.SetTagger(managers.NewTagManager(managers.TagManagerTypeKeeper, cr)) + return cr, sm +} + +// TestCreateServiceHostEmitsPeerAndClient pins the issue #1982 contract: with no +// user-supplied replicaServiceTemplate, each Keeper host gets TWO headless Services — +// - a peer/Raft Service: publishNotReadyAddresses=true, keeps the Raft port, Service tier "host" +// - a client Service: publishNotReadyAddresses=false, no Raft port, Service tier "host-client" +// +// The peer keeps the bare StatefulSet Service name (Raft / pod DNS binding) and the +// client name is that + "-client". The distinct "host-client" tier label is what the keeper-ref +// resolver selects so ClickHouse clients never resolve a not-yet-Ready Keeper. +func TestCreateServiceHostEmitsPeerAndClient(t *testing.T) { + cr, sm := normalizeSingleHostCHK(t) + svcLabel := serviceLabelKey(t) + + hosts := 0 + cr.WalkHosts(func(host *chi.Host) error { + hosts++ + services := sm.CreateService(interfaces.ServiceHost, host) + require.Len(t, services, 2, "a default Keeper host must emit exactly two Services") + + peer, client := services[0], services[1] + + // Peer: bare name, publishNotReady=true, host tier, retains the Raft port. + require.Equal(t, "chk-kpr-keeper-0-0", peer.Name) + require.True(t, peer.Spec.PublishNotReadyAddresses, + "peer/Raft Service must publish not-ready addresses for quorum bootstrap") + require.Equal(t, "host", peer.Labels[svcLabel]) + require.Contains(t, portNames(peer), chi.KpDefaultRaftPortName) + + // Client: peer name + "-client", publishNotReady=false, host-client tier, no Raft port. + require.Equal(t, peer.Name+"-client", client.Name) + require.False(t, client.Spec.PublishNotReadyAddresses, + "client Service must resolve only Ready Keeper endpoints") + require.Equal(t, "host-client", client.Labels[svcLabel]) + require.NotContains(t, portNames(client), chi.KpDefaultRaftPortName, + "client Service must not expose the Raft port") + + // Both are headless and share the host selector (one pod, two readiness views). + require.Equal(t, "None", peer.Spec.ClusterIP) + require.Equal(t, "None", client.Spec.ClusterIP) + return nil + }) + require.Equal(t, 1, hosts, "minimal one-cluster CHK must normalize to exactly one host") +} + +// TestCreateServiceHostHonorsUserTemplate verifies the back-compat escape hatch: when the host +// carries a replicaServiceTemplate the operator emits the single user-controlled Service and does +// NOT inject the second client Service. +func TestCreateServiceHostHonorsUserTemplate(t *testing.T) { + src := chk.NewClickHouseKeeperInstallation("kpr", "ns") + src.Spec.Configuration = &chk.Configuration{ + Clusters: []*chk.Cluster{{ + Name: "keeper", + Templates: &chi.TemplatesList{ReplicaServiceTemplate: "svc-tpl"}, + }}, + } + src.Spec.Templates = &chi.Templates{ + ServiceTemplates: []chi.ServiceTemplate{{ + Name: "svc-tpl", + Spec: core.ServiceSpec{Type: core.ServiceTypeClusterIP}, + }}, + } + cr, err := chkNormalizer.New().CreateTemplated(src, commonNormalizer.NewOptions[chk.ClickHouseKeeperInstallation]()) + require.NoError(t, err) + + sm := managers.NewServiceManager(managers.ServiceManagerTypeKeeper) + sm.SetCR(cr) + sm.SetTagger(managers.NewTagManager(managers.TagManagerTypeKeeper, cr)) + + cr.WalkHosts(func(host *chi.Host) error { + if _, ok := host.GetServiceTemplate(); !ok { + // Template did not attach (normalization specifics) — skip rather than assert a + // false negative; the two-Service default path is covered by the test above. + t.Skip("replicaServiceTemplate did not attach to host; template wiring not exercised") + } + services := sm.CreateService(interfaces.ServiceHost, host) + require.Len(t, services, 1, "a templated host must emit exactly one (user-controlled) Service") + return nil + }) +} diff --git a/pkg/model/chk/namer/const.go b/pkg/model/chk/namer/const.go index 277502b14..415116cf2 100644 --- a/pkg/model/chk/namer/const.go +++ b/pkg/model/chk/namer/const.go @@ -44,4 +44,8 @@ const ( // patternClusterPDBName is a template of cluster scope PDB. "chi-{chi}-{cluster}" patternClusterPDBName = "pdb chk- + macrosList.Get().Get(macro.MacrosCRName) + - + macrosList.Get().Get(macro.MacrosClusterName)" + + // statefulSetClientServiceNameSuffix is appended to the per-host StatefulSet Service name to + // form the client-facing Service name (see createStatefulSetServiceClientName). + statefulSetClientServiceNameSuffix = "-client" ) diff --git a/pkg/model/chk/namer/name.go b/pkg/model/chk/namer/name.go index 605d1c7d0..dbad76a1e 100644 --- a/pkg/model/chk/namer/name.go +++ b/pkg/model/chk/namer/name.go @@ -174,6 +174,14 @@ func (n *Namer) createStatefulSetServiceName(host *api.Host) string { return n.macro.Scope(host).Line(pattern) } +// createStatefulSetServiceClientName returns the name of the client-facing per-host Service — +// the StatefulSet Service name plus a "-client" suffix. The client Service publishes only Ready +// Keeper endpoints (publishNotReadyAddresses=false) so ClickHouse never resolves a not-yet-Ready +// node; the peer Service (createStatefulSetServiceName) keeps the bare name for Raft and pod DNS. +func (n *Namer) createStatefulSetServiceClientName(host *api.Host) string { + return n.createStatefulSetServiceName(host) + statefulSetClientServiceNameSuffix +} + // createPodHostname returns a hostname of a Pod of a ClickHouse instance. // Is supposed to be used where network connection to a Pod is required. // NB: right now Pod's hostname points to a Service, through which Pod can be accessed. diff --git a/pkg/model/chk/namer/name_test.go b/pkg/model/chk/namer/name_test.go new file mode 100644 index 000000000..6a35481d9 --- /dev/null +++ b/pkg/model/chk/namer/name_test.go @@ -0,0 +1,40 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package namer + +import ( + "testing" + + "github.com/stretchr/testify/require" + + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/interfaces" +) + +// TestStatefulSetServiceClientName pins the client Service naming contract (issue #1982): +// the client-facing Service name is the peer/StatefulSet Service name plus a "-client" suffix. +// The peer name MUST stay the bare StatefulSet Service name — Raft and the pod DNS +// (StatefulSet.serviceName) bind to it, so any drift would break quorum on existing clusters. +func TestStatefulSetServiceClientName(t *testing.T) { + n := New() + host := &chi.Host{} + + peer := n.Name(interfaces.NameStatefulSetService, host) + client := n.Name(interfaces.NameStatefulSetServiceClient, host) + + require.Equal(t, peer+"-client", client, + "client Service name must be the peer Service name plus the -client suffix") + require.NotEqual(t, peer, client, "peer and client Service names must differ") +} diff --git a/pkg/model/chk/namer/namer.go b/pkg/model/chk/namer/namer.go index 4912cc524..644c3d95c 100644 --- a/pkg/model/chk/namer/namer.go +++ b/pkg/model/chk/namer/namer.go @@ -83,6 +83,9 @@ func (n *Namer) Name(what interfaces.NameType, params ...any) string { case interfaces.NameStatefulSetService: host := params[0].(*api.Host) return n.createStatefulSetServiceName(host) + case interfaces.NameStatefulSetServiceClient: + host := params[0].(*api.Host) + return n.createStatefulSetServiceClientName(host) case interfaces.NamePodHostname: host := params[0].(*api.Host) return n.createPodHostname(host) @@ -100,10 +103,12 @@ func (n *Namer) Name(what interfaces.NameType, params ...any) string { return n.createClusterPDBName(cluster) default: + // Delegate any type not handled above to the common namer, whose Name() + // has no default case and panics on an unspecified type. That is the + // single fail-loud guard, so an unknown type cannot sneak through to a + // silent zero-value return here. return n.commonNamer.Name(what, params...) } - - panic("unknown name type") } func (n *Namer) Names(what interfaces.NameType, params ...any) []string { diff --git a/pkg/model/chk/normalizer/normalizer.go b/pkg/model/chk/normalizer/normalizer.go index 0a820eb7a..40194bf60 100644 --- a/pkg/model/chk/normalizer/normalizer.go +++ b/pkg/model/chk/normalizer/normalizer.go @@ -35,6 +35,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer/subst" "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer/templates" "github.com/altinity/clickhouse-operator/pkg/model/managers" + "github.com/altinity/clickhouse-operator/pkg/util" ) // Normalizer specifies structures normalizer @@ -268,6 +269,9 @@ func (n *Normalizer) normalizeDefaults(defaults *chi.Defaults) *chi.Defaults { if defaults.StorageManagement == nil { defaults.StorageManagement = chi.NewStorageManagement() } + // Fold casing + validate the default StorageManagement (provisioner/reclaimPolicy), + // matching the CHI normalizer — previously left un-normalized at the defaults level. + templates.NormalizeStorageManagement(defaults.StorageManagement) // Ensure field if defaults.Templates == nil { //defaults.Templates = api.NewChiTemplateNames() @@ -342,16 +346,13 @@ func (n *Normalizer) normalizeReconcile(reconcile *chi.ChiReconcile) *chi.ChiRec reconcile = chi.NewChiReconcile().SetDefaults() } - // Policy - switch strings.ToLower(reconcile.GetPolicy()) { - case strings.ToLower(chi.ReconcilingPolicyWait): - // Known value, overwrite it to ensure case-ness + // Policy — fold any accepted casing to the canonical const; unknown values fall back to default. + switch util.FoldEnum(reconcile.GetPolicy(), chi.ReconcilingPolicyWait, chi.ReconcilingPolicyNoWait) { + case chi.ReconcilingPolicyWait: reconcile.SetPolicy(chi.ReconcilingPolicyWait) - case strings.ToLower(chi.ReconcilingPolicyNoWait): - // Known value, overwrite it to ensure case-ness + case chi.ReconcilingPolicyNoWait: reconcile.SetPolicy(chi.ReconcilingPolicyNoWait) default: - // Unknown value, fallback to default reconcile.SetPolicy(chi.ReconcilingPolicyUnspecified) } @@ -393,7 +394,8 @@ func (n *Normalizer) normalizeReconcileRuntime(runtime chi.ReconcileRuntime) chi } func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) chi.ReconcileStatefulSet { - // Create + // Create — fold casing to canonical const, then default if empty. + sts.Create.OnFailure = chi.NormalizeOnStatefulSetCreateFailureAction(sts.Create.OnFailure) if sts.Create.OnFailure == "" { sts.Create.OnFailure = chi.OnStatefulSetCreateFailureActionDelete } @@ -404,13 +406,16 @@ func (n *Normalizer) normalizeReconcileStatefulSet(sts chi.ReconcileStatefulSet) if sts.Update.PollInterval == 0 { sts.Update.PollInterval = defaultStatefulSetUpdatePollInterval } + sts.Update.OnFailure = chi.NormalizeOnStatefulSetUpdateFailureAction(sts.Update.OnFailure) if sts.Update.OnFailure == "" { sts.Update.OnFailure = chi.OnStatefulSetUpdateFailureActionRollback } // Recreate + sts.Recreate.OnDataLoss = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnDataLoss) if sts.Recreate.OnDataLoss == "" { sts.Recreate.OnDataLoss = chi.OnStatefulSetRecreateOnDataLossActionRecreate } + sts.Recreate.OnUpdateFailure = chi.NormalizeOnStatefulSetRecreateAction(sts.Recreate.OnUpdateFailure) if sts.Recreate.OnUpdateFailure == "" { sts.Recreate.OnUpdateFailure = chi.OnStatefulSetRecreateOnUpdateFailureActionRecreate } @@ -460,15 +465,13 @@ func (n *Normalizer) normalizeCleanup(str *string, value string) { if str == nil { return } - switch strings.ToLower(*str) { - case strings.ToLower(chi.ObjectsCleanupRetain): - // Known value, overwrite it to ensure case-ness + // Fold any accepted casing to the canonical const; unknown values fall back to the supplied default. + switch util.FoldEnum(*str, chi.ObjectsCleanupRetain, chi.ObjectsCleanupDelete) { + case chi.ObjectsCleanupRetain: *str = chi.ObjectsCleanupRetain - case strings.ToLower(chi.ObjectsCleanupDelete): - // Known value, overwrite it to ensure case-ness + case chi.ObjectsCleanupDelete: *str = chi.ObjectsCleanupDelete default: - // Unknown value, fallback to default *str = value } } diff --git a/pkg/model/chk/tags/labeler/labeler.go b/pkg/model/chk/tags/labeler/labeler.go index 6547c46dc..83453a202 100644 --- a/pkg/model/chk/tags/labeler/labeler.go +++ b/pkg/model/chk/tags/labeler/labeler.go @@ -44,9 +44,12 @@ func (l *Labeler) Label(what interfaces.LabelType, params ...any) map[string]str return l.labelConfigMapHost(params...) default: + // Delegate any type not handled above to the base labeler, whose Label() + // has no default case and panics on an unspecified type. That is the + // single fail-loud guard, so an unknown type cannot sneak through to a + // silent zero-value return here. return l.Labeler.Label(what, params...) } - panic("unknown label type") } // Selector diff --git a/pkg/model/chk/tags/labeler/list.go b/pkg/model/chk/tags/labeler/list.go index d3ee92022..497646107 100644 --- a/pkg/model/chk/tags/labeler/list.go +++ b/pkg/model/chk/tags/labeler/list.go @@ -46,6 +46,7 @@ var list = types.List{ labeler.LabelServiceValueCluster: "cluster", labeler.LabelServiceValueShard: "shard", labeler.LabelServiceValueHost: "host", + labeler.LabelServiceValueHostClient: "host-client", labeler.LabelPVCReclaimPolicyName: clickhouse_keeper_altinity_com.APIGroupName + "/" + "reclaimPolicy", // Supplementary service labels - used to cooperate with k8s diff --git a/pkg/model/chk/tags/labeler/list_test.go b/pkg/model/chk/tags/labeler/list_test.go new file mode 100644 index 000000000..afe4326f0 --- /dev/null +++ b/pkg/model/chk/tags/labeler/list_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labeler + +import ( + "testing" + + "github.com/stretchr/testify/require" + + chk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/chop" + commonLabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler" +) + +// The labeler reads the global operator config (chop.Config()) during construction. +func init() { chop.New(nil, nil, "") } + +// TestServiceTierLabelValues guards the Service-tier label-value mapping (issue #1982). The +// client Service tier MUST resolve to a concrete, distinct value: a missing entry in the +// labeler `list` map silently yields an empty value, which makes the client Service unselectable +// by the keeper-ref resolver and leaves clients pointed at the not-ready peer tier. +func TestServiceTierLabelValues(t *testing.T) { + l := New(chk.NewClickHouseKeeperInstallation("kpr", "ns")) + + require.Equal(t, "host", l.Get(commonLabeler.LabelServiceValueHost)) + require.Equal(t, "host-client", l.Get(commonLabeler.LabelServiceValueHostClient), + "host-client tier must map to a concrete value; empty means the resolver can't select it") + require.NotEqual(t, l.Get(commonLabeler.LabelServiceValueHost), l.Get(commonLabeler.LabelServiceValueHostClient), + "peer and client Service tiers must be distinguishable by label") +} diff --git a/pkg/model/clickhouse/connection.go b/pkg/model/clickhouse/connection.go index 88df39e47..05bdd56be 100644 --- a/pkg/model/clickhouse/connection.go +++ b/pkg/model/clickhouse/connection.go @@ -177,7 +177,7 @@ func legacyVerifiedTLSConfig() *tls.Config { // users opting into TLS hardening should not silently get InsecureSkipVerify=true. func (c *Connection) setupTLSAdvanced() { // Nothing to do for HTTP DSNs. - if c.params.scheme != httpsScheme { + if c.params.scheme != api.ChSchemeHTTPS { return } diff --git a/pkg/model/clickhouse/credentials_endpoint.go b/pkg/model/clickhouse/credentials_endpoint.go index 0d2a411ac..112094df1 100644 --- a/pkg/model/clickhouse/credentials_endpoint.go +++ b/pkg/model/clickhouse/credentials_endpoint.go @@ -33,8 +33,6 @@ const ( dsnUsernamePasswordPairPattern = "%s:%s@" dsnUsernamePasswordPairUsernameOnlyPattern = "%s@" - httpsScheme = "https" - // tlsSettingsLegacy is the registry key used when no per-endpoint TLS knobs // are configured (the legacy path). Identical knobs across endpoints share // this key. Endpoints with explicit Verify/MinVersion/ServerName/rootCA get @@ -212,7 +210,7 @@ func (c *EndpointCredentials) makeDSN(hideCredentials bool) string { c.hostname, strconv.Itoa(c.port), ) - if c.scheme == httpsScheme { + if c.scheme == api.ChSchemeHTTPS { baseUrl += "?tls_config=" + c.tlsConfigKey } return baseUrl @@ -228,7 +226,7 @@ func (c *EndpointCredentials) makeDSNLogQueries(hideCredentials bool) string { strconv.Itoa(c.port), ) baseUrl += "?log_queries=1" - if c.scheme == httpsScheme { + if c.scheme == api.ChSchemeHTTPS { baseUrl += "&tls_config=" + c.tlsConfigKey } return baseUrl diff --git a/pkg/model/common/normalizer/templates/host.go b/pkg/model/common/normalizer/templates/host.go index dafe9fcba..ce6fa843b 100644 --- a/pkg/model/common/normalizer/templates/host.go +++ b/pkg/model/common/normalizer/templates/host.go @@ -37,6 +37,8 @@ func NormalizeHostTemplate(template *api.HostTemplate) { // Normalize PortDistribution for i := range template.PortDistribution { portDistribution := &template.PortDistribution[i] + // Fold any accepted casing to the canonical const before validating. + portDistribution.Type = deployment.NormalizePortDistributionType(portDistribution.Type) switch portDistribution.Type { case deployment.PortDistributionUnspecified, diff --git a/pkg/model/common/normalizer/templates/pod.go b/pkg/model/common/normalizer/templates/pod.go index 1a484041e..0541d5c20 100644 --- a/pkg/model/common/normalizer/templates/pod.go +++ b/pkg/model/common/normalizer/templates/pod.go @@ -125,6 +125,11 @@ func normalizePodDistribution(replicasCount int, podDistribution *api.PodDistrib podDistribution.TopologyKey = defaultTopologyKey } + // Fold any accepted casing to the canonical const so the switch below (and downstream + // affinity builders) can compare with plain ==. + podDistribution.Type = deployment.NormalizePodDistributionType(podDistribution.Type) + podDistribution.Scope = deployment.NormalizePodDistributionScope(podDistribution.Scope) + switch podDistribution.Type { case deployment.PodDistributionUnspecified, diff --git a/pkg/model/common/normalizer/templates/volume_claim.go b/pkg/model/common/normalizer/templates/volume_claim.go index ffd6b1f05..066089a3f 100644 --- a/pkg/model/common/normalizer/templates/volume_claim.go +++ b/pkg/model/common/normalizer/templates/volume_claim.go @@ -22,20 +22,26 @@ func NormalizeVolumeClaimTemplate(template *api.VolumeClaimTemplate) { // Skip for now // StorageManagement - normalizeStorageManagement(&template.StorageManagement) + NormalizeStorageManagement(&template.StorageManagement) // Check Spec // Skip for now } -// normalizeStorageManagement normalizes StorageManagement -func normalizeStorageManagement(storage *api.StorageManagement) { - // Check PVCProvisioner +// NormalizeStorageManagement normalizes StorageManagement: it folds the letter-casing +// of PVCProvisioner / PVCReclaimPolicy to their canonical consts (so both humped and +// all-lowercase CRD inputs are accepted), then resets any unrecognized value to +// Unspecified. Exported so callers normalizing a bare StorageManagement (e.g. +// spec.defaults.storageManagement) reuse the same folding+validation. +func NormalizeStorageManagement(storage *api.StorageManagement) { + // PVCProvisioner — fold casing to canonical, then validate. + storage.PVCProvisioner = storage.PVCProvisioner.Normalize() if !storage.PVCProvisioner.IsValid() { storage.PVCProvisioner = api.PVCProvisionerUnspecified } - // Check PVCReclaimPolicy + // PVCReclaimPolicy — fold casing to canonical, then validate. + storage.PVCReclaimPolicy = storage.PVCReclaimPolicy.Normalize() if !storage.PVCReclaimPolicy.IsValid() { storage.PVCReclaimPolicy = api.PVCReclaimPolicyUnspecified } diff --git a/pkg/model/common/tags/labeler/labeler.go b/pkg/model/common/tags/labeler/labeler.go index 6ca7903c8..06c1589cf 100644 --- a/pkg/model/common/tags/labeler/labeler.go +++ b/pkg/model/common/tags/labeler/labeler.go @@ -56,6 +56,8 @@ func (l *Labeler) Label(what interfaces.LabelType, params ...any) map[string]str return l.labelServiceShard(params...) case interfaces.LabelServiceHost: return l.labelServiceHost(params...) + case interfaces.LabelServiceHostClient: + return l.labelServiceHostClient(params...) case interfaces.LabelExistingPV: return l.labelExistingPV(params...) diff --git a/pkg/model/common/tags/labeler/labels.go b/pkg/model/common/tags/labeler/labels.go index a5f48383c..6f7bb9ea1 100644 --- a/pkg/model/common/tags/labeler/labels.go +++ b/pkg/model/common/tags/labeler/labels.go @@ -87,6 +87,27 @@ func (l *Labeler) _labelServiceHost(host *api.Host) map[string]string { }) } +// labelServiceHostClient +func (l *Labeler) labelServiceHostClient(params ...any) map[string]string { + var host *api.Host + if len(params) > 0 { + host = params[0].(*api.Host) + return l._labelServiceHostClient(host) + } + panic("not enough params for labeler") +} + +// _labelServiceHostClient labels the client-facing per-host Service (publishNotReadyAddresses=false). +// Mirrors _labelServiceHost but carries the host-client Service value so the keeper-ref resolver +// can select the ready-only client tier distinctly from the Raft/peer tier. +func (l *Labeler) _labelServiceHostClient(host *api.Host) map[string]string { + return util.MergeStringMapsOverwrite( + l.GetHostScope(host, false), + map[string]string{ + l.Get(LabelService): l.Get(LabelServiceValueHostClient), + }) +} + func (l *Labeler) labelExistingPV(params ...any) map[string]string { var pv *core.PersistentVolume var host *api.Host diff --git a/pkg/model/common/tags/labeler/list.go b/pkg/model/common/tags/labeler/list.go index 250d1bfb4..ce0e83c74 100644 --- a/pkg/model/common/tags/labeler/list.go +++ b/pkg/model/common/tags/labeler/list.go @@ -41,6 +41,7 @@ const ( LabelServiceValueCluster = "cluster" LabelServiceValueShard = "shard" LabelServiceValueHost = "host" + LabelServiceValueHostClient = "host-client" LabelPVCReclaimPolicyName = "APIGroupName" + "/" + "reclaimPolicy" // Supplementary service labels - used to cooperate with k8s diff --git a/pkg/util/enum.go b/pkg/util/enum.go new file mode 100644 index 000000000..8bd0d4c93 --- /dev/null +++ b/pkg/util/enum.go @@ -0,0 +1,30 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import "strings" + +// FoldEnum returns the canonical-cased member of canonical that case-insensitively +// matches value, so callers can accept both humped and all-lowercase enum input and +// then compare downstream with plain ==. If value matches no canonical member it is +// returned unchanged (the caller's default/validation handles unrecognized values). +func FoldEnum(value string, canonical ...string) string { + for _, c := range canonical { + if strings.EqualFold(value, c) { + return c + } + } + return value +} diff --git a/pkg/util/enum_test.go b/pkg/util/enum_test.go new file mode 100644 index 000000000..3602b39d3 --- /dev/null +++ b/pkg/util/enum_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestFoldEnum verifies that any accepted casing folds to the canonical member, +// while empty and unrecognized inputs pass through unchanged. +func TestFoldEnum(t *testing.T) { + canonical := []string{"Abort", "Delete", "Ignore"} + + tests := []struct { + name string + value string + want string + }{ + {"exact canonical preserved", "Abort", "Abort"}, + {"all-lowercase folds up", "abort", "Abort"}, + {"all-uppercase folds", "DELETE", "Delete"}, + {"mixed case folds", "iGnOrE", "Ignore"}, + {"empty passes through", "", ""}, + {"unrecognized passes through unchanged", "bogus", "bogus"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, FoldEnum(tc.value, canonical...)) + }) + } +} + +// TestFoldEnumNoCandidates verifies the value is returned unchanged when no +// canonical members are supplied (defensive — never panics on empty varargs). +func TestFoldEnumNoCandidates(t *testing.T) { + require.Equal(t, "anything", FoldEnum("anything")) +} diff --git a/pkg/xml/xml.go b/pkg/xml/xml.go index 4f58ff45f..dfc7b4b7a 100644 --- a/pkg/xml/xml.go +++ b/pkg/xml/xml.go @@ -202,12 +202,12 @@ func (n *xmlNode) writeTagWithValue(w io.Writer, value string, attributes string // embedded value NB - printed w/o indent // n.writeTagOpen(w, indent, attributes, eol) - n.writeValue(w, value) + n.writeValue(w, value, Raw) // embedded value is a pre-rendered XML fragment n.writeTagClose(w, indent, eol) } else { // value n.writeTagOpen(w, indent, attributes, noEol) - n.writeValue(w, value) + n.writeValue(w, value, Escape) // element text — escape reserved characters n.writeTagClose(w, 0, eol) } } @@ -252,7 +252,36 @@ func (n *xmlNode) writeTag(w io.Writer, indent uint8, attributes string, openTag } } -// writeValue prints XML value into io.Writer -func (n *xmlNode) writeValue(w io.Writer, value string) { - util.Fprintf(w, "%s", value) +// valueEncoding selects how writeValue renders an element's content. +type valueEncoding int + +const ( + // Escape escapes the XML reserved characters &, < and >. Used for leaf element + // text (passwords, scalar setting values) so a value containing them keeps the + // document well-formed. + Escape valueEncoding = iota + // Raw emits the value verbatim. Used for embedded (SetEmbed) settings that are + // already pre-rendered XML fragments — e.g. the CHK keeper_server/raft_configuration + // or the secure-port overlays — where escaping would turn markup into literal text + // and break the generated config. + Raw +) + +// writeValue prints XML value into io.Writer. +// +// With Escape, reserved characters &, < and > are escaped. Tab, \n and \r are +// intentionally left untouched to preserve existing multi-line values +// (encoding/xml.EscapeText would mangle them). With Raw, the value is emitted +// verbatim (see valueEncoding). +func (n *xmlNode) writeValue(w io.Writer, value string, encoding valueEncoding) { + if encoding == Raw { + util.Fprintf(w, "%s", value) + return + } + escaped := strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + ).Replace(value) + util.Fprintf(w, "%s", escaped) } diff --git a/pkg/xml/xml_test.go b/pkg/xml/xml_test.go new file mode 100644 index 000000000..58e312213 --- /dev/null +++ b/pkg/xml/xml_test.go @@ -0,0 +1,81 @@ +package xml + +import ( + "strings" + "testing" +) + +func TestWriteValue(t *testing.T) { + cases := []struct { + name string + input string + encoding valueEncoding // zero value is Escape + expected string + }{ + // Element text (Escape): reserved characters are escaped. + { + name: "ampersand is escaped", + input: "p%X&word", + expected: "p%X&word", + }, + { + name: "less-than is escaped", + input: "ab", + expected: "a>b", + }, + { + name: "generated password with mixed special chars", + input: "l%XubpKqz2y!QsKlsynEEE6#Thknj&fG", + expected: "l%XubpKqz2y!QsKlsynEEE6#Thknj&fG", + }, + { + name: "plain value is unchanged", + input: "plainpassword", + expected: "plainpassword", + }, + { + // CH multi-line settings must survive: tab, newline and CR are preserved. + name: "whitespace control chars are preserved", + input: "a\tb\nc\rd", + expected: "a\tb\nc\rd", + }, + { + // Single-pass escaping: the '&' of a pre-escaped entity is escaped exactly + // once (the replacer never reprocesses its own output). + name: "pre-escaped input is escaped once, not recursively", + input: "a&b", + expected: "a&amp;b", + }, + + // Embedded values (Raw): a pre-rendered XML fragment (SetEmbed) must be + // emitted verbatim — escaping it would turn markup into literal text and break + // the generated config (e.g. CHK keeper_server/raft_configuration). + { + name: "embedded xml fragment is emitted verbatim", + encoding: Raw, + input: "\n 0\n", + expected: "\n 0\n", + }, + { + name: "embedded remove-attribute fragment is emitted verbatim", + encoding: Raw, + input: ``, + expected: ``, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var sb strings.Builder + (&xmlNode{}).writeValue(&sb, tc.input, tc.encoding) + if sb.String() != tc.expected { + t.Errorf("writeValue(%q, encoding=%v) = %q, expected %q", tc.input, tc.encoding, sb.String(), tc.expected) + } + }) + } +} diff --git a/release b/release index 83b473049..3edc695dc 100644 --- a/release +++ b/release @@ -1 +1 @@ -0.27.1 +0.27.2 diff --git a/release_notes.md b/release_notes.md index 5bbb6f22f..6de8a2240 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,3 +1,26 @@ +## Release 0.27.2 +### New Features +* **Keeper split client/peer Services** ([#1982](https://github.com/Altinity/clickhouse-operator/issues/1982)). Each `ClickHouseKeeper` host now exposes two headless Services instead of one: + * a **peer** Service (unchanged name) with `publishNotReadyAddresses: true` — used for intra-Keeper Raft traffic and the StatefulSet pod DNS, so Raft peers can reach each other before pods are `Ready` and bootstrap quorum; + * a **client** Service (`-client`) with `publishNotReadyAddresses: false` — used by ClickHouse, so clients only ever resolve `Ready` Keeper nodes and never connect to a node still starting up. + + A user-supplied `replicaServiceTemplate` still produces a single Service (the split applies only to the operator's default Services). +* **Configurable CHK reconcile concurrency** ([#2032](https://github.com/Altinity/clickhouse-operator/issues/2032)). A new `reconcile.runtime.reconcileCHKsThreadsNumber` option in `ClickHouseOperatorConfiguration` sets the `ClickHouseKeeper` controller's maximum concurrent reconciles, mirroring the existing `reconcileCHIsThreadsNumber` for `ClickHouseInstallation`. Defaults to `1` (serial reconciliation — unchanged behavior); raise it to reconcile many CHK resources in parallel when running at scale. See [PR #2033](https://github.com/Altinity/clickhouse-operator/pull/2033). +* **Helm `watchNamespaces` value** ([#1919](https://github.com/Altinity/clickhouse-operator/issues/1919)). The Helm chart gains a top-level `watchNamespaces` value that wires directly into the operator config's `watch.namespaces.include`, so the set of watched namespaces can be set at install time (`--set` / `values.yaml`) instead of hand-patching the generated ConfigMap. Empty by default (operator watches its own namespace — unchanged); use `[".*"]` to watch all namespaces. Helm installs only. See [PR #2007](https://github.com/Altinity/clickhouse-operator/pull/2007). + +### Behavior Changes +* **OLM install modes now honor the OperatorGroup** ([#2008](https://github.com/Altinity/clickhouse-operator/issues/2008)). The OperatorHub bundle previously hard-wired `WATCH_NAMESPACE` to the operator's own namespace, so `SingleNamespace`, `MultiNamespace`, and `AllNamespaces` installs were all silently scoped to the operator's namespace only. The CSV now reads `metadata.annotations['olm.targetNamespaces']`, so each advertised install mode watches the namespaces the OperatorGroup actually selects. **On upgrade, an OLM install configured for Single/Multi/AllNamespaces will start watching the intended namespaces for the first time** — if you relied on the old own-namespace-only behavior, scope the OperatorGroup (or a `ClickHouseOperatorConfiguration` `watch.namespaces`) accordingly. Affects OLM/OperatorHub installs only; plain manifest and Helm installs are unchanged. +* **One-time ClickHouse rolling restart on upgrade for CHI→CHK keeper references** ([#1982](https://github.com/Altinity/clickhouse-operator/issues/1982)). A `ClickHouseInstallation` that references a `ClickHouseKeeper` via a `keeper:` ref (the default `serviceType: Replicas`) now resolves to the new ready-only Keeper **client** Service. The first reconcile after upgrade rewrites the CHI's `` endpoints from `chk-…` to `chk-…-client`, which the operator treats as a configuration change requiring a restart — so **every ClickHouse pod of an affected CHI restarts once**. This is a one-time event; the resolved endpoints are stable afterwards. No action required. +* **Backward-incompatible config rename** in `ClickHouseOperatorConfiguration`: `reconcile.recovery.from.{aborted,completed}` → `reconcile.recovery.onStatus.{aborted,completed}`. The `from` grouping level is removed; the per-status scopes and their action keys (`onPodReady`/`onPodNotReady`/`onPodNotReadyThreshold`) are unchanged. The obsolete `from` key is silently ignored on load. **If you set `reconcile.recovery.from.aborted.onPodReady: none` on 0.27.0/0.27.1 to disable Aborted auto-recovery, re-apply it as `reconcile.recovery.onStatus.aborted.onPodReady: none`** — otherwise the default (`retry`) silently re-enables it. The `completed` scope (sustained-NotReady host recovery) is new in 0.27.2 and off by default. See [docs/operator_upgrade.md](docs/operator_upgrade.md). + +### Fixed +* **Scaled-up replica no longer stays permanently broken after `remote_servers` is published** ([#2013](https://github.com/Altinity/clickhouse-operator/issues/2013)). When a replica was added to an existing multi-host cluster, it could start ClickHouse before the operator published the full `remote_servers`; cluster-dependent objects (`Distributed`, `DICTIONARY`, refreshable `MATERIALIZED VIEW`) then failed their async startup load with `CLUSTER_DOESNT_EXIST` and — because ClickHouse never re-runs terminal loader jobs after a `SYSTEM RELOAD CONFIG` — stayed broken until a manual pod restart. The operator now detects this on the newly-added host (it recorded a `CLUSTER_DOESNT_EXIST` error) and restarts that host once, after the complete `remote_servers` is published, so those objects re-load against the real cluster. The restart is scoped to a newly-added host that actually hit the failure — pre-existing replicas, fresh installs, single-host clusters, and replicas with no cluster-dependent objects (e.g. a replica still catching up replication) are never restarted. +* **Config values containing `&`, `<`, or `>` no longer break generated ClickHouse config** ([#1578](https://github.com/Altinity/clickhouse-operator/issues/1578)). Setting values such as a generated password that contained XML-reserved characters were injected verbatim into the rendered `users.xml`/`config.xml`, producing malformed XML that prevented ClickHouse from starting. Element text is now XML-escaped; embedded pre-rendered XML fragments (e.g. Keeper `raft_configuration`) are still emitted verbatim. See [PR #2034](https://github.com/Altinity/clickhouse-operator/pull/2034). +* **Lower metrics-exporter memory footprint for excluded metrics** ([PR #2028](https://github.com/Altinity/clickhouse-operator/pull/2028)). The metrics collector fetched every ClickHouse system metric into an in-memory buffer and applied the `excludeRegexp` filter only just before emitting to Prometheus, so excluded (e.g. high-cardinality per-CPU) metrics still inflated peak memory and could OOM the exporter sidecar. Excluded metric names are now dropped during the SQL row scan, before entering the buffer. Prometheus output is unchanged. + +### Security +* **Operator metrics port no longer exposes `/debug/pprof`**. The operator's metrics HTTP listener (default `:9999`) served Go's shared `http.DefaultServeMux`, onto which `net/http/pprof` is transitively registered through a controller-runtime dependency. That unintentionally exposed the `/debug/pprof/*` endpoints — including the CPU `profile` and `trace` handlers — to anything able to reach the operator's metrics Service, enabling unauthenticated profiling/DoS and heap/goroutine dumps. The listener now uses a dedicated mux that serves only the metrics endpoint; `/metrics` is unchanged. + ## Release 0.27.1 ### Behavior Changes * `StatefulSet` create returning `AlreadyExists` (e.g. due to a stale informer cache or a prior failed delete) is no longer silently treated as a successful create. The reconciler now propagates the recreate sentinel so the host correctly enters the recreate path. See https://github.com/Altinity/clickhouse-operator/pull/1993. diff --git a/releases b/releases index f2fb57115..1874fa93d 100644 --- a/releases +++ b/releases @@ -1,3 +1,4 @@ +0.27.1 0.27.0 0.26.3 0.26.2 diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py index 7506cd1be..17993fbf6 100644 --- a/tests/e2e/kubectl.py +++ b/tests/e2e/kubectl.py @@ -111,10 +111,11 @@ def run_shell(cmd, timeout=600, ok_to_fail=False, shell=None, retry_transient=Fa assert code == 0, error() -def delete_kind(kind, name, ns=None, ok_to_fail=False, shell=None): +def delete_kind(kind, name, ns=None, ok_to_fail=False, shell=None, wait=True): with When(f"Delete {kind} {name}"): + wait_flag = "" if wait else "--wait=false" launch( - f"delete {kind} {name} -v 5 --now --timeout=600s", + f"delete {kind} {name} -v 5 --now --timeout=600s {wait_flag}".strip(), ns=ns, timeout=600, ok_to_fail=ok_to_fail, @@ -233,7 +234,7 @@ def delete_all(kind, ns=None): # OR mid-restart — e.g. chopconf onChange=restart in test_030008) # makes `kubectl delete --timeout` exit non-zero; we recover via # the force-clear loop below, so this must not raise here. - delete_kind(kind, name, ns=ns, ok_to_fail=True) + delete_kind(kind, name, ns=ns, ok_to_fail=True, wait=False) # Stuck/re-attached finalizer recovery. The operator can RE-ATTACH # a finalizer after a clear while it is restarting, so a single # wait_object would race the restart and raise. Re-clear + re-delete @@ -244,7 +245,7 @@ def delete_all(kind, ns=None): if get_count(kind, name=name, ns=ns) == 0: break force_clear_finalizers(kind, name, ns=ns) - delete_kind(kind, name, ns=ns, ok_to_fail=True) + delete_kind(kind, name, ns=ns, ok_to_fail=True, wait=False) # Only sleep if another re-check follows; skip on the last # attempt so a genuinely-stuck CR hits the final wait_object # (the authoritative leak assertion) without an extra wait. @@ -252,7 +253,7 @@ def delete_all(kind, ns=None): retry_sleep(attempt, 5, f"{kind}/{name} still terminating") # Final assertion: if the CR survived every force-clear, this # raises — surfacing a real cleanup leak rather than hiding it. - wait_object(kind, name, ns=ns, count=0) + # wait_object(kind, name, ns=ns, count=0) def delete_all_keeper(ns=None): @@ -366,7 +367,7 @@ def get(kind, name, label="", ns=None, ok_to_fail=False, shell=None): raise ValueError(f"Failed to parse JSON from: {stripped}") from e -def get_container_restart_count(pod, container, ns=None, shell=None): +def get_container_restart_count(pod_name, container=None, ns=None, shell=None): """restartCount of a single named container in a pod; None if pod/container absent. Name-scoped on purpose: callers detecting a SPECIFIC container's in-place @@ -374,13 +375,16 @@ def get_container_restart_count(pod, container, ns=None, shell=None): container restarts). Parses the pod JSON in Python to avoid the jsonpath quoting hazard of an inline `[?(@.name=="...")]` filter. """ - pod_obj = get("pod", pod, ns=ns, ok_to_fail=True, shell=shell) - if not pod_obj: + pod = get("pod", pod_name, ns=ns, ok_to_fail=True, shell=shell) + if not pod: return None - statuses = (pod_obj.get("status") or {}).get("containerStatuses") or [] - for cs in statuses: - if cs.get("name") == container: - return int(cs.get("restartCount") or 0) + statuses = (pod.get("status") or {}).get("containerStatuses") or [] + if container != None: + for cs in statuses: + if cs.get("name") == container: + return int(cs.get("restartCount") or 0) + else: + return sum(int(cs.get("restartCount") or 0) for cs in statuses) return None @@ -578,9 +582,19 @@ def get_pod_status(pod, shell=None, ns=None): def wait_container_status(pod, status, shell=None, ns=None): wait_field("pod", pod, ".status.containerStatuses[0].ready", status, ns, shell=shell) -def get_container_status(pod, shell=None, ns=None): - return get_field("pod", pod, ".status.containerStatuses[0].ready", ns, shell=shell) +def get_container_status(pod, container_index=0, shell=None, ns=None): + return get_field("pod", pod, f".status.containerStatuses[{container_index}].ready", ns, shell=shell) + +def get_condition_status(pod_name, condition_type, shell=None, ns=None): + pod = get("pod", pod_name, ns=ns, ok_to_fail=True, shell=shell) + if not pod: + return None + conditions = (pod.get("status") or {}).get("conditions") or [] + for condition in conditions: + if condition.get("type") == condition_type: + return condition.get("status") + return None def wait_field( kind, @@ -938,8 +952,10 @@ def force_reconcile(name, kind, taskID, ns=None, shell=None): # InProgress poll. The accept-set on the Completed wait below tolerates # either "already Completed" or "InProgress → Completed" transitions. if kind == "chi": + wait_chi_status(name, "InProgress", ns=ns, shell=shell) wait_chi_status(name, "Completed", ns=ns, shell=shell) elif kind == "chk": + wait_chk_status(name, "InProgress", ns=ns, shell=shell) wait_chk_status(name, "Completed", ns=ns, shell=shell) else: assert kind == "chi" or kind == "chk" \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-016-settings-07.yaml b/tests/e2e/manifests/chi/test-016-settings-07.yaml new file mode 100644 index 000000000..012dbd94c --- /dev/null +++ b/tests/e2e/manifests/chi/test-016-settings-07.yaml @@ -0,0 +1,13 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" + +metadata: + name: test-016-settings + +spec: + useTemplates: + - name: clickhouse-version + configuration: + settings: + macros/layer: "&$" + models_config: qwertyinjection diff --git a/tests/e2e/manifests/chi/test-018-configmap-1.yaml b/tests/e2e/manifests/chi/test-018-configmap-1.yaml deleted file mode 100644 index 96140c921..000000000 --- a/tests/e2e/manifests/chi/test-018-configmap-1.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: clickhouse.altinity.com/v1 -kind: ClickHouseInstallation -metadata: - name: test-018-configmap -spec: - useTemplates: - - name: clickhouse-version - configuration: - settings: - display_name: "old_display_name" - macros/test: "old_test" - clusters: - - name: default - \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-018-configmap-2.yaml b/tests/e2e/manifests/chi/test-018-configmap-2.yaml deleted file mode 100644 index 6c43a4e22..000000000 --- a/tests/e2e/manifests/chi/test-018-configmap-2.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: clickhouse.altinity.com/v1 -kind: ClickHouseInstallation -metadata: - name: test-018-configmap -spec: - useTemplates: - - name: clickhouse-version - configuration: - settings: - display_name: "new_display_name" - macros/test: "new_test" - clusters: - - name: default - \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml deleted file mode 100644 index c5ec3f271..000000000 --- a/tests/e2e/manifests/chi/test-020-1-multi-volume.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-020-1-multi-volume" -spec: - useTemplates: - - name: clickhouse-version - configuration: - clusters: - - name: simple - layout: - shardsCount: 1 - settings: - storage_configuration/disks/disk2/path: /var/lib/clickhouse2/ - storage_configuration/policies/default/volumes/default/disk: default - storage_configuration/policies/default/volumes/disk2/disk: disk2 - defaults: -# storageManagement: -# provisioner: StatefulSet - templates: - podTemplate: default - templates: - volumeClaimTemplates: - - name: disk1 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 300Mi - - name: disk2 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - volumeMounts: - - name: disk1 - mountPath: /var/lib/clickhouse - - name: disk2 - mountPath: /var/lib/clickhouse2 - command: - - /bin/bash - - '-c' - - chown clickhouse /var/lib/clickhouse2 && /entrypoint.sh diff --git a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml b/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml deleted file mode 100644 index 5013608ce..000000000 --- a/tests/e2e/manifests/chi/test-020-2-multi-volume.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-020-2-multi-volume" -spec: - useTemplates: - - name: clickhouse-version - configuration: - clusters: - - name: simple - layout: - shardsCount: 1 - settings: - storage_configuration/disks/disk2/path: /var/lib/clickhouse2/ - storage_configuration/policies/default/volumes/default/disk: default - storage_configuration/policies/default/volumes/disk2/disk: disk2 - defaults: - storageManagement: - provisioner: Operator - templates: - podTemplate: default - templates: - volumeClaimTemplates: - - name: disk1 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 300Mi - - name: disk2 - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Mi - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - volumeMounts: - - name: disk1 - mountPath: /var/lib/clickhouse - - name: disk2 - mountPath: /var/lib/clickhouse2 - command: - - /bin/bash - - '-c' - - chown clickhouse /var/lib/clickhouse2 && /entrypoint.sh diff --git a/tests/e2e/manifests/chi/test-030003.yaml b/tests/e2e/manifests/chi/test-030003.yaml index c61b0fffc..f7fcc00f7 100644 --- a/tests/e2e/manifests/chi/test-030003.yaml +++ b/tests/e2e/manifests/chi/test-030003.yaml @@ -43,14 +43,14 @@ spec: /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem none - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt false strict - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 diff --git a/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml b/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml new file mode 100644 index 000000000..8f618d26c --- /dev/null +++ b/tests/e2e/manifests/chi/test-030008-permissive-non-fips.yaml @@ -0,0 +1,21 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030008-permissive-non-fips +spec: + configuration: + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 + templates: + podTemplates: + - name: non-fips + spec: + containers: + - name: clickhouse-pod + image: altinity/clickhouse-server:25.8.16.10002.altinitystable + defaults: + templates: + podTemplate: non-fips \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-030016.yaml b/tests/e2e/manifests/chi/test-030016.yaml new file mode 100644 index 000000000..35d9373fd --- /dev/null +++ b/tests/e2e/manifests/chi/test-030016.yaml @@ -0,0 +1,80 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030009 +spec: + defaults: + templates: + podTemplate: test-030009 + templates: + podTemplates: + - name: test-030009 + spec: + containers: + - name: clickhouse + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + security: + clickhouse: + tls: + rootCASecretRef: + name: clickhouse-certs + key: ca.crt + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + layout: + shardsCount: 1 + replicasCount: 1 + settings: + http_port: _removed_ + tcp_port: _removed_ + interserver_http_port: _removed_ + mysql_port: _removed_ + postgresql_port: _removed_ + https_port: 8443 + tcp_port_secure: 9440 + interserver_https_port: 9010 + files: + openssl.xml: | + + + + /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt + /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + none + sslv2,sslv3,tlsv1,tlsv1_1 + ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + true + + + /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt + false + strict + sslv2,sslv3,tlsv1,tlsv1_1 + ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + + + + server.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.crt + server.key: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.key + dhparam.pem: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: dhparam.pem + ca.crt: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: ca.crt diff --git a/tests/e2e/manifests/chi/test-030017-chi.yaml b/tests/e2e/manifests/chi/test-030017-chi.yaml new file mode 100644 index 000000000..d1bf7b0d1 --- /dev/null +++ b/tests/e2e/manifests/chi/test-030017-chi.yaml @@ -0,0 +1,33 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-030017-chi +spec: + # chi-...-0-0..127.0.0.1.nip.io → 127.0.0.1 on the host; operator Ping(:8443) + # hits local openssl s_server, not a real ClickHouse HTTPS listener. + namespaceDomainPattern: "%s.127.0.0.1.nip.io" + reconcile: + host: + wait: + exclude: "no" + queries: "no" + probes: + startup: "no" + readiness: "no" + templates: + podTemplates: + - name: clickhouse + spec: + containers: + - name: clickhouse + image: altinity/clickhouse-server:25.3.8.30001.altinityfips + configuration: + clusters: + - name: default + secure: "yes" + insecure: "no" + templates: + podTemplate: clickhouse + layout: + shardsCount: 1 + replicasCount: 1 diff --git a/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml new file mode 100644 index 000000000..dd548b54a --- /dev/null +++ b/tests/e2e/manifests/chi/test-035-2-sustained-not-ready.yaml @@ -0,0 +1,36 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-035-sustained-not-ready +spec: + configuration: + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 + templates: + podTemplates: + - name: readiness-flap + spec: + containers: + - name: clickhouse-pod + image: altinity/clickhouse-server:25.8.16.10001.altinitystable + - name: readiness-flap + image: altinity/clickhouse-server:25.8.16.10001.altinitystable + command: + - "/bin/bash" + - "-c" + - "touch /tmp/ready; while true; do sleep 5; done" + readinessProbe: + exec: + command: + - "/bin/bash" + - "-c" + - "test -f /tmp/ready" + initialDelaySeconds: 1 + periodSeconds: 2 + failureThreshold: 1 + defaults: + templates: + podTemplate: readiness-flap diff --git a/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml index 18b0b4a8f..60e2de558 100644 --- a/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-0-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" layout: diff --git a/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml index 4bc8ef649..82dcc9a78 100644 --- a/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-1-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml index a46bee7d0..64ea5ef03 100644 --- a/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-2-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml index 5d7fcc2ac..fc7768c79 100644 --- a/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml +++ b/tests/e2e/manifests/chi/test-039-3-communications-with-secret.yaml @@ -8,10 +8,6 @@ spec: configuration: users: default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 clusters: - name: "default" secret: diff --git a/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml b/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml deleted file mode 100644 index 8b9be245a..000000000 --- a/tests/e2e/manifests/chi/test-039-4-communications-with-secret.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: "test-039-secret-communications" -spec: - useTemplates: - - name: clickhouse-version - configuration: - users: - default/password: qkrq - zookeeper: - nodes: - - host: zookeeper - port: 2181 - clusters: - - name: "default" - secure: "yes" - secret: - auto: "yes" - layout: - shardsCount: 2 - replicasCount: 1 - settings: - tcp_port: 9000 # keep for localhost - tcp_port_secure: 9440 - interserver_http_port: _removed_ - interserver_https_port: 9009 - files: - settings.xml: | - - - - /etc/clickhouse-server/secrets.d/server.crt/clickhouse-certs/server.crt - /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key - /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem - none - - AcceptCertificateHandler - - true - true - sslv2,sslv3 - true - - - true - true - sslv2,sslv3 - true - none - - AcceptCertificateHandler - - - - - server.crt: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: server.crt - server.key: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: server.key - dhparam.pem: - valueFrom: - secretKeyRef: - name: clickhouse-certs - key: dhparam.pem - diff --git a/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-2.yaml b/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-2.yaml new file mode 100644 index 000000000..866227b59 --- /dev/null +++ b/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-2.yaml @@ -0,0 +1,23 @@ +apiVersion: "clickhouse.altinity.com/v1" + +kind: "ClickHouseInstallation" + +metadata: + name: test-081-scaleup + +spec: + useTemplates: + - name: clickhouse-version + - name: persistent-volume + configuration: + zookeeper: + nodes: + - host: zookeeper + port: 2181 + session_timeout_ms: 5000 + operation_timeout_ms: 5000 + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 2 diff --git a/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-3.yaml b/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-3.yaml new file mode 100644 index 000000000..3519a4a39 --- /dev/null +++ b/tests/e2e/manifests/chi/test-081-scaleup-cluster-objects-3.yaml @@ -0,0 +1,23 @@ +apiVersion: "clickhouse.altinity.com/v1" + +kind: "ClickHouseInstallation" + +metadata: + name: test-081-scaleup + +spec: + useTemplates: + - name: clickhouse-version + - name: persistent-volume + configuration: + zookeeper: + nodes: + - host: zookeeper + port: 2181 + session_timeout_ms: 5000 + operation_timeout_ms: 5000 + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 3 diff --git a/tests/e2e/manifests/chi/test-082-canary-2.yaml b/tests/e2e/manifests/chi/test-082-canary-2.yaml new file mode 100644 index 000000000..bff9b119d --- /dev/null +++ b/tests/e2e/manifests/chi/test-082-canary-2.yaml @@ -0,0 +1,30 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-082-canary + labels: + clickhouse.altinity.com/chi: test-082-canary +spec: + useTemplates: + - name: clickhouse-version + configuration: + clusters: + - name: default + layout: + shardsCount: 3 + replicasCount: 2 + replicas: + - shards: + - {} + - templates: + podTemplate: test-26.3 + settings: + macros/canary: canary + + templates: + podTemplates: + - name: test-26.3 + spec: + containers: + - name: clickhouse-pod + image: clickhouse/clickhouse-server:26.3 diff --git a/tests/e2e/manifests/chi/test-082-canary.yaml b/tests/e2e/manifests/chi/test-082-canary.yaml new file mode 100644 index 000000000..b88e0a874 --- /dev/null +++ b/tests/e2e/manifests/chi/test-082-canary.yaml @@ -0,0 +1,15 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-082-canary + labels: + clickhouse.altinity.com/chi: test-082-canary +spec: + useTemplates: + - name: clickhouse-version + configuration: + clusters: + - name: default + layout: + shardsCount: 3 + replicasCount: 2 diff --git a/tests/e2e/manifests/chit/test-082-canary.yaml b/tests/e2e/manifests/chit/test-082-canary.yaml new file mode 100644 index 000000000..a8f94dd57 --- /dev/null +++ b/tests/e2e/manifests/chit/test-082-canary.yaml @@ -0,0 +1,28 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallationTemplate" +metadata: + name: test-082-canary + +spec: + templating: + policy: auto + chiSelector: + clickhouse.altinity.com/chi: test-082-canary + configuration: + clusters: + - layout: + replicas: + - shards: + - {} + - templates: + podTemplate: test-26.3 + settings: + macros/canary: canary + + templates: + podTemplates: + - name: test-26.3 + spec: + containers: + - name: clickhouse-pod + image: clickhouse/clickhouse-server:26.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml deleted file mode 100644 index 5df512e53..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-19.11.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:19.11.12.69 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml deleted file mode 100644 index a21442346..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.1.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.1 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml deleted file mode 100644 index b139a30a2..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml deleted file mode 100644 index 0141a880a..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.4.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.4 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml deleted file mode 100644 index f2869d66d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.5.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.5 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml deleted file mode 100644 index e1b2527a0..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.6.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.6 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml deleted file mode 100644 index af13cbd38..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.7.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.7 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml deleted file mode 100644 index 0b6448e9c..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-20.8.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:20.8 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml deleted file mode 100644 index c0ad7a1c1..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.11.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:21.11 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml deleted file mode 100644 index 58bd4bdcb..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.12.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: yandex/clickhouse-server:21.12 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml deleted file mode 100644 index 7c82d105d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:21.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml deleted file mode 100644 index 9a1f371b0..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-21.8.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:21.8 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml deleted file mode 100644 index b057df097..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.1.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.1 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml deleted file mode 100644 index 9e06352dd..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.2.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.2 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml deleted file mode 100644 index 79b94c7a8..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.3 diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml deleted file mode 100644 index 619c6ea7d..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.6.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.6 - imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml deleted file mode 100644 index c04ecbc93..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.7.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: clickhouse/clickhouse-server:22.7 - imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml deleted file mode 100644 index a414a07cb..000000000 --- a/tests/e2e/manifests/chit/tpl-clickhouse-22.8.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallationTemplate" - -metadata: - name: clickhouse-version -spec: - defaults: - templates: - podTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-pod - image: altinity/clickhouse-server:22.8.15.25.altinitystable - imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml b/tests/e2e/manifests/chk/test-020017-chk-two-services.yaml similarity index 62% rename from tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml rename to tests/e2e/manifests/chk/test-020017-chk-two-services.yaml index 1b8e6f4c6..2d571782f 100644 --- a/tests/e2e/manifests/chk/test-020013-chk-insecure-baseline.yaml +++ b/tests/e2e/manifests/chk/test-020017-chk-two-services.yaml @@ -1,13 +1,11 @@ apiVersion: "clickhouse-keeper.altinity.com/v1" kind: "ClickHouseKeeperInstallation" metadata: - name: test-020013-insecure + name: test-020017 spec: - # Legacy CHK shape: no cluster.secure flag. Used by test_020013 as a - # regression sentinel — secure-flag normalizer changes must be dormant here - # (no zk-secure Service port, no 1 in Raft XML). Mirrors - # test-020011 minus the secure flag so any divergence is attributable to - # cluster.secure alone. + # No replicaServiceTemplate is declared, so the operator emits the two default + # per-host Services (issue #1982): a peer/Raft Service (publishNotReadyAddresses=true) + # and a client-facing Service (publishNotReadyAddresses=false, "-client" suffix). defaults: templates: podTemplate: default diff --git a/tests/e2e/manifests/chk/test-030003.yaml b/tests/e2e/manifests/chk/test-030003.yaml index e1c0bc736..0b2d288ae 100644 --- a/tests/e2e/manifests/chk/test-030003.yaml +++ b/tests/e2e/manifests/chk/test-030003.yaml @@ -26,14 +26,14 @@ spec: /etc/clickhouse-server/secrets.d/server.key/clickhouse-certs/server.key none - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 /etc/clickhouse-server/secrets.d/ca.crt/clickhouse-certs/ca.crt false strict - sslv2,sslv3,tlsv1,tlsv1_1 + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384 diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml new file mode 100644 index 000000000..576de0cc3 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-good.yaml @@ -0,0 +1,20 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-good" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Source the operator's ClickHouse-TLS rootCA from a Secret in the operator + # namespace holding the SAN-correct CA that issued the server cert. + rootCASecretRef: + name: test-010080-correct-ca + key: ca.crt + security: + clickhouse: + tls: + # Strict turns off InsecureSkipVerify so the secret-sourced rootCA is + # actually exercised on the wire (otherwise verification is bypassed). + verify: Strict diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml new file mode 100644 index 000000000..e937c5ba6 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-missing.yaml @@ -0,0 +1,17 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-missing" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Fail-open control: the referenced Secret does not exist. The operator + # must log a Warning, leave rootCA empty, and keep running/reconciling + # (no crashloop). scheme stays https but verify is left default (not + # Strict) so the operator still connects and serves — proving the bad + # ref did not brick the operator, isolated from a verification failure. + rootCASecretRef: + name: test-010080-no-such-secret + key: ca.crt diff --git a/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml b/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml new file mode 100644 index 000000000..33a0bf2cc --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-010080-chopconf-wrong.yaml @@ -0,0 +1,18 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-010080-chopconf-wrong" +spec: + clickhouse: + access: + scheme: https + port: 8443 + # Negative control: an unrelated CA that did NOT issue the server cert. + # With verify:Strict the operator -> ClickHouse handshake must fail. + rootCASecretRef: + name: test-010080-wrong-ca + key: ca.crt + security: + clickhouse: + tls: + verify: Strict diff --git a/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml b/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml new file mode 100644 index 000000000..947e3e7ea --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-030008-permissive-chopconf.yaml @@ -0,0 +1,8 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-030008-permissive-chopconf" +spec: + security: + images: + policy: Permissive \ No newline at end of file diff --git a/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml b/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml index 926fd410c..a7bd4b2c7 100644 --- a/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml +++ b/tests/e2e/manifests/chopconf/test-035-1-auto-recovery-disabled.yaml @@ -10,7 +10,7 @@ spec: update: timeout: 30 recovery: - from: + onStatus: aborted: # Opt-out: CHI should STAY Aborted even when pod becomes Ready onPodReady: none diff --git a/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml new file mode 100644 index 000000000..f3920e9b6 --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-035-2-sustained-not-ready.yaml @@ -0,0 +1,14 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-035-2-sustained-not-ready" +spec: + reconcile: + recovery: + onStatus: + # Recovery is OFF by default, so the test must explicitly opt in with + # onPodNotReady: retry. Shorten the sustained-NotReady threshold from the 5m + # default so the recovery fires within the test's 420s window. + completed: + onPodNotReady: retry + onPodNotReadyThreshold: 30s diff --git a/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml new file mode 100644 index 000000000..0a3ae521e --- /dev/null +++ b/tests/e2e/manifests/chopconf/test-035-3-opt-out.yaml @@ -0,0 +1,14 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseOperatorConfiguration" +metadata: + name: "test-035-3-opt-out" +spec: + reconcile: + recovery: + onStatus: + # Explicit opt-out: even with an aggressive 30s threshold, onPodNotReady=none + # must leave a sustained-NotReady pod alone (no force-recreate). Isolates the + # on/off knob from the threshold — proves the default-off contract. + completed: + onPodNotReady: none + onPodNotReadyThreshold: 30s diff --git a/tests/e2e/manifests/chopconf/test-058-chopconf.yaml b/tests/e2e/manifests/chopconf/test-058-chopconf.yaml index 82df14e9c..7d3065474 100644 --- a/tests/e2e/manifests/chopconf/test-058-chopconf.yaml +++ b/tests/e2e/manifests/chopconf/test-058-chopconf.yaml @@ -8,22 +8,21 @@ spec: scheme: https # Use HTTPS for connecting to ClickHouse rootCA: |- -----BEGIN CERTIFICATE----- - MIIDFzCCAf+gAwIBAgIUKPAilo3+YvoeiIkvieiBp5GaXYUwDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE1MDda - Fw0yODA2MTgxMzE1MDdaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi - MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv5k1vd7s1KrPnENFB9Tw0dtYT - wlIzpulIKuXmbEGNXIB9SEV69A7UxUZrwF585kFX91LVq+SOb3WD0/KWjs7N+hXq - RLiLuObBVrehoFFRHUca/JZS3Gz9Wsrlsr8twnX5pxavfmnhXOdw/+3P47e22kQ8 - zeAKaSfJ2wF9U9gic/uQWZmLUohrUaT0AejSQpXMm2dlk4ZhBos5BnDbQ+R+rsXu - WiTR4aS3W5Not/mV9neVJZpv2k5+02G0Wdhwy93Q44kEezSVBnSz3hvzEI5RmA72 - LhqTR/Z3y7wZf57vxkqKjqaampwzIhFxtcpAl3j+UVHPV7WkrJ59OkfaxLdtAgMB - AAGjUzBRMB0GA1UdDgQWBBSPxwYV5VGN3/J5N8ipfzefxQyvAjAfBgNVHSMEGDAW - gBSPxwYV5VGN3/J5N8ipfzefxQyvAjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 - DQEBCwUAA4IBAQCAKisQ/Ez9t88CzbrKkC4MA7rqLvHwV33sC9ttbsILk5kwvlyQ - yeeVYme6+KyK28UzuvUQrge6+JY4a33ki4G+gcltXHUaCxSWMGl20Kx++533uH4W - bLXmEPDsR1iPh+sl+3zJBs/aH3HSovBeaLu0pFRupKW5HWDDxgBz92JLVHfqHfY5 - U4bHqOiLopbcRKOQTRAqP9IQsbCmVr4PU8/LWvdMWrYTpn5IIcq1CD0GpunQBxv1 - N7YosHN5QijMvdHVTdR1B7m3ylJa5cVUPaR6HDrDkXJibaFdZBa0eIXZc3KwTon3 - q/+BIPQNS7JiXY82j8OC2HcPFSuS7t5wqcQN + MIIDGTCCAgGgAwIBAgIUXXjvd9XZQrs8ZAlYd7dCmT3FTnAwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTCC + ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANiokuw/31gySsIwAirYaHGW + 5WaW+qTf3FttUH7hX6nf5K5FoDD6vGEjDIt/nufbgeQDx8x7tNT0kuJW7ocCohaY + rhXHVwpaTRMZPGbfOpX1cPI0HRAEmWEkM7dkUg4kbtKrfcybynMVnN4Ieu4s2ruo + GKETKieb8TWuH3YNjAOMIRZK7foiuqDSyW5O62N9pAH6Y1WFg30kzkAgkrLwHFRU + LeC6Uup7/KB5sqiQMJo8EMo/VxLHfgZW0IgB7Ck0H0mNpEuaWFwXKa6zB0RYTqjY + M8FUCWruT0JxlUF+53HfrCVLFMiHq0vEtK0w1YoygCcgFKyypw2k91SmpMFaFKcC + AwEAAaNTMFEwHQYDVR0OBBYEFFe+dG2Y4vcOX7BtHZDBALSqwb7dMB8GA1UdIwQY + MBaAFFe+dG2Y4vcOX7BtHZDBALSqwb7dMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI + hvcNAQELBQADggEBAFZo6eOhHdgJQll3Nyvn7uXg3oA/Vof4rADESdhxeyY5tSUR + e/JWElXOKMDWAzzOIlTyvc0yzmC27ksYYqQA+/WNq88Q2kT7NHHyERnCPf8dklFl + OPAcAsg5LmH4l52m37RWkN+CIr6Gj6r5zF5uX8RfEMQPOFsj3QdgqLvA/Y3+b9EZ + BSg4GWOHpm4EktdvNVfzrYazt0Ithv4mtS4k6Q2mjUUesilwhAx/7x+3xCtnT7I6 + SeElKpvA3J7JcMjGICT0ke6jxgWpZ2gqyf9C4wQgfawZgt4Y9ZOkSFTSmykJfi+b + 2lekI4vDuoCrKJ+39/8n0FUsN3O93WDnAUg+SHY= -----END CERTIFICATE----- - diff --git a/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml b/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml new file mode 100644 index 000000000..91790b03a --- /dev/null +++ b/tests/e2e/manifests/secret/test-010080-wrong-ca.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Secret +metadata: + name: test-010080-wrong-ca +type: Opaque +stringData: + # Self-signed CA (CN=test-077-unrelated-ca.example) that did NOT issue the + # ClickHouse server cert. Used as the negative control: pointing the operator's + # access.rootCASecretRef at it with verify:Strict must fail chain verification, + # proving the secret-sourced CA is actually used (not bypassed). + ca.crt: |- + -----BEGIN CERTIFICATE----- + MIIDMTCCAhmgAwIBAgIUW3Bc7vzCdyEauj+8RIRKaAPhx4IwDQYJKoZIhvcNAQEL + BQAwKDEmMCQGA1UEAwwddGVzdC0wNzctdW5yZWxhdGVkLWNhLmV4YW1wbGUwHhcN + MjYwNTIwMDg0NzI0WhcNMzYwNTE3MDg0NzI0WjAoMSYwJAYDVQQDDB10ZXN0LTA3 + Ny11bnJlbGF0ZWQtY2EuZXhhbXBsZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC + AQoCggEBANluYJRN5a+2jRSEPKTYbh9zw0Q+F9iSNPFUZ9Wbn8AwAYoerc7I2t5g + 9V6FvN893YKtJ1JitbPYH7Yd7e4j+8kmJS1h0u9d6HEzO0RaB6bOn2jqY92MLPIi + g1u/Pda7hp9hj/5VJSrwF/CehA24TXIdFEObQaoYYDTkqJkjfCTtau9ukvF5p8BI + 6hMHMnP8dymka+jhwF18xv46JGJMpOiJ+joQu79QLbPlEsOItPEV+WLC54iw7Cmb + wnXnvX4DMszB/ZrEcMFk+RAkH5KiU/s3jN1fg8KUmAqzxOs880zsCSvJGpemYYkp + 2B2TlJlJ+fl7QJWcreuUMqnvtCiRUCcCAwEAAaNTMFEwHQYDVR0OBBYEFH6TciYq + /FLps7gEpqnl8oX8yX/3MB8GA1UdIwQYMBaAFH6TciYq/FLps7gEpqnl8oX8yX/3 + MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAEywSmeyPprRvTk4 + +M6HN8oWwMcCCAso3eaCtb41fWsqYrA8wLln0FlNpV/tW7JhzMLUXgPfVcehYLK+ + K/Xt/Qm7gmPAIUlUkS/MSFJ5s3WwK5FV+HJjqVtB+WNpUZEvIhHNfiS/EkRFFoWJ + LnHEYH0ltFM1xnvYfDVFSyQ9B/TBoM+1RRBREWwOsbNLiJI34adcNz9pHImaPmPD + aOPja7/pR9FDJ/9amjcDdUCJ2W5ZkVdqde/Z7LA/cAhitIB2BamHXYslC04Xz2nc + 8QqTsoPhJ8e7f1JaCUvInvqF8brbYXeKS0Zurr8j8m3ddHRhJ40w9dZKCvPFR7m8 + +8l7sxo= + -----END CERTIFICATE----- diff --git a/tests/e2e/manifests/secret/test-058-secret.yaml b/tests/e2e/manifests/secret/test-058-secret.yaml index 917154caa..34bfc011c 100644 --- a/tests/e2e/manifests/secret/test-058-secret.yaml +++ b/tests/e2e/manifests/secret/test-058-secret.yaml @@ -7,70 +7,69 @@ type: Opaque stringData: ca.crt: |- -----BEGIN CERTIFICATE----- - MIIDFzCCAf+gAwIBAgIUKPAilo3+YvoeiIkvieiBp5GaXYUwDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE1MDda - Fw0yODA2MTgxMzE1MDdaMBsxGTAXBgNVBAMMEG1hcnNuZXQubG9jYWwgQ0EwggEi - MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv5k1vd7s1KrPnENFB9Tw0dtYT - wlIzpulIKuXmbEGNXIB9SEV69A7UxUZrwF585kFX91LVq+SOb3WD0/KWjs7N+hXq - RLiLuObBVrehoFFRHUca/JZS3Gz9Wsrlsr8twnX5pxavfmnhXOdw/+3P47e22kQ8 - zeAKaSfJ2wF9U9gic/uQWZmLUohrUaT0AejSQpXMm2dlk4ZhBos5BnDbQ+R+rsXu - WiTR4aS3W5Not/mV9neVJZpv2k5+02G0Wdhwy93Q44kEezSVBnSz3hvzEI5RmA72 - LhqTR/Z3y7wZf57vxkqKjqaampwzIhFxtcpAl3j+UVHPV7WkrJ59OkfaxLdtAgMB - AAGjUzBRMB0GA1UdDgQWBBSPxwYV5VGN3/J5N8ipfzefxQyvAjAfBgNVHSMEGDAW - gBSPxwYV5VGN3/J5N8ipfzefxQyvAjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 - DQEBCwUAA4IBAQCAKisQ/Ez9t88CzbrKkC4MA7rqLvHwV33sC9ttbsILk5kwvlyQ - yeeVYme6+KyK28UzuvUQrge6+JY4a33ki4G+gcltXHUaCxSWMGl20Kx++533uH4W - bLXmEPDsR1iPh+sl+3zJBs/aH3HSovBeaLu0pFRupKW5HWDDxgBz92JLVHfqHfY5 - U4bHqOiLopbcRKOQTRAqP9IQsbCmVr4PU8/LWvdMWrYTpn5IIcq1CD0GpunQBxv1 - N7YosHN5QijMvdHVTdR1B7m3ylJa5cVUPaR6HDrDkXJibaFdZBa0eIXZc3KwTon3 - q/+BIPQNS7JiXY82j8OC2HcPFSuS7t5wqcQN + MIIDGTCCAgGgAwIBAgIUXXjvd9XZQrs8ZAlYd7dCmT3FTnAwDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTCC + ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANiokuw/31gySsIwAirYaHGW + 5WaW+qTf3FttUH7hX6nf5K5FoDD6vGEjDIt/nufbgeQDx8x7tNT0kuJW7ocCohaY + rhXHVwpaTRMZPGbfOpX1cPI0HRAEmWEkM7dkUg4kbtKrfcybynMVnN4Ieu4s2ruo + GKETKieb8TWuH3YNjAOMIRZK7foiuqDSyW5O62N9pAH6Y1WFg30kzkAgkrLwHFRU + LeC6Uup7/KB5sqiQMJo8EMo/VxLHfgZW0IgB7Ck0H0mNpEuaWFwXKa6zB0RYTqjY + M8FUCWruT0JxlUF+53HfrCVLFMiHq0vEtK0w1YoygCcgFKyypw2k91SmpMFaFKcC + AwEAAaNTMFEwHQYDVR0OBBYEFFe+dG2Y4vcOX7BtHZDBALSqwb7dMB8GA1UdIwQY + MBaAFFe+dG2Y4vcOX7BtHZDBALSqwb7dMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI + hvcNAQELBQADggEBAFZo6eOhHdgJQll3Nyvn7uXg3oA/Vof4rADESdhxeyY5tSUR + e/JWElXOKMDWAzzOIlTyvc0yzmC27ksYYqQA+/WNq88Q2kT7NHHyERnCPf8dklFl + OPAcAsg5LmH4l52m37RWkN+CIr6Gj6r5zF5uX8RfEMQPOFsj3QdgqLvA/Y3+b9EZ + BSg4GWOHpm4EktdvNVfzrYazt0Ithv4mtS4k6Q2mjUUesilwhAx/7x+3xCtnT7I6 + SeElKpvA3J7JcMjGICT0ke6jxgWpZ2gqyf9C4wQgfawZgt4Y9ZOkSFTSmykJfi+b + 2lekI4vDuoCrKJ+39/8n0FUsN3O93WDnAUg+SHY= -----END CERTIFICATE----- server.crt: |- -----BEGIN CERTIFICATE----- - MIIDJTCCAg2gAwIBAgIUFZBt5OVfoThrbao6LQ+CA1c2538wDQYJKoZIhvcNAQEL - BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAeFw0yNTA2MTkxMzE2MTNa - Fw0yNjA2MTkxMzE2MTNaMBIxEDAOBgNVBAMMB2Nobm9kZTEwggEiMA0GCSqGSIb3 - DQEBAQUAA4IBDwAwggEKAoIBAQDMqNisnlgEiRTYCk1dEvgcRN7FQiNDxwWKSGjo - zhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTWgBFMPbbkrlGgYisoD8GDRKN0 - Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF5t/NuetMVMqxwe4mA+5RB4hW - rlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs6BkE1uVxFg+UQ38R1wnsE/up - Yj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1HgENnP6n/nN0FG8N6z3yolKk2 - xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2Qs3ntY3/AgMBAAGjajBoMCYG - A1UdEQQfMB2CFWNobm9kZTEubWFyc25ldC5sb2NhbIcEwKgB3TAdBgNVHQ4EFgQU - 3E9tBAiCqqJLSVQKi5sXzHLC9PkwHwYDVR0jBBgwFoAUj8cGFeVRjd/yeTfIqX83 - n8UMrwIwDQYJKoZIhvcNAQELBQADggEBABQIXDzlc/wINnkSfcncEAfIY5WvVTdl - Nilr6nVd1Fgq7JAlVD1WrbZ52xZLK3xg+T99Wezks/Js9x243DWt+qwlCKe/xlrC - ezzI3BunnhRxw/7IRm0soTvPNNImcZ2Fuwhn/ojlOg+37NttdTLKlJu2+RguRWLf - 95sXdxhhfTWrkhe1gWFmmDyl02hFpWRO1A/Ogy+hJ+yuruTrlokcM5zJ1L50kyi5 - iL3/ZXMZxWw1BDSSXlbUUZQnwXJuHQbVTd4NwRO6tLVlQyNj45gRf3WeygVzbQmq - sAjL7iOqF7iCwHEri5FWyDJ0Sj5b2YJKV1LLIEMiE8dJesN0O+OlH9U= + MIIDIzCCAgugAwIBAgIUNO0av7W3Je8qhH5yoNm2HkIxoO4wDQYJKoZIhvcNAQEL + BQAwGzEZMBcGA1UEAwwQbWFyc25ldC5sb2NhbCBDQTAgFw0yNjA2MjAxNTI0NDFa + GA8yMTI2MDUyNzE1MjQ0MVowEjEQMA4GA1UEAwwHY2hub2RlMTCCASIwDQYJKoZI + hvcNAQEBBQADggEPADCCAQoCggEBAKWDZU6xTV2iwFP/bpWh6StKvEuAHYUgJwnm + bYGKoS7ssuj2BRO128JEjjJHCHqzgaIPzb0W3371b0XkJCl6EHFED6cxIvjF65Rv + hVO80rDzk4NOMxpb5naFX+Vb2l0zqwlmjvyvqUyE+sYj3pWZ4930uLddFU9mPxRs + mxJ9Xti2FPwpifL/lQLygxkqZWOnfM/gdxLaOexx/GeVrWfzVXPfxXkYuitUs6Le + RCxmsX+td/+DBmTla6fgStFmC6zt+XsiD1LjacFJ4UCeA3Y9AcNT8WyB2wigHbAm + Eth4Hdu77z0pL4VviTpi5IVmBZ/8tvcPU2vq44cTkNnAgEiNNPsCAwEAAaNmMGQw + YgYDVR0RBFswWYIVY2hub2RlMS5tYXJzbmV0LmxvY2FsgiBjaGktdGVzdC0wNTgt + cm9vdC1jYS1kZWZhdWx0LTAtMIIYKi50ZXN0LnN2Yy5jbHVzdGVyLmxvY2FshwTA + qAHdMA0GCSqGSIb3DQEBCwUAA4IBAQB+CnyOTWV7qoG5shpK/2YWCZF+wPiD5eRP + knVx4m3n+u86hpMoT3pjvDWjdO6KFHpijTO5JRBcnB14jSWPsNqTfnaZZrKTZ5e9 + t43uDUJ7urTTVY3RZZTntr1H/cRlRfgwTcUaNZ+Y50wpqAC/uD68m8imsctX7jhg + afM/f20T268xVYdWRL/+9EPW15lmUJQlbf/iPuBdjr/rJNkqDMcxu0DURogvEjWy + EmmY709MNK8icxlhkNWnI8NFzpe8dTqfxnznAO8Gd3xEHajXGw+EAbL9c2/ZYgEF + 4Nry8iGAqXDULGuaRNgx/458ZvNeDN8P1+LEVsB8pv18GQj8/MLK -----END CERTIFICATE----- server.key: |- - -----BEGIN PRIVATE KEY----- - MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDMqNisnlgEiRTY - Ck1dEvgcRN7FQiNDxwWKSGjozhsbQYmS7ZcRD6wRKQD9Sb62eKtPGvrOrC5dIdTW - gBFMPbbkrlGgYisoD8GDRKN0Bw8E0/g8eC+jl2TjwZ4F+zsJeMZXhchJeXFCrMTF - 5t/NuetMVMqxwe4mA+5RB4hWrlxwJ4PPdgNBb9nJ5X28cDnLkrfexnd4UY+CkpFs - 6BkE1uVxFg+UQ38R1wnsE/upYj1MSbDig3gcz9UmHyAeUFyEoZmyxbJ53b/tVA1H - gENnP6n/nN0FG8N6z3yolKk2xjdHHEJTkuA9dzvO3zRLS+0z/XeJJRed2AJ0SOD2 - Qs3ntY3/AgMBAAECggEAL9/Nc6/Esibo79KVH1EZJe+8VtNuUWQEeUEP/Wl9MMaH - ao3WeUC7xPXdC+MM0D1xAVuz0NW5MMMBuT2TDk0fc+YNJSHhq4joAQ901ubxzfTR - zD9nEXMQQDDiCM8ok8IjT4T1ga59XpXwn8SulL7Jen0ZPzS4wz7HKEBFVdWKvRct - 24mW0H45ea5M6V1XUaMvzWSHbuFpT6C1MhvjdyEoNGvwbvw+xwwEc8W3tnEAP63d - 8wWPQf9kecVWbvg6ck8ufIfJnEMOR7u20V0sNki/JtD9mRCWEEn2wLunzflYW23x - WHrhPtz0wwoZLLGB2mn8+E9BNgXf06V/2Iitvrwz3QKBgQD2xq4/9H9wjsCAqK/0 - sth4JOIpmCcMT77Vz2iwjkrkJehpS6j1By3FOzQSQkaG8G2j/f9Zr8ofiscU1V/n - B+4ghGl21gZcCzxi67Ygp8rSTUN+ocpHODGN26jIUk8Jtf6c0BsiHJU5CrdcDwm5 - voR5Y1Ixaq5PWpegatUc1GUPKwKBgQDUTyqYyuOA4ul/bt8xYlO+puQ3G8ITTLW2 - JB8a7jdjED7+XMOwWh4CW9M81dnIVP6UyfCdPlSC3W6q3p/DgMPbtV1w93L9AXpS - HN5wrFqPUKAuXqxIWrIwDCk6oRmvQMXui/jUGZUHDK47h5EeU/PbDOWvWSSKzlNW - q3985tRyfQKBgQCVuMFryBmx3spoxO/MlN3FNwuIlOnMDG4KJwaraAmEFoPFrsPZ - tftNGLhlA5TqteCviKFudrs5G+fhefvvnd4aGHwsP3ooSiDfG4eqlGL36Sy0HdEu - GKfoG4dx0o5lo+fQmGp97b2TmC7bSbxq125kf6AUn1cWii5Ig8i87xhJdQKBgEmB - RzwzMmUTKshl+Hw+kMP3QBgcUisgaeEvzF0kkKSJoWWrdE0ARleGtzHe0FHdq26U - I+wtAlF0nLYn8aRcVnMg7cMIyRTziAgZ2qGj6o6n2W10da1vSTX9X+DemeflQyH9 - 8B5u5PvV1hTiMMoRQuJaKsN014P/PzdIlREHUhJ5AoGBAOODAt0nM4Jz4u8DykI/ - sPjMZ0yY62FWdAvzbIZAiZ/OPPE+fV1OGAsakot01mAym9xWM7kgIt3oWDPfaLN1 - vyt/evXV9YfnyVbhLfxbonhCWHRCiLE7CTulbQpdDkYx6D87SmEdtKbmJa4lwpVK - zBy5BhOMzamzojiV/4d1B30k - -----END PRIVATE KEY----- + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEApYNlTrFNXaLAU/9ulaHpK0q8S4AdhSAnCeZtgYqhLuyy6PYF + E7XbwkSOMkcIerOBog/NvRbffvVvReQkKXoQcUQPpzEi+MXrlG+FU7zSsPOTg04z + GlvmdoVf5VvaXTOrCWaO/K+pTIT6xiPelZnj3fS4t10VT2Y/FGybEn1e2LYU/CmJ + 8v+VAvKDGSplY6d8z+B3Eto57HH8Z5WtZ/NVc9/FeRi6K1Szot5ELGaxf613/4MG + ZOVrp+BK0WYLrO35eyIPUuNpwUnhQJ4Ddj0Bw1PxbIHbCKAdsCYS2Hgd27vvPSkv + hW+JOmLkhWYFn/y29w9Ta+rjhxOQ2cCASI00+wIDAQABAoIBAErDw+t8I9p8Piyz + YZyt+snXhJ8GTE9qargKIsU1fgHYYijhmQGCULa8iQ8lDyt+ErzGLsWPo32SGKWV + nNAvl2XSvM9lXsrJfNUcWzmsPfA41xWlKWhqwvwe22aby1P2lvg0H7r9DpjGKRF/ + +nfRgCEu/pG1tn6bTTtIo/QCNenl/DUTS5Qi5oXKIKWE5t+N2xAKENZyktMwjfze + zygPPGO8/zYvPdciN65rdLLGEVScPpTfxg8MxEALK6p3bPi75hQq/iH7stB3v9YK + wmPym3nUu2TwCtc4xr9uibhHr3W8Qp5WqtuZQNpCKS3BRlvlnOgWpoJwDocza3Pg + DEBKiAECgYEAzeXkBC3oHflpx202CM4mkcoCIwaqD9AcEa2IfdFyzikDGGVBHZmY + J44PBJdS7LeAmdefE1Ujq/SQ6X7/QbYfH8/3OGMAYD3Kac7SeV8Sb9BGUwjwZSR6 + aHYS07GoiyrB/VISg1+luUEJKohQhZxCBtaTBgpTGDDzvU+akJ61ZfsCgYEAzcnM + LOnOAfHD5TedgMLYPCeMnozDk2McqRAR2vTGBAhVZ2ulE1SdmXZKmojX0s1ifTmm + Nza/9Mn4s0ksgn/77mMtK/w4OalRLsFu52QENwI8gxVC8XzC5u5HDWvoypOK8WCC + ngPsgI9Oa6Smo2ys4TH97D5J/kJbTEyVG2qxPQECgYAFJjWwoRFIBp/Nm/6Y88bl + KH8rLxR7tsGs84ERXHaZj08DgizBt8ClZJkdjUdGokQ2FL1mt19gAorJPCLYGtzm + Z8YQA/HTdlgkk0aSQH1ujG/lzbhtXx8sk59e6feEG3qkgjPyUycK3gSDqssQvFqu + XxloMkPnu/msh1wfN8jjlwKBgQCXfaetpIyAB+9S7UcoQ8eVOPQev7c15+9wUaEj + U6/1xgDA+pByE4dVMqym6Hgg+gs37ll7KfXTiV9o9EQs6XSXwDC/wZPOduOJjOJM + uucTa7UKNnuqdFKyV9S8f6TGhCjzmj1tf6v51AVB3trBUb5OpVOtNwmXgFffaj0W + CsvhAQKBgB4IYNSEie5b8orY/XO5ONkBEOCNxO0kuuv0xTjNx5lRaFePuCAUEQYa + HvzKUwNPvw/ezFkUXv/oWWX3ZhW7ILCF/IWi3mkX9ZJ74HJGbWFsy1pZyH+4tn8b + v9yJTkkLkMWaqiPsbsijAzMCg8Y07+gpcOPjHAoXe8ty6gfgKdAs + -----END RSA PRIVATE KEY----- diff --git a/tests/e2e/merge_dual_results.py b/tests/e2e/merge_dual_results.py new file mode 100755 index 000000000..a2f1a532f --- /dev/null +++ b/tests/e2e/merge_dual_results.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Fuse the result lists of dual-cluster e2e raw logs into ONE unified report. + +Each raw log is a complete, independent TestFlows run, so the logs cannot be +concatenated and replayed: their test-id namespaces both root at the same path, the +ids collide, and the later run's scenarios are silently dropped. Instead we render +each log's ``tfs transform short`` separately, then fuse the per-scenario result lines +into a single Passing/Failing listing and re-tally the scenario/step totals. + +Usage: merge_dual_results.py [ ...] +Prints the unified report; exits 1 if any scenario failed/errored, else 0. +""" +import re +import subprocess +import sys + +ANSI = re.compile(r"\x1b\[[0-9;]*m") +# Leaf scenario result line, e.g. "[ OK ] /regression/.../test_010072. name (8m 58s)". +# Rollup lines (/regression, /regression/e2e.test_operator) lack "/test_" -> excluded. +RESULT = re.compile(r"\[\s*(OK|Fail|Error|Skip)\s*\]\s+(/regression/\S*/test_\S.*)$") +# Scenario counts are derived from deduped RESULT lines (retry-safe), not this summary, +# so there is no SCEN regex. STEP is still read from the summary for the informational tally. +STEP = re.compile(r"(\d+)\s+steps?\s+\(([^)]*)\)") +TOTAL = re.compile(r"Total time\s+(.+)") +BREAKDOWN = re.compile(r"(\d+)\s+(ok|failed|skipped|errored)") +PASS_SYMBOL, FAIL_SYMBOL = "✔", "✘" + + +def transform(raw): + with open(raw, "rb") as fh: + proc = subprocess.run( + ["tfs", "--no-colors", "transform", "short"], + stdin=fh, capture_output=True, text=True, + ) + return ANSI.sub("", proc.stdout) + + +def breakdown(counts): + parts = [f"{v} {k}" for k, v in counts.items() if v] + return ", ".join(parts) if parts else "0" + + +_STATUS_KEY = {"OK": "ok", "Fail": "failed", "Skip": "skipped", "Error": "errored"} +_TIME_SUFFIX = re.compile(r"\s*\([0-9hms .]+\)\s*$") # trailing " (10m 13s)" / " (16s 903ms)" + + +def parse(raw, label): + # Dedup by scenario (path+name, minus the trailing "(time)") so a test that was + # retried fail->...->ok collapses to ONE entry, with OK/Skip winning over Fail/Error + # — i.e. a scenario that eventually passed counts as passing. Scenario counts are + # derived from these deduped results, so --retry never double-counts. Step counts + # come from the transform summary (informational; may include retry attempts). + by_scenario = {} # scenario-key -> (status, formatted entry line) + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + n_steps = 0 + total = "?" + for line in transform(raw).splitlines(): + m = RESULT.search(line) + if m: + status, rest = m.group(1), m.group(2).strip() + key = _TIME_SUFFIX.sub("", rest) + passed = status in ("OK", "Skip") + prev = by_scenario.get(key) + # First sighting, or a later pass that supersedes an earlier fail (retry won). + if prev is None or (passed and prev[0] in ("Fail", "Error")): + symbol = PASS_SYMBOL if passed else FAIL_SYMBOL + by_scenario[key] = (status, f"{symbol} [ {status} ] [{label}] {rest}") + continue + st = STEP.search(line) + if st: + n_steps = int(st.group(1)) + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + steps.update({w: int(n) for n, w in BREAKDOWN.findall(st.group(2)) if w in steps}) + tm = TOTAL.search(line) + if tm: + total = tm.group(1).strip() + passing = [e for s, e in by_scenario.values() if s in ("OK", "Skip")] + failing = [e for s, e in by_scenario.values() if s in ("Fail", "Error")] + scen = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + for s, _ in by_scenario.values(): + scen[_STATUS_KEY[s]] += 1 + return {"label": label, "passing": passing, "failing": failing, + "n_scen": len(by_scenario), "scen": scen, "n_steps": n_steps, "steps": steps, "total": total} + + +def main(argv): + args = argv[1:] + if len(args) < 2 or len(args) % 2: + sys.exit("usage: merge_dual_results.py [ ...]") + runs = [parse(args[i], args[i + 1]) for i in range(0, len(args), 2)] + + passing = [e for r in runs for e in r["passing"]] + failing = [e for r in runs for e in r["failing"]] + scen = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + steps = {"ok": 0, "failed": 0, "skipped": 0, "errored": 0} + n_scen = n_steps = 0 + for r in runs: + n_scen += r["n_scen"] + n_steps += r["n_steps"] + for k in scen: + scen[k] += r["scen"][k] + for k in steps: + steps[k] += r["steps"][k] + + print() + print("==================== COMBINED dual-cluster results ====================") + if passing: + print("\nPassing\n") + print("\n".join(passing)) + if failing: + print("\nFailing\n") + print("\n".join(failing)) + print() + print(f"{n_scen} scenarios ({breakdown(scen)})") + print(f"{n_steps} steps ({breakdown(steps)})") + for r in runs: + print(f" {r['label']}: {r['n_scen']} scenarios ({breakdown(r['scen'])}), total time {r['total']}") + print("=======================================================================") + + # Prominent, scannable headline — the one line to read for the run's outcome. + failed_total = scen["failed"] + scen["errored"] + bar = "#" * 71 + print() + print(bar) + if failed_total == 0: + print(f"### RESULT: ALL OK — {scen['ok']}/{n_scen} scenarios passed, 0 failed") + else: + print(f"### RESULT: FAILED TESTS: {failed_total} " + f"({scen['failed']} failed, {scen['errored']} errored of {n_scen}) — see 'Failing' above") + print(bar) + return 1 if failed_total else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/e2e/run_minikube_dual_reset.sh b/tests/e2e/run_minikube_dual_reset.sh new file mode 100755 index 000000000..ad77a689c --- /dev/null +++ b/tests/e2e/run_minikube_dual_reset.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Reset TWO isolated minikube profiles concurrently for dual-cluster e2e. +# +# k8s-par hosts the PARALLEL workload, k8s-seq the SEQUENTIAL (NO_PARALLEL) workload. +# Each profile gets its own kubeconfig file so the two concurrent `minikube start` +# calls never race on a shared ~/.kube/config, and a split of host CPU/RAM. The +# single-cluster path (run_minikube_reset.sh with no MINIKUBE_PROFILE) is untouched; +# this wrapper just drives it twice with distinct profiles. Used by +# run_tests_operator_dual.sh. +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +PROFILE_PAR="${MINIKUBE_PROFILE_PAR:-k8s-par}" +PROFILE_SEQ="${MINIKUBE_PROFILE_SEQ:-k8s-seq}" +KUBECONFIG_PAR="${KUBECONFIG_PAR:-${HOME}/.kube/${PROFILE_PAR}.config}" +KUBECONFIG_SEQ="${KUBECONFIG_SEQ:-${HOME}/.kube/${PROFILE_SEQ}.config}" +# ASYMMETRIC resource split (override per host). k8s-par runs the full PARALLEL pool +# (POOL_SIZE scenarios at once, each spinning its own operator + ClickHouse pods), so +# it needs the BULK of the host. k8s-seq runs NO_PARALLEL tests SERIALLY (one at a +# time) and needs only a small slice. An equal split starves k8s-par: its control +# plane and operator-restart paths race under load (apiserver transients, +# pod-not-found), which fails operator-mutating tests (e.g. test_010055 chopconf +# restart) and times out others as collateral. Defaults below suit a ~12-CPU / ~31g +# host; tune for yours, and lower POOL_SIZE if k8s-par is still CPU-bound (it can't +# have the whole host like a single-cluster run does). +CPUS_PAR="${CPUS_PAR:-8}" +MEMORY_PAR="${MEMORY_PAR:-16g}" +CPUS_SEQ="${CPUS_SEQ:-4}" +MEMORY_SEQ="${MEMORY_SEQ:-8g}" + +reset_one() { + local profile="$1" kubeconfig="$2" cpus="$3" memory="$4" + # MINIKUBE_PROFILE (non-default) makes run_minikube_reset.sh target this profile + # AND skip the destructive cross-profile prune + k9s; a dedicated KUBECONFIG + # isolates this cluster's context so the concurrent start does not corrupt the + # sibling's kubeconfig. + SKIP_K9S=yes \ + MINIKUBE_PROFILE="${profile}" \ + KUBECONFIG="${kubeconfig}" \ + CPUS="${cpus}" MEMORY="${memory}" \ + "${CUR_DIR}/run_minikube_reset.sh" +} + +echo "Resetting dual minikube clusters concurrently: ${PROFILE_PAR} (${CPUS_PAR} CPU / ${MEMORY_PAR}) + ${PROFILE_SEQ} (${CPUS_SEQ} CPU / ${MEMORY_SEQ})" +reset_one "${PROFILE_PAR}" "${KUBECONFIG_PAR}" "${CPUS_PAR}" "${MEMORY_PAR}" > "/tmp/minikube_reset_${PROFILE_PAR}.log" 2>&1 & +PID_PAR=$! +reset_one "${PROFILE_SEQ}" "${KUBECONFIG_SEQ}" "${CPUS_SEQ}" "${MEMORY_SEQ}" > "/tmp/minikube_reset_${PROFILE_SEQ}.log" 2>&1 & +PID_SEQ=$! + +wait "${PID_PAR}"; RC_PAR=$? +wait "${PID_SEQ}"; RC_SEQ=$? + +echo "=== ${PROFILE_PAR} reset log tail ==="; tail -8 "/tmp/minikube_reset_${PROFILE_PAR}.log" +echo "=== ${PROFILE_SEQ} reset log tail ==="; tail -8 "/tmp/minikube_reset_${PROFILE_SEQ}.log" +echo "dual reset exit codes: ${PROFILE_PAR}=${RC_PAR} ${PROFILE_SEQ}=${RC_SEQ}" + +[[ "${RC_PAR}" -eq 0 && "${RC_SEQ}" -eq 0 ]] diff --git a/tests/e2e/run_minikube_reset.sh b/tests/e2e/run_minikube_reset.sh index bc5621b2c..9b43aa060 100755 --- a/tests/e2e/run_minikube_reset.sh +++ b/tests/e2e/run_minikube_reset.sh @@ -21,6 +21,11 @@ DOCKER_VERSION="${DOCKER_VERSION:-""}" # Whether to prune minikube during reset process MINIKUBE_PRUNE="${MINIKUBE_PRUNE:-""}" +# Minikube profile to operate on. Defaults to "minikube" (single-cluster, unchanged). +# Set to an isolated name for dual-cluster e2e; for a non-default profile the +# destructive cross-profile prune is skipped so a sibling cluster is never nuked. +MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" + echo "Reset kubernetes cluster." echo "k8s version: ${KUBERNETES_VERSION}" echo "nodes: ${NODES}" @@ -28,9 +33,10 @@ echo "cpus: ${CPUS}" echo "memory: ${MEMORY}" echo "docker prune: ${DOCKER_PRUNE}" echo "minikube prune:${MINIKUBE_PRUNE}" +echo "minikube profile:${MINIKUBE_PROFILE}" echo "Delete cluster" -minikube delete +minikube delete -p "${MINIKUBE_PROFILE}" if [[ ! -z "${DOCKER_PRUNE}" ]]; then echo "Docker system prune" docker system prune -f @@ -39,7 +45,10 @@ if [[ ! -z "${DOCKER_PRUNE_ALL}" ]]; then echo "Docker system prune all" docker system prune -f --all fi -if [[ ! -z "${MINIKUBE_PRUNE}" ]]; then +if [[ ! -z "${MINIKUBE_PRUNE}" && "${MINIKUBE_PROFILE}" == "minikube" ]]; then + # `--all --purge` + `rm -rf ~/.minikube` destroy EVERY profile, so this only + # runs for the default profile. A named (dual-cluster) profile skips it to + # avoid nuking the sibling cluster running concurrently. echo "Minikube prune" minikube stop minikube delete --all --purge @@ -76,13 +85,13 @@ echo "-----------------------" echo "-- Starting minikube --" echo "-----------------------" -minikube start --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" -#minikube start --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" --cache-images=false +minikube start -p "${MINIKUBE_PROFILE}" --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" +#minikube start -p "${MINIKUBE_PROFILE}" --kubernetes-version="${KUBERNETES_VERSION}" --nodes="${NODES}" --cpus="${CPUS}" --memory="${MEMORY}" --cache-images=false echo "Enabling metrics-server addon" -minikube addons enable metrics-server +minikube addons enable metrics-server -p "${MINIKUBE_PROFILE}" -if [[ -z "${SKIP_K9S}" ]]; then +if [[ -z "${SKIP_K9S}" && "${MINIKUBE_PROFILE}" == "minikube" ]]; then echo "Launching k9s" k9s -c ns fi diff --git a/tests/e2e/run_tests_acvp_local.sh b/tests/e2e/run_tests_acvp_local.sh index f33175ead..fbb885e08 100755 --- a/tests/e2e/run_tests_acvp_local.sh +++ b/tests/e2e/run_tests_acvp_local.sh @@ -1,18 +1,20 @@ #!/bin/bash -# Runs the e2e ACVP responder smoke tests only. +# Runs the e2e ACVP responder smoke tests only — host-only, NO minikube cluster +# and NO operator image. The host's Go toolchain and `GOFIPS140=v1.0.0` build env +# are all these tests need. # -# The test module (tests/e2e/test_acvp.py) builds the operator and -# metrics-exporter binaries with `-tags acvp_wrapper`, invokes them via argv0 -# dispatch (binary symlinked as `-acvp`), and round-trips ACVP requests -# over stdin/stdout. NO minikube cluster, NO operator image is required — -# the host's Go toolchain and `GOFIPS140=v1.0.0` build env are all the test -# needs. +# The ACVP scenarios were consolidated into test_operator.py by PR #2031 as +# test_030018 (clickhouse-operator) and test_030019 (metrics-exporter) — they +# build each binary with `-tags acvp_wrapper`, invoke it via argv0 dispatch, and +# round-trip ACVP requests over stdin/stdout. This script runs ONLY those two +# scenarios (via --native, no docker-compose Cluster, no minikube) so `WHAT=all` +# keeps its fast fail-fast crypto pre-flight ahead of the metrics/operator suites. +# They also run inside the full operator suite (test_operator), which is what CI +# executes; this script is the standalone host-only entry point. # -# Full BoringSSL acvptool reproducibility (vector-by-vector comparison -# against geomys/acvp-testdata) lives in pkg/util/fips/acvp/run.sh and is -# reproduced locally per release — this script is the fast pre-flight that -# catches build-tag / argv0-dispatch / FIPS-mode regressions before the -# heavier vector-roundtrip run. +# Full BoringSSL acvptool reproducibility (vector-by-vector vs geomys/acvp-testdata) +# lives in pkg/util/fips/acvp/run.sh and is reproduced locally per release — this is +# the fast pre-flight that catches build-tag / argv0-dispatch / FIPS-mode regressions. CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/test_common.sh" @@ -22,7 +24,7 @@ common_export_test_env RUN_ALL_FLAG=$(common_convert_run_all) python3 "${COMMON_DIR}/../regression.py" \ - --only="/regression/e2e.test_acvp/${ONLY}" \ + --only="/regression/e2e.test_operator/test_03001[89]*" \ ${RUN_ALL_FLAG} \ -o short \ --trim-results on \ diff --git a/tests/e2e/run_tests_local.sh b/tests/e2e/run_tests_local.sh index 1d43898f8..feb93163a 100755 --- a/tests/e2e/run_tests_local.sh +++ b/tests/e2e/run_tests_local.sh @@ -87,6 +87,19 @@ case "${WHAT}" in ;; esac +# Dual-cluster opt-in: route the operator suite to the two-cluster orchestrator +# (PARALLEL pool on one minikube, NO_PARALLEL set on another, merged into one +# table). Only the operator suite is split; acvp/metrics stay single-cluster, so +# DUAL_CLUSTER is most useful with WHAT=operator. Default (unset) is unchanged. +if [[ "${DUAL_CLUSTER:-}" == "yes" ]]; then + for i in "${!LOCAL_SCRIPTS[@]}"; do + if [[ "${LOCAL_SCRIPTS[$i]}" == "run_tests_operator_local.sh" ]]; then + LOCAL_SCRIPTS[$i]="run_tests_operator_dual.sh" + fi + done + echo "DUAL_CLUSTER=yes -> operator suite uses run_tests_operator_dual.sh" +fi + # Only wait for confirmation when running interactively (stdin is a terminal) if [ -t 0 ]; then TIMEOUT=30 diff --git a/tests/e2e/run_tests_metrics_local.sh b/tests/e2e/run_tests_metrics_local.sh index c85ae6e08..c67eecc39 100755 --- a/tests/e2e/run_tests_metrics_local.sh +++ b/tests/e2e/run_tests_metrics_local.sh @@ -9,6 +9,6 @@ MINIKUBE_PRELOAD_IMAGES="${MINIKUBE_PRELOAD_IMAGES:-"yes"}" export MINIKUBE_PRELOAD_IMAGES common_minikube_reset -common_preload_images "${PRELOAD_IMAGES_METRICS[@]}" +common_preload_images "${PRELOAD_IMAGES_ALL[@]}" common_build_and_load_images && \ common_run_test_script "run_tests_metrics.sh" diff --git a/tests/e2e/run_tests_operator.sh b/tests/e2e/run_tests_operator.sh index 4126e5c5f..ceaa5c9b9 100755 --- a/tests/e2e/run_tests_operator.sh +++ b/tests/e2e/run_tests_operator.sh @@ -10,15 +10,27 @@ common_export_test_env RUN_ALL_FLAG=$(common_convert_run_all) RETRY_ARGS=() +# Retry failing scenarios in-process (TestFlows native). Applies to single-cluster AND +# dual: a retried fail->pass writes both attempts to the raw log, but merge_dual_results.py +# dedups by scenario (OK wins over Fail), so a rescued test is counted once as passing. if [[ -n "${RETRY_COUNT}" ]]; then RETRY_ARGS+=(--retry "/regression/e2e.test_operator/test_0:,${RETRY_COUNT},,${RETRY_DELAY:-30}") fi +# Optional untrimmed native raw log for result aggregation. When TF_LOG is set, write +# the full TestFlows log so two concurrent runs can be merged into ONE combined table +# via `tfs transform short`. Unset (single-cluster/CI) -> no --log, argv unchanged. +LOG_ARGS=() +if [[ -n "${TF_LOG:-}" ]]; then + LOG_ARGS+=(--log "${TF_LOG}") +fi + python3 "${COMMON_DIR}/../regression.py" \ --only="/regression/e2e.test_operator/${ONLY}" \ ${RUN_ALL_FLAG} \ "${RETRY_ARGS[@]}" \ + "${LOG_ARGS[@]}" \ -o short \ - --trim-results on \ + --trim-results "${TRIM_RESULTS:-on}" \ --debug \ --native diff --git a/tests/e2e/run_tests_operator_dual.sh b/tests/e2e/run_tests_operator_dual.sh new file mode 100755 index 000000000..45da4156a --- /dev/null +++ b/tests/e2e/run_tests_operator_dual.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Dual-cluster operator e2e: run the PARALLEL-safe scenarios and the NO_PARALLEL +# scenarios SIMULTANEOUSLY against two independent minikube clusters, then merge +# both result sets into ONE combined table. +# +# k8s-par (parallel cluster): E2E_PHASE=parallel, POOL_SIZE threads -> the concurrent pool +# k8s-seq (sequential cluster): E2E_PHASE=serial, POOL_SIZE=1 -> NO_PARALLEL scenarios +# +# PAR_ONLY=yes runs ONLY k8s-par (no k8s-seq) so the parallel cluster gets the whole +# host — used to test high parallelism (e.g. POOL_SIZE=25 on all 12 CPU). +# +# Each suite is a normal run_tests_operator.sh process whose kube comms are pinned to +# its cluster via KUBECTL_CMD=--context/--kubeconfig, emitting an untrimmed raw log; +# the logs are rendered into one combined table by merge_dual_results.py. +# The single-cluster scripts are untouched; this is a separate opt-in entry point. +CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source "${CUR_DIR}/test_common.sh" + +PROFILE_PAR="${MINIKUBE_PROFILE_PAR:-k8s-par}" +PROFILE_SEQ="${MINIKUBE_PROFILE_SEQ:-k8s-seq}" +KUBECONFIG_PAR="${KUBECONFIG_PAR:-${HOME}/.kube/${PROFILE_PAR}.config}" +KUBECONFIG_SEQ="${KUBECONFIG_SEQ:-${HOME}/.kube/${PROFILE_SEQ}.config}" +RAW_PAR="${RAW_PAR:-/tmp/e2e_dual_${PROFILE_PAR}.raw}" +RAW_SEQ="${RAW_SEQ:-/tmp/e2e_dual_${PROFILE_SEQ}.raw}" +OUT_PAR="${OUT_PAR:-/tmp/e2e_dual_${PROFILE_PAR}.out}" +OUT_SEQ="${OUT_SEQ:-/tmp/e2e_dual_${PROFILE_SEQ}.out}" + +# PAR_ONLY=yes: run only the parallel cluster. With k8s-seq not competing for the host, +# k8s-par gets the whole machine, so its CPU/RAM defaults jump (override as needed). +PAR_ONLY="${PAR_ONLY:-}" +if [[ -n "${PAR_ONLY}" ]]; then + export CPUS_PAR="${CPUS_PAR:-12}" + export MEMORY_PAR="${MEMORY_PAR:-28g}" +fi + +# Active profiles drive both image preload/load and (for normal mode) the dual reset. +if [[ -n "${PAR_ONLY}" ]]; then + ACTIVE_PROFILES="${PROFILE_PAR}" +else + ACTIVE_PROFILES="${PROFILE_PAR} ${PROFILE_SEQ}" +fi + +# Tear down the profile(s) on ANY exit (normal, early FATAL, Ctrl-C) so a failed +# reset/build or interrupt never leaks clusters. KEEP_CLUSTERS=yes opts out. Deleting +# a non-existent profile is a harmless no-op (safe on preflight exit / PAR_ONLY). +teardown_clusters() { + [[ -n "${KEEP_CLUSTERS:-}" ]] && return + echo "Tearing down ${ACTIVE_PROFILES} (set KEEP_CLUSTERS=yes to keep)" + local p + for p in ${ACTIVE_PROFILES}; do minikube delete -p "${p}" >/dev/null 2>&1; done +} +trap teardown_clusters EXIT + +# PREFLIGHT: the result merge depends on the `tfs` CLI. Fail LOUD now rather than +# after ~an hour of testing. +command -v tfs >/dev/null 2>&1 || { echo "FATAL: tfs CLI not found (needed to render results)"; exit 3; } +tfs transform short --help >/dev/null 2>&1 || { echo "FATAL: 'tfs transform short' unavailable"; exit 3; } + +# Reset cluster(s) unless explicitly opted out (MINIKUBE_RESET=no). +if [[ "${MINIKUBE_RESET:-yes}" != "no" ]]; then + if [[ -n "${PAR_ONLY}" ]]; then + SKIP_K9S=yes MINIKUBE_PROFILE="${PROFILE_PAR}" KUBECONFIG="${KUBECONFIG_PAR}" \ + CPUS="${CPUS_PAR}" MEMORY="${MEMORY_PAR}" \ + "${CUR_DIR}/run_minikube_reset.sh" || { echo "FATAL: ${PROFILE_PAR} reset failed"; exit 2; } + else + MINIKUBE_PROFILE_PAR="${PROFILE_PAR}" MINIKUBE_PROFILE_SEQ="${PROFILE_SEQ}" \ + KUBECONFIG_PAR="${KUBECONFIG_PAR}" KUBECONFIG_SEQ="${KUBECONFIG_SEQ}" \ + "${CUR_DIR}/run_minikube_dual_reset.sh" || { echo "FATAL: dual minikube reset failed"; exit 2; } + fi +fi + +# Preload the ClickHouse/Keeper/Zookeeper images into the cluster(s) BEFORE the run. +# Without this, ~POOL_SIZE parallel tests each pull large images from the registry +# concurrently -> network contention -> pods stuck ContainerCreating/InProgress for +# minutes (the single-cluster runner preloads these; the dual path must too). +MINIKUBE_PRELOAD_IMAGES=yes MINIKUBE_PROFILES="${ACTIVE_PROFILES}" \ + common_preload_images "${PRELOAD_IMAGES_ALL[@]}" || echo "WARNING: image preload had failures (continuing)" + +# Build operator+metrics images ONCE, load into the active profile(s). +MINIKUBE_PROFILES="${ACTIVE_PROFILES}" common_build_and_load_images || { echo "FATAL: image build/load failed"; exit 2; } + +# k8s-par's pool size. Defaults to 25 threads. Image preload (above) removes the +# per-test image-pull stalls that previously made high parallelism flake; the +# remaining limit is host CPU (~1 CH-server-test per core), so 25 needs ~12 cores — +# which PAR_ONLY gives by handing k8s-par the whole host. Override POOL_SIZE to tune. +POOL_SIZE_PAR="${POOL_SIZE:-25}" + +# Retry failing scenarios in-process. Dual default is 5 (matches single-cluster full runs); +# merge_dual_results.py collapses a retried fail->pass to one passing entry, so retries never +# double-count. Exported so both child run_tests_operator.sh suites inherit it. RETRY_DELAY is +# seconds between attempts (run_tests_operator.sh defaults it to 30). Note: retry masks +# transient flakes but cannot rescue a deterministic resource shortage on an undersized cluster. +export RETRY_COUNT="${RETRY_COUNT:-5}" +export RETRY_DELAY="${RETRY_DELAY:-30}" + +# Each suite streams to the console LIVE, line-prefixed by cluster, AND to a per-suite +# file. `> >(sed | tee file)` is process substitution: the sed|tee runs concurrently +# but is NOT $!, so `wait "${PID_PAR}"` still captures run_tests_operator.sh's status. +# IMAGE_PULL_POLICY=IfNotPresent uses the locally built/preloaded images. +KUBECTL_CMD="kubectl --context=${PROFILE_PAR} --kubeconfig=${KUBECONFIG_PAR}" \ +E2E_PHASE=parallel POOL_SIZE="${POOL_SIZE_PAR}" MINIKUBE_PROFILE="${PROFILE_PAR}" \ +IMAGE_PULL_POLICY="${IMAGE_PULL_POLICY:-IfNotPresent}" \ +TF_LOG="${RAW_PAR}" TRIM_RESULTS=off ONLY="${ONLY:-*}" \ + "${CUR_DIR}/run_tests_operator.sh" > >(sed -u "s/^/[${PROFILE_PAR}] /" | tee "${OUT_PAR}") 2>&1 & +PID_PAR=$! + +if [[ -z "${PAR_ONLY}" ]]; then + KUBECTL_CMD="kubectl --context=${PROFILE_SEQ} --kubeconfig=${KUBECONFIG_SEQ}" \ + E2E_PHASE=serial POOL_SIZE=1 MINIKUBE_PROFILE="${PROFILE_SEQ}" \ + IMAGE_PULL_POLICY="${IMAGE_PULL_POLICY:-IfNotPresent}" \ + TF_LOG="${RAW_SEQ}" TRIM_RESULTS=off ONLY="${ONLY:-*}" \ + "${CUR_DIR}/run_tests_operator.sh" > >(sed -u "s/^/[${PROFILE_SEQ}] /" | tee "${OUT_SEQ}") 2>&1 & + PID_SEQ=$! +fi + +wait "${PID_PAR}"; RC_PAR=$? +RC_SEQ=0 +[[ -z "${PAR_ONLY}" ]] && { wait "${PID_SEQ}"; RC_SEQ=$?; } +# Drain the process-substitution tee/sed pipelines before printing the summary. +wait 2>/dev/null + +# Build the (raw, label) pairs for the merge — only the clusters that actually ran. +MERGE_ARGS=("${RAW_PAR}" "${PROFILE_PAR}") +[[ -z "${PAR_ONLY}" ]] && MERGE_ARGS+=("${RAW_SEQ}" "${PROFILE_SEQ}") +for ((i = 0; i < ${#MERGE_ARGS[@]}; i += 2)); do + f="${MERGE_ARGS[$i]}" + [[ -s "${f}" ]] || { echo "FATAL: raw log ${f} missing/empty — cannot render results"; exit 4; } +done + +# Fuse the run(s) into ONE unified report (single Passing/Failing listing + tally). +# Not `cat *.raw | transform`: each raw log is a complete TestFlows stream whose +# test-id namespace roots at the same path, so concatenation collides ids and drops a +# run's scenarios. The merge helper renders each log separately and fuses result lines. +python3 "${CUR_DIR}/merge_dual_results.py" "${MERGE_ARGS[@]}" || true + +# Combined verdict: fail if EITHER cluster failed. Teardown runs via the EXIT trap. +if [[ "${RC_PAR}" -eq 0 && "${RC_SEQ}" -eq 0 ]]; then COMBINED=0; VERDICT="PASS"; else COMBINED=1; VERDICT="FAIL"; fi +echo +echo "==================== COMBINED dual-cluster verdict: ${VERDICT} ====================" +echo " ${PROFILE_PAR} (parallel) exit=${RC_PAR}" +[[ -z "${PAR_ONLY}" ]] && echo " ${PROFILE_SEQ} (no-parallel) exit=${RC_SEQ}" +echo "===================================================================================" + +exit "${COMBINED}" diff --git a/tests/e2e/run_tests_operator_local.sh b/tests/e2e/run_tests_operator_local.sh index e8c889394..b37d21e3c 100755 --- a/tests/e2e/run_tests_operator_local.sh +++ b/tests/e2e/run_tests_operator_local.sh @@ -21,6 +21,6 @@ export MINIKUBE_PRELOAD_IMAGES export RETRY_COUNT common_minikube_reset -common_preload_images "${PRELOAD_IMAGES_OPERATOR[@]}" +common_preload_images "${PRELOAD_IMAGES_ALL[@]}" common_build_and_load_images && \ common_run_test_script "run_tests_operator.sh" diff --git a/tests/e2e/settings.py b/tests/e2e/settings.py index c1aa19bfe..0687b36a9 100644 --- a/tests/e2e/settings.py +++ b/tests/e2e/settings.py @@ -57,13 +57,6 @@ def get_docker_compose_path(): os.getenv("CLICKHOUSE_TEMPLATE") if "CLICKHOUSE_TEMPLATE" in os.environ else "manifests/chit/tpl-clickhouse-stable.yaml" - # "manifests/chit/tpl-clickhouse-19.17.yaml" - # "manifests/chit/tpl-clickhouse-20.3.yaml" - # "manifests/chit/tpl-clickhouse-20.8.yaml" - # "manifests/chit/tpl-clickhouse-21.3.yaml" - # "manifests/chit/tpl-clickhouse-21.8.yaml" - # "manifests/chit/tpl-clickhouse-22.3.yaml" - # "manifests/chit/tpl-clickhouse-22.8.yaml" # "manifests/chit/tpl-clickhouse-23.3.yaml" # "manifests/chit/tpl-clickhouse-23.8.yaml" ) diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py index 30bba785c..13635b30c 100644 --- a/tests/e2e/steps.py +++ b/tests/e2e/steps.py @@ -5,6 +5,7 @@ import uuid import os import re +import shlex import yaml import time import inspect @@ -92,6 +93,28 @@ def set_settings(self): self.context.kubectl_cmd = define("kubectl_cmd", os.getenv("KUBECTL_CMD") if "KUBECTL_CMD" in os.environ else self.context.kubectl_cmd) + # Dual-cluster e2e: extract the --context / --kubeconfig flags from kubectl_cmd so + # direct subprocess kubectl calls (e.g. the port-forward helpers in steps_fips.py) + # hit the SAME cluster as kubectl.launch(). Empty list for single-cluster runs. + # Only the --flag=value form is recognized (the dual wrapper emits exactly that); + # space-separated "--context foo" would drop the value and is not used. + self.context.kubectl_context_args = [ + arg for arg in shlex.split(self.context.kubectl_cmd) + if arg.startswith(("--context=", "--kubeconfig=")) + ] + # minikube profile for direct `minikube` invocations (e.g. decoy image load). + self.context.minikube_profile = define( + "minikube_profile", os.getenv("MINIKUBE_PROFILE") if "MINIKUBE_PROFILE" in os.environ else "minikube" + ) + # Direct-subprocess kube calls invoke the `kubectl` binary natively, so they can + # only carry --context/--kubeconfig when the suite runs --native. A dual-cluster + # run (context flags present) via the docker-compose runner path cannot route them. + if self.context.kubectl_context_args and not current().context.native: + raise ValueError( + "KUBECTL_CMD carries --context/--kubeconfig but the suite is not --native; " + "dual-cluster runs require --native so port-forward/minikube calls reach the right cluster" + ) + self.context.test_namespace = define("test_namespace", os.getenv("TEST_NAMESPACE") if "TEST_NAMESPACE" in os.environ else "test") self.context.operator_version = define("operator_version", ( os.getenv("OPERATOR_VERSION") @@ -124,13 +147,6 @@ def set_settings(self): self.context.image_pull_policy = define("image_pull_policy", os.getenv("IMAGE_PULL_POLICY") if "IMAGE_PULL_POLICY" in os.environ else "Always") # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-stable.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-19.17.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-20.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-20.8.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-21.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-21.8.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-22.3.yaml" - # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-22.8.yaml" # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-23.3.yaml" # self.context.clickhouse_template = "manifests/chit/tpl-clickhouse-23.8.yaml" self.context.clickhouse_template = define("clickhouse_template", os.getenv("CLICKHOUSE_TEMPLATE") if "CLICKHOUSE_TEMPLATE" in os.environ else "manifests/chit/tpl-clickhouse-stable.yaml") diff --git a/tests/e2e/steps_fips.py b/tests/e2e/steps_fips.py index 861a6ed5a..03ea17433 100644 --- a/tests/e2e/steps_fips.py +++ b/tests/e2e/steps_fips.py @@ -12,26 +12,280 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Python 3.8: postpone annotation evaluation so PEP-604 unions (X | None) and +# PEP-585 builtin generics (list[dict]) in this file don't fail at import time. +from __future__ import annotations + +import copy +import json import os import re +import select import shlex import shutil import socket +import ssl import subprocess +import sys import tempfile +import threading import time import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qs, urlparse import yaml import e2e.util as util -from e2e.steps import create_shell_namespace_clickhouse_template +from e2e.steps import create_shell_namespace_clickhouse_template, delete_test_namespace, get_shell from testflows.asserts import error from testflows.core import * import e2e.kubectl as kubectl +import struct +import hashlib + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + +FAKE_OPENSSL_SERVER = "fake-openssl-server" +TLS_REJECT_MARKERS = ( + "Cipher is (NONE)", + "Cipher : 0000", + "handshake failure", + "alert handshake failure", + "no shared cipher", + "no protocols available", + "unsupported protocol", + "wrong version number", + "tlsv1 alert protocol version", + "no peer certificate available", +) + +# Server-originated evidence read BACK from the peer. A client-side local refusal +# (OpenSSL 3.x / hardened openssl.cnf MinProtocol floor) never sends a ClientHello +# and cannot produce these. +TLS_SERVER_REJECT_MARKERS = ( + "sslv3 alert", + "tlsv1 alert", + "alert handshake failure", + "alert protocol version", + "no shared cipher", + "ssl alert number", +) + +approved_tls1_3_ciphers = [ + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", +] +# TLS 1.2 suites permitted by Go's native FIPS 140-3 module +# (crypto/tls/defaults_fips140.go `allowedCipherSuitesFIPS`, the set clickhouse-backup's +# GOFIPS140=v1.0.0 build negotiates). OpenSSL names for the six Go suites; a Go-FIPS +# server offers only these at TLS 1.2, so they are ACCEPTED, not rejected. Kept separate +# from approved_tls1_3_ciphers because only clickhouse-backup (Go, MinVersion 1.2) uses +# them -- the OpenSSL-backed ClickHouse/Keeper listeners are pinned to TLS 1.3. +approved_tls1_2_ciphers = [ + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES128-SHA256", + "ECDHE-ECDSA-AES128-SHA256", +] +ciphers_by_protocol = { + "TLSv1.3": [ + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_GCM_SHA256", + ], + "TLSv1.2": [ + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "DHE-DSS-AES256-GCM-SHA384", + "DHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-RSA-CHACHA20-POLY1305", + "DHE-RSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES256-CCM", + "DHE-RSA-AES256-CCM", + "ECDHE-ECDSA-ARIA256-GCM-SHA384", + "ECDHE-ARIA256-GCM-SHA384", + "DHE-DSS-ARIA256-GCM-SHA384", + "DHE-RSA-ARIA256-GCM-SHA384", + "ADH-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "DHE-DSS-AES128-GCM-SHA256", + "DHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-CCM", + "DHE-RSA-AES128-CCM", + "ECDHE-ECDSA-ARIA128-GCM-SHA256", + "ECDHE-ARIA128-GCM-SHA256", + "DHE-DSS-ARIA128-GCM-SHA256", + "DHE-RSA-ARIA128-GCM-SHA256", + "ADH-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES256-CCM8", + "ECDHE-ECDSA-AES128-CCM8", + "DHE-RSA-AES256-CCM8", + "DHE-RSA-AES128-CCM8", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "DHE-RSA-AES256-SHA256", + "DHE-DSS-AES256-SHA256", + "ECDHE-ECDSA-CAMELLIA256-SHA384", + "ECDHE-RSA-CAMELLIA256-SHA384", + "DHE-RSA-CAMELLIA256-SHA256", + "DHE-DSS-CAMELLIA256-SHA256", + "ADH-AES256-SHA256", + "ADH-CAMELLIA256-SHA256", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "DHE-RSA-AES128-SHA256", + "DHE-DSS-AES128-SHA256", + "ECDHE-ECDSA-CAMELLIA128-SHA256", + "ECDHE-RSA-CAMELLIA128-SHA256", + "DHE-RSA-CAMELLIA128-SHA256", + "DHE-DSS-CAMELLIA128-SHA256", + "ADH-AES128-SHA256", + "ADH-CAMELLIA128-SHA256", + "RSA-PSK-AES256-GCM-SHA384", + "DHE-PSK-AES256-GCM-SHA384", + "RSA-PSK-CHACHA20-POLY1305", + "DHE-PSK-CHACHA20-POLY1305", + "ECDHE-PSK-CHACHA20-POLY1305", + "DHE-PSK-AES256-CCM", + "RSA-PSK-ARIA256-GCM-SHA384", + "DHE-PSK-ARIA256-GCM-SHA384", + "AES256-GCM-SHA384", + "AES256-CCM", + "ARIA256-GCM-SHA384", + "PSK-AES256-GCM-SHA384", + "PSK-CHACHA20-POLY1305", + "PSK-AES256-CCM", + "PSK-ARIA256-GCM-SHA384", + "RSA-PSK-AES128-GCM-SHA256", + "DHE-PSK-AES128-GCM-SHA256", + "DHE-PSK-AES128-CCM", + "RSA-PSK-ARIA128-GCM-SHA256", + "DHE-PSK-ARIA128-GCM-SHA256", + "AES128-GCM-SHA256", + "AES128-CCM", + "ARIA128-GCM-SHA256", + "PSK-AES128-GCM-SHA256", + "PSK-AES128-CCM", + "PSK-ARIA128-GCM-SHA256", + "DHE-PSK-AES256-CCM8", + "DHE-PSK-AES128-CCM8", + "AES256-CCM8", + "AES128-CCM8", + "PSK-AES256-CCM8", + "PSK-AES128-CCM8", + "AES256-SHA256", + "CAMELLIA256-SHA256", + "AES128-SHA256", + "CAMELLIA128-SHA256", + ], + "TLSv1": [ + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA", + "AECDH-AES256-SHA", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "AECDH-AES128-SHA", + "ECDHE-PSK-AES256-CBC-SHA384", + "ECDHE-PSK-AES256-CBC-SHA", + "RSA-PSK-AES256-CBC-SHA384", + "DHE-PSK-AES256-CBC-SHA384", + "ECDHE-PSK-CAMELLIA256-SHA384", + "RSA-PSK-CAMELLIA256-SHA384", + "DHE-PSK-CAMELLIA256-SHA384", + "PSK-AES256-CBC-SHA384", + "PSK-CAMELLIA256-SHA384", + "ECDHE-PSK-AES128-CBC-SHA256", + "ECDHE-PSK-AES128-CBC-SHA", + "RSA-PSK-AES128-CBC-SHA256", + "DHE-PSK-AES128-CBC-SHA256", + "ECDHE-PSK-CAMELLIA128-SHA256", + "RSA-PSK-CAMELLIA128-SHA256", + "DHE-PSK-CAMELLIA128-SHA256", + "PSK-AES128-CBC-SHA256", + "PSK-CAMELLIA128-SHA256", + ], +} + +_OPENSSL_NEGOTIATED_CIPHER = re.compile( + r"(?:^|\n)Cipher is (?!\(NONE\))(?P\S+)", + re.IGNORECASE, +) + +CIPHERS_PROTOCOL_TLS_VERSION = { + "TLSv1.3": "1.3", + "TLSv1.2": "1.2", + "TLSv1": "1.0", +} + +FIPS_REJECTED_PROTOCOL_CASES = ( + {"name": "TLS 1.0 protocol", "tls_version": "1.0", "cipher_suite": None}, + {"name": "TLS 1.1 protocol", "tls_version": "1.1", "cipher_suite": None}, + {"name": "TLS 1.2 protocol", "tls_version": "1.2", "cipher_suite": None}, +) + + +def fips_rejected_cipher_cases_from_ciphers_by_protocol(): + """Every cipher in ciphers_by_protocol except approved TLS 1.3 suites.""" + cases = [] + for protocol, tls_version in CIPHERS_PROTOCOL_TLS_VERSION.items(): + for cipher in ciphers_by_protocol[protocol]: + if protocol == "TLSv1.3" and cipher in approved_tls1_3_ciphers: + continue + cases.append({ + "name": f"TLS {tls_version} {cipher}", + "tls_version": tls_version, + "cipher_suite": cipher, + }) + return tuple(cases) + + +FIPS_LISTENER_REJECTED_TLS_CASES = ( + *FIPS_REJECTED_PROTOCOL_CASES, + *fips_rejected_cipher_cases_from_ciphers_by_protocol(), +) + +# clickhouse-backup's Go FIPS runtime pins ciphers but keeps stdlib default +# MinVersion (TLS 1.2), so at TLS 1.2 it ACCEPTS any FIPS-approved suite it can +# negotiate: bare `-tls1_2` (default → an approved suite) and any explicit cipher +# in approved_tls1_2_ciphers (the RSA server cert negotiates the ECDHE-RSA ones; +# the ECDHE-ECDSA ones fail as "no shared cipher" but are excluded too so the sweep +# stays cert-agnostic). Every non-approved 1.2 cipher and all legacy protocols +# (1.0/1.1) stay in the rejected set. +FIPS_BACKUP_LISTENER_REJECTED_TLS_CASES = tuple( + case for case in FIPS_LISTENER_REJECTED_TLS_CASES + if not ( + case["tls_version"] == "1.2" + and (case["cipher_suite"] is None or case["cipher_suite"] in approved_tls1_2_ciphers) + ) +) + +FIPS_APPROVED_TLS13_CIPHER_CASES = tuple( + { + "name": f"TLS 1.3 {cipher}", + "tls_version": "1.3", + "cipher_suite": cipher, + } + for cipher in approved_tls1_3_ciphers +) +FIPS_OPERATOR_APPROVED_TLS13_CIPHER = "TLS_AES_256_GCM_SHA384" +FIPS_OPERATOR_APPROVED_TLS13_CIPHER_SUITES = ":".join(approved_tls1_3_ciphers) + +OPERATOR_CONTAINER_TLS_FAILURE_NEEDLES = ( + "handshake failure", + "no shared cipher", + "alert handshake failure", + "TLS connect error", + "HTTP:000", +) # --------------------------------------------------------------------------- # Build verification @@ -60,29 +314,42 @@ def fips_extract_shipped_binaries(self): f"{self.context.operator_version}" ) - extract_dir = tempfile.mkdtemp(prefix="fips-shipped-bin-") - op_bin = os.path.join(extract_dir, "clickhouse-operator") - me_bin = os.path.join(extract_dir, "metrics-exporter") - suffix = uuid.uuid1().hex[:8] - - for image, image_path, dest, label in ( - (operator_image, "/clickhouse-operator", op_bin, f"cho-verify-{suffix}"), - ( - metrics_exporter_image, - "/metrics-exporter", - me_bin, - f"me-verify-{suffix}", - ), - ): - container_name = shlex.quote(label) - kubectl.run_shell(f"docker create --name {container_name} {shlex.quote(image)}") + # Concurrent FIPS tests all extract via the host docker daemon at once; under that + # contention the docker create/cp calls intermittently fail mid-extraction (seen as + # a transient IndexError under POOL_SIZE=25). Retry the whole extraction — each + # attempt uses a fresh tempdir + uuid-suffixed containers, so it is idempotent. + attempts = 4 + for attempt in range(1, attempts + 1): + extract_dir = tempfile.mkdtemp(prefix="fips-shipped-bin-") + op_bin = os.path.join(extract_dir, "clickhouse-operator") + me_bin = os.path.join(extract_dir, "metrics-exporter") + suffix = uuid.uuid1().hex[:8] try: - kubectl.run_shell( - f"docker cp {container_name}:{shlex.quote(image_path)} {shlex.quote(dest)}" - ) - finally: - kubectl.run_shell(f"docker rm {container_name}", ok_to_fail=True) - os.chmod(dest, 0o755) + for image, image_path, dest, label in ( + (operator_image, "/clickhouse-operator", op_bin, f"cho-verify-{suffix}"), + ( + metrics_exporter_image, + "/metrics-exporter", + me_bin, + f"me-verify-{suffix}", + ), + ): + container_name = shlex.quote(label) + kubectl.run_shell(f"docker create --name {container_name} {shlex.quote(image)}") + try: + kubectl.run_shell( + f"docker cp {container_name}:{shlex.quote(image_path)} {shlex.quote(dest)}" + ) + finally: + kubectl.run_shell(f"docker rm {container_name}", ok_to_fail=True) + os.chmod(dest, 0o755) + break + except Exception as exc: + shutil.rmtree(extract_dir, ignore_errors=True) + if attempt == attempts: + raise + note(f"FIPS binary extraction failed ({type(exc).__name__}: {exc}); retry {attempt}/{attempts - 1}") + time.sleep(attempt * 3) self.context.fips_extract_dir = extract_dir self.context.fips_op_bin = op_bin @@ -270,45 +537,6 @@ def fips_assert_fips_enforced_coercion_in_logs(self, logs): ) - - -@TestStep(Given) -def fips_apply_operator_godebug(self): - """Apply suite-configured GODEBUG=fips140= on the operator deployment.""" - mode = self.context.fips140_mode - - ns = current().context.operator_namespace - expected = f"fips140={mode}" - # One patch for both containers (kubectl set env defaults to --containers='*', - # and the operator Deployment has exactly clickhouse-operator + metrics-exporter). - # Two separate per-container edits produced two ReplicaSet revisions racing each - # other, so `rollout status` could latch onto an intermediate revision while the - # final pod was still cache-syncing -- one of the test_030008 restart-storm races. - kubectl.launch( - "set env deployment/clickhouse-operator " - f"--overwrite GODEBUG={expected}", - ns=ns, - ) - kubectl.launch( - "rollout status deployment/clickhouse-operator", - ns=ns, - timeout=600, - ) - -@TestStep(Given) -def fips_apply_operator_config(self, chopconf_path): - """Apply a ClickHouseOperatorConfiguration and restart the operator.""" - util.apply_operator_config(chopconf_path) - fips_apply_operator_godebug() - - -@TestStep(Given) -def fips_create_shell_namespace_clickhouse_template(self): - """Create test namespace and apply suite-configured operator GODEBUG.""" - create_shell_namespace_clickhouse_template() - fips_apply_operator_godebug() - - @TestStep(When) def fips_apply_manifest_raw(self, manifest_path): """Apply a CHI/CHK manifest without waiting for reconcile.""" @@ -390,10 +618,12 @@ def fips_assert_chi_admitted(self, chi, reason="FIPSImagePolicyViolation"): @TestStep(Given) def create_tls_secret_for_fips_hosts( self, - chi, - chk, + chi=None, + chk=None, secret_name="clickhouse-certs", replicas=2, + pod_hostnames=None, + extra_dns_names=None, ): """Create a TLS secret whose SANs match this test namespace's pod DNS names.""" ns = self.context.test_namespace @@ -415,13 +645,25 @@ def create_tls_secret_for_fips_hosts( dns_suffixes = ("", f".{ns}", f".{ns}.svc", f".{ns}.svc.cluster.local") dns_names = ["localhost", "clickhouse", "clickhouse1", f"*.{ns}.svc.cluster.local"] - for replica in range(replicas): - for host in ( - f"chi-{chi}-default-0-{replica}", - f"chk-{chk}-keeper-0-{replica}", - ): + if pod_hostnames: + for host in pod_hostnames: for suffix in dns_suffixes: dns_names.append(f"{host}{suffix}") + else: + assert chi and chk, error( + "create_tls_secret_for_fips_hosts requires chi and chk " + "when pod_hostnames is not set" + ) + for replica in range(replicas): + for host in ( + f"chi-{chi}-default-0-{replica}", + f"chk-{chk}-keeper-0-{replica}", + ): + for suffix in dns_suffixes: + dns_names.append(f"{host}{suffix}") + + if extra_dns_names: + dns_names.extend(extra_dns_names) san_entries = ["IP.1 = 127.0.0.1"] san_entries.extend( @@ -571,6 +813,10 @@ def start_external_ch_container(self, ns=None, cipher_suites=None): note(f"external ClickHouse client container started: {container}") self.context.external_chi_container = container + yield + + with Finally("stop external ClickHouse client container"): + stop_external_ch_container() @TestStep(Finally) def stop_external_ch_container(self): @@ -592,11 +838,12 @@ def fips_ch_external_secure_query(self, pod, sql, ns=None): """ ns = ns or self.context.test_namespace container = self.context.external_chi_container - local_port = "9440" + local_port = _free_local_port() pf = subprocess.Popen( [ "kubectl", + *self.context.kubectl_context_args, "-n", ns, "port-forward", f"pod/{pod}", @@ -668,7 +915,7 @@ def fips_ch_external_secure_query(self, pod, sql, ns=None): # --------------------------------------------------------------------------- @TestStep(Given) -def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): +def fips_edit_manifest(self, source_manifest, replicas_count=None, cipher_suites=None, kind="chi"): """Load a CHI/CHK manifest, patch ``replicasCount``, write a temp copy.""" source_path = util.get_full_path(source_manifest) with open(source_path, encoding="utf-8") as f: @@ -678,6 +925,20 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): manifest["spec"]["configuration"]["clusters"][0]["layout"]["replicasCount"] = ( replicas_count ) + if cipher_suites is not None: + xml = manifest["spec"]["configuration"]["files"]["openssl.xml"] + + old = ( + "TLS_AES_128_GCM_SHA256:" + "TLS_AES_256_GCM_SHA384" + ) + + manifest["spec"]["configuration"]["files"]["openssl.xml"] = ( + xml.replace( + old, + ":".join(cipher_suites), + ) + ) fd, temp_path = tempfile.mkstemp(suffix=".yaml", prefix=f"fips-{kind}-") os.close(fd) @@ -688,6 +949,7 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): if replicas_count is not None: note(f" replicasCount={replicas_count}") + return temp_path @@ -695,7 +957,7 @@ def fips_edit_manifest(self, source_manifest, replicas_count=None, kind="chi"): def fips_apply_manifest( self, manifest_path, - expected_pod_count=None, + replica_count=None, kind="chi", apply_templates=None, timeout=None, @@ -712,8 +974,8 @@ def fips_apply_manifest( check = { "do_not_delete": 1, } - if expected_pod_count is not None: - check["pod_count"] = expected_pod_count + if replica_count is not None: + check["pod_count"] = replica_count if expected_status is not None: if kind == "chi": check["chi_status"] = expected_status @@ -744,18 +1006,20 @@ def get_binary_version(self, pod, binary, container=None, ns=None): ns=ns, ) +@TestStep(Then) +def check_fips_binary_version(self, pod, binary, container=None, ns=None): + """Run `` --version`` inside a pod and check it contains altinityfips tag.""" -@TestStep(When) -def fips_read_listening_ports(self, pod, container="clickhouse", ns=None): - """Return TCP ports in LISTEN state inside the container via ``/proc/net/tcp``.""" - ns = ns or self.context.test_namespace - raw = kubectl.launch( - f"exec {pod} -c {container} -- " - f"sh -c 'cat /proc/net/tcp /proc/net/tcp6'", - ns=ns, + version = get_binary_version(pod=pod, binary=binary, container=container, ns=ns) + + assert "altinityfips" in version, error( + f"{pod}: expected altinityfips in {binary} version, got {version!r}" ) +def translate_tcp_port_output(raw): + """Translates raw output to a readable set of ports""" ports = set() + for line in raw.splitlines(): cols = line.split() if len(cols) < 4 or cols[0] == "sl" or cols[3] != "0A": @@ -764,26 +1028,70 @@ def fips_read_listening_ports(self, pod, container="clickhouse", ns=None): ports.add(int(cols[1].split(":")[1], 16)) except (IndexError, ValueError): continue + return ports +@TestStep(When) +def fips_read_listening_ports( + self, + pod, + container, + ns=None, + debug=False, + target=None, +): + """Return TCP ports in LISTEN state from /proc/net/tcp and /proc/net/tcp6.""" + + ns = ns or self.context.test_namespace + + if debug: + target = target or container + raw = kubectl.launch( + f"debug {pod} " + f"--image=busybox:1.36 " + f"--target={target} " + f"--attach " + f"-- sh -c 'cat /proc/1/net/tcp /proc/1/net/tcp6'", + ns=ns, + ) + else: + raw = kubectl.launch( + f"exec {pod} -c {container} -- " + f"sh -c 'cat /proc/net/tcp /proc/net/tcp6'", + ns=ns, + ) + + return translate_tcp_port_output(raw=raw) + @TestStep(Then) -def fips_assert_only_tls_ports( +def fips_assert_only_expected_ports( self, pod, - required, + expected, container="clickhouse", + ns=None, max_iters=1, sleep_s=2, + debug=False ): - """Assert the container listens on exactly ``required`` and nothing else.""" + """Assert the container listens on expected required ports.""" + ports = set() + for attempt in range(max_iters): - ports = fips_read_listening_ports(pod=pod, container=container) + ports = fips_read_listening_ports( + pod=pod, + container=container, + ns=ns, + debug=debug + ) + note(f"listening ports on {pod}: {sorted(ports)}") - missing = required - ports - unexpected = ports - required + missing = expected - ports + unexpected = ports - expected + if not missing and not unexpected: return @@ -795,15 +1103,15 @@ def fips_assert_only_tls_ports( ) time.sleep(sleep_s) - missing = required - ports + missing = expected - ports assert not missing, error( f"{pod}: required {container} TLS ports missing: {sorted(missing)}" ) - unexpected = ports - required + unexpected = ports - expected assert not unexpected, error( f"{pod}: unexpected {container} ports listening " - f"(approved={sorted(required)}): {sorted(unexpected)}" + f"(approved={sorted(expected)}): {sorted(unexpected)}" ) @@ -835,7 +1143,7 @@ def fips_wait_cluster_topology( self, pod, cluster_name, - expected_count, + replica_count, max_iters=30, sleep_s=2, ): @@ -846,682 +1154,3388 @@ def fips_wait_cluster_topology( f"SELECT count() FROM system.clusters " f"WHERE cluster = '{cluster_name}'" ), - expected=expected_count, + expected=replica_count, max_iters=max_iters, sleep_s=sleep_s, ) - note(f"{pod} sees {expected_count} hosts in cluster {cluster_name!r}") - + note(f"{pod} sees {replica_count} hosts in cluster {cluster_name!r}") -@TestStep(Then) -def fips_assert_replicas_healthy( - self, - workload, - expected_count, - kind="chi", - cluster_name="default", -): - """Run essential FIPS/TLS health checks for the current CHI or CHK replica set.""" - if kind == "chi": - pods = sorted(kubectl.get_pod_names(workload)) - binary = "clickhouse" - container = "clickhouse" - tls_ports = {8443, 9440, 9010, 7171} - elif kind == "chk": - pods = sorted(kubectl.get_chk_pod_names(workload)) - binary = "clickhouse-keeper" - container = "clickhouse-keeper" - tls_ports = {2281, 9444, 9182} - else: - raise ValueError(f"unsupported workload kind: {kind}") - note(f"{kind.upper()} pods: {pods}") - assert len(pods) == expected_count, error( - f"expected {expected_count} {kind.upper()} pods, " - f"got {len(pods)}: {pods}" - ) +def openssl_tls_version_args(tls_version): + if tls_version == "1.3": + return ["-tls1_3"] + if tls_version == "1.2": + return ["-tls1_2"] + if tls_version == "1.1": + return ["-tls1_1"] + if tls_version == "1.0": + return ["-tls1"] + if tls_version == "ssl3": + return ["-ssl3"] + if tls_version == "ssl2": + return ["-ssl2"] - for pod in pods: - version = get_binary_version(pod=pod, binary=binary) - assert "altinityfips" in version, error( - f"{pod}: expected altinityfips in {binary} version, got {version!r}" - ) - fips_assert_only_tls_ports( - pod=pod, - required=tls_ports, - container=container, - max_iters=30, - sleep_s=2, - ) + raise ValueError(f"unsupported TLS/SSL version: {tls_version}") - if kind == "chi": - pod0 = pods[0] - out = fips_ch_external_secure_query(pod=pod0, sql="SELECT 1") - assert out == "1", error(f"external secure query failed, got {out!r}") - fips_wait_cluster_topology( - pod=pod0, - cluster_name=cluster_name, - expected_count=expected_count, - ) - return pods +def openssl_cipher_args(tls_version, cipher_suite): + # TLS 1.0/1.1 cipher suites are all SHA1/RSA-based and sit below the host + # openssl's default SECLEVEL=2 floor, so s_server has no cipher it is allowed + # to offer and aborts the handshake with an internal_error alert (SSL alert 80) + # instead of the expected protocol_version alert (SSL alert 70). A min-1.3 + # client (the Go FIPS client under test) then sees `remote error: tls: internal + # error`, which is NOT a rejection marker -> false FAIL. Drop to SECLEVEL=0 for + # the legacy protocols so the server genuinely offers TLS 1.0/1.1: the min-1.3 + # client now gets the clean `protocol version not supported` alert (already a + # marker), while a peer that wrongly ACCEPTED the legacy protocol would complete + # the handshake -- so this preserves the rejection assertion with no false pass. + if tls_version in ("1.0", "1.1"): + return ["-cipher", f"{cipher_suite or 'DEFAULT'}@SECLEVEL=0"] + if not cipher_suite: + return [] -@TestStep(Then) -def fips_check_replication_across_replicas(self, chi_pods, table="repl_test"): - """Verify ReplicatedMergeTree data converges to every replica over TLS.""" - if len(chi_pods) < 2: - note(f"skipping replication check with {len(chi_pods)} replica(s)") - return + if tls_version == "1.3": + return ["-ciphersuites", cipher_suite] - pod0 = chi_pods[0] + return ["-cipher", cipher_suite] - with When("a replicated table is created on the cluster"): - fips_ch_external_secure_query( - pod=pod0, - sql=( - f"CREATE TABLE IF NOT EXISTS {table} ON CLUSTER '{{cluster}}' " - "(a UInt32) " - "ENGINE = ReplicatedMergeTree(" - f"'/clickhouse/{{installation}}/{{cluster}}/tables/{{shard}}/{table}', " - "'{replica}') ORDER BY a" - ), - ) - with And("rows are inserted on replica 0"): - fips_ch_external_secure_query( - pod=pod0, - sql=f"INSERT INTO {table} SELECT number FROM numbers(10)", - ) - with Then("rows are replicated to every other replica over interserver TLS"): - target = 10 - for pod in chi_pods[1:]: - fips_poll_secure_scalar( - pod=pod, - sql=f"SELECT count() FROM {table}", - expected=target, - ) +def openssl_s_client_negotiated_cipher(output): + """Return the negotiated cipher name when s_client completed a handshake.""" + match = _OPENSSL_NEGOTIATED_CIPHER.search(output) + if match: + return match.group("cipher") + return None @TestStep(When) -def fips_read_chop_generated_settings(self, pod, container="clickhouse", ns=None): - """Return the operator-generated ``chop-generated-settings.xml`` from ``pod``.""" +def fips_run_openssl_s_client_on_pod_port( + self, + pod, + port, + cipher_suite="TLS_AES_128_GCM_SHA256", + tls_version="1.3", + ok_to_fail=False, + ns=None, +): + """Run ``openssl s_client`` against a pod listener through ``kubectl port-forward``.""" ns = ns or self.context.test_namespace - return kubectl.launch( - f"exec {pod} -c {container} -- " - f"cat /etc/clickhouse-server/config.d/chop-generated-settings.xml", - ns=ns, - ) + ca_crt = self.context.tls["ca_crt"] + local_port = _free_local_port() + with Given(f"port-forward from localhost:{local_port} to {pod}:{port}"): + pf = subprocess.Popen( + [ + "kubectl", + *self.context.kubectl_context_args, + "-n", ns, + "port-forward", + f"pod/{pod}", + f"{local_port}:{port}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) -@TestStep(Then) -def check_ports_in_chi_settings(self, settings_xml): - """Check approved TLS ports and removed plaintext ports in CHI settings.""" - assert "8443" in settings_xml, error( - "https_port 8443 missing from operator-generated settings" - ) - assert "9440" in settings_xml, error( - "tcp_port_secure 9440 missing from operator-generated settings" - ) - assert "9010" in settings_xml, error( - "interserver_https_port 9010 missing from operator-generated settings" - ) + try: + with When("port-forward is ready on localhost"): + deadline = time.time() + 10 + while time.time() < deadline: + if pf.poll() is not None: + out, err = pf.communicate() + assert False, error( + "kubectl port-forward exited early\n" + f"stdout:\n{out}\n" + f"stderr:\n{err}" + ) + + try: + socket.create_connection( + ("127.0.0.1", int(local_port)), timeout=0.5 + ).close() + break + except OSError: + time.sleep(0.2) + else: + assert False, error( + f"kubectl port-forward to {pod}:{port} " + f"did not become ready on 127.0.0.1:{local_port}" + ) - for port in ( - "http_port", - "tcp_port", - "mysql_port", - "postgresql_port", - "interserver_http_port", - ): - assert f'{port} remove="1"' in settings_xml, error( - f"{port} not marked removed in operator-generated settings" - ) + with And("openssl s_client runs against the forwarded port"): + command = [ + "openssl", "s_client", + "-connect", f"127.0.0.1:{local_port}", + "-servername", "localhost", + "-CAfile", ca_crt, + "-verify_return_error", + ] + command.extend(openssl_tls_version_args(tls_version)) + command.extend(openssl_cipher_args(tls_version, cipher_suite)) -# --------------------------------------------------------------------------- -# clickhouse-backup sidecar -# --------------------------------------------------------------------------- + result = subprocess.run( + command, + input="Q\n", + text=True, + capture_output=True, + check=False, + ) -@TestStep(Then) -def check_clickhouse_backup_embeds_gofips( + output = f"{result.stdout}\n{result.stderr}" + + if not ok_to_fail: + with Then("openssl s_client handshake succeeds"): + assert result.returncode == 0, error( + f"{pod}:{port}: openssl s_client failed for " + f"tls={tls_version}, cipher={cipher_suite}\n" + f"exit code: {result.returncode}\n" + f"output:\n{output}" + ) + + return output + + finally: + pf.terminate() + try: + pf.wait(timeout=3) + except subprocess.TimeoutExpired: + pf.kill() + +@TestStep(Check) +def fips_assert_rejected_tls_cases_on_endpoint( self, - pods, - gofips_version="v1.0.0", + label, + pod, + port, + rejected_cases, ns=None, ): - """Verify each clickhouse-backup sidecar binary embeds GOFIPS140 metadata.""" + """Assert rejected TLS protocol/cipher cases fail on one endpoint.""" ns = ns or self.context.test_namespace - expected = f"GOFIPS140={gofips_version}" - for pod in pods: - backup_bin = f"/tmp/{pod}-clickhouse-backup" - kubectl.launch( - f"cp {pod}:/bin/clickhouse-backup {backup_bin} " - f"-c clickhouse-backup", - ns=ns, - ) - build_info = kubectl.run_shell(f"go version -m {backup_bin}") - assert expected in build_info, error( - f"{pod}: expected {expected} in clickhouse-backup binary" - ) - note(f"{pod} clickhouse-backup embeds {expected}") + for case in rejected_cases: + with Check(f"{label} {pod}:{port} rejects {case['name']}"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, + port=port, + tls_version=case["tls_version"], + cipher_suite=case["cipher_suite"], + ok_to_fail=True, + ns=ns, + ) + output_lower = output.lower() -@TestStep(Then) -def check_clickhouse_backup_https_api_serves_tls(self, pods, ns=None): - """Verify clickhouse-backup HTTPS API accepts clients trusted by the test CA.""" - ns = ns or self.context.test_namespace + negotiated_cipher = openssl_s_client_negotiated_cipher(output) + assert negotiated_cipher is None, error( + f"{label} {pod}:{port}: server negotiated disallowed {case['name']}\n" + f"negotiated cipher: {negotiated_cipher}\n" + f"tls_version={case['tls_version']}\n" + f"cipher_suite={case['cipher_suite']}\n" + f"output:\n{output}" + ) - for pod in pods: - out = kubectl.launch( - f"exec {pod} -c clickhouse-backup -- " - f"curl -sS -o /tmp/backup_tables.out -w 'HTTP:%{{http_code}}' " - f"--cacert /etc/clickhouse-backup/tls/ca.crt " - f"https://127.0.0.1:7171/backup/tables", - ns=ns, - ) - assert out == "HTTP:200", error( - f"{pod}: /backup/tables did not return HTTP 200, got {out!r}" - ) + if case["tls_version"] in ("1.0", "1.1"): + # Downgrade cases: host openssl may refuse the legacy protocol + # LOCALLY and never contact the server -> a client-side refusal + # must NOT count as a pass. Require a SERVER-originated alert; + # else skip (an actually-accepting server is already caught above + # by the negotiated_cipher assert). + if not any(m in output_lower for m in TLS_SERVER_REJECT_MARKERS): + skip( + f"{label} {pod}:{port}: no server-side rejection alert for " + f"{case['name']} - host openssl likely refused locally " + f"(no ClientHello reached the server)\noutput:\n{output}" + ) + else: + assert any( + marker.lower() in output_lower + for marker in TLS_REJECT_MARKERS + ), error( + f"{label} {pod}:{port}: expected TLS rejection for {case['name']}\n" + f"tls_version={case['tls_version']}\n" + f"cipher_suite={case['cipher_suite']}\n" + f"output:\n{output}" + ) -@TestStep(Then) -def check_clickhouse_backup_https_api_rejects_untrusted(self, pods, ns=None): - """Verify clickhouse-backup HTTPS API rejects clients without the test CA.""" +@TestStep(Check) +def fips_assert_all_rejected_tls_cases_on_all_endpoints( + self, + chi_pods, + chk_pods, + ns=None, +): + """Assert all rejected TLS cases fail on every FIPS TLS endpoint.""" ns = ns or self.context.test_namespace - for pod in pods: - out = kubectl.launch( - f"exec {pod} -c clickhouse-backup -- " - f"sh -c 'curl -sS --fail " - f"https://127.0.0.1:7171/backup/tables >/dev/null 2>&1; " - f"echo EXIT:$?'", + endpoints = ( + ("ClickHouse HTTPS", chi_pods[0], 8443, FIPS_LISTENER_REJECTED_TLS_CASES), + ("ClickHouse native TLS", chi_pods[0], 9440, FIPS_LISTENER_REJECTED_TLS_CASES), + ("ClickHouse interserver HTTPS", chi_pods[0], 9010, FIPS_LISTENER_REJECTED_TLS_CASES), + ("Keeper secure client", chk_pods[0], 2281, FIPS_LISTENER_REJECTED_TLS_CASES), + ("Backup API HTTPS", chi_pods[0], 7171, FIPS_BACKUP_LISTENER_REJECTED_TLS_CASES), + ) + + for label, pod, port, endpoint_rejected_cases in endpoints: + fips_assert_rejected_tls_cases_on_endpoint( + label=label, + pod=pod, + port=port, + rejected_cases=endpoint_rejected_cases, ns=ns, ) - assert "EXIT:60" in out, error( - f"{pod}: expected certificate verification failure EXIT:60, got {out!r}" - ) @TestStep(Then) -def check_external_clickhouse_reports_fips_version(self, pod): - """Verify an external strict-TLS client sees a FIPS ClickHouse server.""" - version = fips_ch_external_secure_query(pod=pod, sql="SELECT version()") - note(f"external SELECT version(): {version}") - assert "fips" in version.lower(), error( - f"expected FIPS in ClickHouse version(), got {version!r}" - ) - -@TestStep(Then) -def fips_assert_operator_tls_rejection_in_logs( +def fips_assert_aes256_tls13_probes( self, - workload, - min_version="1.3", - rejection="remote error: tls: protocol version not supported", - operator_namespace=None, - max_iters=60, - sleep_s=5, + chi_pods, + chk_pods, + ns=None, ): - """Poll operator logs until the expected TLS version rejection is observed.""" - - operator_namespace = operator_namespace or current().context.operator_namespace - - expected_setup_parts = ( - "setupTLSAdvanced():TLS setup OK", - f"minVersion={min_version}", + """Assert approved TLS 1.3 AES-256-GCM cipher negotiates on FIPS TLS listeners.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_256_GCM_SHA384" + + endpoints = ( + ("ClickHouse HTTPS", chi_pods[0], 8443), + ("ClickHouse native TLS", chi_pods[0], 9440), + ("ClickHouse interserver HTTPS", chi_pods[0], 9010), + ("Keeper secure client", chk_pods[0], 2281), + ("Backup API HTTPS", chi_pods[0], 7171), ) - last_logs = "" + for label, pod, port in endpoints: + with Then(f"{label} {pod}:{port} accepts approved AES-256 TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, + port=port, + tls_version="1.3", + cipher_suite=approved_cipher, + ok_to_fail=True, + ns=ns, + ) - for attempt in range(max_iters): - operator_pod = kubectl.get_operator_pod(ns=operator_namespace) - last_logs = get_container_logs( - pod=operator_pod, - container="clickhouse-operator", - ns=operator_namespace, - ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{label} {pod}:{port}: expected {approved_cipher} to negotiate\n" + f"output:\n{output}" + ) - setup_found = any( - all(part in line for part in expected_setup_parts) - for line in last_logs.splitlines() - ) +@TestStep(When) +def fips_curl_pod_port(self, pod, port, path="/", ns=None): + """Return the HTTP status code from a plain ``curl`` to a pod listener via port-forward.""" + ns = ns or self.context.test_namespace + local_port = _free_local_port() - rejection_found = any( - "connect():FAILED" in line and rejection in line - for line in last_logs.splitlines() - ) + pf = subprocess.Popen( + [ + "kubectl", + *self.context.kubectl_context_args, + "-n", ns, + "port-forward", + f"pod/{pod}", + f"{local_port}:{port}", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + try: + deadline = time.time() + 10 + while time.time() < deadline: + if pf.poll() is not None: + out, err = pf.communicate() + assert False, error( + "kubectl port-forward exited early\n" + f"stdout:\n{out}\n" + f"stderr:\n{err}" + ) + + try: + socket.create_connection( + ("127.0.0.1", int(local_port)), timeout=0.5 + ).close() + break + except OSError: + time.sleep(0.2) + else: + assert False, error( + f"kubectl port-forward to {pod}:{port} " + f"did not become ready on 127.0.0.1:{local_port}" + ) + + result = subprocess.run( + [ + "curl", "-sS", + "-o", "/dev/null", + "-w", "%{http_code}", + f"http://127.0.0.1:{local_port}{path}", + ], + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, error( + f"{pod}:{port}{path}: curl failed\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + return result.stdout.strip() + + finally: + pf.terminate() + try: + pf.wait(timeout=3) + except subprocess.TimeoutExpired: + pf.kill() + + +@TestStep(Then) +def check_chi_ports(self, pod, ns=None): + """TLS positive/negative probes for ClickHouse HTTPS and native ports.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + + for port in (8443, 9010): + with Then(f"{pod}:{port} accepts approved TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, port=port, cipher_suite=approved_cipher, ns=ns, + ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{pod}:{port}: expected approved cipher negotiation\n{output}" + ) + + with And(f"{pod}:9440 accepts approved native TLS query"): + out = fips_ch_external_secure_query(pod=pod, sql="SELECT 1") + assert out == "1", error( + f"{pod}:9440: expected SELECT 1 over native TLS, got {out!r}" + ) + + + +@TestStep(Then) +def check_chk_ports(self, pod, ns=None): + """TLS and readiness HTTP probes for ClickHouse Keeper listeners.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + port = 2281 + + # raft 9444 doesnt communicate over TLS endpoint + with Then(f"{pod}:{port} accepts approved TLS 1.3 cipher"): + output = fips_run_openssl_s_client_on_pod_port( + pod=pod, port=port, cipher_suite=approved_cipher, ns=ns, + ) + assert f"Cipher is {approved_cipher}" in output, error( + f"{pod}:{port}: expected approved cipher negotiation\n{output}" + ) + + with And(f"{pod}:9182/ready accepts plain HTTP"): + code = fips_curl_pod_port(pod=pod, port=9182, path="/ready", ns=ns) + assert code == "200", error( + f"{pod}:9182/ready: expected HTTP 200, got {code!r}" + ) + + +@TestStep(Then) +def check_backup_ports(self, pod, ns=None): + """TLS positive/negative probes for the clickhouse-backup HTTPS API.""" + ns = ns or self.context.test_namespace + approved_cipher = "TLS_AES_128_GCM_SHA256" + + with Then(f"{pod}:7171 accepts approved TLS 1.3 cipher"): + out = kubectl.launch( + f"exec {pod} -c clickhouse-backup -- " + f"sh -c 'curl -sS -o /dev/null -w HTTP:%{{http_code}} " + f"--cacert /etc/clickhouse-backup/tls/ca.crt " + f"--tlsv1.3 --tls13-ciphers {approved_cipher} " + f"https://127.0.0.1:7171/backup/tables'", + ns=ns, + ) + assert out == "HTTP:200", error( + f"{pod}:7171: expected HTTP 200 with approved cipher, got {out!r}" + ) + + with Then(f"{pod}:7171 rejects plaintext requests"): + out = kubectl.launch( + f"exec {pod} -c clickhouse-backup -- " + f"sh -c 'curl -s -o /dev/null -w %{{http_code}} " + f"http://127.0.0.1:7171/backup/tables'", + ns=ns, + ok_to_fail=True, + ) + assert out != "200", error( + f"{pod}:7171: expected plaintext HTTP request to be rejected, got {out!r}" + ) + +@TestStep(Then) +def check_k8s_api_requires_tls_from_operator_pod(self, ns=None): + """Assert Kubernetes API :443 rejects plaintext HTTP from operator pod containers.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + for container in ("clickhouse-operator", "metrics-exporter"): + with Then(f"{container} cannot use plaintext HTTP to Kubernetes API :443"): + out = kubectl.launch( + f"exec {pod} -c {container} -- " + "curl -skv http://kubernetes.default.svc:443", + ns=ns, + ok_to_fail=True, + ) + + assert "Client sent an HTTP request to an HTTPS server" in out, error( + f"{container}: expected Kubernetes API :443 to reject plaintext HTTP\n{out}" + ) + +@TestStep(Then) +def check_operator_clickhouse_tls_logs(self, ns=None): + """Assert operator uses HTTPS/TLS config when communicating with ClickHouse.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c clickhouse-operator", + ns=ns, + ) + + assert "setupTLSAdvanced():TLS setup OK" in logs, error( + "operator did not log ClickHouse TLS setup" + ) + assert "verify=Strict minVersion=1.3" in logs, error( + "operator ClickHouse TLS config is not Strict / TLS 1.3" + ) + assert "Ping(https://clickhouse_operator:" in logs, error( + "operator did not log HTTPS ClickHouse ping" + ) + assert ":8443?tls_config=" in logs, error( + "operator ClickHouse ping did not use HTTPS port 8443 with TLS config" + ) + +@TestStep(Then) +def check_metrics_exporter_discovers_clickhouse_https(self, ns=None): + """Assert metrics-exporter discovers ClickHouse hosts using HTTPS :8443.""" + + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c metrics-exporter --tail=4000", + ns=ns, + ) + + assert '"httpsPort":8443' in logs, error( + "metrics-exporter did not discover ClickHouse hosts with httpsPort=8443\n" + f"{logs}" + ) + +@TestStep(Then) +def check_clickhouse_uses_secure_keeper_port(self, chi, ns=None): + """Assert ClickHouse replicas use Keeper secure client port 2281 with secure=yes.""" + + ns = ns or current().context.test_namespace + pods = sorted(kubectl.get_pod_names(chi)) + + for pod in pods: + with Then(f"{pod} connects to Keeper on secure port 2281"): + logs = kubectl.launch( + f"logs {pod} -c clickhouse --tail=5000", + ns=ns, + ) + + assert re.search(r"Connected to ZooKeeper at .+:2281\b", logs), error( + f"{pod}: ClickHouse did not connect to Keeper on port 2281\n{logs}" + ) + + +@TestStep(Then) +def check_operator_skips_plaintext_keeper_dial(self, ns=None): + """Assert operator skips plaintext ZK helper when Keeper ensemble is TLS-only.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + logs = kubectl.launch( + f"logs {pod} -c clickhouse-operator", + ns=ns, + ) + + assert 'Port:&2281,Secure:&"yes"' in logs, error( + "operator logs do not show Keeper configured as secure port 2281" + ) + assert "Skip ZK root-path ensure" in logs, error( + "operator did not log skipping ZK root-path ensure" + ) + assert "ensemble is TLS-only and the operator dial is plaintext" in logs, error( + "operator did not log that plaintext Keeper dial was skipped for TLS-only ensemble" + ) + +@TestStep(Then) +def check_operator_ports(self, ns=None): + """Plain HTTP probes for operator Prometheus listener ports.""" + ns = ns or current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + + for port in (9999, 8888): + with Then(f"operator pod:{port}/metrics accepts plain HTTP"): + code = fips_curl_pod_port(pod=pod, port=port, path="/metrics", ns=ns) + assert code == "200", error( + f"operator pod:{port}/metrics: expected HTTP 200, got {code!r}" + ) + + +@TestStep(Then) +def run_operator_fips_checks(self): + """ + Run FIPS validation checks against the operator pod: + + * verify the pod network namespace exposes only expected Prometheus ports + * verify clickhouse-operator and metrics-exporter emit FIPS startup banners + * verify metrics ports accept HTTP and reject disapproved TLS handshakes + """ + ns = current().context.operator_namespace + pod = kubectl.get_operator_pod(ns=ns) + expected_ports = {8888, 9999} + + with Then("operator pod exposes only expected listener ports"): + fips_assert_only_expected_ports( + pod=pod, + container="clickhouse-operator", + ns=ns, + expected=expected_ports, + debug=True, + ) + + with And("both containers report the FIPS startup banner"): + op_logs = get_container_logs( + pod=pod, + container="clickhouse-operator", + ns=ns, + ) + me_logs = get_container_logs( + pod=pod, + container="metrics-exporter", + ns=ns, + ) + fips_startup_banner_ok(container="clickhouse-operator", logs=op_logs) + fips_startup_banner_ok(container="metrics-exporter", logs=me_logs) + + with Then("operator metrics ports accept HTTP and reject disapproved TLS"): + check_operator_ports(ns=ns) + + with Then("Kubernetes API port 443 requires TLS from operator pod containers"): + check_k8s_api_requires_tls_from_operator_pod(ns=ns) + +@TestStep(Then) +def run_operator_reconcile_fips_checks(self): + """Run the fips checks after operator already reconciled CHK and CHI.""" + + ns = current().context.operator_namespace + + with Then("operator communicates with ClickHouse over HTTPS/TLS"): + check_operator_clickhouse_tls_logs(ns=ns) + + with And("operator skips plaintext Keeper helper against TLS-only Keeper"): + check_operator_skips_plaintext_keeper_dial(ns=ns) + + +@TestStep(Then) +def run_chi_fips_checks(self, workload, replica_count, cluster_name="default"): + """ + Run FIPS and TLS validation checks against the ClickHouse cluster: + + * wait for the expected cluster topology to become available + * verify ClickHouse binaries report an Altinity FIPS build + * verify only approved secure listener ports are exposed + * verify external TLS connectivity to ClickHouse succeeds + * verify the server reports a FIPS version string + * verify operator-generated configuration removes plaintext listeners + * verify each listener accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_pod_names(workload)) + binary = "clickhouse" + container = "clickhouse" + expected_ports = {8443, 9440, 9010, 7171} + pod0 = pods[0] + + note(f"CHI pods: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHI pods, got {len(pods)}: {pods}" + ) + + with When("I wait for full cluster deployment"): + fips_wait_cluster_topology( + pod=pod0, + cluster_name=cluster_name, + replica_count=replica_count, + ) + + for pod in pods: + with Then("check the binary version contains altinityfips tag"): + check_fips_binary_version(pod=pod, binary=binary, container=container) + + with And("check the container only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with And("check TLS port behavior on each replica"): + check_chi_ports(pod=pod) + + with And("operator-generated ClickHouse config removes plaintext ports"): + check_ports_in_chi_settings(pod=pod) + + with Then("check connection via external secure query"): + check_external_clickhouse_reports_fips_version(pod=pod0) + + return pods + + +@TestStep(Then) +def run_chk_fips_checks(self, workload, replica_count): + """ + Run FIPS and TLS validation checks against the ClickHouse Keeper cluster: + + * verify the expected number of Keeper pods are running + * verify Keeper binaries report an Altinity FIPS build + * verify only approved secure listener ports are exposed + * verify operator-generated configuration removes plaintext listeners + * verify Raft inter-node communication is configured for TLS + * verify each listener accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_chk_pod_names(workload)) + binary = "clickhouse-keeper" + container = "clickhouse-keeper" + expected_ports = {2281, 9444, 9182} + + note(f"CHK pods: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHK pods, got {len(pods)}: {pods}" + ) + + for pod in pods: + with Then("check the binary version contains altinityfips tag"): + check_fips_binary_version(pod=pod, binary=binary, container=container) + + with And("check the container only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with And("check TLS port behavior on each Keeper node"): + check_chk_ports(pod=pod) + + with And("operator-generated Keeper config removes plaintext listeners"): + check_ports_in_chk_settings(pod=pod) + + return pods + + +@TestStep(Then) +def run_backup_fips_checks(self, workload, replica_count): + """ + Run FIPS and TLS validation checks against clickhouse-backup sidecars: + + * verify the expected number of CHI pods with backup sidecars are running + * verify clickhouse-backup binaries report a FIPS build + * verify only approved secure listener ports are exposed + * verify each sidecar binary embeds GOFIPS metadata + * verify the HTTPS API accepts approved TLS and rejects disapproved TLS + """ + pods = sorted(kubectl.get_pod_names(workload)) + container = "clickhouse-backup" + expected_ports = {8443, 9440, 9010, 7171} + + note(f"CHI pods with backup sidecar: {pods}") + assert len(pods) == replica_count, error( + f"expected {replica_count} CHI pods, got {len(pods)}: {pods}" + ) + + for pod in pods: + with Then("check the backup binary version contains fips tag"): + check_backup_fips_binary_version(pod=pod) + + with And("check the sidecar only listens on expected ports"): + fips_assert_only_expected_ports( + pod=pod, + expected=expected_ports, + container=container, + max_iters=30, + sleep_s=2, + ) + + with Then("check TLS port behavior on each backup sidecar"): + check_backup_ports(pod=pod) + + with And("each sidecar binary embeds GOFIPS metadata"): + check_clickhouse_backup_embeds_gofips(pod=pod) + + with And("clickhouse-backup TLS config is secure"): + check_clickhouse_backup_clickhouse_tls_config(pod=pod) + + return pods + + +@TestStep(Then) +def fips_check_replication_across_replicas(self, chi_pods, table="repl_test"): + """Verify ReplicatedMergeTree data converges to every replica over TLS.""" + if len(chi_pods) < 2: + note(f"skipping replication check with {len(chi_pods)} replica(s)") + return + + pod0 = chi_pods[0] + + with When("a replicated table is created on the cluster"): + fips_ch_external_secure_query( + pod=pod0, + sql=( + f"CREATE TABLE IF NOT EXISTS {table} ON CLUSTER '{{cluster}}' " + "(a UInt32) " + "ENGINE = ReplicatedMergeTree(" + f"'/clickhouse/{{installation}}/{{cluster}}/tables/{{shard}}/{table}', " + "'{replica}') ORDER BY a" + ), + ) + + with And("rows are inserted on replica 0"): + fips_ch_external_secure_query( + pod=pod0, + sql=f"INSERT INTO {table} SELECT number FROM numbers(10)", + ) + + with Then("rows are replicated to every other replica over interserver TLS"): + target = 10 + for pod in chi_pods[1:]: + fips_poll_secure_scalar( + pod=pod, + sql=f"SELECT count() FROM {table}", + expected=target, + ) + + +@TestStep(When) +def fips_read_chop_generated_chi_settings(self, pod, container="clickhouse", ns=None): + """Return the operator-generated ``chop-generated-settings.xml`` from ``pod``.""" + ns = ns or self.context.test_namespace + return kubectl.launch( + f"exec {pod} -c {container} -- " + f"cat /etc/clickhouse-server/config.d/chop-generated-settings.xml", + ns=ns, + ) + + +@TestStep(Then) +def check_ports_in_chi_settings(self, pod): + """Check approved TLS ports and removed plaintext ports in CHI settings.""" + + settings_xml = fips_read_chop_generated_chi_settings(pod=pod) + note(f"chop-generated-settings.xml:\n{settings_xml}") + + assert "8443" in settings_xml, error( + "https_port 8443 missing from operator-generated settings" + ) + assert "9440" in settings_xml, error( + "tcp_port_secure 9440 missing from operator-generated settings" + ) + assert "9010" in settings_xml, error( + "interserver_https_port 9010 missing from operator-generated settings" + ) + + for port in ( + "http_port", + "tcp_port", + "mysql_port", + "postgresql_port", + "interserver_http_port", + ): + assert f'{port} remove="1"' in settings_xml, error( + f"{port} not marked removed in operator-generated settings" + ) + + +@TestStep(When) +def fips_read_chop_generated_chk_settings(self, pod, container="clickhouse-keeper", ns=None): + """Return operator-generated Keeper listener and Raft XML from ``pod``.""" + ns = ns or self.context.test_namespace + common_listeners_xml = kubectl.launch( + f"exec {pod} -c {container} -- " + "cat /etc/clickhouse-keeper/keeper_config.d/chop-generated-common-listeners.xml", + ns=ns, + ) + raft_xml = kubectl.launch( + f"exec {pod} -c {container} -- " + "cat /etc/clickhouse-keeper/keeper_config.d/chop-generated-raft.xml", + ns=ns, + ) + return common_listeners_xml, raft_xml + + +@TestStep(Then) +def check_ports_in_chk_settings(self, pod): + """Check plaintext listener removal and Raft TLS in CHK settings.""" + + common_listeners_xml, raft_xml = fips_read_chop_generated_chk_settings(pod=pod) + note(f"chop-generated-common-listeners.xml:\n{common_listeners_xml}") + note(f"chop-generated-raft.xml:\n{raft_xml}") + + assert '' in common_listeners_xml, error( + "tcp_port not marked removed in operator-generated Keeper settings" + ) + assert "1" in raft_xml, error( + "expected 1 in operator-generated Raft config" + ) + +@TestStep(Then) +def check_backup_fips_binary_version(self, pod, ns=None): + """Run ``clickhouse-backup --version`` and check it contains a fips tag.""" + version = get_binary_version( + pod=pod, + binary="/bin/clickhouse-backup", + container="clickhouse-backup", + ns=ns, + ) + note(f"{pod} clickhouse-backup --version: {version}") + assert "fips" in version.lower(), error( + f"{pod}: expected fips in clickhouse-backup version, got {version!r}" + ) + + +@TestStep(Then) +def check_clickhouse_backup_embeds_gofips( + self, + pod, + gofips_version="v1.0.0", + ns=None, +): + """Verify each clickhouse-backup sidecar binary embeds GOFIPS140 metadata.""" + ns = ns or self.context.test_namespace + expected = f"GOFIPS140={gofips_version}" + + backup_bin = f"/tmp/{pod}-clickhouse-backup" + kubectl.launch( + f"cp {pod}:/bin/clickhouse-backup {backup_bin} " + f"-c clickhouse-backup", + ns=ns, + ) + build_info = kubectl.run_shell(f"go version -m {backup_bin}") + assert expected in build_info, error( + f"{pod}: expected {expected} in clickhouse-backup binary" + ) + note(f"{pod} clickhouse-backup embeds {expected}") + + +@TestStep(Then) +def check_external_clickhouse_reports_fips_version(self, pod): + """Verify an external strict-TLS client sees a FIPS ClickHouse server.""" + version = fips_ch_external_secure_query(pod=pod, sql="SELECT version()") + note(f"external SELECT version(): {version}") + assert "fips" in version.lower(), error( + f"expected FIPS in ClickHouse version(), got {version!r}" + ) + +def _fips_tls_rejection_present_in_logs(logs, min_version, rejection): + """Return True when logs contain coerced TLS setup and a connect rejection.""" + expected_setup_parts = ( + "setupTLSAdvanced():TLS setup OK", + f"minVersion={min_version}", + ) + setup_found = any( + all(part in line for part in expected_setup_parts) + for line in logs.splitlines() + ) + rejection_found = any( + "connect():FAILED" in line and rejection in line + for line in logs.splitlines() + ) + return setup_found and rejection_found + + +def _fips_tls_rejection_log_excerpt(logs): + return "\n".join( + line for line in logs.splitlines() + if ( + "setupTLSAdvanced()" in line + or "connect():FAILED" in line + or "tls:" in line + or "minVersion" in line + ) + ) + + +# Distroless operator/exporter images ship sh/curl only (no cat/base64). Read the +# IPC token with POSIX shell builtins — same file both containers mount. +_IPC_TOKEN_READ_SHELL = ( + 'TOKEN=""; ' + 'while IFS= read -r line || [ -n "$line" ]; do TOKEN="${TOKEN}${line}"; done ' + "< /etc/clickhouse-operator-ipc/token" +) + + +def _kubectl_pod_exec_stdin(ns, pod, container, shell_script, stdin=None, timeout=120): + """kubectl exec -i … sh -c