shubhamkapoor01 · Nov 3, 2026 · Nov 6, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * **EXPERIMENTAL: Operator-managed backup and restore.** Added `ClickHouseBackup`, `ClickHouseBackupSchedule`, and `ClickHouseRestore` resources using `clickhouse-backup`. Supports per-cluster backups and restores, scheduled backups, retention, configurable storage paths, and cross-installation restores. Enable with `spec.backup.enabled: "yes"`. See [backup documentation](docs/backup.md). by [@discostur](https://github.com/discostur) and [@sunsingerus](https://github.com/sunsingerus) in [#2015](https://github.com/Altinity/clickhouse-operator/pull/2015). Related [#1795](https://github.com/Altinity/clickhouse-operator/issues/1795), [#862](https://github.com/Altinity/clickhouse-operator/issues/862)
 
+* **Kubernetes API client QPS and burst can be set in the operator configuration** as `reconcile.runtime.k8sClientQPS` and `reconcile.runtime.k8sClientBurst`. The `OPERATOR_K8S_CLIENT_QPS_LIMIT` and `OPERATOR_K8S_CLIENT_BURST_LIMIT` env vars still override them. The operator now warns at startup when `reconcileShardsThreadsNumber` is above the client QPS. by [@shubhamkapoor01](https://github.com/shubhamkapoor01). Fixes [#2045](https://github.com/Altinity/clickhouse-operator/issues/2045)
+
 * **Volume downscale tool now supports StorageClass migration.** Set `NEW_STORAGE_CLASS` in `tools/volume_downscale.sh` to move a volume to another StorageClass at the same or a smaller size; the script also detects repeated volume attachment and mount failures. See [runbook](tools/volume_downscale.md). by [@lesandie](https://github.com/lesandie) in [#2125](https://github.com/Altinity/clickhouse-operator/pull/2125)
 
 ### Changed

diff --git a/cmd/operator/app/thread_chi.go b/cmd/operator/app/thread_chi.go
@@ -75,6 +75,7 @@ func initClickHouse(ctx context.Context) {
 	chop.New(kubeClient, chopClient, chopConfigFile)
 	log.V(1).F().Info("Config parsed:")
 	log.Info("\n" + chop.Config().String(true))
+	chop.WarnIfK8sClientQPSTooLow(kubeClient, chop.Config())
 
 	// Validate the runtime FIPS posture against chopconf security.policy.
 	// Fatals when chopconf requires FIPS but the binary/runtime can't deliver.

diff --git a/config/config-dev.yaml b/config/config-dev.yaml
@@ -333,6 +333,12 @@ reconcile:
     # Max percentage of concurrent shard reconciles within one cluster in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
+    # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+    # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+    # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+    k8sClientQPS: 0
+    k8sClientBurst: 0
+
   # Reconcile StatefulSet scenario
   statefulSet:
     # Create StatefulSet scenario

diff --git a/config/config.yaml b/config/config.yaml
@@ -429,6 +429,12 @@ reconcile:
     # Max percentage of concurrent shard reconciles within one cluster in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
+    # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+    # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+    # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+    k8sClientQPS: 0
+    k8sClientBurst: 0
+
   # Reconcile StatefulSet scenario
   statefulSet:
     # Create StatefulSet scenario

diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml
@@ -423,6 +423,12 @@ reconcile:
    # Max percentage of concurrent shard reconciles within one cluster in progress
     reconcileShardsMaxConcurrencyPercent: 50
 
+    # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+    # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+    # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+    k8sClientQPS: 0
+    k8sClientBurst: 0
+
   # Reconcile StatefulSet scenario
   statefulSet:
     # Create StatefulSet scenario

diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json
@@ -477,6 +477,12 @@
                                         "runtime": {
                                             "type": "object",
                                             "properties": {
+                                                "k8sClientBurst": {
+                                                    "type": "integer"
+                                                },
+                                                "k8sClientQPS": {
+                                                    "type": "number"
+                                                },
                                                 "reconcileCHIsThreadsNumber": {
                                                     "type": "integer"
                                                 },

diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml
@@ -736,6 +736,11 @@ configs:
           reconcileShardsThreadsNumber: 5
           # Max percentage of concurrent shard reconciles within one cluster in progress
           reconcileShardsMaxConcurrencyPercent: 50
+          # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+          # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+          # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+          k8sClientQPS: 0
+          k8sClientBurst: 0
         # Reconcile StatefulSet scenario
         statefulSet:
           # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml
@@ -7320,6 +7320,12 @@ data:
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml
@@ -7539,6 +7539,12 @@ data:
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml
@@ -7647,6 +7647,12 @@ data:
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml
@@ -7231,6 +7231,12 @@ data:
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml
@@ -7313,6 +7313,12 @@ data:
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml
@@ -7320,6 +7320,12 @@ data:<
10BC8
/td>
         # Max percentage of concurrent shard reconciles within one cluster in progress
         reconcileShardsMaxConcurrencyPercent: 50
 
+        # Kubernetes API client rate limit of the operator: requests per second and burst. Set both.
+        # 0 keeps the client-go defaults (5 and 10 per API group). Read from this file only, at startup.
+        # OPERATOR_K8S_CLIENT_QPS_LIMIT and OPERATOR_K8S_CLIENT_BURST_LIMIT env vars override them.
+        k8sClientQPS: 0
+        k8sClientBurst: 0
+
       # Reconcile StatefulSet scenario
       statefulSet:
         # Create StatefulSet scenario

diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go
@@ -677,6 +677,11 @@ type OperatorConfigReconcileRuntime struct {
 	ReconcileShardsThreadsNumber         int `json:"reconcileShardsThreadsNumber"         yaml:"reconcileShardsThreadsNumber"`
 	ReconcileShardsMaxConcurrencyPercent int `json:"reconcileShardsMaxConcurrencyPercent" yaml:"reconcileShardsMaxConcurrencyPercent"`
 
+	// K8s API client rate limit. Read from the config file only, before the client is built.
+	// 0 keeps the client-go defaults. The OPERATOR_K8S_CLIENT_*_LIMIT env vars win when set.
+	K8sClientQPS   float32 `json:"k8sClientQPS"   yaml:"k8sClientQPS"`
+	K8sClientBurst int     `json:"k8sClientBurst" yaml:"k8sClientBurst"`
+
 	// DEPRECATED, is replaced with reconcileCHIsThreadsNumber
 	ThreadsNumber int `json:"threadsNumber" yaml:"threadsNumber"`
 }

diff --git a/pkg/chop/kube_machinery.go b/pkg/chop/kube_machinery.go
@@ -31,6 +31,7 @@ import (
 	kubeclientcmd "k8s.io/client-go/tools/clientcmd"
 
 	log "github.com/altinity/clickhouse-operator/pkg/announcer"
+	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
 	"github.com/altinity/clickhouse-operator/pkg/apis/deployment"
 	chopclientset "github.com/altinity/clickhouse-operator/pkg/client/clientset/versioned"
 	"github.com/altinity/clickhouse-operator/pkg/util/tlsutil"
@@ -106,6 +107,14 @@ func GetClientset(kubeConfigFile, masterURL, chopConfigFile string) (
 		applyK8sClientTLSMinVersion(kubeConfig, minVer, hardened)
 	}
 
+	qps, burst := resolveK8sClientRateLimits(chopConfigFile)
+	if qps > 0 {
+		kubeConfig.QPS = qps
+	}
+	if burst > 0 {
+		kubeConfig.Burst = burst
+	}
+
 	// Layer on k8s client rate limiting overrides if specified in CHOP config.
 	if maybeQps := os.Getenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT); maybeQps != "" {
 		parsedQps, err := strconv.ParseFloat(maybeQps, 32)
@@ -177,6 +186,33 @@ func resolveK8sTLSMinVersion(chopConfigFile string) (minVersion string, hardened
 	return string(fileConfig.ResolveK8sTLSMinVersion()), fileConfig.Security.RequiresHardening()
 }
 
+// resolveK8sClientRateLimits reads reconcile.runtime.k8sClientQPS and k8sClientBurst from the
+// file-based chopconf. File-only for the same reason as resolveK8sTLSMinVersion.
+func resolveK8sClientRateLimits(chopConfigFile string) (qps float32, burst int) {
+	cm := newConfigManager(nil, nil, chopConfigFile)
+	fileConfig, err := cm.getFileBasedConfig(chopConfigFile)
+	if err != nil || fileConfig == nil {
+		return 0, 0
+	}
+	return fileConfig.Reconcile.Runtime.K8sClientQPS, fileConfig.Reconcile.Runtime.K8sClientBurst
+}
+
+// WarnIfK8sClientQPSTooLow warns when one CHI may reconcile more shards at once than the k8s
+// client allows requests per second, so the reconcile waits on client-side throttling. The QPS
+// is taken from the built client, so env overrides count. Returns whether it warned.
+func WarnIfK8sClientQPSTooLow(kubeClient *kube.Clientset, config *api.OperatorConfig) bool {
+	shardsThreads := config.Reconcile.Runtime.ReconcileShardsThreadsNumber
+	qps := kubeClient.CoreV1().RESTClient().GetRateLimiter().QPS()
+	if float32(shardsThreads) <= qps {
+		return false
+	}
+	log.Warning("k8s client QPS limit %v is below reconcile.runtime.reconcileShardsThreadsNumber %d, "+
+		"reconcile will wait on client-side throttling. Raise %s and %s if set, "+
+		"otherwise reconcile.runtime.k8sClientQPS and k8sClientBurst",
+		qps, shardsThreads, deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT, deployment.OPERATOR_K8S_CLIENT_BURST_LIMIT)
+	return true
+}
+
 // applyK8sClientTLSMinVersion stamps MinVersion onto the rest.Config transport via
 // rest.Config.Wrap, preserving client-go's TLS/proxy/HTTP2 setup. client-go invokes the
 // wrapper on the freshly-built *http.Transport before any request, and crypto/tls reads

diff --git a/pkg/chop/kube_machinery_test.go b/pkg/chop/kube_machinery_test.go
@@ -19,9 +19,16 @@ import (
 	"crypto/x509"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	kube "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/flowcontrol"
+
+	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
+	"github.com/altinity/clickhouse-operator/pkg/apis/deployment"
 )
 
 // notATransport is a RoundTripper that is NOT *http.Transport, used to exercise the
@@ -95,3 +102,114 @@ func TestFloorTransportTLSMinVersion_EnforcesHandshake(t *testing.T) {
 		_ = resp.Body.Close()
 	})
 }
+
+// testKubeconfig points at a port nothing listens on: building the clients sends no request.
+const testKubeconfig = `apiVersion: v1
+kind: Config
+clusters:
+- name: test
+  cluster:
+    server: https://127.0.0.1:1
+contexts:
+- name: test
+  context:
+    cluster: test
+current-context: test
+`
+
+// newClientset builds the clients from a config file with the given reconcile.runtime lines.
+func newClientset(t *testing.T, runtime string) *kube.Clientset {
+	t.Helper()
+	dir := t.TempDir()
+	kubeconfig := filepath.Join(dir, "kubeconfig")
+	require.NoError(t, os.WriteFile(kubeconfig, []byte(testKubeconfig), 0o600))
+	chopconf := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(chopconf, []byte("reconcile:\n  runtime:\n"+runtime), 0o600))
+	kubeClient, _, _, _, _ := GetClientset(kubeconfig, "", chopconf)
+	return kubeClient
+}
+
+// rateLimiters returns the rate limiters of the core/v1 and apps/v1 clients.
+func rateLimiters(t *testing.T, runtime string) (core, apps flowcontrol.RateLimiter) {
+	t.Helper()
+	kubeClient := newClientset(t, runtime)
+	return kubeClient.CoreV1().RESTClient().GetRateLimiter(), kubeClient.AppsV1().RESTClient().GetRateLimiter()
+}
+
+// burst counts the requests a fresh limiter lets through at once.
+func burst(limiter flowcontrol.RateLimiter) int {
+	n := 0
+	for n < 1000 && limiter.TryAccept() {
+		n++
+	}
+	return n
+}
+
+// The client takes QPS and burst from the config file; each env var, when set, wins over its own value.
+func TestGetClientsetRateLimitsFromConfig(t *testing.T) {
+	cases := []struct {
+		name             string
+		envQPS, envBurst string
+		qps              float32
+		burst            int
+	}{
+		{"config", "", "", 3, 7},
+		{"env vars win", "2", "4", 2, 4},
+		{"env burst wins", "", "4", 3, 4},
+		{"env qps wins", "2", "", 2, 7},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Setenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT, c.envQPS)
+			t.Setenv(deployment.OPERATOR_K8S_CLIENT_BURST_LIMIT, c.envBurst)
+			core, _ := rateLimiters(t, "    k8sClientQPS: 3\n    k8sClientBurst: 7\n")
+			require.Equal(t, c.qps, core.QPS())
+			require.Equal(t, c.burst, burst(core))
+		})
+	}
+}
+
+// Unset, 0 or negative keeps the client-go defaults: every API group gets its own 5/10 limiter.
+// Setting 5/10 explicitly would not: the clientset would share one limiter across groups.
+func TestGetClientsetRateLimitsDefault(t *testing.T) {
+	t.Setenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT, "")
+	t.Setenv(deployment.OPERATOR_K8S_CLIENT_BURST_LIMIT, "")
+	for _, runtime := range []string{
+		"",
+		"    k8sClientQPS: 0\n    k8sClientBurst: 0\n",
+		"    k8sClientQPS: -1\n    k8sClientBurst: -1\n",
+	} {
+		core, apps := rateLimiters(t, runtime)
+		require.NotSame(t, core, apps, runtime)
+		require.Equal(t, float32(5), core.QPS(), runtime)
+		require.Equal(t, 10, burst(core), runtime)
+	}
+}
+
+// The warning fires when one CHI may reconcile more shards at once than the client's QPS.
+func TestWarnIfK8sClientQPSTooLow(t *testing.T) {
+	t.Setenv(deployment.OPERATOR_K8S_CLIENT_BURST_LIMIT, "")
+	cases := []struct {
+		name          string
+		envQPS        string
+		runtime       string
+		shardsThreads int
+		warn          bool
+	}{
+		{"code defaults", "", "", 1, false},
+		{"shipped config.yaml", "", "", 5, false},
+		{"shards raised, default limit", "", "", 200, true},
+		{"shards raised, low limit", "", "    k8sClientQPS: 50\n    k8sClientBurst: 100\n", 200, true},
+		{"shards raised, limit raised", "", "    k8sClientQPS: 200\n    k8sClientBurst: 400\n", 200, false},
+		{"env qps 300 over config 3", "300", "    k8sClientQPS: 3\n    k8sClientBurst: 7\n", 200, false},
+		{"env qps 2 over config 200", "2", "    k8sClientQPS: 200\n    k8sClientBurst: 400\n", 200, true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Setenv(deployment.OPERATOR_K8S_CLIENT_QPS_LIMIT, c.envQPS)
+			config := &api.OperatorConfig{}
+			config.Reconcile.Runtime.ReconcileShardsThreadsNumber = c.shardsThreads
+			require.Equal(t, c.warn, WarnIfK8sClientQPSTooLow(newClientset(t, c.runtime), config))
+		})
+	}
+}