Slach · Nov 2, 2026 · Nov 2, 2026 · Nov 6, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * **EXPERIMENTAL: Operator-managed backup and restore.** Added `ClickHouseBackup`, `ClickHouseBackupSchedule`, and `ClickHouseRestore` resources using `clickhouse-backup`. Supports per-cluster backups and restores, scheduled backups, retention, configurable storage paths, and cross-installation restores. Enable with `spec.backup.enabled: "yes"`. See [backup documentation](docs/backup.md). by [@discostur](https://github.com/discostur) and [@sunsingerus](https://github.com/sunsingerus) in [#2015](https://github.com/Altinity/clickhouse-operator/pull/2015). Related [#1795](https://github.com/Altinity/clickhouse-operator/issues/1795), [#862](https://github.com/Altinity/clickhouse-operator/issues/862)
 
+* **Opt-in pprof endpoint for the operator and metrics exporter.** Disabled by default. Enable with `-pprof-address` / `OPERATOR_PPROF_ADDRESS` / `METRICS_EXPORTER_PPROF_ADDRESS`, or the Helm values `operator.pprof.*` and `metrics.pprof.*`. pprof is served on its own loopback listener, never on the metrics ports. The metrics exporter now serves its metrics port from a private mux, as the operator already does. See [profiling](docs/monitoring_setup.md#profiling-the-operator-and-metrics-exporter-pprof). by [@Slach](https://github.com/Slach) in [#2133](https://github.com/Altinity/clickhouse-operator/pull/2133). Fixes [#2016](https://github.com/Altinity/clickhouse-operator/issues/2016)
+
 * **Volume downscale tool now supports StorageClass migration.** Set `NEW_STORAGE_CLASS` in `tools/volume_downscale.sh` to move a volume to another StorageClass at the same or a smaller size; the script also detects repeated volume attachment and mount failures. See [runbook](tools/volume_downscale.md). by [@lesandie](https://github.com/lesandie) in [#2125](https://github.com/Altinity/clickhouse-operator/pull/2125)
 
 ### Changed

diff --git a/cmd/metrics_exporter/app/metrics_exporter.go b/cmd/metrics_exporter/app/metrics_exporter.go
@@ -28,11 +28,13 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
+	"github.com/altinity/clickhouse-operator/pkg/apis/deployment"
 	"github.com/altinity/clickhouse-operator/pkg/chop"
 	chopclientset "github.com/altinity/clickhouse-operator/pkg/client/clientset/versioned"
 	chopinformers "github.com/altinity/clickhouse-operator/pkg/client/informers/externalversions"
 	"github.com/altinity/clickhouse-operator/pkg/metrics/clickhouse"
 	"github.com/altinity/clickhouse-operator/pkg/util/fips"
+	"github.com/altinity/clickhouse-operator/pkg/util/profiling"
 	"github.com/altinity/clickhouse-operator/pkg/version"
 )
 
@@ -70,6 +72,9 @@ var (
 	metricsEP string
 
 	chiListEP string
+
+	// pprofAddress defines pprof listen address. Empty means pprof is disabled
+	pprofAddress string
 )
 
 func init() {
@@ -80,6 +85,7 @@ func init() {
 	flag.StringVar(&masterURL, "master", "", "The address of custom Kubernetes API server. Makes sense if runs outside of the cluster and not being specified in kube config file only.")
 	flag.StringVar(&metricsEP, "metrics-endpoint", defaultMetricsEndpoint, "The Prometheus exporter endpoint.")
 	flag.StringVar(&chiListEP, "chi-list-endpoint", defaultChiListEndPoint, "The CHI list endpoint.")
+	flag.StringVar(&pprofAddress, profiling.FlagName, "", profiling.FlagUsage)
 	flag.Parse()
 }
 
@@ -123,6 +129,9 @@ func Run() {
 
 	log.Infof("Starting metrics exporter. Version:%s GitSHA:%s BuiltAt:%s\n", version.Version, version.GitSHA, version.BuiltAt)
 
+	// Opt-in pprof on its own listener, see pkg/util/profiling
+	profiling.Start(ctx, profiling.ResolveAddress(pprofAddress, deployment.METRICS_EXPORTER_PPROF_ADDRESS), "metrics-exporter")
+
 	// Initialize k8s API clients
 	kubeClient, _, chopClient, _, _ := chop.GetClientset(kubeConfigFile, masterURL, chopConfigFile)
 

diff --git a/cmd/operator/app/main.go b/cmd/operator/app/main.go
@@ -24,8 +24,10 @@ import (
 	"syscall"
 
 	log "github.com/altinity/clickhouse-operator/pkg/announcer"
+	"github.com/altinity/clickhouse-operator/pkg/apis/deployment"
 	"github.com/altinity/clickhouse-operator/pkg/util"
 	"github.com/altinity/clickhouse-operator/pkg/util/fips"
+	"github.com/altinity/clickhouse-operator/pkg/util/profiling"
 	"github.com/altinity/clickhouse-operator/pkg/version"
 )
 
@@ -51,6 +53,9 @@ var (
 
 	// masterURL defines URL of kubernetes master to be used
 	masterURL string
+
+	// pprofAddress defines pprof listen address. Empty means pprof is disabled
+	pprofAddress string
 )
 
 func init() {
@@ -59,6 +64,7 @@ func init() {
 	flag.BoolVar(&debugRequest, "debug", false, "Debug run")
 	flag.StringVar(&chopConfigFile, "config", "", "Path to clickhouse-operator config file.")
 	flag.StringVar(&masterURL, "master", "", "The address of custom Kubernetes API server. Makes sense if runs outside of the cluster and not being specified in kube config file only.")
+	flag.StringVar(&pprofAddress, profiling.FlagName, "", profiling.FlagUsage)
 }
 
 // Run is an entry point of the application
@@ -95,6 +101,10 @@ func Run() {
 	// Setup notification signals with cancel
 	setupSignalsNotification(cancelFunc)
 
+	// Opt-in pprof on its own listener, once per process, see pkg/util/profiling.
+	// controller-runtime managers keep their own PprofBindAddress unset (disabled).
+	profiling.Start(ctx, profiling.ResolveAddress(pprofAddress, deployment.OPERATOR_PPROF_ADDRESS), "clickhouse-operator")
+
 	var wg sync.WaitGroup
 
 	launchClickHouse(ctx, &wg)

diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md
@@ -112,6 +112,8 @@ crdHook:
 | metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository |
 | metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
 | metrics.livenessProbe | string | `nil` | optional liveness probe for the metrics-exporter container check `kubectl explain pod.spec.containers.livenessProbe` for details example:   httpGet:     path: /metrics     port: ch-metrics   initialDelaySeconds: 10   periodSeconds: 10 |
+| metrics.pprof.address | string | `"127.0.0.1:6061"` | pprof listen address; keep it on loopback, a non-loopback address logs a warning. Must differ from `operator.pprof.address`, both containers share the pod network |
+| metrics.pprof.enabled | bool | `false` | serve Go pprof from the metrics-exporter container on a separate listener (sets `METRICS_EXPORTER_PPROF_ADDRESS`). Never served on the metrics port. Access with `kubectl port-forward` |
 | metrics.readinessProbe | string | `nil` | optional readiness probe for the metrics-exporter container check `kubectl explain pod.spec.containers.readinessProbe` for details example:   httpGet:     path: /metrics     port: ch-metrics   initialDelaySeconds: 5   periodSeconds: 5 |
 | metrics.resources | object | `{}` | custom resource configuration |
 | nameOverride | string | `""` | override name of the chart |
@@ -124,6 +126,8 @@ crdHook:
 | operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository |
 | operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) |
 | operator.livenessProbe | string | `nil` | optional liveness probe for the clickhouse-operator container check `kubectl explain pod.spec.containers.livenessProbe` for details example:   httpGet:     path: /metrics     port: op-metrics   initialDelaySeconds: 10   periodSeconds: 10 |
+| operator.pprof.address | string | `"127.0.0.1:6060"` | pprof listen address; keep it on loopback, a non-loopback address logs a warning. Must differ from `metrics.pprof.address`, both containers share the pod network |
+| operator.pprof.enabled | bool | `false` | serve Go pprof from the clickhouse-operator container on a separate listener (sets `OPERATOR_PPROF_ADDRESS`). Never served on the metrics port. Access with `kubectl port-forward` |
 | operator.priorityClassName | string | "" | priority class name for the clickhouse-operator deployment, check `kubectl explain pod.spec.priorityClassName` for details |
 | operator.readinessProbe | string | `nil` | optional readiness probe for the clickhouse-operator container check `kubectl explain pod.spec.containers.readinessProbe` for details example:   httpGet:     path: /metrics     port: op-metrics   initialDelaySeconds: 5   periodSeconds: 5 |
 | operator.resources | object | `{}` | custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details |

diff --git a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml
@@ -150,6 +150,10 @@ spec:
                   containerName: {{ .Chart.Name }}
                   resource: limits.memory
                   divisor: "1Mi"
+            {{- if .Values.operator.pprof.enabled }}
+            - name: OPERATOR_PPROF_ADDRESS
+              value: {{ .Values.operator.pprof.address | quote }}
+            {{- end }}
             {{ with .Values.operator.env }}{{ toYaml . | nindent 12 }}{{ end }}
           ports:
             - containerPort: 9999
@@ -240,6 +244,10 @@ spec:
                   containerName: {{ .Chart.Name }}
                   resource: limits.memory
                   divisor: "1Mi"
+            {{- if .Values.metrics.pprof.enabled }}
+            - name: METRICS_EXPORTER_PPROF_ADDRESS
+              value: {{ .Values.metrics.pprof.address | quote }}
+            {{- end }}
             {{ with .Values.metrics.env }}{{ toYaml . | nindent 12 }}{{ end }}
           ports:
             - containerPort: 8888

diff --git a/deploy/helm/clickhouse-operator/values.schema.json b/deploy/helm/clickhouse-operator/values.schema.json
@@ -799,6 +799,19 @@
                     "description": "optional liveness probe for the metrics-exporter container, check `kubectl explain pod.spec.containers.livenessProbe` for details",
                     "type": ["object", "null"]
                 },
+                "pprof": {
+                    "type": "object",
+                    "properties": {
+                        "enabled": {
+                            "description": "serve Go pprof from the metrics-exporter container on a separate listener (sets `METRICS_EXPORTER_PPROF_ADDRESS`)",
+                            "type": "boolean"
+                        },
+                        "address": {
+                            "description": "pprof listen address, keep it on loopback",
+                            "type": "string"
+                        }
+                    }
+                },
                 "readinessProbe": {
                     "description": "optional readiness probe for the metrics-exporter container, check `kubectl explain pod.spec.containers.readinessProbe` for details",
                     "type": ["object", "null"]
@@ -868,6 +881,19 @@
                         }
                     }
                 },
+                "pprof": {
+                    "type": "object",
+                    "properties": {
+                        "enabled": {
+                            "description": "serve Go pprof from the clickhouse-operator container on a separate listener (sets `OPERATOR_PPROF_ADDRESS`)",
+                            "type": "boolean"
+                        },
+                        "address": {
+                            "description": "pprof listen address, keep it on loopback",
+                            "type": "string"
+                        }
+                    }
+                },
                 "priorityClassName": {
                     "description": "priority class name for the clickhouse-operator deployment, check `kubectl explain pod.spec.priorityClassName` for details",
                     "type": "string"

diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml
@@ -86,6 +86,13 @@ operator:
   # operator.env -- additional environment variables for the clickhouse-operator container in deployment
   # possible format value `[{"name": "SAMPLE", "value": "text"}]`
   env: []
+  pprof:
+    # operator.pprof.enabled -- serve Go pprof from the clickhouse-operator container on a separate listener (sets `OPERATOR_PPROF_ADDRESS`).
+    # Never served on the metrics port. Access with `kubectl port-forward`
+    enabled: false
+    # operator.pprof.address -- pprof listen address; keep it on loopback, a non-loopback address logs a warning.
+    # Must differ from `metrics.pprof.address`, both containers share the pod network
+    address: "127.0.0.1:6060"
   # operator.livenessProbe -- optional liveness probe for the clickhouse-operator container
   # check `kubectl explain pod.spec.containers.livenessProbe` for details
   # example:
@@ -128,6 +135,13 @@ metrics:
   # metrics.env -- additional environment variables for the deployment of metrics-exporter containers
   # possible format value `[{"name": "SAMPLE", "value": "text"}]`
   env: []
+  pprof:
+    # metrics.pprof.enabled -- serve Go pprof from the metrics-exporter container on a separate listener (sets `METRICS_EXPORTER_PPROF_ADDRESS`).
+    # Never served on the metrics port. Access with `kubectl port-forward`
+    enabled: false
+    # metrics.pprof.address -- pprof listen address; keep it on loopback, a non-loopback address logs a warning.
+    # Must differ from `operator.pprof.address`, both containers share the pod network
+    address: "127.0.0.1:6061"
   # metrics.livenessProbe -- optional liveness probe for the metrics-exporter container
   # check `kubectl explain pod.spec.containers.livenessProbe` for details
   # example:

diff --git a/dev/generate_helm_chart.sh b/dev/generate_helm_chart.sh
@@ -239,6 +239,7 @@ function update_deployment_resource() {
   yq e -i '.spec.template.spec.containers[0].resources |= "{{ toYaml .Values.operator.resources | nindent 12 }}"' "${file}"
   yq e -i '.spec.template.spec.containers[0].securityContext |= "{{ toYaml .Values.operator.containerSecurityContext | nindent 12 }}"' "${file}"
   yq e -i '(.spec.template.spec.containers[0].env[] | select(.valueFrom.resourceFieldRef.containerName == "clickhouse-operator") | .valueFrom.resourceFieldRef.containerName) = "{{ .Chart.Name }}"' "${file}"
+  yq e -i '.spec.template.spec.containers[0].env += ["__OPERATOR_PPROF_ENV__"]' "${file}"
   yq e -i '.spec.template.spec.containers[0].env += ["{{ with .Values.operator.env }}{{ toYaml . | nindent 12 }}{{ end }}"]' "${file}"
   yq e -i '.spec.template.spec.containers[0] += {"__OPERATOR_LIVENESS_PROBE__": null}' "${file}"
   yq e -i '.spec.template.spec.containers[0] += {"__OPERATOR_READINESS_PROBE__": null}' "${file}"
@@ -248,14 +249,17 @@ function update_deployment_resource() {
   yq e -i '.spec.template.spec.containers[1].resources |= "{{ toYaml .Values.metrics.resources | nindent 12 }}"' "${file}"
   yq e -i '.spec.template.spec.containers[1].securityContext |= "{{ toYaml .Values.metrics.containerSecurityContext | nindent 12 }}"' "${file}"
   yq e -i '(.spec.template.spec.containers[1].env[] | select(.valueFrom.resourceFieldRef.containerName == "clickhouse-operator") | .valueFrom.resourceFieldRef.containerName) = "{{ .Chart.Name }}"' "${file}"
+  yq e -i '.spec.template.spec.containers[1].env += ["__METRICS_PPROF_ENV__"]' "${file}"
   yq e -i '.spec.template.spec.containers[1].env += ["{{ with .Values.metrics.env }}{{ toYaml . | nindent 12 }}{{ end }}"]' "${file}"
   yq e -i '.spec.template.spec.containers[1] += {"__METRICS_LIVENESS_PROBE__": null}' "${file}"
   yq e -i '.spec.template.spec.containers[1] += {"__METRICS_READINESS_PROBE__": null}' "${file}"
 
   perl -pi -e "s/'{{ if .Values.podAnnotations }}{{ toYaml .Values.podAnnotations \| nindent 8 }}{{ end }}': null/{{ if .Values.podAnnotations }}{{ toYaml .Values.podAnnotations \| nindent 8 }}{{ end }}/g" "${file}"
   perl -pi -e "s/'{{ if .Values.operator.priorityClassName }}priorityClassName: {{ .Values.operator.priorityClassName \| quote }}{{ end }}': null/{{ if .Values.operator.priorityClassName }}priorityClassName: {{ .Values.operator.priorityClassName | quote }}{{ end }}/g" "${file}"
   perl -pi -e "s/- '{{ with .Values.operator.env }}{{ toYaml . \| nindent 12 }}{{ end }}'/{{ with .Values.operator.env }}{{ toYaml . \| nindent 12 }}{{ end }}/g" "${file}"
+  perl -0pi -e 's/\n(\s*)- __OPERATOR_PPROF_ENV__/\n${1}{{- if .Values.operator.pprof.enabled }}\n${1}- name: OPERATOR_PPROF_ADDRESS\n${1}  value: {{ .Values.operator.pprof.address | quote }}\n${1}{{- end }}/g' "${file}"
   perl -pi -e "s/- '{{ with .Values.metrics.env }}{{ toYaml . \| nindent 12 }}{{ end }}'/{{ with .Values.metrics.env }}{{ toYaml . \| nindent 12 }}{{ end }}/g" "${file}"
+  perl -0pi -e 's/\n(\s*)- __METRICS_PPROF_ENV__/\n${1}{{- if .Values.metrics.pprof.enabled }}\n${1}- name: METRICS_EXPORTER_PPROF_ADDRESS\n${1}  value: {{ .Values.metrics.pprof.address | quote }}\n${1}{{- end }}/g' "${file}"
   perl -0pi -e 's/\n\s*__OPERATOR_LIVENESS_PROBE__: null/\n          {{- with .Values.operator.livenessProbe }}\n          livenessProbe: {{- toYaml . | nindent 12 }}\n          {{- end }}/g' "${file}"
   perl -0pi -e 's/\n\s*__OPERATOR_READINESS_PROBE__: null/\n          {{- with .Values.operator.readinessProbe }}\n          readinessProbe: {{- toYaml . | nindent 12 }}\n          {{- end }}/g' "${file}"
   perl -0pi -e 's/\n\s*__METRICS_LIVENESS_PROBE__: null/\n          {{- with .Values.metrics.livenessProbe }}\n          livenessProbe: {{- toYaml . | nindent 12 }}\n          {{- end }}/g' "${file}"

diff --git a/docs/monitoring_setup.md b/docs/monitoring_setup.md
@@ -6,3 +6,44 @@ In order to setup monitoring, we need to:
 
 [prometheus_setup]: ./prometheus_setup.md
 [grafana_setup]: ./grafana_setup.md
+
+## Profiling the operator and metrics-exporter (pprof)
+
+Both `clickhouse-operator` and `metrics-exporter` can serve Go [pprof](https://pkg.go.dev/net/http/pprof) profiles,
+for example to find out why a container is OOM-killed. pprof is **disabled by default**. When enabled it is served
+on its own listener only. It never appears on the metrics ports (`9999` and `8888`) or behind the `clickhouse-operator-metrics` Service.
+
+| Container           | Flag             | Environment variable             | Helm values                                  |
+|---------------------|------------------|----------------------------------|----------------------------------------------|
+| `clickhouse-operator` | `-pprof-address` | `OPERATOR_PPROF_ADDRESS`         | `operator.pprof.enabled`, `operator.pprof.address` (default `127.0.0.1:6060`) |
+| `metrics-exporter`  | `-pprof-address` | `METRICS_EXPORTER_PPROF_ADDRESS` | `metrics.pprof.enabled`, `metrics.pprof.address` (default `127.0.0.1:6061`) |
+
+The value is a `host:port` listen address. An empty value disables pprof, and the flag takes precedence over the environment variable.
+Both containers share the pod network, so give them different ports.
+Profiles expose process memory, which includes credentials, so bind to loopback (`127.0.0.1`). The binaries log a warning
+whenever pprof is enabled, plus another one if the address is not loopback.
+
+Enable with Helm:
+
+```bash
+helm upgrade clickhouse-operator altinity/altinity-clickhouse-operator --reuse-values \
+  --set operator.pprof.enabled=true \
+  --set metrics.pprof.enabled=true
+```
+
+Without Helm, add `OPERATOR_PPROF_ADDRESS=127.0.0.1:6060` / `METRICS_EXPORTER_PPROF_ADDRESS=127.0.0.1:6061` to the `env` of the respective container.
+
+Reach the loopback listener with `kubectl port-forward` and collect profiles:
+
+```bash
+kubectl -n <namespace> port-forward deploy/clickhouse-operator 6060:6060 6061:6061
+
+# operator
+go tool pprof http://localhost:6060/debug/pprof/heap
+# metrics-exporter
+go tool pprof http://localhost:6061/debug/pprof/heap
+go tool pprof "http://localhost:6061/debug/pprof/profile?seconds=30"
+curl -s "http://localhost:6061/debug/pprof/goroutine?debug=2"
+```
+
+Disable pprof again when you are done.
diff --git a/pkg/apis/deployment/env_vars.go b/pkg/apis/deployment/env_vars.go
@@ -53,4 +53,9 @@ const (
 
 	// CHOP_CONFIG path to clickhouse operator configuration file
 	CHOP_CONFIG = "CHOP_CONFIG"
+
+	// OPERATOR_PPROF_ADDRESS enables pprof on the operator at the specified host:port. Overridden by -pprof-address
+	OPERATOR_PPROF_ADDRESS = "OPERATOR_PPROF_ADDRESS"
+	// METRICS_EXPORTER_PPROF_ADDRESS enables pprof on the metrics-exporter at the specified host:port. Overridden by -pprof-address
+	METRICS_EXPORTER_PPROF_ADDRESS = "METRICS_EXPORTER_PPROF_ADDRESS"
 )