sunsingerus · Aug 3, 2026 · Jul 24, 2026 · Jul 24, 2026
diff --git a/pkg/controller/chi/worker-reconciler-chi.go b/pkg/controller/chi/worker-reconciler-chi.go
@@ -52,9 +52,6 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal
 	switch {
 	case w.isAfterFinalizerInstalled(old, new):
 		w.a.M(new).F().Info("isAfterFinalizerInstalled - continue reconcile-1")
-	case w.isGenerationTheSame(old, new):
-		log.V(2).M(new).F().Info("isGenerationTheSame() - nothing to do here, exit")
-		return nil
 	}
 
 	w.a.M(new).S().P()
@@ -64,6 +61,10 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal
 	metrics.CRReconcilesStarted(ctx, new)
 	startTime := time.Now()
 
+	// Capture before buildCR: normalize/fillStatus overwrites status.chop-ip with
+	// the current operator pod IP, which would hide an IP change.
+	prevCHOpIP := new.Status.GetCHOpIP()
+
 	new = w.buildCR(ctx, new)
 
 	// If buildCR's normalize already aborted the CR (e.g. FIPS strict rejected
@@ -83,6 +84,10 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal
 		return nil
 	}
 
+	generationTheSame := w.isGenerationTheSame(old, new)
+	hasUnhealthyHosts := w.hasUnhealthyHosts(ctx, new)
+	operatorIPTheSame := w.isOperatorIPTheSame(prevCHOpIP)
+
 	switch {
 	case new.Spec.Suspend.Value():
 		// if CR is suspended, should skip reconciliation
@@ -107,12 +112,23 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal
 			metrics.CRReconcilesCompleted(ctx, new)
 		}
 		return nil
+	case generationTheSame && !hasUnhealthyHosts && operatorIPTheSame:
+		// Spec unchanged, hosts healthy, operator IP the same — nothing to do.
+		// Do not skip when unhealthy (#1704) or when operator IP changed (need to
+		// refresh clickhouse-operator user networks).
+		w.a.M(new).F().Info("isGenerationTheSame(), all hosts healthy, operator IP the same - nothing to do, exit")
+		metrics.CRReconcilesCompleted(ctx, new)
+		return nil
+	case generationTheSame && !hasUnhealthyHosts && !operatorIPTheSame:
+		w.a.M(new).F().Info("Operator IP changed - continue reconcile to refresh clickhouse-operator user networks")
 	case new.HasReconcileWork():
 		w.a.M(new).F().Info("CR has reconcile work - continue reconcile")
 	case w.isAfterFinalizerInstalled(new.GetAncestorT(), new):
 		w.a.M(new).F().Info("isAfterFinalizerInstalled - continue reconcile-2")
 	case w.crHasHostNeedingStuckRecovery(ctx, new):
 		w.a.M(new).F().Info("CR has a sustained-NotReady host - continue reconcile for stuck-host recovery")
+	case hasUnhealthyHosts:
+		w.a.M(new).F().Warning("Unhealthy hosts detected - continue reconcile for recovery")
 	default:
 		w.a.M(new).F().Info("No reconcile work - abort reconcile")
 		metrics.CRReconcilesCompleted(ctx, new)
@@ -549,7 +565,9 @@ func (w *worker) reconcileHostStatefulSet(ctx context.Context, host *api.Host, o
 	// If a pod-template rollout is already coming via the StatefulSet reconcile below, skip
 	// the pre-rollout software restart — see hostRequiresStatefulSetRollout for why.
 	if w.shouldForceRestartHost(ctx, host) {
-		if hostRequiresStatefulSetRollout(host) {
+		if w.isHostHealthyForReconcile(ctx, host) && !w.isShardSafeToDisruptHost(ctx, host) {
+			w.a.V(1).M(host).F().Warning("Skip force restart: no healthy peer in shard. Host: %s", host.GetName())
+		} else if hostRequiresStatefulSetRollout(host) {
 			w.a.V(1).M(host).F().Info("Pod template changed - skipping software restart; StatefulSet rollout will restart the pod. Host: %s", host.GetName())
 		} else {
 			w.a.V(1).M(host).F().Info("Reconcile host STS force restart: %s", host.GetName())
@@ -900,9 +918,40 @@ func (w *worker) reconcileShardWithHosts(ctx context.Context, shard api.IShard)
 	if err := w.reconcileShard(ctx, shard); err != nil {
 		return err
 	}
-	return shard.WalkHostsAbortOnError(func(host *api.Host) error {
-		return w.reconcileHost(ctx, host)
+
+	// Recovery first, then rollout — avoid disrupting the last healthy peer while
+	// another replica in the shard is still down (#1704).
+	var recovery, rollout []*api.Host
+	shard.WalkHosts(func(host *api.Host) error {
+		if w.isHostHealthyForReconcile(ctx, host) {
+			rollout = append(rollout, host)
+		} else {
+			recovery = append(recovery, host)
+		}
+		return nil
 	})
+
+	for _, host := range recovery {
+		if err := w.reconcileHost(ctx, host); err != nil {
+			return err
+		}
+	}
+
+	deferred := 0
+	for _, host := range rollout {
+		if w.hostMayRequireDisruption(ctx, host) && !w.isShardSafeToDisruptHost(ctx, host) {
+			w.a.V(1).M(host).F().Warning("Deferring host reconcile: no healthy peer in shard. Host: %s", host.GetName())
+			deferred++
+			continue
+		}
+		if err := w.reconcileHost(ctx, host); err != nil {
+			return err
+		}
+	}
+	if deferred > 0 {
+		return common.ErrCRUDAbort
+	}
+	return nil
 }
 
 // reconcileShard reconciles specified shard, excluding nested replicas

diff --git a/pkg/controller/chi/worker-status-helpers.go b/pkg/controller/chi/worker-status-helpers.go
@@ -163,6 +163,74 @@ func (w *worker) isPodOK(ctx context.Context, host *api.Host) bool {
 	return false
 }
 
+// isHostHealthyForReconcile is true when host is safe to count as a live peer for
+// shard-safety checks. Stopped/troubleshoot hosts are intentionally unavailable
+// and do not block disruption of a peer.
+func (w *worker) isHostHealthyForReconcile(ctx context.Context, host *api.Host) bool {
+	if host == nil {
+		return false
+	}
+	if host.IsStopped() || host.IsTroubleshoot() {
+		return true
+	}
+	return w.c != nil && w.c.kube != nil &&
+		w.isPodRunning(ctx, host) &&
+		w.isPodReady(ctx, host) &&
+		!w.isPodCrushed(ctx, host)
+}
+
+func (w *worker) hasUnhealthyHosts(ctx context.Context, cr *api.ClickHouseInstallation) bool {
+	if cr == nil {
+		return false
+	}
+	found := false
+	cr.WalkHosts(func(host *api.Host) error {
+		if !w.isHostHealthyForReconcile(ctx, host) {
+			found = true
+		}
+		return nil
+	})
+	return found
+}
+
+// isOperatorIPTheSame reports whether the operator pod IP still matches the
+// CHOpIP persisted on the CR from the previous reconcile. A changed IP must
+// force reconcile so clickhouse-operator user networks/host_regexp are refreshed.
+// Compare against prevCHOpIP captured before buildCR — normalize/fillStatus
+// overwrites status.chop-ip with the current operator IP.
+func (w *worker) isOperatorIPTheSame(prevCHOpIP string) bool {
+	ip, _ := chop.GetRuntimeParam(deployment.OPERATOR_POD_IP)
+	return ip == prevCHOpIP
+}
+
+// isShardSafeToDisruptHost is true when host may be excluded/restarted/rolled
+// without taking down the last healthy replica in its shard.
+func (w *worker) isShardSafeToDisruptHost(ctx context.Context, host *api.Host) bool {
+	if host == nil || host.GetShard() == nil || host.GetShard().HostsCount() <= 1 {
+		return true
+	}
+	safe := false
+	host.GetShard().WalkHosts(func(peer *api.Host) error {
+		if peer != nil && peer.GetName() != host.GetName() && w.isHostHealthyForReconcile(ctx, peer) {
+			safe = true
+		}
+		return nil
+	})
+	return safe
+}
+
+// hostMayRequireDisruption is true when reconcile is expected to restart or roll the host.
+func (w *worker) hostMayRequireDisruption(ctx context.Context, host *api.Host) bool {
+	if host == nil || host.IsStopped() || host.IsTroubleshoot() {
+		return false
+	}
+	if w.shouldForceRestartHost(ctx, host) {
+		return true
+	}
+	status := host.GetReconcileAttributes().GetStatus()
+	return !status.Is(types.ObjectStatusRequested) && !status.Is(types.ObjectStatusSame)
+}
+
 func (w *worker) isPodRestarted(ctx context.Context, host *api.Host, initialRestartCounters map[string]int) bool {
 	curRestartCounters, _ := w.c.kube.Pod().(interfaces.IKubePodEx).GetRestartCounters(ctx, host)
 	return !util.MapsAreTheSame(initialRestartCounters, curRestartCounters)
@@ -180,66 +248,6 @@ func (w *worker) doesHostHaveNoReplicationDelay(ctx context.Context, host *api.H
 	return delay <= chop.Config().Reconcile.Host.Wait.Replicas.Delay.IntValue()
 }
 
-// isCHIProcessedOnTheSameIP checks whether it is just a restart of the operator on the same IP
-func (w *worker) isCHIProcessedOnTheSameIP(chi *api.ClickHouseInstallation) bool {
-	ip, _ := chop.GetRuntimeParam(deployment.OPERATOR_POD_IP)
-	operatorIpIsTheSame := ip == chi.Status.GetCHOpIP()
-	log.V(1).Info("Operator IPs to process CHI: %s. Previous: %s Cur: %s", chi.Name, chi.Status.GetCHOpIP(), ip)
-
-	if !operatorIpIsTheSame {
-		// Operator has restarted on the different IP address.
-		// We may need to reconcile config files
-		log.V(1).Info("Operator IPs are different. Operator was restarted on another IP since previous reconcile of the CHI: %s", chi.Name)
-		return false
-	}
-
-	log.V(1).Info("Operator IPs are the same as on previous reconcile of the CHI: %s", chi.Name)
-	return w.isCleanRestart(chi)
-}
-
-// isCleanRestart checks whether it is just a restart of the operator and CHI has no changes since last processed
-func (w *worker) isCleanRestart(chi *api.ClickHouseInstallation) bool {
-	// Clean restart may be only in case operator has just recently started
-	if !w.isJustStarted() {
-		log.V(1).Info("Operator is not just started. May not be clean restart")
-		return false
-	}
-
-	log.V(1).Info("Operator just started. May be clean restart")
-
-	// Migration support
-	// Do we have have previously completed CHI?
-	// In case no - this means that CHI has either not completed or we are migrating from
-	// such a version of the operator, where there is no completed CHI at all
-	noCompletedCHI := !chi.HasAncestor()
-	// Having status completed and not having completed CHI suggests we are migrating operator version
-	statusIsCompleted := chi.Status.GetStatus() == api.StatusCompleted
-	if noCompletedCHI && statusIsCompleted {
-		// In case of a restart - assume that normalized is already completed
-		chi.SetAncestor(chi.GetTarget())
-	}
-
-	// Check whether anything has changed in CHI spec
-	// In case the generation is the same as already completed - it is clean restart
-	generationIsOk := false
-	// However, completed CHI still can be missing, for example, in newly requested CHI
-	if chi.HasAncestor() {
-		generationIsOk = chi.Generation == chi.GetAncestor().GetGeneration()
-		log.V(1).Info(
-			"CHI %s has ancestor. Generations. Prev: %d Cur: %d Generation is the same: %t",
-			chi.Name,
-			chi.GetAncestor().GetGeneration(),
-			chi.Generation,
-			generationIsOk,
-		)
-	} else {
-		log.V(1).Info("CHI %s has NO ancestor, meaning reconcile cycle was never completed.", chi.Name)
-	}
-
-	log.V(1).Info("Is CHI %s clean on operator restart: %t", chi.Name, generationIsOk)
-	return generationIsOk
-}
-
 // areUsableOldAndNew checks whether there are old and new usable
 func (w *worker) areUsableOldAndNew(old, new *api.ClickHouseInstallation) bool {
 	if old == nil {

diff --git a/pkg/controller/chi/worker-wait-exclude-include-restart.go b/pkg/controller/chi/worker-wait-exclude-include-restart.go
@@ -381,6 +381,18 @@ func (w *worker) shouldExcludeHost(ctx context.Context, host *api.Host) bool {
 		return false
 
 	case w.shouldForceRestartHost(ctx, host):
+		if !w.isHostHealthyForReconcile(ctx, host) {
+			w.a.V(1).M(host).F().Info(
+				"Host requires restart but is unhealthy - skip exclude. Host/shard/cluster: %d/%d/%s",
+				host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName)
+			return false
+		}
+		if !w.isShardSafeToDisruptHost(ctx, host) {
+			w.a.V(1).M(host).F().Warning(
+				"Host restart exclude deferred: no healthy peer. Host/shard/cluster: %d/%d/%s",
+				host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName)
+			return false
+		}
 		w.a.V(1).
 			M(host).F().
 			Info("Host should be restarted, need to exclude. Host/shard/cluster: %d/%d/%s",

diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go
@@ -353,18 +353,6 @@ func (w *worker) updateCHI(ctx context.Context, old, new *api.ClickHouseInstalla
 		return nil
 	}
 
-	if w.isCHIProcessedOnTheSameIP(new) {
-		// First minute after restart do not reconcile already reconciled generations
-		w.a.V(1).M(new).F().Info("Will not reconcile known generation after restart. Generation %d", new.Generation)
-		return nil
-	}
-
-	if util.IsContextDone(ctx) {
-		log.V(1).Info("Reconcile is aborted.")
-		return nil
-	}
-
-	// CHI is being reconciled
 	return w.reconcileCR(ctx, old, new)
 }
 

diff --git a/tests/e2e/kubectl.py b/tests/e2e/kubectl.py
@@ -932,14 +932,14 @@ def check_pdb(chi, kind, clusters, ns=None, shell=None):
             assert labels[f"{label}/namespace"] == current().context.test_namespace
             assert pdb["spec"]["maxUnavailable"] == max_unavailable
 
-def force_chi_reconcile(chi, taskID="reconcile", ns=None, shell=None):
-    force_reconcile(chi, "chi", taskID, ns, shell)
+def force_chi_reconcile(chi, taskID="reconcile", status="Completed", ns=None, shell=None):
+    force_reconcile(chi, "chi", taskID, status, ns, shell)
 
-def force_chk_reconcile(chk, taskID="reconcile", ns=None, shell=None):
-    force_reconcile(chk, "chk", taskID, ns, shell)
+def force_chk_reconcile(chk, taskID="reconcile", status="Completed", ns=None, shell=None):
+    force_reconcile(chk, "chk", taskID, status, ns, shell)
 
 
-def force_reconcile(name, kind, taskID, ns=None, shell=None):
+def force_reconcile(name, kind, taskID, status="Completed", ns=None, shell=None):
     with Then(f"Trigger {kind} reconcile with taskID:\"{taskID}\""):
         cmd = f'patch {kind} {name} --type=\'json\' --patch=\'[{{"op":"add","path":"/spec/taskID","value":"{taskID}"}}]\''
         launch(cmd, ns=ns, shell=shell)
@@ -953,9 +953,9 @@ def force_reconcile(name, kind, taskID, ns=None, shell=None):
         # either "already Completed" or "InProgress → Completed" transitions.
         if kind == "chi":
             wait_chi_status(name, "InProgress", ns=ns, shell=shell)
-            wait_chi_status(name, "Completed", ns=ns, shell=shell)
+            wait_chi_status(name, status, ns=ns, shell=shell)
         elif kind == "chk":
             wait_chk_status(name, "InProgress", ns=ns, shell=shell)
-            wait_chk_status(name, "Completed", ns=ns, shell=shell)
+            wait_chk_status(name, status, ns=ns, shell=shell)
         else:
-            assert kind == "chi" or kind == "chk"
+            assert kind == "chi" or kind == "chk"
diff --git a/tests/e2e/manifests/chi/test-083-shard-safety-1.yaml b/tests/e2e/manifests/chi/test-083-shard-safety-1.yaml
@@ -0,0 +1,13 @@
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
+metadata:
+  name: test-083-shard-safety
+spec:
+  useTemplates:
+    - name: clickhouse-version
+  configuration:
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 1
+          replicasCount: 2
diff --git a/tests/e2e/manifests/chi/test-083-shard-safety-2.yaml b/tests/e2e/manifests/chi/test-083-shard-safety-2.yaml
@@ -0,0 +1,21 @@
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
+metadata:
+  name: test-083-shard-safety
+spec:
+  templates:
+    podTemplates:
+      - name: clickhouse-broken
+        spec:
+          containers:
+            - name: clickhouse-pod
+              image: clickhouse/clickhouse-server:26.3-broken
+  defaults:
+    templates:
+      podTemplate: clickhouse-broken
+  configuration:
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 1
+          replicasCount: 2
diff --git a/tests/e2e/manifests/chi/test-083-shard-safety-3.yaml b/tests/e2e/manifests/chi/test-083-shard-safety-3.yaml
@@ -0,0 +1,21 @@
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
+metadata:
+  name: test-083-shard-safety
+spec:
+  templates:
+    podTemplates:
+      - name: clickhouse-stable-new
+        spec:
+          containers:
+            - name: clickhouse-pod
+              image: clickhouse/clickhouse-server:26.3
+  defaults:
+    templates:
+      podTemplate: clickhouse-stable-new
+  configuration:
+    clusters:
+      - name: default
+        layout:
+          shardsCount: 1
+          replicasCount: 2