Tyagiquamar · Oct 18, 2026 · Oct 20, 2026 · Oct 20, 2026 · Oct 21, 2026 · Nov 3, 2026
diff --git a/pkg/controller/chi/worker-pod-retry.go b/pkg/controller/chi/worker-pod-retry.go
@@ -56,6 +56,11 @@ var normalizeTimeAbortReasons = []string{
 // Controlled by reconcile.recovery.onStatus.aborted.onPodReady config option (default: retry).
 // The decision to re-enqueue is based on the CHI's Status alone, not on ActionPlan —
 // see shouldTriggerAutoRecovery for the rationale.
+//
+// Peer-safety during the recovery reconcile itself is enforced by
+// isHostHealthyForReconcile, which requires clickhouse.altinity.com/ready=yes so a
+// container-Ready but Service-excluded replica cannot authorize disruption of its
+// last eligible sibling (#2106).
 func (w *worker) recoverAbortedReconcileOnPodReady(ctx context.Context, oldPod, newPod *core.Pod) {
 	if !chop.Config().ShouldRecoverAbortedOnPodReady() {
 		return

diff --git a/pkg/controller/chi/worker-status-helpers.go b/pkg/controller/chi/worker-status-helpers.go
@@ -27,7 +27,9 @@ import (
 	"github.com/altinity/clickhouse-operator/pkg/chop"
 	"github.com/altinity/clickhouse-operator/pkg/interfaces"
 	"github.com/altinity/clickhouse-operator/pkg/model/chi/config"
+	chiLabeler "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler"
 	commonConfig "github.com/altinity/clickhouse-operator/pkg/model/common/config"
+	commonlabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler"
 	"github.com/altinity/clickhouse-operator/pkg/model/k8s"
 	"github.com/altinity/clickhouse-operator/pkg/util"
 )
@@ -187,8 +189,13 @@ func (w *worker) isPodOK(ctx context.Context, host *api.Host) bool {
 }
 
 // isHostHealthyForReconcile is true when host is safe to count as a live peer for
-// shard-safety checks. Stopped/troubleshoot hosts are intentionally unavailable
-// and do not block disruption of a peer.
+// shard-safety checks and recovery ordering. Stopped/troubleshoot hosts are
+// intentionally unavailable and do not block disruption of a peer.
+//
+// Kubelet readiness alone is not enough: after an aborted reconcile a pod can be
+// container-Ready while still excluded from the chop-generated Service
+// (clickhouse.altinity.com/ready=no). Counting that host as healthy lets recovery
+// disrupt its last Service-eligible sibling (#2106).
 func (w *worker) isHostHealthyForReconcile(ctx context.Context, host *api.Host) bool {
 	if host == nil {
 		return false
@@ -199,9 +206,9 @@ func (w *worker) isHostHealthyForReconcile(ctx context.Context, host *api.Host)
 	if w.c == nil || w.c.kube == nil {
 		return false
 	}
-	// One fetch, four pure reads. Pod().Get is a live API call, and this predicate runs per peer
-	// across the shard walk, so asking four times cost four round trips per host - and let the
-	// four answers come from four different versions of the pod, which is exactly the kind of
+	// One fetch, five pure reads. Pod().Get is a live API call, and this predicate runs per peer
+	// across the shard walk, so asking multiple times cost multiple round trips per host - and let
+	// the answers come from different versions of the pod, which is exactly the kind of
 	// inconsistency a health verdict must not be built on.
 	pod, err := w.c.kube.Pod().Get(ctx, host)
 	if err != nil {
@@ -210,7 +217,22 @@ func (w *worker) isHostHealthyForReconcile(ctx context.Context, host *api.Host)
 	return k8s.PodPhaseIsRunning(pod) &&
 		!k8s.PodHasNotReadyContainers(pod) &&
 		!podIsTerminating(pod) &&
-		!k8s.PodHasCrushedContainers(pod)
+		!k8s.PodHasCrushedContainers(pod) &&
+		podHasOperatorServiceReadyLabel(pod, host.GetCR())
+}
+
+// podHasOperatorServiceReadyLabel reports whether the pod carries
+// clickhouse.altinity.com/ready=yes for the given CR's labeler.
+func podHasOperatorServiceReadyLabel(pod *core.Pod, cr api.ICustomResource) bool {
+	if pod == nil || cr == nil {
+		return false
+	}
+	l := chiLabeler.New(cr)
+	labels := pod.GetLabels()
+	if labels == nil {
+		return false
+	}
+	return labels[l.Get(commonlabeler.LabelReadyName)] == l.Get(commonlabeler.LabelReadyValueReady)
 }
 
 func (w *worker) hasUnhealthyHosts(ctx context.Context, cr *api.ClickHouseInstallation) bool {

diff --git a/pkg/controller/chi/worker-status-helpers_test.go b/pkg/controller/chi/worker-status-helpers_test.go
@@ -25,6 +25,8 @@ import (
 
 	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
 	"github.com/altinity/clickhouse-operator/pkg/interfaces"
+	chiLabeler "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler"
+	commonlabeler "github.com/altinity/clickhouse-operator/pkg/model/common/tags/labeler"
 )
 
 // TestPodIsSustainedNotReady covers the pure post-fetch decision used by
@@ -289,10 +291,17 @@ func newHealthyPodWorker(pod *core.Pod) (*worker, *api.Host) {
 func TestIsHostHealthyForReconcileTreatsTerminatingPodAsUnhealthy(t *testing.T) {
 	deleting := meta.NewTime(time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC))
 
-	// Running, single container Ready, not crashing - healthy by every other predicate.
+	// Running, single container Ready, Service-eligible, not crashing - healthy by every predicate.
 	newPod := func(deletionTimestamp *meta.Time) *core.Pod {
+		cr := &api.ClickHouseInstallation{}
+		l := chiLabeler.New(cr)
 		return &core.Pod{
-			ObjectMeta: meta.ObjectMeta{DeletionTimestamp: deletionTimestamp},
+			ObjectMeta: meta.ObjectMeta{
+				DeletionTimestamp: deletionTimestamp,
+				Labels: map[string]string{
+					l.Get(commonlabeler.LabelReadyName): l.Get(commonlabeler.LabelReadyValueReady),
+				},
+			},
 			Status: core.PodStatus{
 				Phase:             core.PodRunning,
 				ContainerStatuses: []core.ContainerStatus{{Ready: true}},
@@ -311,3 +320,42 @@ func TestIsHostHealthyForReconcileTreatsTerminatingPodAsUnhealthy(t *testing.T)
 			"a pod with deletionTimestamp must never count as a shard's healthy peer")
 	})
 }
+
+// TestIsHostHealthyForReconcileRequiresOperatorReadyLabel pins #2106: a kubelet-Ready
+// pod that is still excluded from the chop-generated Service (ready=no) must not count
+// as a healthy peer for recovery ordering or disruption safety.
+func TestIsHostHealthyForReconcileRequiresOperatorReadyLabel(t *testing.T) {
+	cr := &api.ClickHouseInstallation{}
+	l := chiLabeler.New(cr)
+	readyName := l.Get(commonlabeler.LabelReadyName)
+
+	newPod := func(readyValue string) *core.Pod {
+		labels := map[string]string{}
+		if readyValue != "" {
+			labels[readyName] = readyValue
+		}
+		return &core.Pod{
+			ObjectMeta: meta.ObjectMeta{Labels: labels},
+			Status: core.PodStatus{
+				Phase:             core.PodRunning,
+				ContainerStatuses: []core.ContainerStatus{{Ready: true}},
+			},
+		}
+	}
+
+	t.Run("ready=yes is a healthy peer", func(t *testing.T) {
+		w, host := newHealthyPodWorker(newPod(l.Get(commonlabeler.LabelReadyValueReady)))
+		require.True(t, w.isHostHealthyForReconcile(context.Background(), host))
+	})
+
+	t.Run("ready=no is NOT a healthy peer", func(t *testing.T) {
+		w, host := newHealthyPodWorker(newPod(l.Get(commonlabeler.LabelReadyValueNotReady)))
+		require.False(t, w.isHostHealthyForReconcile(context.Background(), host),
+			"Service-excluded host must recover before peers may be disrupted")
+	})
+
+	t.Run("missing ready label is NOT a healthy peer", func(t *testing.T) {
+		w, host := newHealthyPodWorker(newPod(""))
+		require.False(t, w.isHostHealthyForReconcile(context.Background(), host))
+	})
+}
diff --git a/tests/e2e/manifests/chopconf/test-044-chopconf.yaml b/tests/e2e/manifests/chopconf/test-044-chopconf.yaml
@@ -4,6 +4,12 @@ metadata:
   name: "test-044-chopconf"
 spec:
   reconcile:
+    # The slow replica becomes Kubernetes Ready while its Service-ready label is still no.
+    # Automatic recovery would then reconcile before the test checks that the schema is absent.
+    recovery:
+      onStatus:
+        aborted:
+          onPodReady: none
     host:
       wait:
         exclude: "false"