8000 fix(chi): emit event for reconcile error by devmalcevv · Pull Request #2165 · Altinity/clickhouse-operator · GitHub
[go: up one dir, main page]

Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions pkg/controller/chi/worker-service.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package chi

import (
"context"
"errors"
"fmt"

core "k8s.io/api/core/v1"
Expand All @@ -27,6 +28,14 @@ import (
"github.com/altinity/clickhouse-operator/pkg/util"
)

// errServiceTypeChanged marks an update that is not even attempted: Kubernetes rejects a change of
// Service type in place, so the object has to be replaced.
var errServiceTypeChanged = errors.New("service type changed")

func shouldRecreateService(err error) bool {
return apiErrors.IsNotFound(err) || errors.Is(err, errServiceTypeChanged) || apiErrors.IsInvalid(err)
}

// reconcileService reconciles core.Service
func (w *worker) reconcileService(ctx context.Context, cr chi.ICustomResource, service, prevService *core.Service) error {
w.a.V(2).M(cr).S().Info(service.GetName())
Expand All @@ -41,6 +50,21 @@ func (w *worker) reconcileService(ctx context.Context, cr chi.ICustomResource, s
err = w.updateService(ctx, cr, curService, service, prevService)
}

// Recreate is for the situations a recreate actually fixes, see shouldRecreateService. Any other
// error - a read that was Forbidden or spent its retry budget, an update that hit a Conflict or
// an admission webhook that is down - says nothing about whether the Service is healthy, so
// surface it and let the next pass re-read. Falling through would delete and create the Service,
// dropping its endpoints and its address, which for a LoadBalancer is the external one.
if (err != nil) && !shouldRecreateService(err) {
w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed).
WithAction(cr).
WithError(cr).
M(cr).F().
Error("Reconcile Service: %s failed, leaving it alone. err: %v",
util.NamespaceNameString(service), err)
return err
}

if err != nil {
// The Service is either not found or not updated. Try to recreate it
if apiErrors.IsNotFound(err) {
Expand Down Expand Up @@ -79,10 +103,11 @@ func (w *worker) updateService(
prevService *core.Service,
) error {
if curService.Spec.Type != targetService.Spec.Type {
return fmt.Errorf(
"just recreate the service in case of service type change '%s'=>'%s'",
curService.Spec.Type, targetService.Spec.Type,
)
// Sentinel, not a bare string: reconcileService must tell this apart from a transient
// update failure, because only an update that can not be made in place is resolved by
// deleting and recreating.
return fmt.Err 4D35 orf("%w: '%s'=>'%s'",
errServiceTypeChanged, curService.Spec.Type, targetService.Spec.Type)
}

// Updating a Service is a complicated business
Expand Down
0