8000 fix: count events in the window, share the object budget, trust the Lease's own times, keep CronJob runs in one incident by abahmed · Pull Request #589 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions internal/detection/detectors/event_recovered_custom_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,3 +193,69 @@ func TestCustomFindingClearsWhenReadyUnknownTurnsTrue(t *testing.T) {
})
assert.Empty(t, r.Evaluate(m, t0.Add(time.Minute), id).Findings)
}

// noteEvent records one Event object, identified by its origin.
func noteEvent(
m *inventory.Model, id inventory.EntityID, origin string, at time.Time,
) {
m.Apply(inventory.Observation{
Kind: inventory.Noted, Source: "test", At: at, Entity: id,
Note: inventory.Note{
At: at, Source: "operator", Reason: "SyncFailed", Message: "failed",
Count: 1, Warning: true, Origin: origin,
},
})
}

func TestUnusualEventsReportsDistinctEventsOnNoStatusCustomResource(
t *testing.T,
) {
m, id := noStatusCustom("ops")
for i, origin := range []string{"a", "b", "c"} {
noteEvent(m, id, origin, t0.Add(time.Duration(i)*time.Minute))
}

got := Event{}.Detect(
testDetectorContext(m, t0.Add(3*time.Minute)), entityOf(m, id))

require.Len(t, got, 1)
assert.Contains(t, got[0].Summary, "SyncFailed 3 times")
}

func TestUnusualEventsCountsOnlyDistinctEventsInsideTheWindow(t *testing.T) {
m, id := noStatusCustom("ops")
noteEvent(m, id, "a", t0)
noteEvent(m, id, "b", t0.Add(time.Hour))
last := t0.Add(2 * time.Hour)
noteEvent(m, id, "c", last)

got := Event{}.Detect(
testDetectorContext(m, last.Add(time.Minute)), entityOf(m, id))

assert.Empty(t, got, "three events hours apart are no pattern")
}

func TestUnusualEventsAddsUpSightingsWithoutOrigin(t *testing.T) {
m, id := noStatusCustom("ops")
for i := range unusualEventMin {
noteEntity(m, id, "SyncFailed", "failed", 1,
t0.Add(time.Duration(i)*time.Minute))
}

got := Event{}.Detect(
testDetectorContext(m, t0.Add(5*time.Minute)), entityOf(m, id))

require.Len(t, got, 1)
}

func TestUnusualEventsStaySilentOnHealthyCustomResourceWithDistinctEvents(
t *testing.T,
) {
m, id := customModel("ops", "True", t0.Add(-time.Hour))
for i, origin := range []string{"a", "b", "c"} {
noteEvent(m, id, origin, t0.Add(time.Duration(i)*time.Minute))
}

assert.Empty(t, Event{}.Detect(
testDetectorContext(m, t0.Add(3*time.Minute)), entityOf(m, id)))
}
23 changes: 14 additions & 9 deletions internal/detection/detectors/event_unusual.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,19 @@ func unusualEvents(
}
var order []string
byReason := map[string]*eventGroup{}
since := ctx.Now.Add(-EventWindow)
// recent counts each reason's occurrences inside the window: a note's
// own count also holds Events of hours ago.
recent := map[string]int{}
// recurred marks a reason seen again in a later quarter hour than
// its first sighting. It counts as repeating even when each sighting
// came as a fresh Event with a count of one (the model keeps one
// note per source and reason, so such counts do not add up). A
// failure that recurs after its first finding cleared is reported
// again this way. A custom resource needs unusualEventMin in the
// window: controllers there write one fresh event per item.
// came as a fresh Event with a count of one, too few to reach
// unusualEventMin. A failure that recurs after its first finding
// cleared is reported again this way. A custom resource needs
// unusualEventMin in the window: controllers there write one fresh
// event per item.
recurred := map[string]bool{}
for _, note := range ctx.Model.Notes(e.ID, ctx.Now.Add(-EventWindow)) {
for _, note := range ctx.Model.Notes(e.ID, since) {
if !note.Warning || knownEvent(note.Reason, e.ID.Kind) ||
quotedByService(e, note.Reason) || recoveredBefore(e, note) {
continue
Expand All @@ -58,24 +62,25 @@ func unusualEvents(
order = append(order, note.Reason)
}
group.add(note, detection.Info)
recent[note.Reason] += note.CountSince(since)
if !custom && note.At.Sub(note.FirstSeen) > EventWindow {
recurred[note.Reason] = true
}
}
var out []detection.Finding
for _, reason := range order {
group := byReason[reason]
if group.count < unusualEventMin && !recurred[reason] {
if recent[reason] < unusualEventMin && !recurred[reason] {
continue
}
out = append(out, detection.Finding{
Reason: reasons.UnusualEvent(reason),
Severity: detection.Info, Mode: detection.ModeUnusualEvent,
Since: group.newest.At,
Summary: unusualSummary(reason, group.count),
Summary: unusualSummary(reason, recent[reason]),
Evidence: []detection.Evidence{
{Label: "event", Value: group.newest.Message},
{Label: "occurrences", Value: strconv.Itoa(group.count)},
{Label: "occurrences", Value: strconv.Itoa(recent[reason])},
},
})
}
Expand Down
41 changes: 32 additions & 9 deletions internal/detection/detectors/lease.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,16 @@ import (
// a younger limit would flag a healthy holder whose latest renewal is
// simply not read yet. It is two scans plus a margin, taken from the scan
// period itself so the two cannot drift apart.
//
// leaseGapMin is the least gap between two observed renewals that still
// counts as a rhythm: scans are far apart, so a Lease of a few seconds is
// seen to renew one scan later at best, and the allowance has room for one
// scan missed.
const (
leaseStaleFactor = 3
leaseStaleMargin = time.Minute
leaseStaleMin = 2*kube.LeaseScanPeriod + leaseStaleMargin
leaseGapMin = 2 * kube.LeaseScanPeriod
)

// Lease detects a controller or operator that holds its leader Lease but
Expand All @@ -33,8 +39,10 @@ const (
// uninstalled controller leaves behind.
//
// Some programs take a Lease once at startup and renew it only when they
// restart, so a healthy holder counts only once kwatch saw it renew: an
// old renew time alone says nothing about that holder.
// restart, so a healthy holder counts only once kwatch saw it renew in a
// rhythm (see leaseRenews): an old renew time alone says nothing about
// that holder. A restart of kwatch forgets the rhythm, and a healthy
// holder is then left alone until it is seen to renew again.
type Lease struct{}

// Name implements detection.Detector.
Expand All @@ -54,10 +62,10 @@ func (Lease) Detect(
if renewed.IsZero() || holder == "" {
return nil
}
renews := renewalSeen(ctx, holder, renewed)
seconds, _ := number(e, kube.AttrLeaseDuration)
stale := max(time.Duration(seconds*leaseStaleFactor)*time.Second,
leaseStaleMin)
renews := leaseRenews(ctx, holder, renewed, seconds)
age := ctx.Now.Sub(renewed)
if age < stale {
ctx.RecheckAfter(stale - age)
Expand All @@ -82,13 +90,28 @@ func (Lease) Detect(
}}
}

// renewalSeen reports whether holder renewed the Lease since kwatch first
// saw it hold it: the first renew time is kept for as long as the holder
// stays, and a later one proves it renews.
func renewalSeen(
ctx detection.Context, holder string, renewed time.Time,
// leaseRenews reports whether kwatch saw holder renew its Lease in a
// rhythm: two renew times in a row, read by two evaluations, no further
// apart than three durations (at least leaseGapMin). One slow bump, such
// as a program taking the Lease again when it restarts, sets nothing.
//
// The rhythm is kept for as long as the holder stays, in memory per holder.
// A late renewal after a long stall does not undo it: the holder renewed
// before and stalled, which is what is reported. A new holder starts over,
// and the state of the one before is forgotten with it.
func leaseRenews(
ctx detection.Context, holder string, renewed time.Time, seconds float64,
) bool {
return renewed.After(ctx.Onset("lease-renewal/"+holder, renewed))
allowed := max(time.Duration(seconds*leaseStaleFactor)*time.Second,
leaseGapMin)
before, seen := ctx.Track("lease-renewed/"+holder, renewed)
gap := renewed.Sub(before)
const proven = "lease-rhythm/"
if ctx.Ongoing(proven+holder) || (seen && gap > 0 && gap <= allowed) {
ctx.Onset(proven+holder, renewed)
return true
}
return false
}

// holderPod finds the pod a holder identity names. state is what is
Expand Down
127 changes: 123 additions & 4 deletions internal/detection/detectors/lease_renewal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,28 @@ import (

const leaseHolderName = "operator-7d9f_1a2b"

// scanLease records one scan of the Lease and evaluates it.
func scanLease(
// scanLeaseOf records one scan of a Lease of the given duration in seconds
// and evaluates it.
func scanLeaseOf(
r *detection.Registry, m *inventory.Model, lease inventory.EntityID,
holder string, renewed, now time.Time,
holder string, seconds float64, renewed, now time.Time,
) []detection.Finding {
put(m, lease, now, map[string]inventory.Value{
kube.AttrLeaseHolder: inventory.Text(holder),
kube.AttrLeaseRenewed: inventory.Time(renewed),
kube.AttrLeaseDuration: inventory.Number(15),
kube.AttrLeaseDuration: inventory.Number(seconds),
})
return r.Evaluate(m, now, lease).Findings
}

// scanLease records one scan of a 15 second Lease and evaluates it.
func scanLease(
r *detection.Registry, m *inventory.Model, lease inventory.EntityID,
holder string, renewed, now time.Time,
) []detection.Finding {
return scanLeaseOf(r, m, lease, holder, 15, renewed, now)
}

func TestLeaseStaysSilentForOneShotLeaseOfHealthyHolder(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
Expand All @@ -37,6 +46,20 @@ func TestLeaseStaysSilentForOneShotLeaseOfHealthyHolder(t *testing.T) {
t0.Add(24*time.Hour)), "a Lease never seen renewing stays silent")
}

func TestLeaseBumpedOnlyOnRestartStaysQuietForTheSameHolder(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
assert.Empty(t, scanLease(r, m, lease, leaseHolderName,
t0.Add(-10*time.Hour), t0))

// Hours later the container restarts and the same identity takes the
// Lease again: one slow bump is no rhythm.
again := t0.Add(kube.LeaseScanPeriod)
assert.Empty(t, scanLease(r, m, lease, leaseHolderName, again, again))
assert.Empty(t, scanLease(r, m, lease, leaseHolderName, again,
again.Add(24*time.Hour)))
}

func TestLeaseReportsHealthyHolderSeenToRenewThenStop(t *testing.T) {
m, lease := leaseModel(10*time.Second, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
Expand All @@ -51,6 +74,81 @@ func TestLeaseReportsHealthyHolderSeenToRenewThenStop(t *testing.T) {
assert.Equal(t, "LeaseStale", found[0].Reason)
}

func TestLeaseShorterThanTheScanPeriodIsStillDetected(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
last := t0.Add(kube.LeaseScanPeriod)

// Three durations are 15s: far less than the time between two scans.
assert.Empty(t, scanLeaseOf(r, m, lease, leaseHolderName, 5, t0, t0))
assert.Empty(t, scanLeaseOf(r, m, lease, leaseHolderName, 5, last, last))

found := scanLeaseOf(r, m, lease, leaseHolderName, 5, last,
last.Add(leaseStaleMin))
require.Len(t, found, 1)
}

func TestLeaseOfLongDurationIsDetectedAfterThreeDurations(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
const seconds = 3600
last := t0.Add(50 * time.Minute)

assert.Empty(t, scanLeaseOf(r, m, lease, leaseHolderName, seconds,
t0, t0))
assert.Empty(t, scanLeaseOf(r, m, lease, leaseHolderName, seconds,
last, last))
assert.Empty(t, scanLeaseOf(r, m, lease, leaseHolderName, seconds,
last, last.Add(2*time.Hour+59*time.Minute)))

found := scanLeaseOf(r, m, lease, leaseHolderName, seconds, last,
last.Add(3*time.Hour))
require.Len(t, found, 1)
}

func TestLeaseRenewalGapAllowanceIsTwoScansForShortLeases(t *testing.T) {
tests := []struct {
name string
gap time.Duration
want int
}{
{"at the allowance", 2 * kube.LeaseScanPeriod, 1},
{"past the allowance", 2*kube.LeaseScanPeriod + time.Second, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
last := t0.Add(tt.gap)
scanLease(r, m, lease, leaseHolderName, t0, t0)
scanLease(r, m, lease, leaseHolderName, last, last)

found := scanLease(r, m, lease, leaseHolderName, last,
last.Add(24*time.Hour))

assert.Len(t, found, tt.want)
})
}
}

// Once a holder was seen to renew in rhythm it stays proven: a late
// renewal after a long stall is a holder that stalled, and silence then
// would hide the stall that matters.
func TestLeaseRhythmSurvivesALateRenewal(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
second := t0.Add(kube.LeaseScanPeriod)
late := second.Add(10 * time.Hour)
scanLease(r, m, lease, leaseHolderName, t0, t0)
scanLease(r, m, lease, leaseHolderName, second, second)
assert.Empty(t, scanLease(r, m, lease, leaseHolderName, late, late))

found := scanLease(r, m, lease, leaseHolderName, late,
late.Add(leaseStaleMin))

assert.Len(t, found, 1)
}

func TestLeaseNewHolderMustBeSeenToRenewItself(t *testing.T) {
m, lease := leaseModel(10*time.Second, leaseHolderName, "Running")
r := detection.NewRegistry(nil, Lease{})
Expand All @@ -65,6 +163,27 @@ func TestLeaseNewHolderMustBeSeenToRenewItself(t *testing.T) {
"the renewal belonged to the previous holder")
}

func TestLeaseAfterKwatchRestartStaysQuietUntilARhythmIsSeen(t *testing.T) {
m, lease := leaseModel(time.Hour, leaseHolderName, "Running")
before := detection.NewRegistry(nil, Lease{})
last := t0.Add(kube.LeaseScanPeriod)
scanLease(before, m, lease, leaseHolderName, t0, t0)
scanLease(before, m, lease, leaseHolderName, last, last)

// A new process remembers nothing: the stalled Lease is not reported.
after := detection.NewRegistry(nil, Lease{})
later := last.Add(24 * time.Hour)
assert.Empty(t, scanLease(after, m, lease, leaseHolderName, last, later))

first := later.Add(time.Minute)
second := first.Add(kube.LeaseScanPeriod)
assert.Empty(t, scanLease(after, m, lease, leaseHolderName, first, first))
assert.Empty(t, scanLease(after, m, lease, leaseHolderName, second,
second))
assert.Len(t, scanLease(after, m, lease, leaseHolderName, second,
second.Add(leaseStaleMin)), 1, "a rhythm seen again counts")
}

func TestLeaseNeverSeenRenewingIsReportedWhileHolderIsUnhealthy(
t *testing.T,
) {
Expand Down
17 changes: 17 additions & 0 deletions internal/detection/onset.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,23 @@ func (c Context) Onset(key string, firstSeen time.Time) time.Time {
return at
}

// Track records value under key for the evaluated entity and returns the
// value the previous evaluation recorded, false when there was none. It
// lets a detector compare what it reads now with what it read before. Like
// an onset, a tracked value is kept only while every evaluation keeps
// asking for it, and without a registry nothing is remembered. Keep keys
// of tracked values apart from those of onsets.
func (c Context) Track(
key string, value time.Time,
) (previous time.Time, found bool) {
if c. nil {
return time.Time{}, false
}
previous, found = c.onsets.known[key]
c.onsets.kept[key] = value
return previous, found
}

// Ongoing reports whether the previous evaluation of the entity asked
// for the state named key, that is, whether the state held then. It does
// not keep the state; call Onset for that. Detectors use it for
Expand Down
Loading
Loading
0