8000 fix: reduce false alerts and unblock incident resolves by abahmed · Pull Request #585 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions internal/app/pipeline_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ func (p *pipelineBuild) incidentConfig() incident.Config {
Verifiable: func(kind inventory.Kind) bool {
return p.source == nil || p.source.Verifiable(kind)
},
VerifiableObject: func(id inventory.EntityID) bool {
return p.source == nil || p.source.VerifiableObject(id)
},
}
if deps.has(feature.SeverityOverrides) {
policy := deps.runtime.Policy()
Expand Down
99 changes: 99 additions & 0 deletions internal/detection/detectors/claim_unused_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package detectors

import (
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/abahmed/kwatch/internal/detection"
"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// boundClaim is a claim bound a month before t0.
func boundClaim(m *inventory.Model) inventory.EntityID {
claim := newID(kube.KindPVC, "shop", "data")
put(m, claim, t0.Add(-30*24*time.Hour), map[string]inventory.Value{
kube.AttrPhase: inventory.Text("Bound"),
})
return claim
}

func unusedFindings(
r *detection.Registry, m *inventory.Model, claim inventory.EntityID,
now time.Time,
) []detection.Finding {
return r.Evaluate(m, now, claim).Findings
}

func TestClaimUnusedClockStartsWhenTheLastMountGoes(t *testing.T) {
m := newTestModel()
claim := boundClaim(m)
pod := newID(kube.KindPod, "shop", "nightly-1")
put(m, pod, t0, nil)
link(m, pod, inventory.Mounts, claim)
r := detection.NewRegistry(nil, Claim{})
assert.Empty(t, unusedFindings(r, m, claim, t0))

gone := t0.Add(time.Hour)
m.Apply(inventory.Observation{
Kind: inventory.Gone, Source: "test", At: gone, Entity: pod,
})
assert.Empty(t, unusedFindings(r, m, claim, gone),
"a long-bound claim must not count as unused at once")
assert.Empty(t, unusedFindings(r, m, claim,
gone.Add(DefaultUnusedAfter-time.Second)))

found := unusedFindings(r, m, claim, gone.Add(DefaultUnusedAfter))
require.Len(t, found, 1)
assert.Equal(t, "ClaimUnused", found[0].Reason)
assert.Equal(t, gone, found[0].Since)
}

func TestClaimUnusedClockRestartsWithEachMount(t *testing.T) {
m := newTestModel()
claim := boundClaim(m)
r := detection.NewRegistry(nil, Claim{})
assert.Empty(t, unusedFindings(r, m, claim, t0))

pod := newID(kube.KindPod, "shop", "nightly-2")
put(m, pod, t0.Add(20*time.Hour), nil)
link(m, pod, inventory.Mounts, claim)
assert.Empty(t, unusedFindings(r, m, claim, t0.Add(20*time.Hour)))

gone := t0.Add(21 * time.Hour)
m.Apply(inventory.Observation{
Kind: inventory.Gone, Source: "test", At: gone, Entity: pod,
})
assert.Empty(t, unusedFindings(r, m, claim, gone))
assert.Empty(t, unusedFindings(r, m, claim, t0.Add(DefaultUnusedAfter)),
"the mount in between restarts the day")
}

func TestClaimUnusedWaitsADayAfterARestart(t *testing.T) {
m := newTestModel()
claim := boundClaim(m)
first := t0

restarted := detection.NewRegistry(nil, Claim{})
assert.Empty(t, unusedFindings(restarted, m, claim, first))
assert.Empty(t, unusedFindings(restarted, m, claim,
first.Add(DefaultUnusedAfter-time.Second)))
assert.Len(t, unusedFindings(restarted, m, claim,
first.Add(DefaultUnusedAfter)), 1)
}

func TestClaimUnusedSkipsClaimOfACronJobTemplate(t *testing.T) {
m := newTestModel()
claim := boundClaim(m)
cron := newID(kube.KindCronJob, "shop", "nightly")
put(m, cron, t0, nil)
link(m, cron, inventory.References, claim)
r := detection.NewRegistry(nil, Claim{})

assert.Empty(t, unusedFindings(r, m, claim, t0))
assert.Empty(t, unusedFindings(r, m, claim,
t0.Add(2*DefaultUnusedAfter)))
}
17 changes: 16 additions & 1 deletion internal/detection/detectors/event_recovered.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ func recoveredBefore(e inventory.Entity, note inventory.Note) bool {

// readySince says whether e is fully ready, and since when: for a
// workload, since its ready count took the value it has; for a pod,
// since it became ready.
// since it became ready; for a custom resource, since its Ready condition
// turned True, provided no other condition reports a failure.
func readySince(e inventory.Entity) (time.Time, bool) {
switch e.ID.Kind {
case kube.KindDaemonSet:
Expand All @@ -43,5 +44,19 @@ func readySince(e inventory.Entity) (time.Time, bool) {
}
return valueSince(e, kube.AttrReady), true
}
if flag(e, kube.AttrCustom) {
return customReadySince(e)
}
return time.Time{}, false
}

func customReadySince(e inventory.Entity) (time.Time, bool) {
status, _, since := condition(e, "Ready")
if status != "True" || len(failingConditions(e)) > 0 {
return time.Time{}, false
}
if since.IsZero() {
since = valueSince(e, kube.ConditionKey("Ready"))
}
return since, true
}
195 changes: 195 additions & 0 deletions internal/detection/detectors/event_recovered_custom_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
package detectors

import (
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/abahmed/kwatch/internal/detection"
"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// customModel is a resource with one Ready condition. namespace "" makes
// it cluster-scoped, as a ClusterPolicy is.
func customModel(
namespace, status string, since time.Time,
) (*inventory.Model, inventory.EntityID) {
m := newTestModel()
id := inventory.NewEntityID("kyverno.io", "clusterpolicy", namespace,
"require-labels")
key := kube.ConditionKey("Ready")
put(m, id, t0, map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
key: inventory.Text(status),
key + kube.AttrConditionSince: inventory.Time(since),
key + kube.AttrConditionReason: inventory.Text("Reconciled"),
})
return m, id
}

func noteUpdateFailed(
m *inventory.Model, id inventory.EntityID, at time.Time, count int,
) {
m.Apply(inventory.Observation{
Kind: inventory.Noted, Source: "test", At: at, Entity: id,
Note: inventory.Note{
At: at, FirstSeen: at, Source: "controller",
Reason: "UpdateFailed", Message: "conflict", Count: count,
Warning: true,
},
})
}

func eventFindings(
m *inventory.Model, id inventory.EntityID, now time.Time,
) []detection.Finding {
return detection.NewRegistry(nil, Event{}).Evaluate(m, now, id).Findings
}

func TestUnusualEventsIgnoresWarningsOlderThanCustomRecovery(t *testing.T) {
for name, namespace := range map[string]string{
"namespaced": "ops", "cluster-scoped": "",
} {
t.Run(name, func(t *testing.T) {
ready := t0.Add(5 * time.Minute)
m, id := customModel(namespace, "True", ready)
noteUpdateFailed(m, id, t0, 3)

assert.Empty(t, eventFindings(m, id, t0.Add(6*time.Minute)))
})
}
}

func TestUnusualEventsIgnoresNewWarningStreamOnHealthyCustomResource(
t *testing.T,
) {
for name, namespace := range map[string]string{
"namespaced": "ops", "cluster-scoped": "",
} {
t.Run(name, func(t *testing.T) {
m, id := customModel(namespace, "True", t0.Add(-time.Hour))
noteUpdateFailed(m, id, t0, 12)

assert.Empty(t, eventFindings(m, id, t0.Add(time.Minute)))
})
}
}

func TestUnusualEventsClearAsCustomResourceTurnsHealthy(t *testing.T) {
m, id := customModel("ops", "False", t0)
noteUpdateFailed(m, id, t0, 3)
r := detection.NewRegistry(nil, Event{})
require.Len(t, r.Evaluate(m, t0, id).Findings, 1)

ready := kube.ConditionKey("Ready")
later := t0.Add(time.Minute)
put(m, id, later, map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
ready: inventory.Text("True"),
ready + kube.AttrConditionSince: inventory.Time(later),
})
noteUpdateFailed(m, id, later.Add(time.Second), 3)

assert.Empty(t, r.Evaluate(m, later.Add(time.Minute), id).Findings)
}

// noStatusCustom is a custom resource that reports no health at all.
func noStatusCustom(namespace string) (*inventory.Model, inventory.EntityID) {
m := newTestModel()
id := inventory.NewEntityID("example.com", "order", namespace, "o")
put(m, id, t0, map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
})
return m, id
}

func TestUnusualEventsSingleLaterEventIsNoRecurrenceOnCustomResource(
t *testing.T,
) {
for name, namespace := range map[string]string{
"namespaced": "ops", "cluster-scoped": "",
} {
t.Run(name, func(t *testing.T) {
m, id := noStatusCustom(namespace)
noteUpdateFailed(m, id, t0, 1)
later := t0.Add(EventWindow + time.Minute)
noteUpdateFailed(m, id, later, 1)

assert.Empty(t, eventFindings(m, id, later))
})
}
}

func TestUnusualEventsReportsThreeInWindowOnNoStatusCustomResource(
t *testing.T,
) {
for name, namespace := range map[string]string{
"namespaced": "ops", "cluster-scoped": "",
} {
t.Run(name, func(t *testing.T) {
m, id := noStatusCustom(namespace)
noteUpdateFailed(m, id, t0, 3)

found := eventFindings(m, id, t0.Add(time.Minute))
require.Len(t, found, 1)
assert.Equal(t, detection.Info, found[0].Severity)
})
}
}

func TestUnusualEventsCountsWarningsWhileCustomReadyIsNotTrue(t *testing.T) {
for _, status := range []string{"Unknown", "False"} {
t.Run(status, func(t *testing.T) {
m, id := customModel("ops", status, t0.Add(5*time.Minute))
noteUpdateFailed(m, id, t0, 3)

assert.Len(t, eventFindings(m, id, t0.Add(6*time.Minute)), 1)
})
}
}

func TestUnusualEventsCountsWarningsWhileAnotherConditionFails(t *testing.T) {
m, id := customModel("ops", "True", t0.Add(5*time.Minute))
stalled := kube.ConditionKey("Stalled")
put(m, id, t0, map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
kube.ConditionKey("Ready"): inventory.Text("True"),
stalled: inventory.Text("True"),
stalled + kube.AttrConditionSince: inventory.Time(t0),
})
noteUpdateFailed(m, id, t0, 3)

assert.Len(t, eventFindings(m, id, t0.Add(6*time.Minute)), 1)
}

func TestReadySinceFallsBackToWhenCustomReadyWasSeen(t *testing.T) {
m := newTestModel()
id := newID("widget", "ops", "w")
put(m, id, t0, map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
kube.ConditionKey("Ready"): inventory.Text("True"),
})

since, ok := readySince(entityOf(m, id))
assert.True(t, ok)
assert.Equal(t, t0, since)
}

// Ready=Unknown past the reconcile window is a finding, and it ends the
// moment the controller says True, whatever the wait so far.
func TestCustomFindingClearsWhenReadyUnknownTurnsTrue(t *testing.T) {
m, id := customModel("ops", "Unknown", t0.Add(-time.Hour))
r := detection.NewRegistry(nil, Custom{})
require.Len(t, r.Evaluate(m, t0, id).Findings, 1)

key := kube.ConditionKey("Ready")
put(m, id, t0.Add(time.Minute), map[string]inventory.Value{
kube.AttrCustom: inventory.Bool(true),
key: inventory.Text("True"),
key + kube.AttrConditionSince: inventory.Time(t0.Add(time.Minute)),
})
assert.Empty(t, r.Evaluate(m, t0.Add(time.Minute), id).Findings)
}
15 changes: 12 additions & 3 deletions internal/detection/detectors/event_unusual.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/abahmed/kwatch/internal/detection"
"github.com/abahmed/kwatch/internal/detection/reasons"
"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// unusualEventMin is how many times a Warning event kwatch has no
Expand All @@ -23,18 +24,26 @@ const unusualEventMin = 3
func unusualEvents(
ctx detection.Context, e inventory.Entity,
) []detection.Finding {
custom := flag(e, kube.AttrCustom)
if _, healthy := readySince(e); custom && healthy {
// Operators write Warnings about what they process onto their
// own object. A resource that reports itself healthy is not what
// is wrong, however many there are.
return nil
}
var order []string
byReason := map[string]*eventGroup{}
// recurred marks a reason seen again in a later quarter hour than
// its first sighting. It counts as repeating even when each sighting
// came as a fresh Event with a count of one (the model keeps one
// note per source and reason, so such counts do not add up). A
// failure that recurs after its first finding cleared is reported
// again this way.
// again this way. A custom resource needs unusualEventMin in the
// window: controllers there write one fresh event per item.
recurred := map[string]bool{}
for _, note := range ctx.Model.Notes(e.ID, ctx.Now.Add(-EventWindow)) {
if !note.Warning || knownEvent(note.Reason, e.ID.Kind) ||
quotedByService(e, note.Reason) {
quotedByService(e, note.Reason) || recoveredBefore(e, note) {
continue
}
// The window includes a note exactly EventWindow old, so the
Expand All @@ -49,7 +58,7 @@ func unusualEvents(
order = append(order, note.Reason)
}
group.add(note, detection.Info)
if note.At.Sub(note.FirstSeen) > EventWindow {
if !custom && note.At.Sub(note.FirstSeen) > EventWindow {
recurred[note.Reason] = true
}
}
Expand Down
Loading
0