FFFF feat: show what else is grouped in an incident by abahmed · Pull Request #593 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -303,14 +303,14 @@ Prefer Helm? See the [chart guide](./deploy/chart/README.md).
## ⭐ Editions

One binary, two editions. Community needs no account and no key. Pro is free
during early access.
during early access. No credit card required.

| | Community | Pro |
| --- | --- | --- |
| Detection | Pod crash loops, OOM kills and Pending pods; node NotReady | Adds node pressure and draining, readiness, rollouts, Jobs and CronJobs, PDBs, HPAs, storage, network, admission, certificates, quota, custom resources, control plane, probes and unusual events |
| Root cause | One-line headline | Full proof and confidence, shared-factor suspects, recent changes and who made them, affected services, log excerpts |
| Channels | 1 provider (pick with `app.communityProvider`), namespace filters and silences | All providers, routing by scope, threads, text templates, audit log |
| Noise control | Dedup and settling; announced incidents survive a restart, so their resolves are still sent | Flapping detection, startup summary, state kept on disk across restarts, changes made while kwatch was down, gap report |
| Noise control | Related alerts arrive as one incident, in Community and Pro. Dedup and settling; announced incidents survive a restart, so their resolves are still sent | Flapping detection, startup summary, state kept on disk across restarts, changes made while kwatch was down, gap report |
| Operations | RBAC audit, secret redaction, health, text `/status` | Prometheus `/metrics`, severity overrides, maintenance holds, runbook links, heartbeat monitor, `/status` JSON, flight recorder and `replay-export` |
| Account | None | Email only |
| Support | GitHub issues and Discussions | Dedicated support: installation help, and your feature requests and fixes come first |
Expand Down
2 changes: 1 addition & 1 deletion deploy/feature-catalog.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ analysis.shared-factors|runtime|Suspect a node that workloads failing together s
analysis.changes|runtime|Relate incidents to recent changes and who made them|analysis.root-cause|pro
analysis.impact|runtime|Show which workloads and services an incident affects|analysis.root-cause|pro
analysis.investigation|runtime|Attach a redacted log excerpt to announcements||pro
incidents.noise-control|runtime|Settle, merge symptoms and send only material changes|analysis.root-cause|community
incidents.noise-control|runtime|Settle, group related alerts into one incident and send only material changes|analysis.root-cause|community
incidents.flapping|runtime|Recognise flapping and routine recurring incidents|incidents.noise-control|pro
incidents.startup-summary|startup|Summarise pre-existing incidents once at cold start||pro
incidents.state|startup|Keep incidents and history on disk across restarts||pro
Expand Down
3 changes: 2 additions & 1 deletion internal/feature/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ var definitions = []Definition{
Runtime, nil,
},
{
NoiseControl, "Settle, merge symptoms and send only material changes",
NoiseControl, "Settle, group related alerts into one incident and " +
"send only material changes",
Runtime, []ID{RootCause},
},
{
Expand Down
9 changes: 5 additions & 4 deletions internal/notification/compose/community_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ func TestCommunityMessageKeepsTheHeadlineAndDropsTheDetail(t *testing.T) {
}

func firstSentence(note string) string {
first, _, _ := strings.Cut(note, ". ")
return first
line, _, _ := strings.Cut(note, "\n")
first, _, _ := strings.Cut(line, ". ")
return strings.TrimSuffix(first, ".")
}

func TestCommunityNamesWhatProWouldAdd(t *testing.T) {
Expand Down Expand Up @@ -106,8 +107,8 @@ func TestCommunityResolveHasNoProNote(t *testing.T) {
}

func noteFooter(note string) string {
_, last, _ := strings.Cut(note, "\n")
return last
lines := strings.Split(note, "\n")
return lines[len(lines)-1]
}

func TestCommunityOmitsInvestigationOutput(t *testing.T) {
Expand Down
14 changes: 13 additions & 1 deletion internal/notification/compose/golden_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestWriterNoteProperties(t *testing.T) {
if labelStart.MatchString(m.Note) {
t.Errorf("note uses a label: %q", m.Note)
}
if strings.Contains(m.Note, "\n") ||
if strings.Contains(withoutGrouped(m.Note), "\n") ||
strings.Contains(m.Note, "• ") {
t.Errorf("note has lines or bullets: %q", m.Note)
}
Expand Down Expand Up @@ -113,3 +113,15 @@ func assertSentences(t *testing.T, note string) {
t.Errorf("broken punctuation in %q", note)
}
}

// withoutGrouped is the note without its "Also grouped" line, which sits
// on a line of its own.
func withoutGrouped(note string) string {
var out []string
for _, l := range strings.Split(note, "\n") {
if !strings.HasPrefix(l, groupedPrefix) {
out = append(out, l)
}
}
return strings.Join(out, " ")
}
239 changes: 239 additions & 0 deletions internal/notification/compose/grouped_line.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
package compose

import (
"fmt"
"sort"
"strings"
"unicode"

"github.com/abahmed/kwatch/internal/detection"
"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// This file writes the "Also grouped" line: the other findings that
// joined an incident, counted by type. All of its wording lives here so
// a wording change touches one file.

const (
groupedPrefix = "Also grouped: "
// groupedMaxTypes is how many types the line names before it says
// how many more there are.
groupedMaxTypes = 4
)

// groupedLabel is a type of finding in words, singular and plural.
type groupedLabel struct{ one, many string }

var (
labelProbe = groupedLabel{"probe failure", "probe failures"}
labelRollout = groupedLabel{"rollout stall", "rollout stalls"}
labelCrash = groupedLabel{"crash loop", "crash loops"}
labelOOM = groupedLabel{"out-of-memory kill", "out-of-memory kills"}
labelSchedule = groupedLabel{"scheduling failure", "scheduling failures"}
labelPull = groupedLabel{"image pull failure", "image pull failures"}
labelNode = groupedLabel{"node problem", "node problems"}
labelMissing = groupedLabel{"missing reference", "missing references"}
labelCreate = groupedLabel{"failed pod creation", "failed pod creations"}
labelNoEnds = groupedLabel{"service without endpoints",
"services without endpoints"}
labelDown = groupedLabel{"unavailable workload",
"unavailable workloads"}
labelPending = groupedLabel{"pending pod", "pending pods"}
labelThrottle = groupedLabel{"CPU throttling alert",
"CPU throttling alerts"}
labelNotReady = groupedLabel{"not-ready pod", "not-ready pods"}
labelEvicted = groupedLabel{"eviction", "evictions"}
labelNetwork = groupedLabel{"network problem", "network problems"}
)

// groupedExact label a mode as a whole. These modes share a family with
// others that mean something else (Unavailable.APIServer is not an
// unavailable workload).
var groupedExact = map[detection.Mode]groupedLabel{
detection.ModeUnavailable: labelDown,
detection.ModeNoEndpoints: labelNoEnds,
detection.ModeFailedCreate: labelCreate,
detection.ModeEvicted: labelEvicted,
detection.ModeCPUThrottled: labelThrottle,
detection.ModeNetworkErrors: labelNetwork,
detection.ModeStatefulSetRolloutStuck: labelRollout,
}

// groupedLabels name the failure families that have a label of their
// own. A family not listed reads as "<words> alert".
var groupedLabels = map[detection.Mode]groupedLabel{
detection.ModeProbe: labelProbe,
detection.ModeRolloutStuck: labelRollout,
detection.ModeCrashLoop: labelCrash,
detection.ModeOOMKilled: labelOOM,
detection.ModeUnschedulable: labelSchedule,
detection.ModeSchedulingGated: labelSchedule,
detection.ModeImagePull: labelPull,
detection.ModeMemoryPressure: labelNode,
detection.ModeDiskPressure: labelNode,
detection.ModePIDPressure: labelNode,
detection.ModeNetworkUnavailable: labelNode,
detection.ModeDraining: labelNode,
detection.ModeKubeletUnreachable.Family(): labelNode,
detection.ModeHeartbeatStale.Family(): labelNode,
detection.ModeNodeCondition.Family(): labelNode,
detection.ModeNetwork: labelNetwork,
}

func modeOf(f detection.Finding) detection.Mode {
if f.Mode != "" {
return f.Mode
}
return detection.ModeFor(f.Reason)
}

// groupedLabelOf is the label of a finding: by its kind when it is a
// node's, else by its mode, else by its failure family.
func groupedLabelOf(f detection.Finding) groupedLabel {
if f.Entity.Kind == kube.KindNode {
return labelNode
}
mode := modeOf(f)
if l, ok := groupedExact[mode]; ok {
return l
}
if mode.Family() == "Missing" || strings.HasSuffix(string(mode), "Missing") {
return labelMissing
}
if l, ok := podMode(f, mode); ok {
return l
}
if l, ok := groupedLabels[mode.Family()]; ok {
return l
}
words := modeWords(mode.Family())
return groupedLabel{words + " alert", words + " alerts"}
}

// podMode labels the modes whose noun depends on what the finding is
// about: a Pending phase is also reported for other objects, and only a
// pod or container is a "not-ready pod".
func podMode(f detection.Finding, mode detection.Mode) (groupedLabel, bool) {
isPod := f.Entity.Kind == kube.KindPod ||
f.Entity.Kind == kube.KindContainer
switch {
case mode == detection.ModePending && isPod:
return labelPending, true
case mode == detection.ModeNotReady && isPod:
return labelNotReady, true
}
return groupedLabel{}, false
}

// modeWords spells a mode family as lowercase words: "NoEndpoints" is
// "no endpoints" and "CPUThrottled" is "cpu throttled". A run of
// capitals is one word. A raw identifier is never shown.
func modeWords(m detection.Mode) string {
runes := []rune(string(m))
var b strings.Builder
for i, r := range runes {
if i > 0 && unicode.IsUpper(r) && wordStart(runes, i) {
b.WriteRune(' ')
}
b.WriteRune(unicode.ToLower(r))
}
return b.String()
}

// wordStart reports a capital at i that begins a new word: after a
// lowercase letter, or the last capital of a run that a lowercase
// letter follows ("CPUThrottled" splits before the T).
func wordStart(runes []rune, i int) bool {
if unicode.IsLower(runes[i-1]) {
return true
}
return unicode.IsUpper(runes[i-1]) && i+1 < len(runes) &&
unicode.IsLower(runes[i+1])
}

// groupedSentences is the line that counts the other findings grouped
// under the incident, by type. Not counted: the lead's own finding,
// advisories, and findings of the lead's failure family, which the lead
// (or a storm sentence, which names the lead's state) already says.
// Two or more others are shown; one alone is shown unless it is only a
// symptom of the lead's own workload.
func groupedSentences(f caseFacts) []sentence {
others := groupedOthers(f)
if len(others) == 0 ||
(len(others) == 1 && ownSymptom(f.p.Root, others[0])) {
return nil
}
counts := map[groupedLabel]int{}
for _, m := range others {
counts[groupedLabelOf(m)]++
}
return []sentence{{part: partGrouped, text: groupedText(counts)}}
}

// groupedOthers are the members the line may count.
func groupedOthers(f caseFacts) []detection.Finding {
var out []detection.Finding
for _, m := range f.members {
if m.Advisory || (f.ok && (m.Key() == f.lead.Key() ||
modeOf(m).Family() == modeOf(f.lead).Family())) {
continue
}
out = append(out, m)
}
return out
}

// ownSymptom reports a symptom on the root's own workload, such as its
// Service without endpoints: it restates the failure, it adds none.
func ownSymptom(root inventory.EntityID, f detection.Finding) bool {
return f.Symptom && ownedBy(root, f)
}

func groupedText(counts map[groupedLabel]int) string {
labels := make([]groupedLabel, 0, len(counts))
for l := range counts {
labels = append(labels, l)
}
sort.Slice(labels, func(i, j int) bool {
if counts[labels[i]] != counts[labels[j]] {
return counts[labels[i]] > counts[labels[j]]
}
return labels[i].many < labels[j].many
})
var parts []string
for i, l := range labels {
if i == groupedMaxTypes {
break
}
parts = append(parts, countedLabel(counts[l], l))
}
text := strings.Join(parts, ", ")
if more := len(labels) - groupedMaxTypes; more > 0 {
noun := "types"
if more == 1 {
noun = "type"
}
text += fmt.Sprintf(", and %d more %s", more, noun)
}
return groupedPrefix + text
}

func countedLabel(n int, l groupedLabel) string {
if n == 1 {
return "1 " + l.one
}
return fmt.Sprintf("%d %s", n, l.many)
}

// ownedBy reports a finding of the root's own workload: its pod or
// container, or a Service of the same name.
func ownedBy(root inventory.EntityID, f detection.Finding) bool {
e := f.Entity
if e == root {
return true
}
return e.Namespace == root.Namespace && root.Name != "" &&
(e.Name == root.Name || strings.HasPrefix(e.Name, root.Name+"-") ||
strings.HasPrefix(e.Name, root.Name+"/"))
}
Loading
0