8000 fix: word a CronJob's failing run as its last run, and say what points to a cause by abahmed · Pull Request #590 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ say, in this order of strength:
4. **What was checked.** The solver records the upstream objects it reached
that showed nothing wrong (`Trace.Checked`), and the message says "Its
node, image and configuration are healthy and unchanged, so nothing
outside it explains this", followed by the quoted crash output.
outside it points to a cause", followed by the quoted crash output.

Two kinds of finding are not failures. A *symptom* (a Deployment below its
replicas) restates its pods' failures and never leads a message. An
Expand Down
14 changes: 14 additions & 0 deletions internal/incident/recent_changes.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
"github.com/abahmed/kwatch/internal/rootcause"
)

// Recent changes are what a message says when it can name no cause:
Expand Down Expand Up @@ -86,6 +87,9 @@ func rankedBetween(
ranks := map[inventory.EntityID]int{}
var out []inventory.Change
for id, change := range newestPerObject(model, from, to) {
if ownScheduledRun(model, root, change) {
continue
}
if rank := near.rank(id); rank < below {
ranks[id] = rank
out = append(out, change)
Expand All @@ -107,6 +111,16 @@ func rankedBetween(
return out
}

// ownScheduledRun reports the creation of a Job by the schedule of the
// CronJob root: what the root does on its own, which no one changed and
// which says nothing next to its failure.
func ownScheduledRun(
model inventory.Reader, root inventory.EntityID, change inventory.Change,
) bool {
return scheduledRun(root, &change) &&
rootcause.TopOwner(model, change.Entity) == root
}

// worthNaming reports a change a person made from from to to: not
// noise, not a status write, not an autoscaler's replicas.
func worthNaming(change inventory.Change, from, to time.Time) bool {
Expand Down
45 changes: 45 additions & 0 deletions internal/incident/recent_changes_cron_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package incident

import (
"testing"
"time"

"github.com/stretchr/testify/assert"

"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// A CronJob starting its own Job is what it does, not a change next to its
// failure. A Job somebody else created, and any other edit, still count.
func TestRecentChangesLeavesOutTheCronJobsOwnScheduledRun(t *testing.T) {
r := newRig(t, Config{})
cron := entity(kube.KindCronJob, "stats-rollup")
run := entity(kube.KindJob, "stats-rollup-292510")
manual := entity(kube.KindJob, "backfill")
cm := entity(kube.KindConfigMap, "rollup-config")
for _, id := range []inventory.EntityID{cron, run, manual, cm} {
r.observe(id)
}
r.relate(run, inventory.OwnedBy, cron)
r.change(run, 2*time.Minute, inventory.Change{Created: true})
r.change(manual, 3*time.Minute, inventory.Change{Created: true})
r.change(cm, 4*time.Minute, dataEdit("bob"))

got := RecentChanges(r.model, cron, at(10*time.Minute))

assert.Equal(t, []inventory.EntityID{cm, manual}, entitiesOf(got))
}

// With nothing else changed, the list is empty and the message has no line.
func TestRecentChangesIsEmptyWhenOnlyTheScheduledRunChanged(t *testing.T) {
r := newRig(t, Config{})
cron := entity(kube.KindCronJob, "stats-rollup")
run := entity(kube.KindJob, "stats-rollup-292510")
r.observe(cron)
r.observe(run)
r.relate(run, inventory.OwnedBy, cron)
r.change(run, 2*time.Minute, inventory.Change{Created: true})

assert.Empty(t, RecentChanges(r.model, cron, at(10*time.Minute)))
}
10 changes: 7 additions & 3 deletions internal/notification/compose/cause_unclear.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ var checkedWords = map[inventory.Kind]string{
// the root's own: a workload failing without anything outside it to
// blame. Instead of leaving the reader with no cause, they say what was
// checked and found healthy and unchanged. The quoted output and the
// exit facts that follow are then the strongest leads.
// exit facts that follow are then the strongest leads. They say what
// points to a cause, not that nothing explains the failure: the checks
// cover the upstream kinds the solver reached, and a candidate it weighed
// and dropped (CauseUnclear) or could not see (Unverified) is not ruled
// out.
func checkedSentences(f caseFacts) []sentence {
p := f.p
if p.Cause != nil || rootFinding(p, f.members) != nil ||
Expand All @@ -52,11 +56,11 @@ func checkedSentences(f caseFacts) []sentence {
// The node is named as the cause by the sentences that follow.
return recentChangeSentence(f)
}
text := "Nothing outside it explains this."
text := "Nothing outside it points to a cause."
if words := checkedWordsFor(p.Checked); len(words) > 0 {
text = "Its " + joinWords(words) + " " +
verb(len(words), "is", "are") + " healthy and unchanged, " +
"so nothing outside it explains this."
"so nothing outside it points to a cause."
}
return append([]sentence{{part: partCause, text: text}},
recentChangeSentence(f)...)
Expand Down
4 changes: 2 additions & 2 deletions internal/notification/compose/cause_unclear_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func TestUnclearCauseIsNotRepeatedOrContradicted(t *testing.T) {
}
text := notification.Text(Writer{}.Write(announce(noRootFinding),
at(6, 0)))
if !strings.Contains(text, "Nothing outside it explains this.") {
if !strings.Contains(text, "Nothing outside it points to a cause.") {
t.Errorf("the note must still say nothing outside explains it:\n%s",
text)
}
Expand All @@ -88,7 +88,7 @@ func TestCheckedSentencesNameWhatWasHealthy(t *testing.T) {
text := notification.Text(Writer{}.Write(announce(p), at(6, 0)))

want := "Its configuration, image and node are healthy and " +
"unchanged, so nothing outside it explains this."
"unchanged, so nothing outside it points to a cause."
if !strings.Contains(text, want) {
t.Fatalf("note = %s\nwant %q", text, want)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/notification/compose/example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func ExampleWriter_Write() {
fmt.Println(lines.Replace(message.Note))
// Output:
// 🔴 payments in shop is crash looping.
// Nothing outside it explains this.
// Nothing outside it points to a cause.
// It fails with "panic: STRIPE_KEY is not set".
// To read the output of the last crash,
// run kubectl logs payments-7d9f -c app -n shop --previous
Expand Down
2 changes: 1 addition & 1 deletion internal/notification/compose/testdata/flapping.golden
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
🟠 search in shop keeps failing and recovering. Nothing outside it explains this. It has recovered and failed again three times recently. To see the pod's state and events, run kubectl describe pod search-9b -n shop
🟠 search in shop keeps failing and recovering. Nothing outside it points to a cause. It has recovered and failed again three times recently. To see the pod's state and events, run kubectl describe pod search-9b -n shop

🟠 search in shop keeps failing and recovering.
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
🟠 invoices in billing is crash looping. Nothing outside it explains this. It fails with "invalid credentials for database invoices". kwatch can't see secrets in billing, so a changed secret can't be ruled out. To read the output of the last crash, run kubectl logs invoices-5c8 -c app -n billing --previous
🟠 invoices in billing is crash looping. Nothing outside it points to a cause. It fails with "invalid credentials for database invoices". kwatch can't see secrets in billing, so a changed secret can't be ruled out. To read the output of the last crash, run kubectl logs invoices-5c8 -c app -n billing --previous

🟠 invoices in billing is crash looping.
64 changes: 64 additions & 0 deletions internal/notification/compose/update_cron_run_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package compose

import (
"testing"

"github.com/stretchr/testify/assert"

"github.com/abahmed/kwatch/internal/detection"
"github.com/abahmed/kwatch/internal/incident"
"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// cronRunUpdate is an update about a CronJob whose latest run, a Job it
// started, failed. Events lists which members the update reports.
func cronRunUpdate(withCronJob bool, failure string) string {
cron := inventory.CoreID(kube.KindCronJob, "ops", "stats-rollup")
run := inventory.CoreID(kube.KindJob, "ops", "stats-rollup-292510")
last := detection.Finding{Entity: cron, Reason: "CronJobLastRunFailed",
Severity: detection.Warning, Since: at(2, 0),
Summary: "Cron job stats-rollup has a last run that failed"}
job := detection.Finding{Entity: run, Reason: "JobBackoffLimitExceeded",
Severity: detection.Warning, Since: at(2, 0),
Summary: "Job stats-rollup-292510 " + failure}
events := []incident.Event{{At: at(2, 0), Text: job.Summary,
Entity: aboutEntity(run)}}
if withCronJob {
events = append(events, incident.Event{At: at(2, 0),
Text: last.Summary, Entity: aboutEntity(cron)})
}
p := incident.Incident{ID: "inc-1", Root: cron, Tier: incident.Notify,
State: incident.Open, Opened: at(0, 0), Revision: 2,
Members: members(last, job), Timeline: events}
d := incident.Decision{Action: incident.Update, Incident: p,
Reason: incident.ReasonMaterialChange}
return Writer{Cluster: "prod-eu-1"}.Write(d, at(3, 0)).Note
}

// A CronJob's run is the CronJob doing its work: the update says the last
// run failed and quotes why, not that the failure is spreading to a Job.
func TestCronJobUpdateSaysItsLastRunFailed(t *testing.T) {
note := cronRunUpdate(true, "failed after 3 failed pod(s)")

assert.Contains(t, note, "Its last run, job stats-rollup-292510, "+
"failed: failed after 3 failed pod(s).")
assert.NotContains(t, note, "spreading")
}

// The failure text is the detector's own, whatever the count.
func TestCronJobUpdateQuotesTheRunsOwnFailure(t *testing.T) {
note := cronRunUpdate(true, "failed after 5 failed pod(s)")

assert.Contains(t, note, "failed: failed after 5 failed pod(s).")
assert.NotContains(t, note, "3 failed")
}

// When the update has not yet named the CronJob, the sentence does.
func TestCronJobUpdateNamesTheCronJobWhenNothingElseDid(t *testing.T) {
note := cronRunUpdate(false, "failed after 3 failed pod(s)")

assert.Contains(t, note, "The last run of cron job stats-rollup in ops "+
"(prod-eu-1), job stats-rollup-292510, failed: failed after 3 "+
"failed pod(s).")
}
38 changes: 38 additions & 0 deletions internal/notification/compose/update_run.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package compose

import (
"strings"

"github.com/abahmed/kwatch/internal/inventory"
"github.com/abahmed/kwatch/internal/inventory/kube"
)

// runOf reports a Job that a CronJob started: the CronJob controller names
// its Jobs after it.
func runOf(cron, job inventory.EntityID) bool {
return cron.Kind == kube.KindCronJob && job.Kind == kube.KindJob &&
job.Namespace == cron.Namespace &&
strings.HasPrefix(job.Name, cron.Name+"-")
}

// lastRunSentence says that the run of a CronJob failed, and why in the
// words of the run's own finding. A run is the CronJob doing its work, so
// the failure is not "spreading" to it. named says an earlier sentence
// already led with the CronJob.
func lastRunSentence(
home inventory.EntityID, homeName string, named bool, g spreadGroup,
) (string, bool) {
if len(g.ids) != 1 || len(g.states) != 1 || !runOf(home, g.ids[0]) {
return "", false
}
why, failing := strings.CutPrefix(g.states[0], "is failing: ")
if !failing {
return "", false
}
text := "Its last run, "
if !named {
text = "The last run of " + homeName + ", "
}
text += nameList(home, g.ids) + ", failed: " + why
return endSentence(text), true
}
5 changes: 5 additions & 0 deletions internal/notification/compose/update_spread.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ func spreadGroupSentences(
homeName, many := f.subjectName(home)
var out []sentence
for i, g := range groups {
if run, ok := lastRunSentence(home, homeName, named || i > 0,
g); ok {
out = append(out, sentence{part: partLead, text: run})
continue
}
text := capitalName(home, homeName+" "+beWord(many)+
" spreading to ")
switch {
Expand Down
2 changes: 1 addition & 1 deletion internal/notification/compose/writer_revision_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func TestWriteNoCauseQuotesOutputOnlyWhenPresent(t *testing.T) {
d.Facts.Output = []string{"panic: missing key", ""}
with := Writer{}.Write(d, revisionNow)

if !strings.Contains(without.Note, "Nothing outside it explains this.") {
if !strings.Contains(without.Note, "Nothing outside it points to a cause.") {
t.Fatalf("note should say nothing outside explains it: %q",
without.Note)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/pipeline/scenario_local_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestEngineAppCrashHasNoInventedCause(t *testing.T) {
t.Fatalf("root = %s, want deployment reports", p.Root)
}
text := notification.Text(h.messages[0])
if !strings.Contains(text, "nothing outside it explains this") ||
if !strings.Contains(text, "nothing outside it points to a cause") ||
!strings.Contains(text, "kubectl logs") {
t.Fatalf("message should rule out the outside and show logs:\n%s",
text)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
10:05:15 settled
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it explains this. In the last 30 minutes in shop: db: image registry.example.com/db:15.1 → registry.example.com/db:15.2 at 10:03 UTC. It fails with "panic: assignment to entry in nil map". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-30157dd2-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it points to a cause. In the last 30 minutes in shop: db: image registry.example.com/db:15.1 → registry.example.com/db:15.2 at 10:03 UTC. It fails with "panic: assignment to entry in nil map". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-30157dd2-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing.
| 🟠 **api** in **shop** (**prod-eu-1**) keeps crashing.
| Its node and registry are healthy and unchanged, so nothing outside it explains this.
| Its node and registry are healthy and unchanged, so nothing outside it points to a cause.
| In the last 30 minutes in shop: db: image registry.example.com/db:15.1 → registry.example.com/db:15.2 at 10:03 UTC.
| It fails with `panic: assignment to entry in nil map`.
| Its last run lasted ten seconds; two restarts since its pod was created one hour ago.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
10:02:15 settled
🟠 checkout in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it explains this. It fails with "dial tcp: lookup sandbox.partner-gateway.example on 10.96.0.10:53: no such host". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs checkout-50cd6817-a -c app -n shop --previous
🟠 checkout in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it points to a cause. It fails with "dial tcp: lookup sandbox.partner-gateway.example on 10.96.0.10:53: no such host". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs checkout-50cd6817-a -c app -n shop --previous
🟠 checkout in shop (prod-eu-1) keeps crashing.
| 🟠 **checkout** in **shop** (**prod-eu-1**) keeps crashing.
| Its node and registry are healthy and unchanged, so nothing outside it explains this.
| Its node and registry are healthy and unchanged, so nothing outside it points to a cause.
| It fails with `dial tcp: lookup sandbox.partner-gateway.example on 10.96.0.10:53: no such host`.
| Its last run lasted ten seconds; two restarts since its pod was created one hour ago.
| To read the output of the last crash, run
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
10:04:15 settled
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it explains this. Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-f04b9c58-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it points to a cause. Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-f04b9c58-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing.
| 🟠 **api** in **shop** (**prod-eu-1**) keeps crashing.
| Its node and registry are healthy and unchanged, so nothing outside it explains this.
| Its node and registry are healthy and unchanged, so nothing outside it points to a cause.
| Its last run lasted ten seconds; two restarts since its pod was created one hour ago.
| To read the output of the last crash, run
| ```
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
10:02:55 settled
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it explains this. In the last 30 minutes in shop: role binding metrics-reader was deleted at 10:01 UTC. It fails with "level=fatal msg="POST https://api.partner.example/v2/orders: 403 Forbidden: {\"error\":\"forbidden: merchant key is not…". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-626da116-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it points to a cause. In the last 30 minutes in shop: role binding metrics-reader was deleted at 10:01 UTC. It fails with "level=fatal msg="POST https://api.partner.example/v2/orders: 403 Forbidden: {\"error\":\"forbidden: merchant key is not…". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs api-626da116-a -c app -n shop --previous
🟠 api in shop (prod-eu-1) keeps crashing.
| 🟠 **api** in **shop** (**prod-eu-1**) keeps crashing.
| Its node and registry are healthy and unchanged, so nothing outside it explains this.
| Its node and registry are healthy and unchanged, so nothing outside it points to a cause.
| In the last 30 minutes in shop: role binding metrics-reader was deleted at 10:01 UTC.
| It fails with
| ```
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
10:04:15 settled
🟠 cart in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it explains this. It fails with "redis: dial tcp redis-prod.abc123.cache.example.com:6379: connect: connection refused". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs cart-a3db1c95-a -c app -n shop --previous
🟠 cart in shop (prod-eu-1) keeps crashing. Its node and registry are healthy and unchanged, so nothing outside it points to a cause. It fails with "redis: dial tcp redis-prod.abc123.cache.example.com:6379: connect: connection refused". Its last run lasted ten seconds; two restarts since its pod was created one hour ago. To read the output of the last crash, run kubectl logs cart-a3db1c95-a -c app -n shop --previous
🟠 cart in shop (prod-eu-1) keeps crashing.
| 🟠 **cart** in **shop** (**prod-eu-1**) keeps crashing.
| Its node and registry are healthy and unchanged, so nothing outside it explains this.
| Its node and registry are healthy and unchanged, so nothing outside it points to a cause.
| It fails with
| ```
| redis: dial tcp redis-prod.abc123.cache.example.com:6379: connect: connection refused
Expand Down
Loading
0