8000 ✨ more accurate root cause, dashboard improvements by abahmed · Pull Request #617 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
38 changes: 30 additions & 8 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
${{ github.event_name == 'pull_request'
&& format('origin/{0}', github.base_ref)
|| github.event.before || 'origin/main' }}
run: make ci
run: make ci-core

# The dashboard in the runner's Google Chrome (headless): console
# errors, CSP violations, sideways scroll and basic accessibility.
Expand All @@ -75,6 +75,35 @@ jobs:
CHROME_PATH: google-chrome
run: make verify-dashboard-browser

- name: Upload coverage to Codecov
if: success() && github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false

# The labelled scenario library and its alert-quality gates: the slowest
# package, so it runs beside Verify instead of inside it.
scenarios:
name: Scenarios
runs-on: ubuntu-24.04
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false

- name: Install Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod

- name: Run scenarios and alert-quality gates
run: make scenarios-gate

- name: Publish alert-quality report
if: always()
run: |
Expand All @@ -90,13 +119,6 @@ jobs:
path: _output/alert-quality.md
if-no-files-found: warn

- name: Upload coverage to Codecov
if: success() && github.event_name == 'push' && github.ref == 'refs/heads/main'
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false

# publish.yml builds the release image only after a tag exists. This job
# catches a broken Dockerfile, a failing binary, or a new HIGH/CRITICAL
# finding before merge.
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,6 @@ jobs:
# A run filtered to some scenarios is a debugging run: log every
# finding and incident membership change.
KWATCH_VERBOSITY: "${{ inputs.scenario != '' && '4' || '' }}"
KWATCH_E2E_RUN_KNOWN_GAPS: "${{ inputs.scenario != '' }}"
REPORTED_REF: "${{ inputs.reported_ref }}"
REPORTED_VERSION: "${{ inputs.reported_version }}"
COMPARE_WITH_MAIN: "${{ inputs.compare_with_main || false }}"
Expand Down
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ Tooling guards (scoped `-update`, hygiene, pinned linter, golden/registry
match) are described under "Guards" in `docs/contributing-scenario.md`.
Regenerate scenario files only with `-run 'Test.../<name>$' -update`.

`make ci` is exactly what the required CI job runs: `make verify`, the
`make ci` is the whole gate. CI runs it as two parallel jobs: `make ci-core`
(Verify) and `make scenarios-gate` (Scenarios). It is `make verify`, the
`licensetest`-tagged tests (`make test-licensetest`), plus
actionlint, ShellCheck and a `go mod tidy` drift check. Locally a missing
actionlint, ShellCheck or Helm skips its check with a notice; under `CI=true`
Expand Down
36 changes: 25 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# wall-clock latency and memory budgets then run once more without -race.

.PHONY: help build test test-short vet lint verify-fmt clean \
coverage coverage-check verify ci verify-focused verify-fast \
coverage coverage-check scenarios-gate verify-core ci-core verify ci verify-focused verify-fast \
verify-latency verify-race verify-security verify-manifests \
verify-operational verify-scenarios verify-negative-regressions \
verify-dashboard-browser \
Expand Down Expand Up @@ -161,22 +161,36 @@ verify-fmt:
fi
@echo "All files are properly formatted."

# The single test run of the gate: race-enabled, with coverage, and with the
# alert-quality gates enforced inside it.
# The test run of the gate: race-enabled, with coverage. The scenario
# library has no production code and its tests do not use -coverpkg, so it
# adds no coverage; it runs in scenarios-gate instead of here.
coverage:
@mkdir -p $(dir $(ALERT_QUALITY_REPORT))
KWATCH_SCORECARD_GATE=1 $(ALERT_QUALITY_ENV) \
$(GOTEST) -race -timeout $(TEST_TIMEOUT) \
--coverprofile=coverage.txt --covermode=atomic ./...
$(GOTEST) -race -timeout $(TEST_TIMEOUT) \
--coverprofile=coverage.txt --covermode=atomic \
$$($(GOCMD) list ./... | grep -v /internal/scenarios$$)

coverage-check: coverage
./scripts/check-coverage.sh coverage.txt

verify: build vet lint coverage-check verify-latency line-check diff-check \
architecture-check test-layout-check hygiene-check verify-catalogs \
docs-verify verify-manifests verify-negative-regressions deadcode-check
# The labelled scenarios with the alert-quality gates enforced, under
# -race. It is the slowest package, so CI runs it as its own parallel job.
scenarios-gate:
@mkdir -p $(dir $(ALERT_QUALITY_REPORT))
KWATCH_SCORECARD_GATE=1 $(ALERT_QUALITY_ENV) \
$(GOTEST) -race -count=1 -timeout $(TEST_TIMEOUT) \
./internal/scenarios/

verify-core: build vet lint coverage-check verify-latency line-check \
diff-check architecture-check test-layout-check hygiene-check \
verify-catalogs docs-verify verify-manifests \
verify-negative-regressions deadcode-check

verify: verify-core scenarios-gate

# What the CI Verify job runs; the Scenarios job runs scenarios-gate.
ci-core: verify-core test-licensetest lint-actions lint-shell mod-tidy-check

ci: verify test-licensetest lint-actions lint-shell mod-tidy-check
ci: ci-core scenarios-gate

# Package-scoped gate for incremental refactors, for example:
# make verify-focused PKGS="./internal/incident ./internal/delivery/..."
Expand Down
79 changes: 78 additions & 1 deletion RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

- **Browser notifications with the dashboard closed.** Turning notifications on in the dashboard subscribes the browser to Web Push: kwatch sends each change itself, encrypted for that browser, and every notification stays on screen until closed, so macOS shows it.

- **Downtime is visible, and kwatch says when it is degraded.** The dashboard marks the hours kwatch was not running: hatched days with "6 h off" on the Overview's week chart (and its table), "Based on 5½ of 7 days of watching" under the week's numbers, and a "kwatch was not running from ... to ..." marker in Activity and issue timelines. `/health`, `/api/status` and the kwatch page always show the `heartbeat` component (not set up is information, not a failure). When one of its own parts stays degraded for 10 minutes (kubelet reads, refused kinds, pod logs (an optional permission you never granted stays quiet; only one lost after it worked is told), the state file, an alert channel, heartbeat pings, API throttling, decision lag, unsent alerts), kwatch sends one low-priority message, updated in place at most once per 10 minutes (the newest reason is never lost), and "back to normal" when it recovers: never paged, at most 6 a day, route reason `kwatch_degraded`, off with `app.disableSelfAlerts`.

- Built with Go 1.27.4 (GO-2026-6859).

### Editions
Expand Down Expand Up @@ -150,6 +152,9 @@

### Dashboard

- A change links to where it came from: "Source: Argo CD app shop · commit 1a2b3c4 · image api:1.5", in "What changed just before", on the release line of Why? and on a blamed change's proof. Built only from what the objects carry: the GitOps application kwatch already names (linked when the new `app.gitOpsLinks` setting gives its web address, e.g. `argocd: https://argocd.example.com/applications/{app}`), the `org.opencontainers.image.source` and `.revision` annotations on the workload or its pod template (a commit is linked on github.com, gitlab.com and bitbucket.org; elsewhere the repository is linked and the revision is text; invalid values are ignored) and the image (linked on Docker Hub and Quay only).
- Why? on an issue without a cause lists what kwatch checked for that issue and found fine ("its node is healthy", "its ConfigMaps and Secrets are there and did not change", "nothing near it was changed in the 30 minutes before it started") and what it could not check ("kwatch cannot read pod logs here."), instead of "kwatch checked 3 things and could read them all". A possible cause gets the same list as "Also ruled out: …". Nothing checked shows no list.
- "What changed just before" says under each change why it is listed, from the links kwatch already holds: "The failing pods mount this ConfigMap", "This is the workload's own template", "This Service routes to the failing pods", "This is the node the failing pods run on". A change with no known link says nothing.
- The issue page and the drawer head with a short title; the whole one is under "Full title" (and in the tooltip). At most three lines on a phone.
- A row that settled on its own shows a muted "Was Urgent" (no red chip) on Issues and Activity.
- Copy summary adds the proof line, who was alerted and when, and the cluster name (when the page knows it).
Expand Down Expand Up @@ -341,6 +346,10 @@

### Incidents and messages

- A message that blames or lists a change adds one line linking it to its source when an address is known: "Source: Argo CD app shop · commit 1a2b3c4" (the GitOps app only with `app.gitOpsLinks`; the commit from the `org.opencontainers.image.source` and `.revision` annotations). Providers without links write "label: address"; the short note never carries a link, and Community messages none.
- The evidence of a Pro message about an issue without a sure cause (Teams, email, the generic webhook's `evidence.checked`) says what kwatch checked for it, at most five items ("kwatch checked: its node is healthy; its image shows no pull errors and did not change; nothing in shop was changed in the last 30 minutes.", or "Also ruled out: its node and its image." under a possible cause), and what it could not check ("kwatch cannot read pod logs here.").
- Changes now record the repository and commit a CI or GitOps tool set on the object; the state file gains two optional fields, read as empty from older files.

- Recovered CronJobs close. Finished helper containers in Job pods, a Job's brief create refusal, progressing ordered StatefulSet scale-ups and suspended CronJobs and Jobs do not alert.

- A failure that outlives its maintenance window is announced when the window ends.
Expand Down Expand Up @@ -699,7 +708,7 @@
hop repeated, "The pod-security changed.", a workload named "storage"
capitalized, one workload named twice in an update.

- Fixed: an update that only raised the tier says "It is now critical."
- Fixed: an update that only raised the tier says "This is now critical."

- Fixed: messages speak of kwatch in the third person ("kwatch can't see ...",
"kwatch will write again ...").
Expand Down Expand Up @@ -808,6 +817,62 @@

### Detection

- **Newer restart semantics are not read as crashes.** A container
restarted by its own restart rule (`restartPolicyRules`, operator `In`)
on the exit code it exited with is not a crash loop, unless it keeps
ending that way within seconds of each start; a memory kill is always
reported. Containers stopped while a `RestartAllContainers` rule
restarts the pod (condition `AllContainersRestarting`) are not
reported. A native sidecar that runs but has not passed its startup
probe yet keeps its pod from counting as stuck Pending within that
probe's budget. A crash loop is held through the runs between crashes
for twice the last run (up to 5 minutes), so on a kubelet with a short
`crashLoopBackOff.maxContainerRestartPeriod` a container that crashes
every minute stays one crash loop instead of clearing on every run.
Messages name a failing sidecar or init container ("has a sidecar
container proxy that keeps crashing") and say to look at it, not at
the app.

- **Failed or stuck Helm releases (Pro, `core.workloads.rollouts`).** A
Helm release whose latest revision is `failed`, or has stayed
`pending-install`, `pending-upgrade`, `pending-rollback` or
`uninstalling` for 15 minutes, is reported ("Helm release payments in
shop has failed to upgrade or roll back at revision 12"), at notify
level, never paged. kwatch reads only the labels Helm writes on its
release Secrets, through a metadata-only watch (the release payload is
never fetched; no new permission). Steps: `helm history`, `helm status`
and `helm rollback <release> <last deployed revision>`, marked as a
change, with the note to change it in Git when a GitOps controller
manages the release. With `watch.secrets: false` releases are not
watched. A `deployed` or `superseded` release, or a pending one younger
than 15 minutes, raises nothing.

- **Memory context on OOM kills and memory evictions.** From the usage
history kwatch already keeps (hourly peaks for 24 hours and the last
runs): which limit was reached (the container's own or the node's),
how much history there is ("in the last 5h, all the history kwatch
has"), a run that stayed near one level, how many hours came near the
limit, where the last runs ended, the request beside the limit, and for
a node kill or a memory eviction the node's requested and used memory,
its biggest users with their requests, and the victim's use against its
request. A container killed before its first reading says so in the
Proof tab and nothing is invented.

- **Observed sizing at failure time.** On a kill by the container's own
limit the message states the peak against limit and request over the
history kwatch really has, and when the peak is at the limit that the
real need is not known. The Pro memory fix is checked first against the
namespace's LimitRange maximum, ResourceQuota `limits.memory` room for
every replica and the largest node; a value the cluster refuses is not
offered and the step says what blocks it; kinds kwatch cannot read are
named. The fix line names a LimitRange that already caps the limit.

- **CPU at its limit as probe evidence.** A probe that fails while its
container runs at its CPU limit (90% of it, even when lightly
throttled) says so: "Liveness timed out while the container used 198m
of its 200m CPU limit". No line when the container is under its limit;
the throttling finding itself stays silent.

- A custom resource kind whose watch is stale or whose access was lost no longer raises its kind checks from the frozen cache (an approved CSR without a certificate, an unallocated ResourceClaim, a CRD not established, a missing route backend): they hold, like its conditions, until kwatch can read the kind again.

- Namespace label scope waits (up to 30 s) for a namespace it has not seen yet.
Expand Down Expand Up @@ -1081,6 +1146,10 @@

### Root cause

- A link that ends is a change kwatch can blame. When a Service stops routing to a workload, a NetworkPolicy stops covering one, a workload stops mounting or naming a ConfigMap, Secret or volume claim, or an Ingress, route or webhook stops pointing at a Service, and the link stays gone for a minute, the object that held it records the change ("network policy orders-egress stopped covering orders"). It counts against the change history caps, is listed among the changes next to a failure and on the issue page with why it is shown, and a policy that stopped covering a workload is blamed for its connection failures. Links of pods, containers and node assignment, and those of a deleted object, are never recorded.

- StatefulSets and DaemonSets get the same rollback suggestion as Deployments: `kubectl rollout undo statefulset/<name> --to-revision=N` to the ControllerRevision before the newest, only for a cause stated at least likely and only when a ready pod of that revision shows it served; GitOps and Helm-managed ones are changed in Git or rolled back with Helm. ControllerRevisions are now watched with their revision number (status mode, the saved template dropped from the cache); the RBAC rules are unchanged.

- Causes are looked for back to the start of the failure, up to 7 days (`MaxChangeAge`). An unrelated older failure of the same workload does not widen that.

- A missing image pull secret is blamed only when a pull fails for it, including Docker Hub's "may require authorization" answer.
Expand Down Expand Up @@ -1324,6 +1393,8 @@

### Delivery and providers

- New provider: ServiceNow (`alert.servicenow`). One incident per kwatch incident through the Table API: kwatch finds it again by `correlation_id` before it files one, so a retry, a restart or a reset never files a duplicate; updates are work notes; a resolve sets the resolve state, resolution code and notes, and a failure within the reopen window reopens it. Basic or OAuth client-credentials authentication, urgency and impact per tier, assignment group, caller, category and extra columns. See docs/providers.md.

- AWS SES and SNS also take the JSON APIs' `ExpiredTokenException` and `UnrecognizedClientException` refusals as an expired or invalid token, and fetch new role credentials for them as for `ExpiredToken` and `InvalidClientTokenId`.

- An SES provider with a China region (`cn-…`) and no `endpoint` does not start and the log says why: SES is not offered there. SNS in China regions is unchanged.
Expand Down Expand Up @@ -1825,6 +1896,12 @@

- Changing the dashboard password restarts the pod.

- Optional `priorityClass`: with `priorityClass.create: true` (off by
default) the chart creates a PriorityClass (`priorityClass.name`, value
1000000, `PreemptLowerPriority`, never the global default) and the pod
uses it, so the pod that reports failures is not the first one evicted
under node pressure. An explicit `priorityClassName` still wins.

- `KWATCH_DASHBOARD_ENABLED` and `KWATCH_DASHBOARD_TRUSTED_PROXIES` are always set,
on or off and with the list of trusted proxies (empty when off).
- A trusted-proxy range wider than /16 (IPv4) or /48 (IPv6) logs a warning at
Expand Down
18 changes: 18 additions & 0 deletions cmd/coveragedocs/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ func render(root string) ([]byte, error) {
}
fmt.Fprintf(&b, "| %s | %s | %s |\n", e.Kind, group, e.Mode)
}
b.WriteString("\nHelm's release Secrets (type `helm.sh/release.v1`) " +
"are left out of the Secret watch and watched for their labels " +
"only (`metadata`), never their payload; with `watch.secrets: " +
"false` they are not watched either.\n")
b.WriteString("\n## Failure modes\n\n")
b.WriteString("Health is taken from the severity each detector " +
"assigns: warning is degraded, critical is failing; `varies` " +
Expand All @@ -47,9 +51,23 @@ func render(root string) ([]byte, error) {
usage.health(m.Reasons), codeList(m.Reasons),
codeList(usage.detectors(m.Reasons)))
}
b.WriteString(restartNote)
return []byte(b.String()), nil
}

// restartNote says how restarts that Kubernetes makes on purpose are
// read.
const restartNote = "\nRestarts Kubernetes makes on purpose are not " +
"crashes. A native sidecar (an init container with `restartPolicy: " +
"Always`) is reported as a sidecar, never as a pod stuck in init, " +
"and while it runs within its startup probe's budget its Pending " +
"pod is starting. An exit that a container's own restart rule " +
"(`restartPolicyRules`, operator `In`) restarts is no crash loop " +
"unless the container keeps ending that way within seconds of each " +
"start. A crash loop is judged by the container's own run lengths, " +
"not a fixed back-off, so a kubelet with a short " +
"`crashLoopBackOff.maxContainerRestartPeriod` reads the same.\n"

func sortedCatalog() []kube.CoverageEntry {
entries := kube.CoverageCatalog()
sort.SliceStable(entries, func(i, j int) bool {
Expand Down
Loading
Loading
0