8000 dashboard: read-only incident API (Pro only) by abahmed · Pull Request #579 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions deploy/chart/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,14 @@ instance Secret, its Role and the HTTPS egress the license needs.
true
{{- end -}}
{{- end }}

{{/*
Secret that holds the dashboard bearer token (key token). An existing
Secret wins; otherwise it is the Secret this chart creates when
dashboard.enabled is true. The deployment reads it with optional: true,
so the pod starts without the dashboard when the Secret is missing.
*/}}
{{- define "kwatch.dashboardSecretName" -}}
{{- $dash := default dict .Values.dashboard -}}
{{- default (printf "%s-dashboard" .Release.Name) $dash.existingSecret -}}
{{- end }}
20 changes: 20 additions & 0 deletions deploy/chart/templates/dashboard-secret.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{{- $dash := default dict .Values.dashboard -}}
{{- if and $dash.enabled (not $dash.existingSecret) }}
{{- $name := include "kwatch.dashboardSecretName" . }}
{{- /* Keep the token across upgrades: reuse the existing Secret's. */}}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
{{- $token := randAlphaNum 32 | b64enc }}
{{- if and $existing $existing.data (index $existing.data "token") }}
{{- $token = index $existing.data "token" }}
{{- end }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "kwatch.labels" . | nindent 4 }}
type: Opaque
data:
token: {{ $token }}
{{- end }}
25 changes: 25 additions & 0 deletions deploy/chart/templates/dashboard-service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{{- $dash := default dict .Values.dashboard -}}
{{- if $dash.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "kwatch.fullname" . }}-dashboard
namespace: {{ .Release.Namespace }}
labels:
{{- include "kwatch.labels" . | nindent 4 }}
annotations:
# Read-only dashboard API. No write endpoints. Access is bearer-token
# protected; do not expose without TLS.
{{- if .Values.networkPolicy.enabled }}
networking.k8s.io/policy: "true"
{{- end }}
spec:
type: ClusterIP
selector:
{{- include "kwatch.selectorLabels" . | nindent 6 }}
ports:
- name: dashboard
port: {{ $dash.port | default 8061 }}
targetPort: {{ $dash.port | default 8061 }}
protocol: TCP
{{- end }}
13 changes: 13 additions & 0 deletions deploy/chart/templates/deployment.yaml
21 changes: 21 additions & 0 deletions deploy/chart/templates/networkpolicy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ spec:
ports:
- name: health
containerPort: {{ include "kwatch.healthPort" . }}
{{- $dash := default dict .Values.dashboard -}}
{{- if $dash.enabled }}
- name: dashboard
containerPort: {{ $dash.port | default 8061 }}
{{- end }}
volumeMounts:
- name: config-volume
mountPath: /config
Expand Down Expand Up @@ -136,6 +141,14 @@ spec:
name: {{ include "kwatch.licenseSecretName" . }}
key: license-key
optional: true
# The dashboard bearer token. optional: true lets the pod start
# without the dashboard (Community, or dashboard disabled).
- name: KWATCH_DASHBOARD_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "kwatch.dashboardSecretName" . }}
key: token
optional: true
{{- if eq (include "kwatch.watchSecrets" .) "false" }}
# watch.secrets=false: matches the RBAC, which grants no
# Secret access, even when config.yaml comes from a Secret.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
{{- fail "networkPolicy.apiServerPorts must list at least one port: an empty list would open all egress" }}
{{- end }}
{{- $port := include "kwatch.healthPort" . | int }}
{{- $dash := default dict .Values.dashboard }}
{{- $dashPort := $dash.port | default 8061 }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
Expand Down Expand Up @@ -39,6 +41,25 @@ spec:
ports:
- protocol: TCP
port: {{ $port }}
{{- if $dash.enabled }}
# Dashboard API (/api/*): bearer-token protected, ClusterIP only.
# Reachable from this namespace and healthIngressFrom sources.
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
{{- range .Values.networkPolicy.kubeletCIDRs }}
- ipBlock:
cidr: {{ . }}
{{- end }}
{{- range .Values.networkPolicy.healthIngressFrom }}
-
{{- toYaml . | nindent 10 }}
{{- end }}
ports:
- protocol: TCP
port: {{ $dashPort }}
{{- end }}
{{- end }}
{{- with .Values.networkPolicy.ingress }}
{{- toYaml . | nindent 4 }}
Expand Down
22 changes: 22 additions & 0 deletions deploy/chart/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,28 @@
},
"additionalProperties": false
},
"dashboard": {
"type": "object",
"description": "Read-only incident dashboard API. Pro only, off by default.",
"properties": {
"enabled": {
"type": "boolean",
"default": false,
"description": "Enable the dashboard API listener and Service"
},
"port": {
"type": "integer",
"default": 8061,
"description": "Port for the dashboard API listener; must not equal healthCheck.port"
},
"existingSecret": {
"type": "string",
"default": "",
"description": "Existing Secret with the key token"
}
},
"additionalProperties": false
},
"persistence": {
"type": "object",
"description": "State store volume",
Expand Down
13 changes: 13 additions & 0 deletions deploy/chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,19 @@ license:
# or an extra environment variable, so the license can activate.
instanceSecret: false

# -- Dashboard read-only API. Pro only, off by default. Runs on its own
# listener and ClusterIP Service. The bearer token lives in the
# kwatch-dashboard Secret (key token); the chart auto-generates one when
# dashboard.enabled is true and no existingSecret is set.
dashboard:
# -- Enable the dashboard API listener and Service.
enabled: false
# -- Port for the dashboard API listener (must not equal healthCheck.port).
port: 8061
# -- Name of an existing Secret with the key `token`. When empty and
# enabled is true, the chart creates a Secret with an auto-generated token.
existingSecret: ""

# -- Use an existing Secret containing config.yaml and credential files.
# Required for alert providers and heartbeatMonitor.url. The Secret's
# config.yaml replaces `config` above; it is not merged.
Expand Down
2 changes: 2 additions & 0 deletions deploy/config-catalog.meta.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ app.communityProvider|Operations|Alert provider the Community edition keeps when
healthCheck.enabled|Operations|Expose the built-in health endpoint.|active|
healthCheck.port|Operations|Port for health and optional diagnostic endpoints.|active|
healthCheck.statusEndpoint|Security|Serve the read-only /status view (object names, no authentication) on the health port; off by default.|active|
dashboard.enabled|Operations|Serve the read-only incident dashboard API on a separate listener; Pro only, off by default.|active|
dashboard.port|Operations|Port for the dashboard API listener; must not equal healthCheck.port.|active|
crd.enabled|Operations|Watch KwatchConfig and supported CRD status conditions; restart kwatch when configuration changes; enabled by the interactive installer after installing the CRD.|active|
auditLog.enabled|Operations|Write structured incident lifecycle records to the configured audit sink.|active|
auditLog.output|Operations|Audit output: stdout or a supported output sink.|active|
Expand Down
2 changes: 2 additions & 0 deletions deploy/config-catalog.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,5 @@ app.hideProHints|boolean|false|Operations|Remove the note that names what kwatch
license.key|string|empty|Operations|kwatch Pro license key; reference it as ${KWATCH_LICENSE_KEY}. The KWATCH_LICENSE_KEY variable wins over this setting. Empty runs the Community edition.|active|
app.communityProvider|string|empty|Operations|Alert provider the Community edition keeps when several are configured; empty keeps the first by name.|active|
clusterCheckins|boolean|empty|Compatibility|Removed: kwatch no longer sends cluster check-ins. Accepted and ignored with a deprecation notice; use heartbeatMonitor to learn when kwatch stops.|deprecated|heartbeatMonitor
dashboard.enabled|boolean|false|Operations|Serve the read-only incident dashboard API on a separate listener; Pro only, off by default.|active|
dashboard.port|integer|8061|Operations|Port for the dashboard API listener; must not equal healthCheck.port.|active|
58 changes: 56 additions & 2 deletions internal/app/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ import (
"context"
"fmt"
"net"
"os"
"time"

"k8s.io/klog/v2"

"github.com/abahmed/kwatch/internal/alert/catalog"
"github.com/abahmed/kwatch/internal/clock"
"github.com/abahmed/kwatch/internal/config"
"github.com/abahmed/kwatch/internal/dashboard"
"github.com/abahmed/kwatch/internal/delivery"
"github.com/abahmed/kwatch/internal/feature"
"github.com/abahmed/kwatch/internal/health"
Expand All @@ -27,6 +29,7 @@ type bootstrap struct {
runtime config.RuntimeConfig
clients kubeclient.ClientSet
healthServer *health.HealthServer
dashboardServer *dashboard.Server
deliveryManager *delivery.Manager
securityMonitor *rbac.Monitor
heartbeat *heartbeat.HeartbeatMonitor
Expand Down Expand Up @@ -86,22 +89,42 @@ func newBootstrap(
}
healthServer := newHealthServer(
runtime, clockSource, overlay.invalid, entitlements)
dashboardServer := newDashboardServer(runtime, clockSource, entitlements)
return newBootstrapDeps(
runtime, clients, healthServer, dashboardServer, deliveryManager,
clockSource, wake, entitlements, inactive, notice, overlay,
processStart,
), nil
}

// newBootstrapDeps assembles the bootstrap struct once the servers,
// delivery manager, and clients are built. Keeping it separate bounds
// newBootstrap's line count.
func newBootstrapDeps(
runtime config.RuntimeConfig, clients kubeclient.ClientSet,
healthServer *health.HealthServer, dashboardServer *dashboard.Server,
deliveryManager *delivery.Manager, clockSource clock.Clock,
threadWake *threadWake, entitlements feature.Entitlements,
inactive []string, notice editionNotice, overlay overlayOutcome,
processStart time.Time,
) *bootstrap {
return &bootstrap{
runtime: runtime,
clients: clients,
healthServer: healthServer,
dashboardServer: dashboardServer,
deliveryManager: deliveryManager,
securityMonitor: rbac.NewMonitor(clients.Kubernetes,
permissionChecks(runtime), clockSource,
reportPermissions(healthServer)),
// The heartbeat pings only while monitoring is ready.
heartbeat: heartbeat.NewHeartbeatMonitorWithRuntime(
runtime, clients.HTTP, healthServer.Ready),
clock: clockSource, threadWake: wake,
clock: clockSource, threadWake: threadWake,
entitlements: entitlements, inactivePro: inactive, notice: notice,
license: newLicenseSetup(runtime, processStart),
overlay: overlay,
}, nil
}
}

// newDeliveryManager builds the delivery manager and its providers.
Expand Down Expand Up @@ -179,3 +202,34 @@ func applyEditionEndpoints(
server.DenyMetrics()
}
}

// newDashboardServer builds the optional dashboard API server. It is Pro
// only and off by default, so Community never opens its port. The
// listener is opened later by the generation. The bearer token comes
// from KWATCH_DASHBOARD_TOKEN (set by the chart from its Secret).
func newDashboardServer(
runtime config.RuntimeConfig, now clock.Clock, e feature.Entitlements,
) *dashboard.Server {
if !feature.OrCommunity(e).Limits().Dashboard {
return nil
}
dc := runtime.Lifecycle().Dashboard()
if !dc.Enabled {
return nil
}
token := os.Getenv(dashboardTokenEnv)
if token == "" {
klog.InfoS("WARNING: dashboard is enabled but "+dashboardTokenEnv+
" is empty; every dashboard request is refused with 401",
"component", "dashboard-server")
}
scope := runtime.Scope()
return dashboard.NewServer(dc.Port, token,
string(feature.OrCommunity(e).Edition()), dashboard.Scope{
Watched: scope.AllowedNamespaces(),
Excluded: scope.ForbiddenNamespaces(),
}, now)
}

// dashboardTokenEnv names the dashboard bearer token variable.
const dashboardTokenEnv = "KWATCH_DASHBOARD_TOKEN"
61 changes: 61 additions & 0 deletions internal/app/dashboard_wiring.go
10BC8
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package app

import (
"strings"

"github.com/abahmed/kwatch/internal/dashboard"
"github.com/abahmed/kwatch/internal/delivery"
"github.com/abahmed/kwatch/internal/incident"
"github.com/abahmed/kwatch/internal/notification"
)

// runbookStepPrefix starts the step compose writes for a configured
// runbook: "Runbook for <reason>: <url>".
const runbookStepPrefix = "Runbook for "

// dashboardDetail is what a sent decision leaves on its incident for the
// dashboard detail page: the composer's title, the runbook link and the
// captured output and evidence, all already redacted.
func dashboardDetail(
d incident.Decision, m notification.Message,
) incident.Detail {
out := incident.Detail{Headline: m.Title, DocsLink: runbookLink(m)}
out.Facts = append(out.Facts, d.Facts.Output...)
for _, f := range d.Facts.Evidence {
out.Facts = append(out.Facts, f.Text)
}
return out
}

// runbookLink is the first runbook link among the message's steps.
func runbookLink(m notification.Message) string {
for _, s := range m.Steps {
rest, ok := strings.CutPrefix(s.Text, runbookStepPrefix)
if !ok {
continue
}
if _, url, ok := strings.Cut(rest, ": "); ok {
return url
}
}
return ""
}

// dashboardHealth reports each provider's delivery health to the
// dashboard.
func dashboardHealth(
m *delivery.Manager,
) func() []dashboard.ChannelHealth {
return func() []dashboard.ChannelHealth {
out := make([]dashboard.ChannelHealth, 0)
for _, h := range m.ProviderHealth() {
out = append(out, dashboard.ChannelHealth{
Channel: h.Name,
LastSuccess: h.LastSuccess,
LastError: h.LastError,
LastErrorMessage: h.LastErrorMessage,
})
}
return out
}
}
9 changes: 9 additions & 0 deletions internal/app/generation.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,15 @@ func (g generationEnv) prepare(
return nil, fail(retry, "failed to open health server",
"health listener", err)
}
// The dashboard API is optional: a failure to open its listener does
// not stop kwatch. It is not required for readiness.
if deps.dashboardServer != nil {
if err := deps.dashboardServer.Open(); err != nil {
klog.ErrorS(err, "failed to open dashboard listener",
"component", "dashboard-server")
deps.dashboardServer = nil
}
}
return deps, nil
}

Expand Down
9 changes: 9 additions & 0 deletions internal/app/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@ func runPipeline(
deps.healthServer.SetStatusSource(engine.Status)
defer deps.healthServer.SetStatusSource(nil)
}
if deps.dashboardServer != nil {
deps.dashboardServer.SetIncidents(p.incidents)
deps.dashboardServer.SetStatus(engine.Status)
deps.dashboardServer.SetDeliveryHealth(
dashboardHealth(deps.deliveryManager))
defer deps.dashboardServer.SetDeliveryHealth(nil)
defer deps.dashboardServer.SetStatus(nil)
defer deps.dashboardServer.SetIncidents(nil)
}
runners, err := p.newRunners(engine)
if err != nil {
return err
Expand Down
1 change: 1 addition & 0 deletions internal/app/pipeline_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ func (p *pipelineBuild) newInvestigator() investigate.Investigator {
func (p *pipelineBuild) recordDecision(
_ context.Context, d incident.Decision, m notification.Message,
) {
p.incidents.StoreDetail(d.Incident.ID, dashboardDetail(d, m))
klog.V(1).InfoS("pipeline decision", "component", "pipeline",
"incident", d.Incident.ID, "reason", d.Reason)
counted := countDecision(metrics.DefaultRegistry(), d)
Expand Down
Loading
0