8000 Harden the license client: offline license end, clock skew, lease copy by abahmed · Pull Request #568 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion deploy/chart/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,10 @@ helm uninstall kwatch --namespace kwatch

The Secret `kwatch-license-instance` (the license instance identity) is
annotated `helm.sh/resource-policy: keep`, so uninstall leaves it in place and
a reinstall keeps the same identity. Delete it yourself to start fresh:
a reinstall keeps the same identity. kwatch also keeps a copy of its signed
Pro lease and clock guard there (field `lease-cache`), so Pro offline grace
survives a lost data volume, including `persistence.emptyDir`. Delete the
Secret yourself to start fresh:

```bash
kubectl delete secret kwatch-license-instance --namespace kwatch
Expand Down
3 changes: 2 additions & 1 deletion deploy/chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@ resources:
# more. Set
# existingClaim to reuse a PVC, or emptyDir for evaluation clusters without
# storage. emptyDir loses all state when the pod is rescheduled or the node
# fails, so open incidents may be announced again after a restart.
# fails, so open incidents may be announced again after a restart. The
# Pro lease is not lost: a signed copy lives in kwatch-license-instance.
persistence:
size: 2Gi
storageClass: ""
Expand Down
4 changes: 4 additions & 0 deletions internal/app/checkins.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const (
type checkinPinger interface {
Status() license.Status
Ping(context.Context, license.Ping) error
// ResyncOnVerdict renews at once after an expired, revoked or
// suspended answer instead of at the next daily renewal.
ResyncOnVerdict(error) bool
}

// checkinRecord is whether the server last heard check-ins on, so turning
Expand Down Expand Up @@ -121,6 +124,7 @@ func (c *checkinRunner) tick(ctx context.Context) time.Duration {
c.remember(true)
case license.StopsCheckins(err):
c.blocked = st.LID
c.pinger.ResyncOnVerdict(err)
klog.InfoS("cluster check-ins paused until the license changes",
"reason", err.Error())
case errors.As(err, &limited):
Expand Down
10 changes: 10 additions & 0 deletions internal/app/checkins_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ type fakePinger struct {
status license.Status
err error
pings []license.Ping
resync []error
}

func (f *fakePinger) ResyncOnVerdict(err error) bool {
f.mu.Lock()
defer f.mu.Unlock()
f.resync = append(f.resync, err)
return true
}

func (f *fakePinger) Status() license.Status {
Expand Down Expand Up @@ -125,6 +133,8 @@ func TestCheckinRunnerStopsUntilTheLicenseChanges(t *testing.T) {
c.tick(context.Background())
c.tick(context.Background())
require.Len(t, p.sent(), 1, "a refused license is not retried")
require.Equal(t, []error{license.ErrRevoked}, p.resync,
"the verdict reaches the license manager once")

p.status, p.err = activePro("lic_…2"), nil
c.tick(context.Background())
Expand Down
181 changes: 181 additions & 0 deletions internal/app/license_lease_store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
package app

import (
"context"
"encoding/json"
"sync"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"

"github.com/abahmed/kwatch/internal/kubeclient"
"github.com/abahmed/kwatch/internal/license"
)

const (
// licenseLeaseCacheField mirrors the lease and the clock guard in the
// instance Secret, so Pro offline grace survives an emptyDir restart.
// Both are kept in one field: deleting it removes the lease too.
licenseLeaseCacheField = "lease-cache"
// leaseMirrorEvery is how far the clock guard may advance before the
// Secret copy is written again; the lease is written on every change.
leaseMirrorEvery = time.Hour
leaseMirrorTimeout = 5 * time.Second
)

// leaseRecord keeps the lease and the clock guard as one value, so the
// guard cannot be deleted on its own to make an old lease usable again.
type leaseRecord struct {
Lease string `json:"lease"`
MaxSeen time.Time `json:"max_seen"`
}

// boltLeaseStore keeps the lease and the clock guard in the state file.
type boltLeaseStore struct{ disk diskState }

// LoadLease implements license.LeaseStore. It also reads the older
// layout: the lease as a bare string with the guard under its own key.
func (s boltLeaseStore) LoadLease() (string, license.ClockGuard, error) {
var raw json.RawMessage
var rec leaseRecord
found, err := s.disk.get(stateLicenseLease, &raw)
if err != nil {
return "", license.ClockGuard{}, err
}
if found && json.Unmarshal(raw, &rec) != nil {
var legacy string
if json.Unmarshal(raw, &legacy) == nil {
rec.Lease = legacy
}
}
var old license.ClockGuard
if _, err := s.disk.get(stateLicenseClock, &old); err != nil {
return "", license.ClockGuard{}, err
}
clock := license.ClockGuard{MaxSeen: rec.MaxSeen}
clock.Observe(old.MaxSeen)
return rec.Lease, clock, nil
}

// SaveLease implements license.LeaseStore.
func (s boltLeaseStore) SaveLease(
lease string, clock license.ClockGuard,
) error {
return s.disk.put(stateLicenseLease,
leaseRecord{Lease: lease, MaxSeen: clock.MaxSeen})
}

// LoadLeaseCache reads the Secret copy of the lease; empty when absent.
func (s kubeInstanceStore) LoadLeaseCache(
ctx context.Context,
) (leaseRecord, error) {
var rec leaseRecord
secret, err := s.secrets().Get(
ctx, licenseInstanceSecret, metav1.GetOptions{})
if err != nil {
return rec, err
}
raw := secret.Data[licenseLeaseCacheField]
if len(raw) == 0 || json.Unmarshal(raw, &rec) != nil {
return leaseRecord{}, nil
}
return rec, nil
}

// SaveLeaseCache writes the Secret copy of the lease. The lease is signed
// and bound to this cluster and instance, so it is safe to keep there.
func (s kubeInstanceStore) SaveLeaseCache(
ctx context.Context, rec leaseRecord,
) error {
raw, err := json.Marshal(rec)
if err != nil {
return err
}
existing, err := s.secrets().Get(
ctx, licenseInstanceSecret, metav1.GetOptions{})
if err != nil {
return err
}
if existing.Data == nil {
existing.Data = map[string][]byte{}
}
existing.Data[licenseLeaseCacheField] = raw
_, err = s.secrets().Update(ctx, existing, metav1.UpdateOptions{})
return err
}

// mirroredLeaseStore keeps the lease in the state file and a copy in the
// instance Secret. Loading takes the newer verified lease and the later
// guard of the two, so a lost data volume (emptyDir) keeps offline grace.
type mirroredLeaseStore struct {
local boltLeaseStore
secret kubeInstanceStore
verifier *license.Verifier

mu sync.Mutex
last leaseRecord // what the Secret is known to hold
}

func newMirroredLeaseStore(
disk diskState, kube kubernetes.Interface, v *license.Verifier,
) *mirroredLeaseStore {
return &mirroredLeaseStore{
local: boltLeaseStore{disk: disk},
secret: kubeInstanceStore{
client: kube, namespace: kubeclient.GetNamespace(),
},
verifier: v,
}
}

// LoadLease implements license.LeaseStore.
func (s *mirroredLeaseStore) LoadLease() (
string, license.ClockGuard, error,
) {
lease, clock, localErr := s.local.LoadLease()
ctx, cancel := context.WithTimeout(
context.Background(), leaseMirrorTimeout)
defer cancel()
rec, err := s.secret.LoadLeaseCache(ctx)
if err != nil {
klog.V(2).InfoS("read license lease copy failed",
"component", "license", "err", err.Error())
return lease, clock, localErr
}
s.mu.Lock()
s.last = rec
s.mu.Unlock()
if localErr != nil {
lease, clock = "", license.ClockGuard{}
}
clock.Observe(rec.MaxSeen)
return license.NewerLease(s.verifier, lease, rec.Lease), clock, nil
}

// SaveLease implements license.LeaseStore. A failed Secret write is
// logged and retried on the next save; only the state file's error is
// returned.
func (s *mirroredLeaseStore) SaveLease(
lease string, clock license.ClockGuard,
) error {
localErr := s.local.SaveLease(lease, clock)
rec := leaseRecord{Lease: lease, MaxSeen: clock.MaxSeen}
s.mu.Lock()
defer s.mu.Unlock()
moved := rec.MaxSeen.Sub(s.last.MaxSeen).Abs()
if rec.Lease == s.last.Lease && moved < leaseMirrorEvery {
return localErr
}
ctx, cancel := context.WithTimeout(
context.Background(), leaseMirrorTimeout)
defer cancel()
if err := s.secret.SaveLeaseCache(ctx, rec); err != nil {
klog.ErrorS(err, "mirror license lease to its Secret",
"component", "license")
return localErr
}
s.last = rec
return localErr
}
95 changes: 95 additions & 0 deletions internal/app/license_lease_store_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package app

import (
"context"
"testing"
"time"

"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"

"github.com/abahmed/kwatch/internal/license"
)

var leaseStoreNow = time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)

func TestBoltLeaseStoreKeepsLeaseAndGuardTogether(t *testing.T) {
disk := diskState{store: openTestStore(t)}
store := boltLeaseStore{disk: disk}
guard := license.ClockGuard{MaxSeen: leaseStoreNow}
require.NoError(t, store.SaveLease("KWL1-token", guard))
require.NoError(t, disk.put(stateLicenseClock, license.ClockGuard{}))

lease, got, err := store.LoadLease()
require.NoError(t, err)
require.Equal(t, "KWL1-token", lease)
require.True(t, got.MaxSeen.Equal(leaseStoreNow),
"an emptied old clock entry does not lower the guard")
}

func TestBoltLeaseStoreReadsTheOlderLayout(t *testing.T) {
disk := diskState{store: openTestStore(t)}
require.NoError(t, disk.put(stateLicenseLease, "KWL1-old"))
require.NoError(t, disk.put(stateLicenseClock,
license.ClockGuard{MaxSeen: leaseStoreNow}))

lease, got, err := boltLeaseStore{disk: disk}.LoadLease()
require.NoError(t, err)
require.Equal(t, "KWL1-old", lease)
require.True(t, got.MaxSeen.Equal(leaseStoreNow))
}

// With persistence.emptyDir the state file is new after a restart; the
// Secret copy brings the guard (and a verified lease) back.
func TestMirroredLeaseStoreSurvivesALostDataVolume(t *testing.T) {
client := fake.NewSimpleClientset(emptyInstanceSecret("kwatch"))
secret := kubeInstanceStore{client: client, namespace: "kwatch"}
first := &mirroredLeaseStore{
local: boltLeaseStore{disk: diskState{store: openTestStore(t)}},
secret: secret,
}
guard := license.ClockGuard{MaxSeen: leaseStoreNow}
require.NoError(t, first.SaveLease("KWL1-token", guard))

raw, err := client.CoreV1().Secrets("kwatch").Get(context.Background(),
licenseInstanceSecret, metav1.GetOptions{})
require.NoError(t, err)
require.Contains(t, string(raw.Data[licenseLeaseCacheField]),
"KWL1-token")

fresh := &mirroredLeaseStore{
local: boltLeaseStore{disk: diskState{store: openTestStore(t)}},
secret: secret,
}
_, got, err := fresh.LoadLease()
require.NoError(t, err)
require.True(t, got.MaxSeen.Equal(leaseStoreNow),
"the guard comes back from the Secret")
}

func TestMirroredLeaseStoreWritesTheSecretOnlyOnChange(t *testing.T) {
client := fake.NewSimpleClientset(emptyInstanceSecret("kwatch"))
store := &mirroredLeaseStore{
local: boltLeaseStore{disk: diskState{store: openTestStore(t)}},
secret: kubeInstanceStore{client: client, namespace: "kwatch"},
}
guard := license.ClockGuard{MaxSeen: leaseStoreNow}
require.NoError(t, store.SaveLease("KWL1-a", guard))
writes := func() int {
n := 0
for _, a := range client.Actions() {
if a.GetVerb() == "update" {
n++
}
}
return n
}
guard.MaxSeen = guard.MaxSeen.Add(time.Minute)
require.NoError(t, store.SaveLease("KWL1-a", guard))
require.Equal(t, 1, writes(), "a small guard move is not mirrored")
guard.MaxSeen = guard.MaxSeen.Add(leaseMirrorEvery)
require.NoError(t, store.SaveLease("KWL1-a", guard))
require.NoError(t, store.SaveLease("KWL1-b", guard))
require.Equal(t, 3, writes())
}
1 change: 1 addition & 0 deletions internal/app/license_session.go
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ func (l *licenseSession) managerConfig(setup *licenseSetup) license.Config {
cfg.InstanceStore = kubeInstanceStore{
client: kube, namespace: kubeclient.GetNamespace(),
}
cfg.LeaseStore = newMirroredLeaseStore(l.disk, kube, setup.verifier)
cfg.KubeSystemUID = kubeSystemUID(kube)
cfg.NodeCount = nodeCount(kube)
// The cluster id is derived from the live kube-system UID on every
Expand Down
28 changes: 0 additions & 28 deletions internal/app/license_stores.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"

"github.com/abahmed/kwatch/internal/license"
)

const (
Expand Down Expand Up @@ -147,32 +145,6 @@ func (s kubeInstanceStore) DeactivatedAt(
return time.Parse(time.RFC3339, string(raw))
}

// boltLeaseStore keeps the lease and the clock guard in the state file.
type boltLeaseStore struct{ disk diskState }

// LoadLease implements license.LeaseStore.
func (s boltLeaseStore) LoadLease() (string, license.ClockGuard, error) {
var lease string
var clock license.ClockGuard
if _, err := s.disk.get(stateLicenseLease, &lease); err != nil {
return "", license.ClockGuard{}, err
}
if _, err := s.disk.get(stateLicenseClock, &clock); err != nil {
return "", license.ClockGuard{}, err
}
return lease, clock, nil
}

// SaveLease implements license.LeaseStore.
func (s boltLeaseStore) SaveLease(
lease string, clock license.ClockGuard,
) error {
if err := s.disk.put(stateLicenseLease, lease); err != nil {
return err
}
return s.disk.put(stateLicenseClock, clock)
}

// kubeSystemUID reads the kube-system namespace UID live. It is never
// cached by the caller into a proof.
func kubeSystemUID(
Expand Down
Loading
0