8000 🐛 Stop the notice watcher from stealing the provider health signal by abahmed · Pull Request #605 · abahmed/kwatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10000
1 change: 1 addition & 0 deletions internal/delivery/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ type Manager struct {
providerReasons map[string]string
providerOutcomes map[string]providerOutcome
healthEvents chan struct{}
noticeEvents chan struct{}
// notices tracks what was said about failing providers; see
// provider_notice.go. sendNotice replaces the queue in tests.
notices map[string]*providerNotice
Expand Down
29 changes: 26 additions & 3 deletions internal/delivery/provider_health.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,24 @@ func (m *Manager) setProviderHealth(
}
}

// signalProviderHealth wakes every consumer. Each has its own channel:
// a shared one lets the notice watcher steal the signal from the health
// publisher, which then never shows the change.
func (m *Manager) signalProviderHealth() {
select {
case m.healthEventsChannel() <- struct{}{}:
default:
m.healthMu.Lock()
if m.healthEvents == nil {
m.healthEvents = make(chan struct{}, 1)
}
if m.noticeEvents == nil {
m.noticeEvents = make(chan struct{}, 1)
}
events, notices := m.healthEvents, m.noticeEvents
m.healthMu.Unlock()
for _, ch := range []chan struct{}{events, notices} {
select {
case ch <- struct{}{}:
default:
}
}
}

Expand All @@ -85,6 +99,15 @@ func (m *Manager) healthEventsChannel() chan struct{} {
return m.healthEvents
}

func (m *Manager) noticeEventsChannel() chan struct{} {
m.healthMu.Lock()
defer m.healthMu.Unlock()
if m.noticeEvents == nil {
m.noticeEvents = make(chan struct{}, 1)
}
return m.noticeEvents
}

// ProviderHealthEvents is signalled when a provider's health changed or
// the configured providers did. Read ProviderHealth after a signal.
func (m *Manager) ProviderHealthEvents() <-chan struct{} {
Expand Down
18 changes: 18 additions & 0 deletions internal/delivery/provider_health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,21 @@ func TestOutboxSweepDropsExpiredRecords(t *testing.T) {
assert.True(t, freshLive)
assert.Equal(t, dropped+1, metrics.DefaultRegistry().OutboxDropped.Load())
}

func TestProviderHealthSignalReachesPublisherAndNoticeWatcher(t *testing.T) {
manager := &Manager{}
manager.signalProviderHealth()

// The notice watcher taking its signal must not consume the
// publisher's.
select {
case <-manager.noticeEventsChannel():
default:
t.Fatal("notice watcher was not signalled")
}
select {
case <-manager.ProviderHealthEvents():
default:
t.Fatal("health publisher was not signalled")
}
}
2 changes: 1 addition & 1 deletion internal/delivery/provider_notice.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func (m *Manager) watchProviders(ctx context.Context) {
return
case <-ticker.C:
m.CheckNotices()
case <-m.healthEventsChannel():
case <-m.noticeEventsChannel():
m.CheckNotices()
}
}
Expand Down
0