|
| 1 | +package executor |
| 2 | + |
| 3 | +import ( |
| 4 | + "errors" |
| 5 | + "sync" |
| 6 | + "time" |
| 7 | + |
| 8 | + "github.com/felinics/twilight/agentcore/executor/notice" |
| 9 | + "github.com/felinics/twilight/agentcore/run/effect" |
| 10 | +) |
| 11 | + |
| 12 | +// backendNotices is the Worker's subscriber to its Backends' settled Refs |
| 13 | +// (notice.Source): one stream per Backend for every execution the Worker |
| 14 | +// observes on it, the way effect.Watcher keeps one stream per executor for |
| 15 | +// the Owner (RUN-EXE-17). A waiter registers a Ref and is woken when the |
| 16 | +// Backend announces it; a stream that drops or changes epoch wakes every |
| 17 | +// waiter so each re-reads, which is what the notice protocol asks of a |
| 18 | +// subscriber that lost track. |
| 19 | +type backendNotices struct { |
| 20 | + worker *Worker |
| 21 | + |
| 22 | + mu sync.Mutex |
| 23 | + subs map[ExecutionBackend]*noticeSubscription |
| 24 | +} |
| 25 | + |
| 26 | +type noticeSubscription struct { |
| 27 | + mu sync.Mutex |
| 28 | + waiters map[string][]chan struct{} |
| 29 | + epoch string |
| 30 | + after uint64 |
| 31 | +} |
| 32 | + |
| 33 | +// noticeWait is one waiter's view of a Backend's notice stream: the channel |
| 34 | +// a notice for its ref (or a stream reset) signals on, whether the stream |
| 35 | +// has delivered anything yet, and how to unregister. |
| 36 | +type noticeWait struct { |
| 37 | + signal <-chan struct{} |
| 38 | + established func() bool |
| 39 | + cancel func() |
| 40 | +} |
| 41 | + |
| 42 | +// await registers ref on backend's subscription. A Backend without notices |
| 43 | +// yields a nil signal and a never-established stream: the caller reads at |
| 44 | +// intervals. A stream that has not yet delivered a notice may have started |
| 45 | +// after the caller's first read, so until it has, the caller keeps a short |
| 46 | +// interval too; the notice protocol asks a subscriber to re-read what it |
| 47 | +// waits on until its subscription is known to be live. |
| 48 | +func (n *backendNotices) await(backend ExecutionBackend, ref string) noticeWait { |
| 49 | + source, ok := backend.(notice.Source) |
| 50 | + if !ok { |
| 51 | + return noticeWait{established: func() bool { return false }, cancel: func() {}} |
| 52 | + } |
| 53 | + n.mu.Lock() |
| 54 | + if n.subs == nil { |
| 55 | + n.subs = make(map[ExecutionBackend]*noticeSubscription) |
| 56 | + } |
| 57 | + sub, started := n.subs[backend] |
| 58 | + if !started { |
| 59 | + sub = ¬iceSubscription{waiters: make(map[string][]chan struct{})} |
| 60 | + n.subs[backend] = sub |
| 61 | + n.worker.spawn(func() { n.stream(source, sub) }) |
| 62 | + } |
| 63 | + n.mu.Unlock() |
| 64 | + ch := make(chan struct{}, 1) |
| 65 | + sub.mu.Lock() |
| 66 | + sub.waiters[ref] = append(sub.waiters[ref], ch) |
| 67 | + sub.mu.Unlock() |
| 68 | + return noticeWait{ |
| 69 | + signal: ch, |
| 70 | + established: func() bool { |
| 71 | + sub.mu.Lock() |
| 72 | + defer sub.mu.Unlock() |
| 73 | + return sub.epoch != "" |
| 74 | + }, |
| 75 | + cancel: func() { |
| 76 | + sub.mu.Lock() |
| 77 | + defer sub.mu.Unlock() |
| 78 | + list := sub.waiters[ref] |
| 79 | + for i := range list { |
| 80 | + if list[i] == ch { |
| 81 | + sub.waiters[ref] = append(list[:i], list[i+1:]...) |
| 82 | + break |
| 83 | + } |
| 84 | + } |
| 85 | + if len(sub.waiters[ref]) == 0 { |
| 86 | + delete(sub.waiters, ref) |
| 87 | + } |
| 88 | + }, |
| 89 | + } |
| 90 | +} |
| 91 | + |
| 92 | +// stream keeps one Settled subscription open for the Worker's lifetime, |
| 93 | +// reconnecting after the epoch it last saw; an eviction or an epoch change |
| 94 | +// wakes every waiter. |
| 95 | +func (n *backendNotices) stream(source notice.Source, sub *noticeSubscription) { |
| 96 | + ctx := n.worker.lifecycle |
| 97 | + for ctx.Err() == nil { |
| 98 | + sub.mu.Lock() |
| 99 | + epoch, after := sub.epoch, sub.after |
| 100 | + sub.mu.Unlock() |
| 101 | + err := source.Settled(ctx, epoch, after, func(r notice.Ref) bool { |
| 102 | + sub.mu.Lock() |
| 103 | + if sub.epoch != "" && r.Epoch != sub.epoch { |
| 104 | + sub.wakeAllLocked() |
| 105 | + } |
| 106 | + sub.epoch, sub.after = r.Epoch, r.Sequence |
| 107 | + for _, ch := range sub.waiters[r.Ref] { |
| 108 | + select { |
| 109 | + case ch <- struct{}{}: |
| 110 | + default: |
| 111 | + } |
| 112 | + } |
| 113 | + sub.mu.Unlock() |
| 114 | + return true |
| 115 | + }) |
| 116 | + if ctx.Err() != nil { |
| 117 | + return |
| 118 | + } |
| 119 | + sub.mu.Lock() |
| 120 | + if errors.Is(err, effect.ErrSettlementsEvicted) { |
| 121 | + sub.epoch, sub.after = "", 0 |
| 122 | + } |
| 123 | + sub.wakeAllLocked() |
| 124 | + sub.mu.Unlock() |
| 125 | + timer := time.NewTimer(time.Second) |
| 126 | + select { |
| 127 | + case <-ctx.Done(): |
| 128 | + timer.Stop() |
| 129 | + return |
| 130 | + case <-timer.C: |
| 131 | + } |
| 132 | + } |
| 133 | +} |
| 134 | + |
| 135 | +func (s *noticeSubscription) wakeAllLocked() { |
| 136 | + for _, list := range s.waiters { |
| 137 | + for _, ch := range list { |
| 138 | + select { |
| 139 | + case ch <- struct{}{}: |
| 140 | + default: |
| 141 | + } |
| 142 | + } |
| 143 | + } |
| 144 | +} |
0 commit comments