~james/core

core/jetstream/jetstream.go -rw-r--r-- 7.0 KiB
452f17b7 — james chore(import): snapshot james.dad.town/core at f70c71c7696a 16 hours ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
package jetstream

import (
	"context"
	"fmt"
	"log/slog"
	"sync"
	"sync/atomic"
	"time"

	"tangled.org/core/log"
)

const (
	gapRewind = time.Second
	// shutdown saves the cursor after this even if events are still held
	heldGrace = 30 * time.Second
)

type DB interface {
	GetLastTimeUs() (int64, error)
	SaveLastTimeUs(int64) error
}

type Set[T comparable] map[T]struct{}

type JetstreamClient struct {
	cfg StreamConfig
	l   *slog.Logger

	logDids         bool
	wantedDids      Set[string]
	unfilteredNsids Set[string]
	db              DB
	waitForDid      bool
	mu              sync.RWMutex

	lastSeenUs atomic.Int64
	marks      watermark
	holding    atomic.Int64

	// the first event of each connection clears it, so the gap check runs once
	anchorUs atomic.Int64
	onGap    func(context.Context)

	done chan struct{}

	saveEvery time.Duration
}

func (j *JetstreamClient) AddDid(did string) {
	if did == "" {
		return
	}

	if j.logDids {
		j.l.Info("adding did to in-memory filter", "did", did)
	}
	j.mu.Lock()
	j.wantedDids[did] = struct{}{}
	j.mu.Unlock()
}

// call before StartJetstream, fn runs on the reader so don't block in it
func (j *JetstreamClient) SetGapHandler(fn func(context.Context)) {
	j.onGap = fn
}

func (j *JetstreamClient) ExemptCollection(nsid string) {
	j.mu.Lock()
	j.unfilteredNsids[nsid] = struct{}{}
	j.mu.Unlock()
}

func (j *JetstreamClient) RemoveDid(did string) {
	if did == "" {
		return
	}

	if j.logDids {
		j.l.Info("removing did from in-memory filter", "did", did)
	}
	j.mu.Lock()
	delete(j.wantedDids, did)
	j.mu.Unlock()
}

type processor func(context.Context, *Event) error

func (j *JetstreamClient) withDidFilter(processFunc processor) processor {
	// since this closure references j.WantedDids; it should auto-update
	// existing instances of the closure when j.WantedDids is mutated
	return func(ctx context.Context, evt *Event) error {
		if anchor := j.anchorUs.Swap(0); anchor != 0 && evt.TimeUS > anchor && j.onGap != nil {
			j.l.Warn("jetstream history no longer reaches the last event seen", "last", anchor, "first", evt.TimeUS)
			j.onGap(ctx)
		}

		ticket := j.ticket(evt.TimeUS)
		h := &hold{ticket: ticket}
		ctx = context.WithValue(ctx, holdKey{}, h)

		j.mu.RLock()
		// empty filter => all dids allowed
		matches := len(j.wantedDids) == 0
		if !matches {
			if _, ok := j.wantedDids[evt.Did]; ok {
				matches = true
			}
		}
		if !matches && evt.Commit != nil {
			if _, ok := j.unfilteredNsids[evt.Commit.Collection]; ok {
				matches = true
			}
		}
		j.mu.RUnlock()

		var err error
		if matches {
			err = processFunc(ctx, evt)
		}
		if err != nil && ctx.Err() != nil {
			// shutdown cut this event short, so the cursor stays before it and the next start replays it
			return err
		}

		if !h.held.Load() {
			ticket.Done()
		}
		return err
	}
}

// StartJetstream replaces cfg.Cursor with the db one, so don't set it
func NewJetstreamClient(endpoint string, collections []string, cfg *StreamConfig, logger *slog.Logger, db DB, waitForDid, logDids bool) (*JetstreamClient, error) {
	if cfg == nil {
		cfg = &StreamConfig{Endpoint: endpoint, Collections: collections, Compress: true}
	}

	return &JetstreamClient{
		cfg:             *cfg,
		db:              db,
		l:               logger,
		wantedDids:      make(map[string]struct{}),
		unfilteredNsids: make(map[string]struct{}),

		logDids: logDids,

		// This will make the goroutine in StartJetstream wait until
		// j.wantedDids has been populated, typically using addDids.
		waitForDid: waitForDid,

		done: make(chan struct{}),

		saveEvery: time.Minute,
	}, nil
}

// StartJetstream starts the jetstream client and processes events using the provided processFunc.
// The client persists the last time_us cursor itself via the DB it was constructed with.
func (j *JetstreamClient) StartJetstream(ctx context.Context, processFunc func(context.Context, *Event) error) error {
	cfg := j.cfg
	cfg.TailWhenNoCursor = true
	cfg.Cursor = func() (int64, bool, error) { return j.cursor(ctx) }
	stream, err := NewStream(cfg, j.l)
	if err != nil {
		return fmt.Errorf("failed to create jetstream client: %w", err)
	}

	handle := j.withDidFilter(processFunc)
	go func() {
		defer close(j.done)
		if j.waitForDid && !j.waitForWantedDid(ctx) {
			return
		}
		j.l.Info("done waiting for did")

		ticked := make(chan struct{})
		go func() {
			defer close(ticked)
			j.periodicLastTimeSave(ctx)
		}()

		// returning the error would replay it forever, so log and skip
		stream.Run(ctx, func(ctx context.Context, e *Event) error {
			if err := handle(ctx, e); err != nil {
				if ctx.Err() != nil {
					return err
				}
				j.l.Error("error processing event", "error", err, "time_us", e.TimeUS)
			}
			return nil
		})
		<-ticked
		j.waitForHeld()
		// save last, nothing may move the cursor once it runs
		j.saveLastSeen(ctx)
	}()

	return nil
}

func (j *JetstreamClient) waitForHeld() {
	for deadline := time.Now().Add(heldGrace); j.holding.Load() > 0 && time.Now().Before(deadline); {
		time.Sleep(10 * time.Millisecond)
	}
}

// one past the last finished event, not the event itself
func (j *JetstreamClient) LastSeenUs() int64 {
	return j.lastSeenUs.Load()
}

// wait on it before closing the db, the final cursor save needs it open
// it never closes if StartJetstream wasn't called
func (j *JetstreamClient) Done() <-chan struct{} {
	return j.done
}

func (j *JetstreamClient) waitForWantedDid(ctx context.Context) bool {
	for {
		j.mu.RLock()
		hasDid := len(j.wantedDids) != 0
		j.mu.RUnlock()
		if hasDid {
			return true
		}
		select {
		case <-ctx.Done():
			return false
		case <-time.After(time.Second):
		}
	}
}

// save cursor periodically
func (j *JetstreamClient) periodicLastTimeSave(ctx context.Context) {
	ticker := time.NewTicker(j.saveEvery)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			j.saveLastSeen(ctx)
		}
	}
}

func (j *JetstreamClient) resume(ctx context.Context) *int64 {
	cursor := j.resumeCursor(ctx)
	j.anchorUs.Store(0)
	if cursor == nil || j.onGap == nil || *cursor <= 0 {
		return cursor
	}
	// cursor sits one past its event, hence the -1
	j.anchorUs.Store(*cursor - 1)
	rewound := max(*cursor-gapRewind.Microseconds(), 0)
	return &rewound
}

func (j *JetstreamClient) cursor(ctx context.Context) (int64, bool, error) {
	cursor := j.resume(ctx)
	if cursor == nil {
		return 0, false, nil
	}
	return *cursor, true, nil
}

func (j *JetstreamClient) resumeCursor(ctx context.Context) *int64 {
	if seen := j.lastSeenUs.Load(); seen != 0 {
		return &seen
	}
	return j.getLastTimeUs(ctx)
}

// the saved cursor is always an event time_us, never a wall clock
func (j *JetstreamClient) getLastTimeUs(ctx context.Context) *int64 {
	l := log.FromContext(ctx)
	lastTimeUs, err := j.db.GetLastTimeUs()
	if err != nil {
		l.Warn("couldn't get last time us, starting from now", "error", err)
		return nil
	}

	l.Info("found last time_us", "time_us", lastTimeUs)
	return &lastTimeUs
}

func (j *JetstreamClient) saveLastSeen(ctx context.Context) {
	if seen := j.lastSeenUs.Load(); seen != 0 {
		if err := j.db.SaveLastTimeUs(seen); err != nil {
			log.FromContext(ctx).Error("failed to save cursor", "error", err)
		}
	}
}