package jetstream
import (
"context"
"fmt"
"log/slog"
"sync"
"sync/atomic"
"time"
"tangled.org/core/log"
)
const (
gapRewind = time.Second
// shutdown saves the cursor after this even if events are still held
heldGrace = 30 * time.Second
)
type DB interface {
GetLastTimeUs() (int64, error)
SaveLastTimeUs(int64) error
}
type Set[T comparable] map[T]struct{}
type JetstreamClient struct {
cfg StreamConfig
l *slog.Logger
logDids bool
wantedDids Set[string]
unfilteredNsids Set[string]
db DB
waitForDid bool
mu sync.RWMutex
lastSeenUs atomic.Int64
marks watermark
holding atomic.Int64
// the first event of each connection clears it, so the gap check runs once
anchorUs atomic.Int64
onGap func(context.Context)
done chan struct{}
saveEvery time.Duration
}
func (j *JetstreamClient) AddDid(did string) {
if did == "" {
return
}
if j.logDids {
j.l.Info("adding did to in-memory filter", "did", did)
}
j.mu.Lock()
j.wantedDids[did] = struct{}{}
j.mu.Unlock()
}
// call before StartJetstream, fn runs on the reader so don't block in it
func (j *JetstreamClient) SetGapHandler(fn func(context.Context)) {
j.onGap = fn
}
func (j *JetstreamClient) ExemptCollection(nsid string) {
j.mu.Lock()
j.unfilteredNsids[nsid] = struct{}{}
j.mu.Unlock()
}
func (j *JetstreamClient) RemoveDid(did string) {
if did == "" {
return
}
if j.logDids {
j.l.Info("removing did from in-memory filter", "did", did)
}
j.mu.Lock()
delete(j.wantedDids, did)
j.mu.Unlock()
}
type processor func(context.Context, *Event) error
func (j *JetstreamClient) withDidFilter(processFunc processor) processor {
// since this closure references j.WantedDids; it should auto-update
// existing instances of the closure when j.WantedDids is mutated
return func(ctx context.Context, evt *Event) error {
if anchor := j.anchorUs.Swap(0); anchor != 0 && evt.TimeUS > anchor && j.onGap != nil {
j.l.Warn("jetstream history no longer reaches the last event seen", "last", anchor, "first", evt.TimeUS)
j.onGap(ctx)
}
ticket := j.ticket(evt.TimeUS)
h := &hold{ticket: ticket}
ctx = context.WithValue(ctx, holdKey{}, h)
j.mu.RLock()
// empty filter => all dids allowed
matches := len(j.wantedDids) == 0
if !matches {
if _, ok := j.wantedDids[evt.Did]; ok {
matches = true
}
}
if !matches && evt.Commit != nil {
if _, ok := j.unfilteredNsids[evt.Commit.Collection]; ok {
matches = true
}
}
j.mu.RUnlock()
var err error
if matches {
err = processFunc(ctx, evt)
}
if err != nil && ctx.Err() != nil {
// shutdown cut this event short, so the cursor stays before it and the next start replays it
return err
}
if !h.held.Load() {
ticket.Done()
}
return err
}
}
// StartJetstream replaces cfg.Cursor with the db one, so don't set it
func NewJetstreamClient(endpoint string, collections []string, cfg *StreamConfig, logger *slog.Logger, db DB, waitForDid, logDids bool) (*JetstreamClient, error) {
if cfg == nil {
cfg = &StreamConfig{Endpoint: endpoint, Collections: collections, Compress: true}
}
return &JetstreamClient{
cfg: *cfg,
db: db,
l: logger,
wantedDids: make(map[string]struct{}),
unfilteredNsids: make(map[string]struct{}),
logDids: logDids,
// This will make the goroutine in StartJetstream wait until
// j.wantedDids has been populated, typically using addDids.
waitForDid: waitForDid,
done: make(chan struct{}),
saveEvery: time.Minute,
}, nil
}
// StartJetstream starts the jetstream client and processes events using the provided processFunc.
// The client persists the last time_us cursor itself via the DB it was constructed with.
func (j *JetstreamClient) StartJetstream(ctx context.Context, processFunc func(context.Context, *Event) error) error {
cfg := j.cfg
cfg.TailWhenNoCursor = true
cfg.Cursor = func() (int64, bool, error) { return j.cursor(ctx) }
stream, err := NewStream(cfg, j.l)
if err != nil {
return fmt.Errorf("failed to create jetstream client: %w", err)
}
handle := j.withDidFilter(processFunc)
go func() {
defer close(j.done)
if j.waitForDid && !j.waitForWantedDid(ctx) {
return
}
j.l.Info("done waiting for did")
ticked := make(chan struct{})
go func() {
defer close(ticked)
j.periodicLastTimeSave(ctx)
}()
// returning the error would replay it forever, so log and skip
stream.Run(ctx, func(ctx context.Context, e *Event) error {
if err := handle(ctx, e); err != nil {
if ctx.Err() != nil {
return err
}
j.l.Error("error processing event", "error", err, "time_us", e.TimeUS)
}
return nil
})
<-ticked
j.waitForHeld()
// save last, nothing may move the cursor once it runs
j.saveLastSeen(ctx)
}()
return nil
}
func (j *JetstreamClient) waitForHeld() {
for deadline := time.Now().Add(heldGrace); j.holding.Load() > 0 && time.Now().Before(deadline); {
time.Sleep(10 * time.Millisecond)
}
}
// one past the last finished event, not the event itself
func (j *JetstreamClient) LastSeenUs() int64 {
return j.lastSeenUs.Load()
}
// wait on it before closing the db, the final cursor save needs it open
// it never closes if StartJetstream wasn't called
func (j *JetstreamClient) Done() <-chan struct{} {
return j.done
}
func (j *JetstreamClient) waitForWantedDid(ctx context.Context) bool {
for {
j.mu.RLock()
hasDid := len(j.wantedDids) != 0
j.mu.RUnlock()
if hasDid {
return true
}
select {
case <-ctx.Done():
return false
case <-time.After(time.Second):
}
}
}
// save cursor periodically
func (j *JetstreamClient) periodicLastTimeSave(ctx context.Context) {
ticker := time.NewTicker(j.saveEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
j.saveLastSeen(ctx)
}
}
}
func (j *JetstreamClient) resume(ctx context.Context) *int64 {
cursor := j.resumeCursor(ctx)
j.anchorUs.Store(0)
if cursor == nil || j.onGap == nil || *cursor <= 0 {
return cursor
}
// cursor sits one past its event, hence the -1
j.anchorUs.Store(*cursor - 1)
rewound := max(*cursor-gapRewind.Microseconds(), 0)
return &rewound
}
func (j *JetstreamClient) cursor(ctx context.Context) (int64, bool, error) {
cursor := j.resume(ctx)
if cursor == nil {
return 0, false, nil
}
return *cursor, true, nil
}
func (j *JetstreamClient) resumeCursor(ctx context.Context) *int64 {
if seen := j.lastSeenUs.Load(); seen != 0 {
return &seen
}
return j.getLastTimeUs(ctx)
}
// the saved cursor is always an event time_us, never a wall clock
func (j *JetstreamClient) getLastTimeUs(ctx context.Context) *int64 {
l := log.FromContext(ctx)
lastTimeUs, err := j.db.GetLastTimeUs()
if err != nil {
l.Warn("couldn't get last time us, starting from now", "error", err)
return nil
}
l.Info("found last time_us", "time_us", lastTimeUs)
return &lastTimeUs
}
func (j *JetstreamClient) saveLastSeen(ctx context.Context) {
if seen := j.lastSeenUs.Load(); seen != 0 {
if err := j.db.SaveLastTimeUs(seen); err != nil {
log.FromContext(ctx).Error("failed to save cursor", "error", err)
}
}
}