~james/core

core/spindle/mill/session.go -rw-r--r-- 4.5 KiB
452f17b7 — james chore(import): snapshot james.dad.town/core at f70c71c7696a 15 hours ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
package mill

import (
	"context"
	"errors"
	"log/slog"
	"sync"
	"time"

	millproto "tangled.org/core/spindle/mill/proto"
	millv1 "tangled.org/core/spindle/mill/proto/gen"
)

var errSessionClosed = errors.New("mill: executor session closed")

// one live websocket to an executor. many leases and async streams share
// it, so one reader goroutine demuxes by message type and correlates by
// lease ID. never hold locks across decodes
type millSession struct {
	nodeID         string
	epoch          string
	arch           string
	labels         []string
	cacheStoreID   string
	cacheNamespace string
	legacyProtocol bool
	enc            messageEncoder
	l              *slog.Logger
	closeTransport func() error

	// recovery and cancellation state is guarded by Mill.mu
	snapshot        *millv1.NodeSnapshot
	disconnected    bool
	recovering      bool
	recoveryReady   bool
	recoveryExpired bool
	graceDeadline   time.Time
	graceTimer      *time.Timer
	lastSeen        time.Time
	pendingCancels  map[string]struct{}

	mu      sync.Mutex
	pending map[string]chan *millproto.Message // maps lease ID to response waiter

	ctx    context.Context
	cancel context.CancelFunc

	closeOnce sync.Once
	closed    chan struct{}
}

type messageEncoder interface {
	Encode(*millproto.Message) error
}

func newSession(nodeID string, epoch string, labels []string, enc messageEncoder, l *slog.Logger) *millSession {
	ctx, cancel := context.WithCancel(context.Background())
	return &millSession{
		nodeID:         nodeID,
		epoch:          epoch,
		labels:         labels,
		enc:            enc,
		l:              l,
		pending:        make(map[string]chan *millproto.Message),
		pendingCancels: make(map[string]struct{}),
		closed:         make(chan struct{}),
		lastSeen:       time.Now(),
		ctx:            ctx,
		cancel:         cancel,
	}
}

// caller holds Mill.mu
func (s *millSession) live(grace time.Duration) bool {
	return !s.disconnected && time.Since(s.lastSeen) <= grace
}

func (s *millSession) send(msg *millproto.Message) error {
	return s.enc.Encode(msg)
}

func (s *millSession) close() {
	s.closeOnce.Do(func() {
		s.cancel()
		close(s.closed)
		if s.closeTransport != nil {
			_ = s.closeTransport()
		}
	})
}

// one-shot waiter for the next response on the lease, cancel unregisters it
func (s *millSession) await(leaseID string) (<-chan *millproto.Message, func()) {
	ch := make(chan *millproto.Message, 1)
	s.mu.Lock()
	s.pending[leaseID] = ch
	s.mu.Unlock()
	return ch, func() {
		s.mu.Lock()
		if s.pending[leaseID] == ch {
			delete(s.pending, leaseID)
		}
		s.mu.Unlock()
	}
}

func (s *millSession) deliver(leaseID string, msg *millproto.Message) {
	s.mu.Lock()
	ch := s.pending[leaseID]
	delete(s.pending, leaseID)
	s.mu.Unlock()
	if ch != nil {
		select {
		case ch <- msg:
		default:
		}
	}
}

// sends a message and waits for its response, respecting ctx and session closure
func (s *millSession) request(ctx context.Context, leaseID string, msg *millproto.Message) (*millproto.Message, error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	ch, cancel := s.await(leaseID)
	defer cancel()

	if err := s.send(msg); err != nil {
		return nil, err
	}

	select {
	case resp := <-ch:
		return resp, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-s.closed:
		return nil, errSessionClosed
	}
}

// demuxes frames until the decoder errors (connection gone)
func (s *millSession) readLoop(m *Mill, dec *millproto.Decoder) error {
	for {
		msg, err := dec.Decode()
		if err != nil {
			return err
		}
		if err := s.dispatch(m, msg); err != nil {
			return err
		}
	}
}

func (s *millSession) dispatch(m *Mill, msg *millproto.Message) error {
	if !m.touchSession(s) {
		return errSessionClosed
	}
	switch {
	case msg.GetNodeSnapshot() != nil:
		return m.onSnapshot(s, msg.GetNodeSnapshot())
	case msg.GetReserveResult() != nil:
		s.deliver(msg.GetReserveResult().GetLeaseId(), msg)
	case msg.GetCommitted() != nil:
		s.deliver(msg.GetCommitted().GetLeaseId(), msg)
	case msg.GetEventBatch() != nil:
		return m.onEventBatch(s, msg.GetEventBatch())
	case msg.GetCancelAck() != nil:
		m.onCancelAck(s, msg.GetCancelAck())
	case msg.GetLiveLog() != nil:
		return m.onLiveLog(s, msg.GetLiveLog())
	case msg.GetQuotaReq() != nil:
		go m.handleQuotaRequest(s, msg.GetQuotaReq())
	case msg.GetNixCacheUsed() != nil:
		m.startNixCacheUsed(s, msg.GetNixCacheUsed())
	case msg.GetDebugKeyReq() != nil:
		go m.handleDebugKeyRequest(s, msg.GetDebugKeyReq())
	default:
		s.l.Warn("session received unexpected message", "node", s.nodeID)
	}
	return nil
}