package mill
import (
"fmt"
"maps"
"slices"
"time"
"tangled.org/core/spindle/db"
"tangled.org/core/spindle/engine"
"tangled.org/core/spindle/models"
millv1 "tangled.org/core/spindle/mill/proto/gen"
)
const (
leaseRowReserved = "reserved"
leaseRowRunning = "running"
)
func (m *Mill) persistLease(lease *RemoteLease, state string) error {
if m.db == nil {
return nil
}
lease.mu.Lock()
quotaID := lease.quotaID
lease.mu.Unlock()
return m.db.SaveMillLease(db.MillLease{
LeaseID: lease.id,
NodeID: lease.nodeID,
Epoch: lease.epoch,
Engine: lease.engine,
PipelineID: string(lease.wid.PipelineId),
Workflow: lease.wid.Name,
State: state,
QuotaReservationID: quotaID,
OwnerDID: lease.ownerDID,
RepoDID: lease.repoDID,
MillRecordsTerminalMetrics: lease.millRecordsTerminalMetrics,
})
}
// reservation ids the fleet still has live leases for. startup recovery
// keeps these and reclaims every other row
func (m *Mill) LiveQuotaReservationIDs() []string {
m.mu.Lock()
leases := slices.Collect(maps.Values(m.leases))
m.mu.Unlock()
var ids []string
for _, lease := range leases {
lease.mu.Lock()
if lease.quotaID != "" {
ids = append(ids, lease.quotaID)
}
ids = append(ids, slices.Collect(maps.Keys(lease.remoteQuotas))...)
lease.mu.Unlock()
}
return ids
}
func (m *Mill) RestoreState() error {
if m.db == nil {
return nil
}
cursors, err := m.db.ListExecutorCursors()
if err != nil {
return err
}
rows, err := m.db.ListMillLeases()
if err != nil {
return err
}
m.mu.Lock()
for _, c := range cursors {
m.nodeSeqno[c.NodeID+"/"+c.Epoch] = c.AckedSeqno
}
for _, r := range rows {
lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine)
lease.wid = models.WorkflowId{
PipelineId: models.PipelineId(r.PipelineID),
Name: r.Workflow,
}
// the charge outlives the mill process. only the id survives, so
// release goes through the manager by id rather than a held lease
lease.quotaID = r.QuotaReservationID
lease.ownerDID = r.OwnerDID
lease.repoDID = r.RepoDID
lease.millRecordsTerminalMetrics = r.MillRecordsTerminalMetrics
// restored leases start as orphans, an executor must reclaim it via
// its first snapshot, or the sweep will fail it
lease.orphaned = true
lease.claimed = false
if r.State == leaseRowRunning {
lease.state = leaseRunning
}
m.leases[r.LeaseID] = lease
}
restored := len(rows)
m.mu.Unlock()
if restored > 0 {
m.l.Info("restored mill leases from previous run", "leases", restored, "cursors", len(cursors))
// executors get a grace window to reconnect and claim their leases
time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans)
}
return nil
}
func (m *Mill) sweepUnclaimedOrphans() {
m.mu.Lock()
var unclaimed []*RemoteLease
for _, lease := range m.leases {
if !lease.orphaned {
continue
}
if lease.claimed {
continue
}
if lease.getState() == leaseDone {
continue
}
if sess := m.sessions[lease.nodeID]; sess == nil || sess.disconnected {
unclaimed = append(unclaimed, lease)
}
}
m.mu.Unlock()
retry := false
reason := "executor did not reconnect after mill restart"
for _, lease := range unclaimed {
m.l.Warn("failing unclaimed restored lease", "lease", lease.id, "node", lease.nodeID)
if err := m.finishOrphan(lease, string(models.StatusKindFailed), &reason, nil); err != nil {
m.l.Error("finish unclaimed restored lease", "lease", lease.id, "err", err)
retry = true
}
}
if retry {
time.AfterFunc(5*time.Second, m.sweepUnclaimedOrphans)
}
}
func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error {
active := make(map[string]struct{}, len(activeLeaseIDs))
for _, id := range activeLeaseIDs {
active[id] = struct{}{}
}
known := make(map[string]struct{})
m.mu.Lock()
var gone []*RemoteLease
for _, lease := range m.leases {
if lease.nodeID == sess.nodeID {
known[lease.id] = struct{}{}
if lease.epoch != sess.epoch {
gone = append(gone, lease)
} else if _, ok := active[lease.id]; !ok {
gone = append(gone, lease)
}
}
}
for _, lease := range m.reservations {
if lease.nodeID == sess.nodeID && lease.epoch == sess.epoch {
known[lease.id] = struct{}{}
}
}
m.mu.Unlock()
var unknown []string
for id := range active {
if _, ok := known[id]; !ok {
unknown = append(unknown, id)
}
}
for _, lease := range gone {
status := string(models.StatusKindFailed)
reason := "executor no longer holds lease"
if cancelled, cancelReason := lease.cancelRequested(); cancelled {
status = string(models.StatusKindCancelled)
reason = cancelReason
}
m.l.Warn("finishing reconciled lease", "lease", lease.id, "node", sess.nodeID, "leaseInc", lease.epoch, "sessInc", sess.epoch)
if lease.orphaned {
if err := m.finishOrphan(lease, status, &reason, nil); err != nil {
return err
}
} else if err := m.finishLiveLease(lease, status, reason); err != nil {
return err
}
}
for _, id := range unknown {
if err := m.sendUntrackedCancel(sess, id, "lease is not owned by this mill"); err != nil {
return fmt.Errorf("cancel unknown executor lease %q: %w", id, err)
}
}
return nil
}
func (m *Mill) completeLeaseRow(lease *RemoteLease, status string, errMsg *string, exitCode *int64) (string, error) {
if m.db == nil {
return status, nil
}
return m.db.CompleteMillLease(
lease.id,
lease.wid,
status,
errMsg,
exitCode,
m.n,
)
}
func (m *Mill) finishLiveLease(lease *RemoteLease, status, reason string) error {
lease.finishMu.Lock()
defer lease.finishMu.Unlock()
if lease.getState() == leaseDone {
return nil
}
status, err := m.completeLeaseRow(lease, status, &reason, nil)
if err != nil {
return err
}
lease.deliverTerminal(&millv1.AttemptResult{
Status: mapTerminalStatusString(status),
Error: reason,
})
m.recordSyntheticTerminal(lease, status)
if err := m.cleanupLeaseLocked(lease); err != nil {
return err
}
return nil
}
func (m *Mill) finishOrphan(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error {
lease.finishMu.Lock()
defer lease.finishMu.Unlock()
if lease.getState() == leaseDone {
return nil
}
status, err := m.completeLeaseRow(lease, status, errMsg, exitCode)
if err != nil {
return err
}
lease.markDone()
m.recordSyntheticTerminal(lease, status)
if err := m.cleanupLeaseLocked(lease); err != nil {
return err
}
return nil
}
func (m *Mill) recordSyntheticTerminal(lease *RemoteLease, status string) {
if !lease.millRecordsTerminalMetrics {
return
}
result := "failure"
class := engine.FailureClassInfrastructure
reason := engine.FailureReasonExecutorLost
switch status {
case string(models.StatusKindSuccess):
result = "success"
class = engine.FailureClassNone
reason = engine.FailureReasonSuccess
case string(models.StatusKindTimeout):
result = "timeout"
class = engine.FailureClassUser
reason = engine.FailureReasonTimeout
case string(models.StatusKindCancelled):
result = "cancelled"
class = engine.FailureClassUser
reason = engine.FailureReasonCancelled
}
m.metrics.RecordWorkflowTerminal(lease.engine, result, string(class), string(reason))
}
func mapTerminalStatusString(s string) millv1.TerminalStatus {
switch s {
case "success":
return millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS
case "failed":
return millv1.TerminalStatus_TERMINAL_STATUS_FAILED
case "timeout":
return millv1.TerminalStatus_TERMINAL_STATUS_TIMEOUT
case "cancelled":
return millv1.TerminalStatus_TERMINAL_STATUS_CANCELLED
default:
return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED
}
}