package mill
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"tangled.org/core/notifier"
"tangled.org/core/spindle/config"
"tangled.org/core/spindle/db"
"tangled.org/core/spindle/engine"
"tangled.org/core/spindle/engines/dummy"
"tangled.org/core/spindle/mill/executor"
"tangled.org/core/spindle/models"
"tangled.org/core/spindle/secrets"
"tangled.org/core/workflow"
)
// the outcome is published before teardown. a held teardown keeps the lease open after it
type heldTeardownEngine struct {
*dummy.DummyEngine
stepErr error
teardown chan struct{}
}
func (e *heldTeardownEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, s []secrets.UnlockedSecret, wl models.WorkflowLogger) error {
if e.stepErr != nil {
return e.stepErr
}
return e.DummyEngine.RunStep(ctx, wid, w, idx, s, wl)
}
func (e *heldTeardownEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error {
<-e.teardown
return nil
}
type lostExecutorWorld struct {
t *testing.T
ctx context.Context
mill *Mill
bdb *db.DB
wsURL string
wid models.WorkflowId
result chan error
}
func newLostExecutorWorld(t *testing.T, grace time.Duration) *lostExecutorWorld {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
l := slog.New(slog.NewTextHandler(io.Discard, nil))
millDir := t.TempDir()
bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db"))
if err != nil {
t.Fatalf("mill db: %v", err)
}
bn := notifier.New()
m := New(l, Config{LogDir: millDir, ReconnectGrace: grace, BidTimeout: 2 * time.Second})
m.Attach(bdb, &bn, testQuotaManager(t, bdb))
registerTestExecutor(t, bdb, "exec-1", HashToken("test-token"), nil)
srv := httptest.NewServer(http.HandlerFunc(m.HandleExecutorConn))
t.Cleanup(srv.Close)
return &lostExecutorWorld{
t: t,
ctx: ctx,
mill: m,
bdb: bdb,
wsURL: "ws" + strings.TrimPrefix(srv.URL, "http"),
wid: models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"},
}
}
func (w *lostExecutorWorld) startExecutor(dbPath string, eng models.Engine) context.CancelFunc {
w.t.Helper()
ctx, cancel := context.WithCancel(w.ctx)
l := slog.New(slog.NewTextHandler(io.Discard, nil))
edb, err := db.Make(ctx, dbPath)
if err != nil {
w.t.Fatalf("exec db: %v", err)
}
en := notifier.New()
cfg := &config.Config{}
cfg.Server.Dev = true
cfg.Server.LogDir = filepath.Dir(dbPath)
cfg.ArtifactStores.Disk.Dir = filepath.Join(filepath.Dir(dbPath), "artifacts")
cfg.Server.Hostname = "exec-1"
cfg.Mill.URL = w.wsURL
cfg.Mill.Seats = 1
cfg.Mill.SharedSecret = "test-token"
exec, err := executor.New(cfg, map[string]models.Engine{"dummy": eng}, edb, &en, l, nil, nil)
if err != nil {
w.t.Fatalf("executor.New: %v", err)
}
go exec.Connect(ctx)
w.waitFor("executor session", func() bool { return w.mill.sessionForNode("exec-1") != nil })
return cancel
}
func (w *lostExecutorWorld) runUntilOutcomeThenKill(dbPath string, stepErr error) {
w.t.Helper()
eng := &heldTeardownEngine{DummyEngine: dummy.New(slog.New(slog.NewTextHandler(io.Discard, nil))), stepErr: stepErr, teardown: make(chan struct{})}
kill := w.startExecutor(dbPath, eng)
wf, err := NewEngine("dummy", w.mill).InitWorkflow(workflow.WorkflowSpec{
Name: "build",
Raw: "steps:\n - name: hello\n command: echo hi\n",
}, testPipeline())
if err != nil {
w.t.Fatalf("InitWorkflow: %v", err)
}
placeCtx, placeCancel := context.WithTimeout(w.ctx, 10*time.Second)
defer placeCancel()
slot, err := w.mill.place(placeCtx, "dummy", w.wid, wf)
if err != nil {
w.t.Fatalf("place: %v", err)
}
w.t.Cleanup(slot.Release)
w.result = make(chan error, 1)
go func() {
ctx, cancel := context.WithTimeout(w.ctx, time.Minute)
defer cancel()
w.result <- w.mill.commitAndWait(ctx, wf, nil)
}()
outcome := models.StatusKindSuccess
if stepErr != nil {
outcome = models.StatusKindFailed
}
w.waitFor("recorded outcome", func() bool {
st, err := w.bdb.GetStatus(w.wid)
return err == nil && st == outcome
})
kill()
w.waitFor("session loss", func() bool { return w.mill.sessionForNode("exec-1") == nil })
}
func (w *lostExecutorWorld) waitFor(what string, cond func() bool) {
w.t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(10 * time.Millisecond)
}
w.t.Fatalf("timed out waiting for %s", what)
}
func (w *lostExecutorWorld) runStepResult() error {
w.t.Helper()
select {
case err := <-w.result:
return err
case <-time.After(5 * time.Second):
w.t.Fatal("RunStep did not return after the lease was finished")
return nil
}
}
type statusRow struct {
status string
err string
}
func (w *lostExecutorWorld) statusHistory() []statusRow {
w.t.Helper()
rows, err := w.bdb.Query(
`select status, coalesce(error, '') from workflow_statuses where pipeline_id = ? and workflow = ? order by id`,
string(w.wid.PipelineId), w.wid.Name,
)
if err != nil {
w.t.Fatal(err)
}
defer rows.Close()
var out []statusRow
for rows.Next() {
var r statusRow
if err := rows.Scan(&r.status, &r.err); err != nil {
w.t.Fatal(err)
}
out = append(out, r)
}
return out
}
func (w *lostExecutorWorld) assertSuccessKept() {
w.t.Helper()
if err := w.runStepResult(); err != nil {
w.t.Fatalf("RunStep = %v, want success", err)
}
h := w.statusHistory()
if last := h[len(h)-1]; last.status != string(models.StatusKindSuccess) {
w.t.Fatalf("status history %+v, want it to end in success", h)
}
}
func TestExecutorRestartAfterOutcomeKeepsSuccess(t *testing.T) {
w := newLostExecutorWorld(t, time.Minute)
dbPath := filepath.Join(t.TempDir(), "exec.db")
w.runUntilOutcomeThenKill(dbPath, nil)
w.startExecutor(dbPath, dummy.New(slog.New(slog.NewTextHandler(io.Discard, nil))))
w.assertSuccessKept()
}
func TestWipedExecutorAfterOutcomeKeepsSuccess(t *testing.T) {
w := newLostExecutorWorld(t, time.Minute)
w.runUntilOutcomeThenKill(filepath.Join(t.TempDir(), "exec.db"), nil)
w.startExecutor(filepath.Join(t.TempDir(), "exec.db"), dummy.New(slog.New(slog.NewTextHandler(io.Discard, nil))))
w.assertSuccessKept()
}
func TestExecutorLostAfterOutcomeKeepsSuccess(t *testing.T) {
w := newLostExecutorWorld(t, time.Second)
w.runUntilOutcomeThenKill(filepath.Join(t.TempDir(), "exec.db"), nil)
w.assertSuccessKept()
}
func TestExecutorRestartAfterFailedOutcomeKeepsItsError(t *testing.T) {
w := newLostExecutorWorld(t, time.Minute)
dbPath := filepath.Join(t.TempDir(), "exec.db")
w.runUntilOutcomeThenKill(dbPath, errors.New("make: *** [test] Error 2"))
recorded := w.statusHistory()
w.startExecutor(dbPath, dummy.New(slog.New(slog.NewTextHandler(io.Discard, nil))))
if err := w.runStepResult(); !errors.Is(err, engine.ErrWorkflowFailed) {
t.Fatalf("RunStep = %v, want %v", err, engine.ErrWorkflowFailed)
}
h := w.statusHistory()
last := h[len(h)-1]
if len(h) != len(recorded) || last != recorded[len(recorded)-1] || !strings.Contains(last.err, "Error 2") {
t.Fatalf("status history %+v, want it unchanged from %+v", h, recorded)
}
}