~james/core

core/gitclient/gitclient.go -rw-r--r-- 11.8 KiB
452f17b7 — james chore(import): snapshot james.dad.town/core at f70c71c7696a 14 hours ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
// Package gitclient is a small git protocol v2 client over smart http, just
// enough to read refs, trees and blobs from a knot without cloning.
package gitclient

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"maps"
	"net/http"
	"path"
	"slices"
	"strings"
	"sync"
)

var (
	ErrRefNotFound  = errors.New("branch or tag not found")
	ErrPathNotFound = errors.New("path not found at the commit")
	ErrNotDir       = fmt.Errorf("%w: not a directory", ErrPathNotFound)
	ErrTooLarge     = errors.New("pack exceeds the read limit")
	ErrUnsupported  = errors.New("server lacks a required v2 capability")
)

// hex object id, sha1 or sha256 depending on the repo's object-format
type Oid string

type Client struct {
	http     *http.Client
	url      string // repo base, e.g. https://knot.example/did:plc:abc
	maxBytes int64  // cap on one fetch's pack

	mu   sync.Mutex
	caps *Caps
}

func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes}
}

type Caps struct {
	ObjectFormat string // "sha1" or "sha256"
	LsRefs       bool
	Fetch        []string // fetch= features: shallow, filter, ...
}

func (c *Client) Caps(ctx context.Context) (Caps, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.caps != nil {
		return *c.caps, nil
	}
	u := c.url + "/info/refs?service=git-upload-pack"
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	if err != nil {
		return Caps{}, err
	}
	req.Header.Set("Git-Protocol", "version=2")
	resp, err := c.http.Do(req)
	if err != nil {
		return Caps{}, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return Caps{}, httpError(resp)
	}
	caps := Caps{ObjectFormat: "sha1"}
	p := newPktReader(io.LimitReader(resp.Body, 1<<20))
	v2 := false
	for {
		kind, line, err := p.next()
		if err != nil && !v2 {
			return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u)
		}
		if err != nil {
			return Caps{}, fmt.Errorf("reading %s: %w", u, err)
		}
		if kind != pktData {
			// a v0-style "# service=" banner ends in a flush before the v2 advert
			if v2 {
				break
			}
			continue
		}
		key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=")
		switch key {
		case "version 2":
			v2 = true
		case "ls-refs":
			caps.LsRefs = true
		case "fetch":
			caps.Fetch = strings.Fields(val)
		case "object-format":
			caps.ObjectFormat = val
		}
	}
	if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" {
		return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat)
	}
	c.caps = &caps
	return caps, nil
}

type Ref struct {
	Name   string
	Oid    Oid
	Peeled Oid    // set for annotated tags when Peel was asked
	Symref string // target of HEAD when Symrefs was asked
}

type LsRefsOpts struct {
	Prefixes []string // ref-prefix lines
	Peel     bool
	Symrefs  bool
}

// command=ls-refs
func (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) {
	caps, err := c.Caps(ctx)
	if err != nil {
		return nil, err
	}
	if !caps.LsRefs {
		return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported)
	}
	var args []string
	if opts.Peel {
		args = append(args, "peel")
	}
	if opts.Symrefs {
		args = append(args, "symrefs")
	}
	for _, prefix := range opts.Prefixes {
		args = append(args, "ref-prefix "+prefix)
	}
	resp, err := c.command(ctx, caps, "ls-refs", args)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes})
	var refs []Ref
	for {
		kind, line, err := p.next()
		if err != nil {
			return nil, fmt.Errorf("ls-refs: %w", err)
		}
		if kind != pktData {
			return refs, nil
		}
		text := strings.TrimSuffix(string(line), "\n")
		if msg, ok := strings.CutPrefix(text, "ERR "); ok {
			return nil, fmt.Errorf("remote: %s", msg)
		}
		fields := strings.Split(text, " ")
		if len(fields) < 2 || !isOid(fields[0]) {
			return nil, fmt.Errorf("ls-refs: bad line %q", text)
		}
		ref := Ref{Oid: Oid(fields[0]), Name: fields[1]}
		for _, attr := range fields[2:] {
			if v, ok := strings.CutPrefix(attr, "peeled:"); ok {
				ref.Peeled = Oid(v)
			} else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok {
				ref.Symref = v
			}
		}
		refs = append(refs, ref)
	}
}

type FetchOpts struct {
	Wants  []Oid
	Deepen int    // 0 means no deepen line
	Filter string // e.g. "tree:0", "blob:none"; empty means none
}

// command=fetch with done and no haves, since there's no local object store
func (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) {
	caps, err := c.Caps(ctx)
	if err != nil {
		return nil, err
	}
	if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") {
		return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported)
	}
	if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") {
		return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported)
	}
	if len(opts.Wants) == 0 {
		return Objects{}, nil
	}
	args := []string{"no-progress", "ofs-delta"}
	for _, want := range opts.Wants {
		if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) {
			return nil, fmt.Errorf("bad object id %q", want)
		}
		args = append(args, "want "+string(want))
	}
	if opts.Deepen > 0 {
		args = append(args, fmt.Sprintf("deepen %d", opts.Deepen))
	}
	if opts.Filter != "" {
		args = append(args, "filter "+opts.Filter)
	}
	args = append(args, "done")
	resp, err := c.command(ctx, caps, "fetch", args)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	pack, err := packfileSection(newPktReader(resp.Body))
	if err != nil {
		return nil, fmt.Errorf("fetch: %w", err)
	}
	objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat)
	if err != nil {
		return nil, fmt.Errorf("fetch: %w", err)
	}
	return objs, nil
}

// branches before tags, like git; tags peeled
func (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) {
	if ref == "" || strings.ContainsAny(ref, "\x00\n ") {
		return "", ErrRefNotFound
	}
	candidates := []string{ref}
	if !strings.HasPrefix(ref, "refs/") {
		candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref}
	}
	refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true})
	if err != nil {
		return "", err
	}
	for _, name := range candidates {
		for _, r := range refs {
			if r.Name != name {
				continue
			}
			if r.Peeled != "" {
				return r.Peeled, nil
			}
			return r.Oid, nil
		}
	}
	return "", ErrRefNotFound
}

type File struct {
	Name string
	Mode string // octal, as in the tree
	Oid  Oid
	Data []byte // nil for subtrees and submodules
}

// one level of dir, blobs filled in
func (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) {
	caps, objs, oid, err := c.walk(ctx, commit, dir)
	if err != nil {
		return nil, err
	}
	entries, err := c.tree(ctx, caps, objs, oid, "tree:1")
	if err != nil {
		return nil, err
	}
	files := make([]File, 0, len(entries))
	for _, e := range entries {
		files = append(files, File{Name: e.Name, Mode: e.Mode, Oid: e.Oid})
	}
	if err := c.fillBlobs(ctx, objs, files); err != nil {
		return nil, err
	}
	return files, nil
}

// everything under dir, named relative to it; the dir tree is wanted unfiltered
// so one pack brings it all
func (c *Client) ReadTree(ctx context.Context, commit Oid, dir string) ([]File, error) {
	caps, objs, oid, err := c.walk(ctx, commit, dir)
	if err != nil {
		return nil, err
	}
	var files []File
	queue := []File{{Oid: oid}}
	for len(queue) > 0 {
		t := queue[0]
		queue = queue[1:]
		entries, err := c.tree(ctx, caps, objs, t.Oid, "")
		if err != nil {
			return nil, err
		}
		for _, e := range entries {
			f := File{Name: path.Join(t.Name, e.Name), Mode: e.Mode, Oid: e.Oid}
			if isTreeMode(e.Mode) {
				queue = append(queue, f)
			}
			files = append(files, f)
		}
	}
	if err := c.fillBlobs(ctx, objs, files); err != nil {
		return nil, err
	}
	return files, nil
}

// middle trees come alone (tree:0) so sibling blobs stay on the server; servers
// differ on filter depth for a wanted tree, hence fillBlobs
func (c *Client) walk(ctx context.Context, commit Oid, dir string) (Caps, Objects, Oid, error) {
	caps, err := c.Caps(ctx)
	if err != nil {
		return Caps{}, nil, "", err
	}
	objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"})
	if err != nil {
		return Caps{}, nil, "", err
	}
	obj, ok := objs[commit]
	if !ok || obj.Kind != KindCommit {
		return Caps{}, nil, "", fmt.Errorf("fetch of commit %s didn't return it", commit)
	}
	tree, err := commitTree(obj.Data)
	if err != nil {
		return Caps{}, nil, "", err
	}
	for _, name := range strings.Split(dir, "/") {
		if name == "" {
			continue
		}
		entries, err := c.tree(ctx, caps, objs, tree, "tree:0")
		if err != nil {
			return Caps{}, nil, "", err
		}
		i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name })
		if i < 0 {
			return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrPathNotFound)
		}
		if !isTreeMode(entries[i].Mode) {
			return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrNotDir)
		}
		tree = entries[i].Oid
	}
	return caps, objs, tree, nil
}

func (c *Client) fillBlobs(ctx context.Context, objs Objects, files []File) error {
	var missing []Oid
	for _, f := range files {
		if _, ok := objs[f.Oid]; isBlobMode(f.Mode) && !ok && !slices.Contains(missing, f.Oid) {
			missing = append(missing, f.Oid)
		}
	}
	if len(missing) > 0 {
		// wanted objects pass any filter; it keeps the server on its selective path
		more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"})
		if err != nil {
			return err
		}
		maps.Copy(objs, more)
	}
	for i, f := range files {
		if !isBlobMode(f.Mode) {
			continue
		}
		obj, ok := objs[f.Oid]
		if !ok || obj.Kind != KindBlob {
			return fmt.Errorf("fetch of blob %s didn't return it", f.Oid)
		}
		files[i].Data = obj.Data
	}
	return nil
}

func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) {
	if _, ok := have[oid]; !ok {
		objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter})
		if err != nil {
			return nil, err
		}
		maps.Copy(have, objs)
	}
	obj, ok := have[oid]
	if !ok || obj.Kind != KindTree {
		return nil, fmt.Errorf("fetch of tree %s didn't return it", oid)
	}
	return parseTree(obj.Data, caps.ObjectFormat)
}

func isBlobMode(mode string) bool {
	mode = strings.TrimLeft(mode, "0")
	return mode != "40000" && mode != "160000"
}

func isTreeMode(mode string) bool {
	return strings.TrimLeft(mode, "0") == "40000"
}

func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) {
	var body bytes.Buffer
	writePkt(&body, "command="+command+"\n")
	// git refuses non-sha1 unless the format is echoed back
	if caps.ObjectFormat != "sha1" {
		writePkt(&body, "object-format="+caps.ObjectFormat+"\n")
	}
	writeDelim(&body)
	for _, arg := range args {
		if err := writePkt(&body, arg+"\n"); err != nil {
			return nil, err
		}
	}
	writeFlush(&body)

	u := c.url + "/git-upload-pack"
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Git-Protocol", "version=2")
	req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
	req.Header.Set("Accept", "application/x-git-upload-pack-result")
	resp, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		defer resp.Body.Close()
		return nil, httpError(resp)
	}
	return resp, nil
}

// knot2 says why in a plain text body, like a want it won't serve
func httpError(resp *http.Response) error {
	body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
	err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status)
	if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
		err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?"))
	}
	return err
}