package git
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"sync"
"syscall"
"time"
)
var (
ErrDiskLimitExceeded = errors.New("scratch disk limit exceeded")
ErrPackLimitExceeded = errors.New("git pack limit exceeded")
ErrAuthRequired = errors.New("authorization required")
ErrRepoNotFound = errors.New("repository not found")
)
type CommandRunner interface {
Run(ctx context.Context, dir string, env []string, name string, args ...string) (string, error)
LookPath(file string) (string, error)
}
type RealCommandRunner struct {
MaxOutputBytes int
MaxDiskBytes int64
WatchDir string
DiskPollInterval time.Duration
}
func (r RealCommandRunner) LookPath(file string) (string, error) {
return exec.LookPath(file)
}
func (r RealCommandRunner) Run(ctx context.Context, dir string, env []string, name string, args ...string) (string, error) {
cmdCtx, cmdCancel := context.WithCancel(ctx)
defer cmdCancel()
cmd := exec.CommandContext(cmdCtx, name, args...)
cmd.Dir = dir
cmd.Env = env
// git spawns child helpers like git-remote-https, kill the process group
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
if cmd.Process == nil {
return nil
}
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
}
maxOut := r.MaxOutputBytes
if maxOut <= 0 {
maxOut = 256 * 1024
}
var stdoutBuf, stderrBuf bytes.Buffer
cmd.Stdout = &boundedWriter{w: &stdoutBuf, limit: maxOut}
cmd.Stderr = &boundedWriter{w: &stderrBuf, limit: maxOut}
var stopMonitor chan struct{}
var monitorDone chan struct{}
var diskMu sync.Mutex
var diskErr error
if r.WatchDir != "" && r.MaxDiskBytes > 0 {
stopMonitor = make(chan struct{})
monitorDone = make(chan struct{})
baseUsed, baseErr := fsUsedBytes(r.WatchDir)
go func() {
defer close(monitorDone)
interval := r.DiskPollInterval
if interval <= 0 {
interval = time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
ticks := diskSweepEvery - 1
for {
select {
case <-stopMonitor:
return
case <-cmdCtx.Done():
return
case <-ticker.C:
ticks++
grown := baseErr == nil
if used, err := fsUsedBytes(r.WatchDir); err != nil || used-baseUsed <= r.MaxDiskBytes {
grown = false
}
if !grown && ticks%diskSweepEvery != 0 {
continue
}
size, err := dirSize(r.WatchDir)
if err == nil && size > r.MaxDiskBytes {
diskMu.Lock()
diskErr = fmt.Errorf("%w: current %d bytes > max %d bytes", ErrDiskLimitExceeded, size, r.MaxDiskBytes)
diskMu.Unlock()
cmdCancel()
return
}
}
}
}()
}
err := cmd.Run()
if stopMonitor != nil {
close(stopMonitor)
<-monitorDone
}
diskMu.Lock()
capturedDiskErr := diskErr
diskMu.Unlock()
outStr := stdoutBuf.String()
if stderr := stderrBuf.String(); stderr != "" {
outStr += stderr
}
if capturedDiskErr != nil {
return outStr, capturedDiskErr
}
if err != nil {
return outStr, fmt.Errorf("command %s failed: %w (output: %s)", name, err, sanitizeOutput(outStr))
}
return outStr, nil
}
type boundedWriter struct {
w io.Writer
limit int
total int
}
func (b *boundedWriter) Write(p []byte) (int, error) {
if b.total >= b.limit {
return len(p), nil
}
remaining := b.limit - b.total
toWrite := p
if len(toWrite) > remaining {
toWrite = toWrite[:remaining]
}
n, err := b.w.Write(toWrite)
b.total += n
return len(p), err
}
const diskSweepEvery = 15
func fsUsedBytes(path string) (int64, error) {
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return 0, err
}
return int64(st.Blocks-st.Bavail) * int64(st.Bsize), nil
}
func dirSize(path string) (int64, error) {
var size int64
err := filepath.WalkDir(path, func(_ string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if !d.IsDir() {
info, err := d.Info()
if err == nil {
size += info.Size()
}
}
return nil
})
return size, err
}
func RedactSecrets(s string, secrets ...string) string {
res := s
for _, sec := range secrets {
if sec == "" || len(sec) < 4 {
continue
}
res = strings.ReplaceAll(res, sec, "[REDACTED]")
if encoded := url.QueryEscape(sec); encoded != sec {
res = strings.ReplaceAll(res, encoded, "[REDACTED]")
}
if encoded := url.PathEscape(sec); encoded != sec {
res = strings.ReplaceAll(res, encoded, "[REDACTED]")
}
}
return res
}
var credentialPattern = regexp.MustCompile(`(Bearer|Basic)\s+[A-Za-z0-9._~+/-]+=*`)
func sanitizeOutput(s string) string {
return credentialPattern.ReplaceAllString(s, "$1 [REDACTED]")
}
var authErrorMarkers = []string{
"401",
"403",
"authentication failed",
"invalid username or password",
"could not read username",
"device not configured",
"terminal prompts disabled",
}
func ClassifyGitError(output string, exitErr error) error {
outLower := strings.ToLower(output)
// github answers a missing repository (and one the token cannot see)
// with "repository not found"; that is not proof of a bad credential,
// so it stays retryable instead of ending the job as authorization
// required and scrubbing the batch token
if strings.Contains(outLower, "repository not found") {
return fmt.Errorf("%w: %s", ErrRepoNotFound, strings.TrimSpace(output))
}
if slices.ContainsFunc(authErrorMarkers, func(m string) bool { return strings.Contains(outLower, m) }) {
return fmt.Errorf("%w: %s", ErrAuthRequired, strings.TrimSpace(output))
}
return exitErr
}
func HardenedGitEnv(proxyAddr, askpassBin, homeDir string, askpassHost, askpassToken string) []string {
env := []string{
"PATH=" + os.Getenv("PATH"),
"HOME=" + homeDir,
"GIT_CONFIG_NOSYSTEM=1",
"GIT_CONFIG_GLOBAL=/dev/null",
"GIT_TERMINAL_PROMPT=0",
// without this a source url of file:///etc/passwd is a readable repository
"GIT_ALLOW_PROTOCOL=https",
"GIT_PROTOCOL_FROM_USER=0",
}
if proxyAddr != "" {
proxyURL := "http://" + proxyAddr
env = append(env,
"http_proxy="+proxyURL,
"HTTP_PROXY="+proxyURL,
"https_proxy="+proxyURL,
"HTTPS_PROXY="+proxyURL,
"all_proxy="+proxyURL,
"ALL_PROXY="+proxyURL,
"NO_PROXY=",
"no_proxy=",
)
}
// git gets none of our env, but a trust store set for the whole process is the operator's
if ca := os.Getenv("SSL_CERT_FILE"); ca != "" {
env = append(env, "GIT_SSL_CAINFO="+ca)
}
if askpassBin != "" {
env = append(env,
"GIT_ASKPASS="+askpassBin,
"MIGRATOR_ASKPASS_HOST="+askpassHost,
"MIGRATOR_ASKPASS_TOKEN="+askpassToken,
)
}
return env
}
var quotedURLRegex = regexp.MustCompile(`'([^']+)'`)
func ExtractHostFromPrompt(prompt string) string {
matches := quotedURLRegex.FindStringSubmatch(prompt)
if len(matches) > 1 {
u, err := url.Parse(matches[1])
if err == nil {
return strings.ToLower(u.Hostname())
}
}
re := regexp.MustCompile(`https?://(?:[^@/\s]+@)?([^/':@\s]+)`)
m := re.FindStringSubmatch(prompt)
if len(m) > 1 {
return strings.ToLower(m[1])
}
return ""
}
func HandleAskpass(prompt, expectedHost, token string) (string, error) {
promptHost := ExtractHostFromPrompt(prompt)
if promptHost == "" {
return "", fmt.Errorf("no host found in askpass prompt %q", prompt)
}
expectedHost = strings.ToLower(expectedHost)
if !strings.EqualFold(promptHost, expectedHost) {
return "", fmt.Errorf("host mismatch: prompt host %q != expected host %q", promptHost, expectedHost)
}
promptLower := strings.ToLower(prompt)
if strings.Contains(promptLower, "username") {
return "x-access-token", nil
}
if strings.Contains(promptLower, "password") {
return token, nil
}
return "", fmt.Errorf("unknown askpass prompt: %q", prompt)
}
func CreateAskpassScript(dir string) (string, error) {
migratorBin, err := os.Executable()
if err != nil {
return "", fmt.Errorf("getting migrator executable path: %w", err)
}
scriptPath := filepath.Join(dir, "askpass.sh")
content := fmt.Sprintf(`#!/bin/sh
exec "%s" askpass "$@"
`, migratorBin)
if err := os.WriteFile(scriptPath, []byte(content), 0700); err != nil {
return "", fmt.Errorf("writing askpass script: %w", err)
}
return scriptPath, nil
}