import { execFile } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { promisify } from "node:util";
import { getOk, pdsCall } from "./api";
import { config, type Account } from "./config";
import type { PublicKey } from "./lexicons";
import { sleep, xrpc } from "./net";
import type { Ctx } from "./vocabulary";
const exec = promisify(execFile);
const KEYS = resolve(".auth/keys");
const keygen = async (who: Account) => {
const path = join(KEYS, who.name);
await mkdir(KEYS, { recursive: true });
await exec("ssh-keygen", [
"-q",
"-t",
"ed25519",
"-N",
"",
"-C",
`e2e-${who.name}`,
"-f",
path
]);
return path;
};
// keyed by the key too, makeKey rewrites the same path with a new one
const registered = new Set<string>();
// the knot picks up org.tangled.key.sshKey from the pds when it connects
const keyFileOf = async (who: Account) => {
const path = join(KEYS, who.name);
await readFile(path).catch(() => keygen(who));
const line = (await readFile(`${path}.pub`, "utf8")).trim();
if (registered.has(`${who.did} ${line}`)) return path;
const listed = await getOk(
xrpc(config.pdsUrl, "com.atproto.repo.listRecords", {
repo: who.did,
collection: "org.tangled.key.sshKey"
})
);
if (!listed.records.some((r: any) => r.value.publicKey === line)) {
await pdsCall(who, "com.atproto.repo.createRecord", {
collection: "org.tangled.key.sshKey",
record: {
$type: "org.tangled.key.sshKey",
publicKey: line,
name: `e2e-${who.name}`,
createdAt: new Date().toISOString()
} satisfies PublicKey
});
}
registered.add(`${who.did} ${line}`);
return path;
};
// a key is only worth keeping while its account exists
export const forgetKey = async (who: Account) => {
const path = join(KEYS, who.name);
await rm(path, { force: true });
await rm(`${path}.pub`, { force: true });
};
// BatchMode and ConnectTimeout make a dead host fail fast instead of prompting
// -F /dev/null so that a personal ssh config isn't used
const sshFor = (key: string) =>
`ssh -F /dev/null -i ${key} -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR`;
// as the ssh user the knot reads keys on the spot, plain "git" only sees the ones gathered so far
const remoteOf = (who: Account, repoDid: string) => {
const [host, port] = config.knotSsh.split(":");
return `ssh://${who.handle}@${host}:${port}/${repoDid}`;
};
export const sshAs = async (who: Account) => sshFor(await keyFileOf(who));
const gitTo = async (who: Account, repoDid: string) => ({
ssh: await sshAs(who),
remote: remoteOf(who, repoDid)
});
// the knot allows four git ops at once per address, and every worker shares one
const retryWhenBusy = async <T>(run: () => Promise<T>): Promise<T> => {
for (let attempt = 1; ; attempt++) {
try {
return await run();
} catch (cause: any) {
const busy = /too many concurrent operations/.test(String(cause.stderr ?? cause));
if (!busy || attempt >= 30) throw cause;
await sleep(200 + Math.random() * 400);
}
}
};
const GIT_TIMEOUT = 120_000;
const git = async (cwd: string, ssh: string, ...args: string[]) =>
retryWhenBusy(() =>
exec("git", ["-c", "user.name=e2e", "-c", "user.email=e2e@example.com", ...args], {
cwd,
env: { ...process.env, GIT_SSH_COMMAND: ssh },
timeout: GIT_TIMEOUT
})
);
const inScratch = async <T>(run: (dir: string) => Promise<T>) => {
const dir = await mkdtemp(join(tmpdir(), "e2e-git-"));
try {
return await run(dir);
} finally {
await rm(dir, { recursive: true, force: true });
}
};
export const push = async (
who: Account,
repoDid: string,
files: Record<string, string>,
{
branch = "main",
base,
options = []
}: { branch?: string; base?: string; options?: string[] } = {}
) =>
inScratch(async (dir) => {
const { ssh, remote } = await gitTo(who, repoDid);
const heads = Object.keys(await lsRemote(who, repoDid, "refs/heads/"));
if (heads.length > 0) {
const known = heads.includes(`refs/heads/${branch}`);
const start = known ? branch : base;
await git(
dir,
ssh,
"clone",
"-q",
"--depth=1",
...(start ? ["--single-branch", "--branch", start] : []),
remote,
"."
);
if (!known) await git(dir, ssh, "checkout", "-q", "-b", branch);
} else {
// knot repos are sha256, and git refuses a sha1 history pushed into one
await git(dir, ssh, "init", "-q", "--object-format=sha256", "-b", branch);
}
for (const [file, text] of Object.entries(files)) {
await mkdir(dirname(join(dir, file)), { recursive: true });
await writeFile(join(dir, file), text);
}
await git(dir, ssh, "add", "-A");
await git(dir, ssh, "commit", "-q", "-m", "e2e");
const sha = (await git(dir, ssh, "rev-parse", "HEAD")).stdout.trim();
const { stderr } = await git(
dir,
ssh,
"push",
...options.flatMap((option) => ["-o", option]),
remote,
`${branch}:${branch}`
);
return { sha, output: stderr };
});
export const lsRemote = async (who: Account, repoDid: string, prefix = "") => {
const { ssh, remote } = await gitTo(who, repoDid);
const { stdout } = await git(tmpdir(), ssh, "ls-remote", remote);
return Object.fromEntries(
stdout
.trim()
.split("\n")
.filter(Boolean)
.map((line) => line.split("\t").reverse())
.filter(([ref]) => ref.startsWith(prefix))
) as Record<string, string>;
};
// git can't resolve the stack's names or trust its dev ca either
const overHttps = (cwd: string, ...args: string[]) => {
const resolve = [config.webUrl, config.knotUrl].flatMap((url) => {
const { hostname, port } = new URL(url);
return ["-c", `http.curloptResolve=${hostname}:${port || 443}:127.0.0.1`];
});
return retryWhenBusy(() =>
exec("git", ["-c", "http.sslVerify=false", ...resolve, ...args], {
cwd,
timeout: GIT_TIMEOUT
})
);
};
export const lsRemoteOverHttps = async (url: string) =>
(await overHttps(tmpdir(), "ls-remote", url)).stdout;
export const cloneOverHttps = (url: string) =>
inScratch(async (dir) => {
await overHttps(dir, "clone", "-q", url, ".");
return (await overHttps(dir, "rev-parse", "HEAD")).stdout.trim();
});
export const headOf = async (who: Account, repoDid: string) => {
const { ssh, remote } = await gitTo(who, repoDid);
const { stdout } = await git(tmpdir(), ssh, "ls-remote", "--symref", remote, "HEAD");
return /^ref: refs\/heads\/(\S+)\tHEAD/m.exec(stdout)?.[1];
};
export const deleteBranch = async (who: Account, repoDid: string, branch: string) =>
inScratch(async (dir) => {
const { ssh, remote } = await gitTo(who, repoDid);
await git(dir, ssh, "init", "-q", "--object-format=sha256");
await git(dir, ssh, "push", remote, "--delete", branch);
});
export const pushAnnotatedTag = async (
who: Account,
repoDid: string,
tag: string,
message: string
) =>
inScratch(async (dir) => {
const { ssh, remote } = await gitTo(who, repoDid);
await git(dir, ssh, "clone", "-q", "--depth=1", remote, ".");
await git(dir, ssh, "tag", "-a", tag, "-m", message);
await git(dir, ssh, "push", "-q", remote, tag);
});
interface Key {
path: string;
pub: string;
fingerprint: string;
}
const made = new WeakMap<Ctx, Key>();
// writes to the path pushes read, so a push uses this key
export const makeKey = async (ctx: Ctx) => {
const path = await keygen(ctx.actor);
const { stdout } = await exec("ssh-keygen", ["-lf", `${path}.pub`]);
const pub = (await readFile(`${path}.pub`, "utf8")).trim();
made.set(ctx, { path, pub, fingerprint: stdout.split(" ")[1] });
};
export const keyOf = (ctx: Ctx) => made.get(ctx)!;
// retryWhenBusy keeps a busy knot from reading as a bad key
export const lsRemoteWith = async (ctx: Ctx) => {
try {
await retryWhenBusy(() =>
exec("git", ["ls-remote", remoteOf(ctx.actor, ctx.repo.did)], {
env: { ...process.env, GIT_SSH_COMMAND: sshFor(keyOf(ctx).path) },
timeout: 60_000
})
);
return "listed";
} catch (cause: any) {
return String(cause.stderr ?? cause);
}
};