9ef6813 · ab1390f · 6cc5501 · f9968ec
@@ -228,6 +228,7 @@
export const GATE: readonly string[] = [
  "test/counts.test.ts",
  "test/crew-loop.test.ts",
  "test/crew-totals.test.ts",
  "test/crew-locks.test.ts",
  "test/ssh-site.test.ts",
  "test/web-code.test.ts",
  "test/web-origins.test.ts",
@@ -0,0 +1,61 @@
/**
 * Stale git locks in a kept clone. Git never removes another process's lock,
 * and a run the forge interrupted leaves its `config.lock` or `index.lock`
 * behind in a clone the forge keeps; every later visit would fail on it.
 * The runner calls this before it touches a clone it is about to own, so no
 * git process of this forge holds it; the age guard keeps a lock a person's
 * own git just took.
 */
import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
import { join } from "node:path";
export const STALE_LOCK_MS = 60_000;
/** Whether a failed git command died on a lock file. */
export const lockedOut = (stderr: string): boolean =>
  /\.lock'?: File exists|Unable to create '[^']*\.lock'|could not lock config file/i.test(
    stderr,
  );
function locksUnder(dir: string, deep: boolean): string[] {
  const found: string[] = [];
  let entries;
  try {
    entries = readdirSync(dir, { withFileTypes: true });
  } catch {
    return found;
  }
  for (const entry of entries) {
    const path = join(dir, entry.name);
    if (entry.isFile() && entry.name.endsWith(".lock")) found.push(path);
    else if (deep && entry.isDirectory())
      
        junipercrew
        1mo ago
        outdated
      
      !entry.isSymbolicLink() is redundant. Dirent.isDirectory() is false for symlinks. Remove it.
    
      found.push(...locksUnder(path, true));
  }
  return found;
}
/** Remove `.git/*.lock` and `.git/refs/**\/*.lock` older than a minute. */
export function clearStaleLocks(
  clone: string,
  write: (line: string) => void,
  now = Date.now(),
): string[] {
  const git = join(clone, ".git");
  if (!existsSync(git)) return [];
  const removed: string[] = [];
  for (const path of [
    ...locksUnder(git, false),
    ...locksUnder(join(git, "refs"), true),
  ]) {
    try {
      if (now - statSync(path).mtimeMs < STALE_LOCK_MS) continue;
      rmSync(path, { force: true });
      removed.push(path);
      write(`removed stale git lock ${path.slice(clone.length + 1)}`);
    } catch {
      // Gone already, or not ours to remove: git will say so if it matters.
    }
  }
  return removed;
}
@@ -81,6 +81,7 @@
import {
  resumeAt,
  whenText,
} from "./limits.ts";
import { clearStaleLocks, lockedOut } from "./locks.ts";
import { interruptedSummary } from "./interrupted.ts";
import { laterGate } from "./later.ts";
import { parkedTwice } from "./parks.ts";
@@ -1261,6 +1262,7 @@
async function prepareClone(
  write: (line: string) => void,
  signal: AbortSignal,
  keep = false,
  fresh = false,
): Promise<{ ok: true } | { ok: false; error: string }> {
  // Two addresses for one forge. The forge itself clones and fetches over
  // `localUrl`, on this machine. The agent's remotes carry `forgeUrl`, the
@@ -1287,6 +1289,9 @@
async function prepareClone(
        error: `clone failed: ${cloned.error.stderr.slice(0, 200)}`,
      };
  } else {
    // A kept clone may carry the locks of a run the forge cut off. This run
    // owns the clone now, so a lock a minute old is nobody's.
    clearStaleLocks(dir, write);
    const fetched = await runGit(
      [
        "fetch",
@@ -1359,6 +1364,37 @@
async function prepareClone(
    cwd: dir,
    env,
  });
  if (!checked.ok && !fresh && lockedOut(checked.error.stderr)) {
    // A lock too young to clear, or one the sweep missed: start over once
    // from a fresh clone rather than failing the follow-up.
    // Only when nothing would be lost: a cut-off run may hold commits that
    // were never pushed, and those live nowhere else.
    const unpushed = await runGit(
      ["rev-list", "--branches", "--not", "--remotes=origin"],
      { cwd: dir, env },
    );
    if (!unpushed.ok || unpushed.value.trim() !== "")
      return {
        ok: false,
        error:
          "checkout blocked by a git lock, and the kept clone holds commits that were never pushed; left as it is",
      };
    write("checkout blocked by a git lock; cloning afresh");
    rmSync(dir, { recursive: true, force: true });
    return prepareClone(
      app,
      options,
      repo,
      fork,
      branch,
      dir,
      env,
      write,
      signal,
      false,
      true,
    );
  }
  if (!checked.ok)
    return {
      ok: false,
@@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { chmodSync, writeFileSync } from "node:fs";
import { chmodSync, existsSync, utimesSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -15,7 +15,11 @@
import {
  createRun,
  getRun,
  listQueued,
  listRuns,
} from "../src/core/runs.ts";
import { listPulls } from "../src/core/pulls.ts";
import { getAgent } from "../src/core/agents.ts";
import { workdirFor } from "../src/crew/workdir.ts";
import { STALE_AFTER_MS } from "../src/runs/lease.ts";
import { createUser, type User } from "../src/core/users.ts";
import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
@@ -33,7 +37,8 @@
describe("a run the last forge left running", () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-interrupted-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    const a = await createUser(app.db, { trusted: true,
    const a = await createUser(app.db, {
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
@@ -179,7 +184,9 @@
describe("a run the last forge left running", () => {
    if (!issued.ok) throw new Error(String(issued.error));
    const issue = issued.value;
    // The word the follow-up answers: a follow-up with nothing new never starts.
    expect(addComment(app, alice, repo, issue, "The low branches first.").ok).toBe(true);
    expect(
      addComment(app, alice, repo, issue, "The low branches first.").ok,
    ).toBe(true);
    // A stand-in that never finishes on its own.
    const slow = join(dataDir, "slow-claude");
    writeFileSync(
@@ -296,6 +303,124 @@
echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while
      listQueued(app.db, "agent").filter((r) => r.issueId === issue.id),
    ).toHaveLength(0);
  }, 40_000);
  const keptClone = async (name: string) => {
    await crew?.stop();
    crew = undefined;
    const created = await createRepo(app, alice, {
      name,
      visibility: "private",
    });
    if (!created.ok) throw new Error(String(created.error));
    const repo = created.value;
    const seeded = await commitToBranch(app, alice, repo, {
      message: "Start",
      files: [{ path: "README.md", content: "# meadow\n" }],
    });
    if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
    expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
    const issued = openIssue(app, alice, repo, { title: "Mow", body: "" });
    if (!issued.ok) throw new Error(String(issued.error));
    const issue = issued.value;
    const fake = join(dataDir, "lock-claude");
    writeFileSync(
      fake,
      `#!/bin/sh
if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
cat > /dev/null &
echo '{"type":"system","subtype":"init","session_id":"lock-1","model":"fake"}'
echo "mowed $(date +%s%N)" >> CREW.md
git add -A && git commit -qm "mowed" && git push -q -u origin HEAD
echo '{"type":"result","subtype":"success","is_error":false,"result":"Done.","total_cost_usd":0.01,"num_turns":2,"session_id":"lock-1","terminal_reason":"completed"}'
`,
    );
    chmodSync(fake, 0o755);
    const options = {
      forgeUrl: servers.url,
      localUrl: servers.url,
      binary: fake,
      concurrency: 1,
      budgetUsd: 1,
      runTimeoutMs: 60_000,
    };
    const first = createRun(app.db, {
      repoId: repo.id,
      kind: "agent",
      issueId: issue.id,
      actorId: bot.id,
      reason: "assigned",
    });
    crew = await startCrew(app, options);
    const wait = async (id: string) => {
      const deadline = Date.now() + 25_000;
      while (
        !["ok", "failed", "cancelled"].includes(getRun(app.db, id)!.state) &&
        Date.now() < deadline
      )
        await Bun.sleep(100);
      return getRun(app.db, id)!;
    };
    expect((await wait(first.id)).state).toBe("ok");
    // The clone is kept while the pull is open; a cut-off run left a lock.
    const agent = getAgent(app.db, bot.id)!;
    const clone = workdirFor(app.config.dataDir, agent, repo, issue.number);
    expect(existsSync(join(clone, ".git"))).toBe(true);
    return { repo, issue, clone };
  };
  const followUp = async (
    repo: Awaited<ReturnType<typeof keptClone>>["repo"],
  ) => {
    const pull = listPulls(app.db, repo.id)[0];
    expect(pull).toBeDefined();
    expect(
      addComment(app, alice, repo, pull!, "Closer to the fence, please.").ok,
    ).toBe(true);
    const deadline = Date.now() + 25_000;
    let next;
    while (Date.now() < deadline) {
      next = listRuns(app.db, repo.id, { issueId: pull!.id })[0];
      if (next && ["ok", "failed", "cancelled"].includes(next.state)) break;
      await Bun.sleep(100);
    }
    expect(next).toBeDefined();
    return next!;
  };
  test("a kept clone with a stale git lock resumes cleanly and the lock is gone", async () => {
    const { repo, clone } = await keptClone("meadow");
    const lock = join(clone, ".git", "config.lock");
    writeFileSync(lock, "");
    const old = new Date(Date.now() - 10 * 60_000);
    utimesSync(lock, old, old);
    const next = await followUp(repo);
    expect(next.summary ?? "").not.toContain("checkout failed");
    expect(existsSync(lock)).toBe(false);
  }, 60_000);
  test("a young lock that blocks checkout is recovered by a fresh clone", async () => {
    const { repo, clone } = await keptClone("orchard");
    const lock = join(clone, ".git", "index.lock");
    writeFileSync(lock, "");
    const next = await followUp(repo);
    expect(next.state).toBe("ok");
    expect(existsSync(lock)).toBe(false);
  }, 60_000);
  test("a young lock is not worked around when the clone holds unpushed commits", async () => {
    const { repo, clone } = await keptClone("paddock");
    writeFileSync(join(clone, "LOST.md"), "unpushed\n");
    const git = (...args: string[]) =>
      Bun.spawnSync(["git", ...args], { cwd: clone });
    git("add", "LOST.md");
    git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "local");
    writeFileSync(join(clone, ".git", "index.lock"), "");
    const next = await followUp(repo);
    expect(next.state).toBe("failed");
    expect(next.summary ?? "").toContain("never pushed");
    expect(existsSync(join(clone, "LOST.md"))).toBe(true);
  }, 60_000);
});
describe("an issue handed to someone else", () => {
@@ -311,7 +436,8 @@
describe("an issue handed to someone else", () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-reassign-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    const a = await createUser(app.db, { trusted: true,
    const a = await createUser(app.db, {
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
@@ -0,0 +1,64 @@
import { describe, expect, test } from "bun:test";
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  rmSync,
  utimesSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { clearStaleLocks, lockedOut } from "../src/crew/locks.ts";
function clone() {
  const dir = mkdtempSync(join(tmpdir(), "forge-locks-"));
  mkdirSync(join(dir, ".git", "refs", "heads", "scout"), { recursive: true });
  return dir;
}
const plant = (path: string, ageMs: number) => {
  writeFileSync(path, "");
  const when = new Date(Date.now() - ageMs);
  utimesSync(path, when, when);
};
describe("stale git locks in a kept clone", () => {
  test("old locks go, one line each; young locks and other files stay", () => {
    const dir = clone();
    const old = [
      join(dir, ".git", "config.lock"),
      join(dir, ".git", "index.lock"),
      join(dir, ".git", "refs", "heads", "scout", "x.lock"),
    ];
    for (const path of old) plant(path, 10 * 60_000);
    const young = join(dir, ".git", "HEAD.lock");
    plant(young, 1_000);
    const config = join(dir, ".git", "config");
    plant(config, 10 * 60_000);
    const lines: string[] = [];
    const removed = clearStaleLocks(dir, (l) => lines.push(l));
    expect(removed.sort()).toEqual([...old].sort());
    expect(lines).toHaveLength(3);
    for (const path of old) expect(existsSync(path)).toBe(false);
    expect(existsSync(young)).toBe(true);
    expect(existsSync(config)).toBe(true);
    rmSync(dir, { recursive: true, force: true });
  });
  test("a directory with no .git is left alone", () => {
    const dir = mkdtempSync(join(tmpdir(), "forge-locks-"));
    expect(clearStaleLocks(dir, () => {})).toEqual([]);
    rmSync(dir, { recursive: true, force: true });
  });
  test("a lock failure is recognised from git's words", () => {
    expect(
      lockedOut("error: could not lock config file .git/config: File exists"),
    ).toBe(true);
    expect(
      lockedOut("fatal: Unable to create '/x/.git/index.lock': File exists."),
    ).toBe(true);
    expect(lockedOut("fatal: not a git repository")).toBe(false);
    expect(lockedOut("error: could not lock ref: cannot lock the thing")).toBe(false);
  });
});
		@@ -228,6 +228,7 @@ export const GATE: readonly string[] = [
228	228	"test/counts.test.ts",
229	229	"test/crew-loop.test.ts",
230	230	"test/crew-totals.test.ts",
	231	"test/crew-locks.test.ts",
231	232	"test/ssh-site.test.ts",
232	233	"test/web-code.test.ts",
233	234	"test/web-origins.test.ts",
		@@ -0,0 +1,61 @@
	1	/**
	2	* Stale git locks in a kept clone. Git never removes another process's lock,
	3	* and a run the forge interrupted leaves its `config.lock` or `index.lock`
	4	* behind in a clone the forge keeps; every later visit would fail on it.
	5	* The runner calls this before it touches a clone it is about to own, so no
	6	* git process of this forge holds it; the age guard keeps a lock a person's
	7	* own git just took.
	8	*/
	9
	10	import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
	11	import { join } from "node:path";
	12
	13	export const STALE_LOCK_MS = 60_000;
	14
	15	/** Whether a failed git command died on a lock file. */
	16	export const lockedOut = (stderr: string): boolean =>
	17	/\.lock'?: File exists\|Unable to create '[^']*\.lock'\|could not lock config file/i.test(
	18	stderr,
	19	);
	20
	21	function locksUnder(dir: string, deep: boolean): string[] {
	22	const found: string[] = [];
	23	let entries;
	24	try {
	25	entries = readdirSync(dir, { withFileTypes: true });
	26	} catch {
	27	return found;
	28	}
	29	for (const entry of entries) {
	30	const path = join(dir, entry.name);
	31	if (entry.isFile() && entry.name.endsWith(".lock")) found.push(path);
	32	else if (deep && entry.isDirectory())
		junipercrew 1mo ago outdated `!entry.isSymbolicLink()` is redundant. Dirent.isDirectory() is false for symlinks. Remove it.
	33	found.push(...locksUnder(path, true));
	34	}
	35	return found;
	36	}
	37
	38	/** Remove `.git/.lock` and `.git/refs/\/.lock` older than a minute. */
	39	export function clearStaleLocks(
	40	clone: string,
	41	write: (line: string) => void,
	42	now = Date.now(),
	43	): string[] {
	44	const git = join(clone, ".git");
	45	if (!existsSync(git)) return [];
	46	const removed: string[] = [];
	47	for (const path of [
	48	...locksUnder(git, false),
	49	...locksUnder(join(git, "refs"), true),
	50	]) {
	51	try {
	52	if (now - statSync(path).mtimeMs < STALE_LOCK_MS) continue;
	53	rmSync(path, { force: true });
	54	removed.push(path);
	55	write(`removed stale git lock ${path.slice(clone.length + 1)}`);
	56	} catch {
	57	// Gone already, or not ours to remove: git will say so if it matters.
	58	}
	59	}
	60	return removed;
	61	}
		@@ -81,6 +81,7 @@ import {
81	81	resumeAt,
82	82	whenText,
83	83	} from "./limits.ts";
	84	import { clearStaleLocks, lockedOut } from "./locks.ts";
84	85	import { interruptedSummary } from "./interrupted.ts";
85	86	import { laterGate } from "./later.ts";
86	87	import { parkedTwice } from "./parks.ts";
		@@ -1261,6 +1262,7 @@ async function prepareClone(
1261	1262	write: (line: string) => void,
1262	1263	signal: AbortSignal,
1263	1264	keep = false,
	1265	fresh = false,
1264	1266	): Promise<{ ok: true } \| { ok: false; error: string }> {
1265	1267	// Two addresses for one forge. The forge itself clones and fetches over
1266	1268	// `localUrl`, on this machine. The agent's remotes carry `forgeUrl`, the
		@@ -1287,6 +1289,9 @@ async function prepareClone(
1287	1289	error: `clone failed: ${cloned.error.stderr.slice(0, 200)}`,
1288	1290	};
1289	1291	} else {
	1292	// A kept clone may carry the locks of a run the forge cut off. This run
	1293	// owns the clone now, so a lock a minute old is nobody's.
	1294	clearStaleLocks(dir, write);
1290	1295	const fetched = await runGit(
1291	1296	[
1292	1297	"fetch",
		@@ -1359,6 +1364,37 @@ async function prepareClone(
1359	1364	cwd: dir,
1360	1365	env,
1361	1366	});
	1367	if (!checked.ok && !fresh && lockedOut(checked.error.stderr)) {
	1368	// A lock too young to clear, or one the sweep missed: start over once
	1369	// from a fresh clone rather than failing the follow-up.
	1370	// Only when nothing would be lost: a cut-off run may hold commits that
	1371	// were never pushed, and those live nowhere else.
	1372	const unpushed = await runGit(
	1373	["rev-list", "--branches", "--not", "--remotes=origin"],
	1374	{ cwd: dir, env },
	1375	);
	1376	if (!unpushed.ok \|\| unpushed.value.trim() !== "")
	1377	return {
	1378	ok: false,
	1379	error:
	1380	"checkout blocked by a git lock, and the kept clone holds commits that were never pushed; left as it is",
	1381	};
	1382	write("checkout blocked by a git lock; cloning afresh");
	1383	rmSync(dir, { recursive: true, force: true });
	1384	return prepareClone(
	1385	app,
	1386	options,
	1387	repo,
	1388	fork,
	1389	branch,
	1390	dir,
	1391	env,
	1392	write,
	1393	signal,
	1394	false,
	1395	true,
	1396	);
	1397	}
1362	1398	if (!checked.ok)
1363	1399	return {
1364	1400	ok: false,
		@@ -1,5 +1,5 @@
1	1	import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2		import { chmodSync, writeFileSync } from "node:fs";
	2	import { chmodSync, existsSync, utimesSync, writeFileSync } from "node:fs";
3	3	import { mkdtemp, rm } from "node:fs/promises";
4	4	import { tmpdir } from "node:os";
5	5	import { join } from "node:path";
		@@ -15,7 +15,11 @@ import {
15	15	createRun,
16	16	getRun,
17	17	listQueued,
	18	listRuns,
18	19	} from "../src/core/runs.ts";
	20	import { listPulls } from "../src/core/pulls.ts";
	21	import { getAgent } from "../src/core/agents.ts";
	22	import { workdirFor } from "../src/crew/workdir.ts";
19	23	import { STALE_AFTER_MS } from "../src/runs/lease.ts";
20	24	import { createUser, type User } from "../src/core/users.ts";
21	25	import { startCrew, type CrewHandle } from "../src/crew/dispatch.ts";
		@@ -33,7 +37,8 @@ describe("a run the last forge left running", () => {
33	37	dataDir = await mkdtemp(join(tmpdir(), "forge-interrupted-"));
34	38	app = createApp({ dataDir, port: 0 });
35	39	servers = await serve(app);
36		const a = await createUser(app.db, { trusted: true,
	40	const a = await createUser(app.db, {
	41	trusted: true,
37	42	username: "alice",
38	43	email: "[email protected]",
39	44	password: "a-good-passphrase",
		@@ -179,7 +184,9 @@ describe("a run the last forge left running", () => {
179	184	if (!issued.ok) throw new Error(String(issued.error));
180	185	const issue = issued.value;
181	186	// The word the follow-up answers: a follow-up with nothing new never starts.
182		expect(addComment(app, alice, repo, issue, "The low branches first.").ok).toBe(true);
	187	expect(
	188	addComment(app, alice, repo, issue, "The low branches first.").ok,
	189	).toBe(true);
183	190	// A stand-in that never finishes on its own.
184	191	const slow = join(dataDir, "slow-claude");
185	192	writeFileSync(
		@@ -296,6 +303,124 @@ echo '{"type":"result","subtype":"success","is_error":false,"result":"Done while
296	303	listQueued(app.db, "agent").filter((r) => r.issueId === issue.id),
297	304	).toHaveLength(0);
298	305	}, 40_000);
	306
	307	const keptClone = async (name: string) => {
	308	await crew?.stop();
	309	crew = undefined;
	310	const created = await createRepo(app, alice, {
	311	name,
	312	visibility: "private",
	313	});
	314	if (!created.ok) throw new Error(String(created.error));
	315	const repo = created.value;
	316	const seeded = await commitToBranch(app, alice, repo, {
	317	message: "Start",
	318	files: [{ path: "README.md", content: "# meadow\n" }],
	319	});
	320	if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
	321	expect(addToCrew(app, alice, repo, "bot").ok).toBe(true);
	322	const issued = openIssue(app, alice, repo, { title: "Mow", body: "" });
	323	if (!issued.ok) throw new Error(String(issued.error));
	324	const issue = issued.value;
	325	const fake = join(dataDir, "lock-claude");
	326	writeFileSync(
	327	fake,
	328	`#!/bin/sh
	329	if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
	330	cat > /dev/null &
	331	echo '{"type":"system","subtype":"init","session_id":"lock-1","model":"fake"}'
	332	echo "mowed $(date +%s%N)" >> CREW.md
	333	git add -A && git commit -qm "mowed" && git push -q -u origin HEAD
	334	echo '{"type":"result","subtype":"success","is_error":false,"result":"Done.","total_cost_usd":0.01,"num_turns":2,"session_id":"lock-1","terminal_reason":"completed"}'
	335	`,
	336	);
	337	chmodSync(fake, 0o755);
	338	const options = {
	339	forgeUrl: servers.url,
	340	localUrl: servers.url,
	341	binary: fake,
	342	concurrency: 1,
	343	budgetUsd: 1,
	344	runTimeoutMs: 60_000,
	345	};
	346	const first = createRun(app.db, {
	347	repoId: repo.id,
	348	kind: "agent",
	349	issueId: issue.id,
	350	actorId: bot.id,
	351	reason: "assigned",
	352	});
	353	crew = await startCrew(app, options);
	354	const wait = async (id: string) => {
	355	const deadline = Date.now() + 25_000;
	356	while (
	357	!["ok", "failed", "cancelled"].includes(getRun(app.db, id)!.state) &&
	358	Date.now() < deadline
	359	)
	360	await Bun.sleep(100);
	361	return getRun(app.db, id)!;
	362	};
	363	expect((await wait(first.id)).state).toBe("ok");
	364
	365	// The clone is kept while the pull is open; a cut-off run left a lock.
	366	const agent = getAgent(app.db, bot.id)!;
	367	const clone = workdirFor(app.config.dataDir, agent, repo, issue.number);
	368	expect(existsSync(join(clone, ".git"))).toBe(true);
	369	return { repo, issue, clone };
	370	};
	371
	372	const followUp = async (
	373	repo: Awaited<ReturnType<typeof keptClone>>["repo"],
	374	) => {
	375	const pull = listPulls(app.db, repo.id)[0];
	376	expect(pull).toBeDefined();
	377	expect(
	378	addComment(app, alice, repo, pull!, "Closer to the fence, please.").ok,
	379	).toBe(true);
	380	const deadline = Date.now() + 25_000;
	381	let next;
	382	while (Date.now() < deadline) {
	383	next = listRuns(app.db, repo.id, { issueId: pull!.id })[0];
	384	if (next && ["ok", "failed", "cancelled"].includes(next.state)) break;
	385	await Bun.sleep(100);
	386	}
	387	expect(next).toBeDefined();
	388	return next!;
	389	};
	390
	391	test("a kept clone with a stale git lock resumes cleanly and the lock is gone", async () => {
	392	const { repo, clone } = await keptClone("meadow");
	393	const lock = join(clone, ".git", "config.lock");
	394	writeFileSync(lock, "");
	395	const old = new Date(Date.now() - 10 * 60_000);
	396	utimesSync(lock, old, old);
	397	const next = await followUp(repo);
	398	expect(next.summary ?? "").not.toContain("checkout failed");
	399	expect(existsSync(lock)).toBe(false);
	400	}, 60_000);
	401
	402	test("a young lock that blocks checkout is recovered by a fresh clone", async () => {
	403	const { repo, clone } = await keptClone("orchard");
	404	const lock = join(clone, ".git", "index.lock");
	405	writeFileSync(lock, "");
	406	const next = await followUp(repo);
	407	expect(next.state).toBe("ok");
	408	expect(existsSync(lock)).toBe(false);
	409	}, 60_000);
	410
	411	test("a young lock is not worked around when the clone holds unpushed commits", async () => {
	412	const { repo, clone } = await keptClone("paddock");
	413	writeFileSync(join(clone, "LOST.md"), "unpushed\n");
	414	const git = (...args: string[]) =>
	415	Bun.spawnSync(["git", ...args], { cwd: clone });
	416	git("add", "LOST.md");
	417	git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "local");
	418	writeFileSync(join(clone, ".git", "index.lock"), "");
	419	const next = await followUp(repo);
	420	expect(next.state).toBe("failed");
	421	expect(next.summary ?? "").toContain("never pushed");
	422	expect(existsSync(join(clone, "LOST.md"))).toBe(true);
	423	}, 60_000);
299	424	});
300	425
301	426	describe("an issue handed to someone else", () => {
		@@ -311,7 +436,8 @@ describe("an issue handed to someone else", () => {
311	436	dataDir = await mkdtemp(join(tmpdir(), "forge-reassign-"));
312	437	app = createApp({ dataDir, port: 0 });
313	438	servers = await serve(app);
314		const a = await createUser(app.db, { trusted: true,
	439	const a = await createUser(app.db, {
	440	trusted: true,
315	441	username: "alice",
316	442	email: "[email protected]",
317	443	password: "a-good-passphrase",
		@@ -0,0 +1,64 @@
	1	import { describe, expect, test } from "bun:test";
	2	import {
	3	existsSync,
	4	mkdirSync,
	5	mkdtempSync,
	6	rmSync,
	7	utimesSync,
	8	writeFileSync,
	9	} from "node:fs";
	10	import { tmpdir } from "node:os";
	11	import { join } from "node:path";
	12	import { clearStaleLocks, lockedOut } from "../src/crew/locks.ts";
	13
	14	function clone() {
	15	const dir = mkdtempSync(join(tmpdir(), "forge-locks-"));
	16	mkdirSync(join(dir, ".git", "refs", "heads", "scout"), { recursive: true });
	17	return dir;
	18	}
	19	const plant = (path: string, ageMs: number) => {
	20	writeFileSync(path, "");
	21	const when = new Date(Date.now() - ageMs);
	22	utimesSync(path, when, when);
	23	};
	24
	25	describe("stale git locks in a kept clone", () => {
	26	test("old locks go, one line each; young locks and other files stay", () => {
	27	const dir = clone();
	28	const old = [
	29	join(dir, ".git", "config.lock"),
	30	join(dir, ".git", "index.lock"),
	31	join(dir, ".git", "refs", "heads", "scout", "x.lock"),
	32	];
	33	for (const path of old) plant(path, 10 * 60_000);
	34	const young = join(dir, ".git", "HEAD.lock");
	35	plant(young, 1_000);
	36	const config = join(dir, ".git", "config");
	37	plant(config, 10 * 60_000);
	38	const lines: string[] = [];
	39	const removed = clearStaleLocks(dir, (l) => lines.push(l));
	40	expect(removed.sort()).toEqual([...old].sort());
	41	expect(lines).toHaveLength(3);
	42	for (const path of old) expect(existsSync(path)).toBe(false);
	43	expect(existsSync(young)).toBe(true);
	44	expect(existsSync(config)).toBe(true);
	45	rmSync(dir, { recursive: true, force: true });
	46	});
	47
	48	test("a directory with no .git is left alone", () => {
	49	const dir = mkdtempSync(join(tmpdir(), "forge-locks-"));
	50	expect(clearStaleLocks(dir, () => {})).toEqual([]);
	51	rmSync(dir, { recursive: true, force: true });
	52	});
	53
	54	test("a lock failure is recognised from git's words", () => {
	55	expect(
	56	lockedOut("error: could not lock config file .git/config: File exists"),
	57	).toBe(true);
	58	expect(
	59	lockedOut("fatal: Unable to create '/x/.git/index.lock': File exists."),
	60	).toBe(true);
	61	expect(lockedOut("fatal: not a git repository")).toBe(false);
	62	expect(lockedOut("error: could not lock ref: cannot lock the thing")).toBe(false);
	63	});
	64	});