228015c · a5c44a8 · 726858c · bc5e571 · 2da791d · fb6f20c
@@ -0,0 +1,6 @@
# A forge killed in the middle of a merge
- A merge writes its intent (`pull_landing`: the commit, who lands it, on what) before it moves the base ref. At boot (`core/reconcile.ts`) and every 30 s (`main.ts`, lease tick; intents under a minute old are left to their own merge) `settleLandings` closes as merged any open pull whose landing commit is already in its base, with a `pull.merged` event; an intent whose ref never moved is dropped.
- Lock files a kill left under `refs/` (and `HEAD.lock`, `packed-refs.lock`) older than 5 s are removed at boot and when a merge meets one (`git/locks.ts`). A lock still held answers `ref_locked`, not `base_moved`.
- A run records its holder (`host:pid:start`) at each heartbeat. At boot a running run whose holder was a process of this host that is gone has its lease lapsed, so the existing sweeps settle it at once instead of after 90 s.
- Not done: the slow tick does not walk every repository for locks; saying why an app did not answer within 90 s of a restart.
@@ -31,6 +31,8 @@
export function takeBackWords(error: RevertPullError): string {
      return "git refused it";
    case "base_moved":
      return "the base moved under it";
    case "ref_locked":
      return "the branch is locked by another git operation";
    case "git":
      return `git said ${error.message}`;
    case "octopus":
@@ -0,0 +1,130 @@
/**
 * The intent to merge, and its settling (#12143). A merge moves the base ref
 * in git and then closes the pull in the database; a kill between the two
 * left main ahead of its own record. `recordLanding` writes what is about to
 * land before the ref moves; `clearLanding` drops it once the pull is closed
 * or the move failed; `settleLandings` closes, as merged, every open pull
 * whose recorded landing commit is already in its base — at boot and on a
 * slow tick — and forgets intents whose ref never moved.
 */
import type { Database } from "bun:sqlite";
import type { App } from "../app.ts";
import { isAncestor } from "../git/builds.ts";
import { repoDir } from "../git/repos.ts";
import { resolveOid, writeRef } from "../git/merge.ts";
import { closeAsMerged } from "./pulls.ts";
export interface Landing {
  readonly mergeOid: string;
  readonly actorId: string;
  readonly baseOid: string;
  readonly headOid: string;
}
export function recordLanding(db: Database, pullId: string, landing: Landing): void {
  db.query(
    `INSERT OR REPLACE INTO pull_landing (issue_id, merge_oid, actor_id, base_oid, head_oid, created_at)
     VALUES (?1, ?2, ?3, ?4, ?5, ?6)`,
  ).run(pullId, landing.mergeOid, landing.actorId, landing.baseOid, landing.headOid, new Date().toISOString());
}
export function clearLanding(db: Database, pullId: string): void {
  db.query("DELETE FROM pull_landing WHERE issue_id = ?1").run(pullId);
}
interface Row {
  issue_id: string;
  number: number;
  repo_id: string;
  owner_name: string;
  repo_name: string;
  base_branch: string;
  merged_at: string | null;
  merge_oid: string;
  actor_id: string;
  base_oid: string;
  head_oid: string;
  created_at: string;
}
/**
 * An intent younger than this may be a merge still in flight in this process: the slow tick
 * leaves it.
 */
const IN_FLIGHT_MS = 60_000;
/**
 * Settle every recorded landing. `all` (boot) treats each as no longer in
 * flight; otherwise intents younger than a minute are left to their merge.
 * Returns the numbers of pulls closed as merged.
 */
let settling = false;
/** One settle at a time: a tick that meets boot's, or its own predecessor, does nothing. */
export async function settleLandings(app: App, all: boolean): Promise<number[]> {
  if (settling) return [];
  settling = true;
  try {
    return await settle(app, all);
  } finally {
    settling = false;
  }
}
async function settle(app: App, all: boolean): Promise<number[]> {
  const rows = app.db
    .query<Row, []>(
      `SELECT pl.*, issue.number, issue.repo_id, user.username AS owner_name, repo.name AS repo_name,
              pull.base_branch, pull.merged_at
         FROM pull_landing pl
         JOIN pull ON pull.issue_id = pl.issue_id
         JOIN issue ON issue.id = pl.issue_id
         JOIN repo ON repo.id = issue.repo_id
         JOIN user ON user.id = repo.owner_id`,
    )
    .all();
  const closed: number[] = [];
  const now = Date.now();
  for (const row of rows) {
    if (!all && now - Date.parse(row.created_at) < IN_FLIGHT_MS) continue;
    if (row.merged_at !== null) {
      clearLanding(app.db, row.issue_id);
      continue;
    }
    const dir = repoDir(app.reposRoot, row.owner_name, row.repo_name);
    const tip = await resolveOid(dir, `refs/heads/${row.base_branch}`);
    if (tip === undefined) continue; // unreadable now: keep the intent for the next tick
    if (!(await isAncestor(dir, row.merge_oid, tip))) {
      // The ref never moved: nothing landed, nothing to settle.
      clearLanding(app.db, row.issue_id);
      continue;
    }
    // Re-read: only the call that closes the pull announces it.
    const fresh = app.db.query<{ merged_at: string | null }, [string]>("SELECT merged_at FROM pull WHERE issue_id = ?1").get(row.issue_id);
    if (fresh?.merged_at != null) {
      clearLanding(app.db, row.issue_id);
      continue;
    }
    closeAsMerged(app.db, row.issue_id, {
      mergeOid: row.merge_oid,
      actorId: row.actor_id,
      baseOid: row.base_oid,
      headOid: row.head_oid,
    });
    clearLanding(app.db, row.issue_id);
    await writeRef(dir, `refs/pull/${row.number}/merged`, row.merge_oid);
    app.bus.append({
      kind: "pull.merged",
      repoId: row.repo_id,
      actorId: row.actor_id,
      payload: { number: row.number, strategy: "recovered", mergeOid: row.merge_oid },
    });
    app.log.warn("settled a pull whose merge was cut off: its commit is on the base", {
      repo: `${row.owner_name}/${row.repo_name}`,
      pull: row.number,
    });
    closed.push(row.number);
  }
  return closed;
}
@@ -0,0 +1,1 @@
5e60c7d4d17e29f4a6d2293882fdc1207ccd48fd96621ccbc0e444f3fc60d827
@@ -0,0 +1,17 @@
// A merge moves the base ref in git and then closes the pull in this database; a kill between the
// two left main ahead and the pull open for good (#12143). The intent is now written first:
// `pull_landing` holds the commit about to land, who lands it and on what, until the pull is
// closed; boot and a slow tick close any pull whose landing commit is already on its base.
// `run.holder` names the process (host:pid:start) that last renewed a run's lease, so a boot can
// settle at once a run whose process is gone. Additive.
export default `
  CREATE TABLE IF NOT EXISTS pull_landing (
    issue_id   TEXT PRIMARY KEY REFERENCES issue(id) ON DELETE CASCADE,
    merge_oid  TEXT NOT NULL,
    actor_id   TEXT NOT NULL,
    base_oid   TEXT NOT NULL,
    head_oid   TEXT NOT NULL,
    created_at TEXT NOT NULL
  );
  ALTER TABLE run ADD COLUMN holder TEXT;
`;
@@ -8,6 +8,7 @@
import { getRepoById } from "./repos.ts";
import { repoDir } from "../git/repos.ts";
import { clearLanding, recordLanding } from "./landing.ts";
import { sealedByNumber } from "./sealed.ts";
import type { Database } from "bun:sqlite";
import { diffRange, type Diff, type DiffLaps } from "../git/diff.ts";
@@ -1247,8 +1248,20 @@
export async function mergePull(
    author: input.author,
    ...(input.committer === undefined ? {} : { committer: input.committer }),
    ...(input.expect === undefined ? {} : { treeOid: input.expect.treeOid }),
    // The intent, before the ref moves: a kill between the move and the close
    // is settled from it at boot (`settleLandings`, #12143).
    beforeMove: (mergeOid) =>
      recordLanding(db, pull.id, {
        mergeOid,
        actorId: input.actorId,
        baseOid: status.baseOid,
        headOid: status.headOid,
      }),
  });
  if (!landed.ok) return landed;
  if (!landed.ok) {
    clearLanding(db, pull.id);
    return landed;
  }
  closeAsMerged(db, pull.id, {
    mergeOid: landed.value.mergeOid,
@@ -1256,6 +1269,7 @@
export async function mergePull(
    baseOid: status.baseOid,
    headOid: status.headOid,
  });
  clearLanding(db, pull.id);
  // The merge commit is kept reachable forever, independent of the base branch.
  await writeRef(dir, `refs/pull/${pull.number}/merged`, landed.value.mergeOid);
@@ -1291,7 +1305,7 @@
async function carriedBy(
/** Record the landing on the row and close the pull — once: a pull already
 * merged keeps the commit it first landed as. */
function closeAsMerged(
export function closeAsMerged(
  db: Database,
  pullId: string,
  landed: {
@@ -20,10 +20,13 @@
 */
import type { App } from "../app.ts";
import { sweepLocks } from "../git/locks.ts";
import { installHooks } from "../git/hooks.ts";
import { listRefs } from "../git/read.ts";
import { repoDir } from "../git/repos.ts";
import { repoExists } from "../git/repos.ts";
import { settleLandings } from "./landing.ts";
import { lapseOrphans } from "./runs.ts";
import { getPull, trackPullHead } from "./pulls.ts";
/** Repos reconciled at once. Filesystem work, so wider than the git limit. */
@@ -66,6 +69,9 @@
export async function reconcile(app: App): Promise<void> {
    if (await repoExists(dir)) {
      await installHooks(dir);
      refreshed++;
      // Locks a kill -9 left (#12143) would refuse every later move of that ref.
      const swept = await sweepLocks(dir);
      if (swept.length > 0) app.log.warn("removed stale git lock files", { repo: `${row.owner_name}/${row.name}`, files: swept.length });
    } else {
      app.log.warn("repo row without a git directory", {
        repo: `${row.owner_name}/${row.name}`,
@@ -73,12 +79,17 @@
export async function reconcile(app: App): Promise<void> {
    }
  });
  // A merge cut off between the ref and the row, and runs whose process is gone (#12143).
  const settled = await settleLandings(app, true);
  const orphaned = lapseOrphans(app.db);
  const heads = await reconcileHeads(app);
  app.log.info("reconciled", {
    repos: rows.length,
    hooks_refreshed: refreshed,
    heads_repaired: heads,
    landings_settled: settled.length,
    orphaned_runs: orphaned,
  });
}
@@ -9,6 +9,7 @@
 */
import type { Database } from "bun:sqlite";
import { hostname } from "node:os";
import { BUDGET_PREFIX } from "../kernel/budgets.ts";
import { id } from "../kernel/id.ts";
@@ -490,13 +491,59 @@
export function lastAgentRun(
  return row ? toRun(row) : undefined;
}
/** The process working these runs says it still is. */
/**
 * This process, as a run's lease names it: host, pid and start, so a reused pid is not mistaken
 * for it.
 */
export const HOLDER = `${hostname()}:${process.pid}:${Date.now()}`;
/** The process working these runs says it still is, and who it is. */
export function heartbeatRuns(db: Database, runIds: readonly string[]): void {
  if (runIds.length === 0) return;
  db.query(
    `UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
     WHERE state = 'running' AND id IN (${runIds.map((_, n) => `?${n + 1}`).join(",")})`,
  ).run(...runIds);
    `UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'), holder = ?1
     WHERE state = 'running' AND id IN (${runIds.map((_, n) => `?${n + 2}`).join(",")})`,
  ).run(HOLDER, ...runIds);
}
/**
 * Lapse the lease of every running run whose holder was a process of this
 * host that is gone (#12143): the next sweep settles it at once, instead of
 * after the 90 s a silent lease waits. A holder on another host, or one
 * that cannot be told apart, is left to its lease. `alive` is a seam for tests.
 */
export function lapseOrphans(
  db: Database,
  self: string = HOLDER,
  alive: (pid: number) => boolean = pidAlive,
): number {
  const host = self.split(":")[0]!;
  const rows = db
    .query<{ id: string; holder: string }, []>(
      "SELECT id, holder FROM run WHERE state = 'running' AND holder IS NOT NULL",
    )
    .all();
  let lapsed = 0;
  for (const row of rows) {
    if (row.holder === self) continue;
    const [h, pid] = row.holder.split(":");
    if (h !== host || !/^\d+$/.test(pid ?? "")) continue;
      
        sentinelcrew
        1d ago
        
      
      A same-hostname holder in another pid namespace, or another instance, reads as ESRCH and its live run is lapsed. Now that this runs every tick, restrict it to boot or add an instance id to the holder.
    
    // Our own pid under another start is a predecessor (a container's pid 1 again): gone.
    if (Number(pid) !== process.pid && alive(Number(pid))) continue;
    lapsed += db
      .query("UPDATE run SET heartbeat_at = '1970-01-01T00:00:00.000Z' WHERE id = ?1 AND state = 'running' AND holder = ?2")
      .run(row.id, row.holder).changes;
  }
  return lapsed;
}
function pidAlive(pid: number): boolean {
  try {
    process.kill(pid, 0);
    return true;
  } catch (e) {
    return (e as NodeJS.ErrnoException).code === "EPERM";
  }
}
/**
@@ -0,0 +1,84 @@
/**
 * Lock files a killed git process left behind (#12143). git takes `<ref>.lock`
 * for the milliseconds a ref moves; a `kill -9` in that window leaves it, and
 * every later update of that ref answers "File exists" — through a reboot —
 * until someone deletes it. A lock older than any git operation could last is
 * nobody's: `sweepLocks` removes those, under `refs/` and the two at the top,
 * at boot only, when no git of ours is alive; a merge removes only its own
 * ref's (`sweepRefLock`).
 */
import { readdir, rm, stat } from "node:fs/promises";
import { join } from "node:path";
/**
 * What boot and a landing treat as older than any live ref update (git holds one for
 * milliseconds).
 */
export const STALE_LOCK_MS = 5_000;
/** The message git gives when a ref's lock is held. */
export const LOCK_HELD = /unable to create '[^']*\.lock'/i;
export function isLockHeld(stderr: string): boolean {
  return LOCK_HELD.test(stderr);
}
async function walk(dir: string, out: string[], depth: number): Promise<void> {
  if (depth > 12) return;
  let names: import("node:fs").Dirent[];
  try {
    names = await readdir(dir, { withFileTypes: true });
  } catch {
    return;
  }
  for (const entry of names) {
    const path = join(dir, entry.name);
    if (entry.isDirectory()) await walk(path, out, depth + 1);
    else if (entry.isFile() && entry.name.endsWith(".lock")) out.push(path);
  }
}
/** Remove the `*.lock` files of a git directory older than `minAgeMs`; the paths removed. */
export async function sweepLocks(
  gitDir: string,
  minAgeMs: number = STALE_LOCK_MS,
  now: number = Date.now(),
): Promise<string[]> {
  const found: string[] = [join(gitDir, "HEAD.lock"), join(gitDir, "packed-refs.lock")];
  await walk(join(gitDir, "refs"), found, 0);
  const removed: string[] = [];
  for (const path of found) {
    try {
      const s = await stat(path);
      
        sentinelcrew
        1d ago
        outdated
      
      Removes locks held by live git processes if they are older than 5 s (packed-refs.lock during pack-refs/gc, HEAD.lock). That breaks git's mutual exclusion. Scope it to the one ref at merge time and sweep the whole repository only at boot.
    
      if (!s.isFile() || now - s.mtimeMs < minAgeMs) continue;
      await rm(path, { force: true });
      removed.push(path);
    } catch {
      // not there, or gone as we looked
    }
  }
  return removed;
}
/**
 * The lock of one ref, when it is older than `minAgeMs`: what a merge may
 * remove while other git processes run. Never another file's lock: a live
 * `pack-refs` holds `packed-refs.lock` for longer than any ref update.
 */
export async function sweepRefLock(
  gitDir: string,
  ref: string,
  minAgeMs: number = STALE_LOCK_MS,
  now: number = Date.now(),
): Promise<boolean> {
  if (!/^refs\/heads\/[A-Za-z0-9._\/-]+$/.test(ref) || ref.includes("..")) return false;
  const path = join(gitDir, `${ref}.lock`);
  try {
    const s = await stat(path);
    if (!s.isFile() || now - s.mtimeMs < minAgeMs) return false;
    await rm(path, { force: true });
    return true;
  } catch {
    return false;
  }
}
@@ -8,6 +8,7 @@
import { err, ok, type Result } from "../kernel/result.ts";
import { diffMemoryKey, recalledBase, rememberBase } from "./diff-memory.ts";
import { runGit, runGitCapture } from "./env.ts";
import { isLockHeld, sweepRefLock } from "./locks.ts";
import { isSafeOid, isSafeRef, isSafeTreePath } from "./guard.ts";
export type MergeStrategy = "merge" | "squash";
@@ -21,6 +22,11 @@
export type MergeError =
  | { kind: "conflict"; files: string[] }
  | { kind: "unsafe" }
  | { kind: "base_moved" }
  /**
   * A ref's lock file is held (by a live git process, or a recent one's leftover): not the base
   * moving.
   */
  | { kind: "ref_locked" }
  | { kind: "git"; message: string };
export interface MergeResult {
@@ -221,6 +227,11 @@
export async function merge(
     * nothing is committed.
     */
    treeOid?: string;
    /**
     * Called with the commit about to land, before the ref moves and in the
     * same tick: the intent to merge, recorded where a kill can find it (#12143).
     */
    beforeMove?: (mergeOid: string) => void;
  },
): Promise<Result<MergeResult, MergeError>> {
  if (
@@ -256,10 +267,20 @@
export async function merge(
  const mergeOid = commit.value.trim();
  // CAS the branch forward: fails if someone pushed between our read and now.
  const update = await runGit(
    ["update-ref", `refs/heads/${input.baseBranch}`, mergeOid, input.baseOid],
    { cwd: dir },
  );
  input.beforeMove?.(mergeOid);
  const move = () =>
    runGit(
      ["update-ref", `refs/heads/${input.baseBranch}`, mergeOid, input.baseOid],
      { cwd: dir },
    );
  let update = await move();
  // A lock a killed git left behind is not the base moving (#12143): one
  // of this ref older than any live update is removed and the move tried once more; a
  // lock still held is reported as held.
      
        sentinelcrew
        1d ago
        
      
      This sweeps every old lock in the repo on any lock-held error, not just the lock of this ref. Remove only refs/heads/<base>.lock, and only when it is stale.
    
  if (!update.ok && isLockHeld(update.error.stderr)) {
    if (await sweepRefLock(dir, `refs/heads/${input.baseBranch}`)) update = await move();
    if (!update.ok && isLockHeld(update.error.stderr)) return err({ kind: "ref_locked" });
  }
  if (!update.ok) return err({ kind: "base_moved" });
  return ok({ mergeOid, previousOid: input.baseOid });
@@ -48,6 +48,8 @@
export function takeBackRefusal(error: RevertPullError, log: Log): Response {
      );
    case "base_moved":
      return apiError("conflict", 409, "the default branch moved under it; try again");
    case "ref_locked":
      return apiError("conflict", 409, "the default branch is locked by another git operation; try again shortly");
    case "octopus":
      return apiError("conflict", 409, `it has ${error.parents} parents; take it back by hand`);
    case "unsafe":
@@ -75,6 +75,7 @@
export const ALLOWED_REMOVALS: readonly { unit: string; why: string }[] = [
    "src/apps/checkout.ts:exportTree",
  ]),
  { unit: "src/core/repos.ts:rowOrUndo", why: "removes the directory git has just made for the repository being created, when its row is refused" },
  { unit: "src/git/locks.ts:sweepRefLock", why: "removes the one lock file of the ref a merge moves, when older than any live update (issue 12143): a name checked to be under refs/heads, never another file" },
  { unit: "src/git/write.ts:commitFiles", why: "removes its own temporary index file" },
  { unit: "src/kernel/file_modes.ts:writePrivate", why: "removes the temporary file it has just made itself, under a name nobody else chose, when the write or the rename failed" },
  { unit: "src/git/repos.ts:detachFork", why: "removes the alternates file of the fork just made" },
@@ -70,6 +70,7 @@
import { resolveDestination } from "./snapshot/destination.ts";
import { resolveMirror } from "./snapshot/mirror.ts";
import { releaseOrphans } from "./core/run_secret.ts";
import { failAbandonedRuns } from "./core/runs.ts";
import { settleLandings } from "./core/landing.ts";
import { DRAIN_LINGER_MAX_S, drainLingerMs, startDraining } from "./http/health.ts";
import { takePresence, warmBehind } from "./apps/presence.ts";
import { STALE_AFTER_MS } from "./runs/lease.ts";
@@ -890,6 +891,7 @@
if (ssh)
// agent runs are the crew's to queue again, unless there is no crew here to
// do it.
const leaseTimer = setInterval(() => during("lease", () => {
  void settleLandings(app, false).catch((e) => app.log.warn("settling cut-off merges failed", { error: String(e) }));
      
        sentinelcrew
        1d ago
        outdated
      
      Ticks and boot can overlap, and two overlapping settles can both emit pull.merged. Add an in-flight guard, or re-check merged_at right before closing.
    
  failAbandonedRuns(
    app.db,
    STALE_AFTER_MS,
@@ -1314,6 +1314,8 @@
export function mergeFailure(error: AccessError | MergePullError): string {
      return `The branches conflict in ${error.files.length} file${error.files.length === 1 ? "" : "s"} — merge the base branch in and push again.`;
    if (error.kind === "base_moved")
      return "The base branch moved while merging. Try again.";
    if (error.kind === "ref_locked")
      return "The base branch is locked by another git operation. Try again in a moment.";
    if (error.kind === "unsafe") return "That merge was refused as unsafe.";
    if (error.kind === "guarded") return guardedRefusal(error.paths);
    if (error.kind === "team_review_required")
@@ -0,0 +1,214 @@
/**
 * A forge killed in the middle of a merge comes back consistent (#12143):
 * a pull whose merge commit reached the base but not the database is closed
 * as merged at boot; a lock file a kill left behind does not block main; a
 * run whose process is gone is settled at once.
 */
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { mergePull, openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { getIssueById } from "../src/core/issues.ts";
import { recordLanding, settleLandings } from "../src/core/landing.ts";
import { getPull, type Pull } from "../src/core/pulls.ts";
import type { Repo } from "../src/core/repos.ts";
import { createRun, lapseOrphans, failAbandonedRuns } from "../src/core/runs.ts";
import type { User } from "../src/core/users.ts";
import { merge, resolveOid } from "../src/git/merge.ts";
import { repoDir } from "../src/git/repos.ts";
import { sweepLocks } from "../src/git/locks.ts";
import { promote } from "./helpers/members.ts";
import { signUpProven as signUp } from "./helpers/verified.ts";
describe("a merge cut off by a kill", () => {
  let dataDir: string;
  let app: App;
  let alice: User;
  let repo: Repo;
  let dir: string;
  let n = 0;
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-merge-crash-"));
    app = createApp({ dataDir, port: 0 });
    const made = await signUp(app, { username: "alice", email: "[email protected]", password: "a-long-passphrase" });
    if (!made.ok) throw new Error("sign-up");
    alice = made.value;
    promote(app.db, alice);
    const r = await createRepo(app, alice, { owner: "alice", name: "site", visibility: "public" });
    if (!r.ok) throw new Error(String(r.error));
    repo = r.value;
    dir = repoDir(app.reposRoot, "alice", "site");
    await commitToBranch(app, alice, repo, { message: "seed", files: [{ path: "README.md", content: "hi\n" }] });
  }, 30_000);
  afterAll(async () => {
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  const newPull = async (): Promise<Pull> => {
    const branch = `f${++n}`;
    await commitToBranch(app, alice, repo, {
      branch,
      message: branch,
      files: [{ path: `${branch}.txt`, content: "x\n" }],
    });
    const pull = await openPull(app, alice, repo, { title: branch, body: "", baseBranch: "main", headBranch: branch });
    if (!pull.ok) throw new Error(String(pull.error));
    return pull.value;
  };
  test("killed between the ref move and the database commit: boot closes the pull as merged, with its event", async () => {
    const pull = await newPull();
    const base = (await resolveOid(dir, "refs/heads/main"))!;
    // What mergePull does, stopped where the kill fell: intent recorded, ref moved, row never
    // closed.
    const landed = await merge(dir, {
      baseBranch: "main",
      baseOid: base,
      headOid: pull.headOid,
      strategy: "squash",
      message: "cut off",
      author: { name: "A", email: "[email protected]" },
      beforeMove: (mergeOid) =>
        recordLanding(app.db, pull.id, { mergeOid, actorId: alice.id, baseOid: base, headOid: pull.headOid }),
    });
    expect(landed.ok).toBe(true);
    expect(getIssueById(app.db, pull.id)?.state).toBe("open");
    // A merge still in flight is not the slow tick's to settle.
    expect(await settleLandings(app, false)).toEqual([]);
    expect(getIssueById(app.db, pull.id)?.state).toBe("open");
    const before = app.db.query<{ n: number }, []>("SELECT count(*) AS n FROM event WHERE kind = 'pull.merged'").get()!.n;
    expect(await settleLandings(app, true)).toEqual([pull.number]);
    const after = getPull(app.db, repo.id, pull.number)!;
    expect(after.mergedAt).not.toBeNull();
    expect(after.mergedOid).toBe(landed.ok ? landed.value.mergeOid : "");
    expect(getIssueById(app.db, pull.id)?.state).toBe("closed");
    expect(app.db.query<{ n: number }, []>("SELECT count(*) AS n FROM event WHERE kind = 'pull.merged'").get()!.n).toBe(before + 1);
    // Once: a second boot finds nothing.
    expect(await settleLandings(app, true)).toEqual([]);
  });
  test("an intent whose ref never moved is dropped and the pull stays open", async () => {
    const pull = await newPull();
    const base = (await resolveOid(dir, "refs/heads/main"))!;
    recordLanding(app.db, pull.id, { mergeOid: pull.headOid, actorId: alice.id, baseOid: base, headOid: pull.headOid });
    expect(await settleLandings(app, true)).toEqual([]);
    expect(getIssueById(app.db, pull.id)?.state).toBe("open");
    expect(app.db.query("SELECT 1 FROM pull_landing WHERE issue_id = ?1").get(pull.id)).toBeNull();
  });
  test("a stale main.lock left by a kill does not block the next merge", async () => {
    const pull = await newPull();
    const lock = join(dir, "refs", "heads", "main.lock");
    await mkdir(join(dir, "refs", "heads"), { recursive: true });
    await writeFile(lock, "");
    const old = new Date(Date.now() - 60_000);
    await utimes(lock, old, old);
    const merged = await mergePull(app, alice, repo, pull, "squash");
    expect(merged.ok).toBe(true);
    expect(getPull(app.db, repo.id, pull.number)?.mergedAt).not.toBeNull();
  });
  test("a merge removes only its own ref's stale lock: an old lock of another file is a live git's", async () => {
    const pull = await newPull();
    const mine = join(dir, "refs", "heads", "main.lock");
    const packed = join(dir, "packed-refs.lock");
    const other = join(dir, "refs", "heads", "other.lock");
    const old = new Date(Date.now() - 60_000);
    for (const f of [mine, packed, other]) {
      await writeFile(f, "");
      await utimes(f, old, old);
    }
    expect((await mergePull(app, alice, repo, pull, "squash")).ok).toBe(true);
    expect(await Bun.file(mine).exists()).toBe(false);
    expect(await Bun.file(packed).exists()).toBe(true);
    expect(await Bun.file(other).exists()).toBe(true);
    await rm(packed);
    await rm(other);
  });
  test("two settles at once announce the merge once", async () => {
    const pull = await newPull();
    const base = (await resolveOid(dir, "refs/heads/main"))!;
    const landed = await merge(dir, {
      baseBranch: "main",
      baseOid: base,
      headOid: pull.headOid,
      strategy: "squash",
      message: "cut off",
      author: { name: "A", email: "[email protected]" },
      beforeMove: (mergeOid) =>
        recordLanding(app.db, pull.id, { mergeOid, actorId: alice.id, baseOid: base, headOid: pull.headOid }),
    });
    expect(landed.ok).toBe(true);
    const count = () => app.db.query<{ n: number }, []>("SELECT count(*) AS n FROM event WHERE kind = 'pull.merged'").get()!.n;
    const before = count();
    const [a, b] = await Promise.all([settleLandings(app, true), settleLandings(app, true)]);
    expect([...a, ...b]).toEqual([pull.number]);
    expect(count()).toBe(before + 1);
  });
  test("a lock held now is reported as held, not as the base moving", async () => {
    const pull = await newPull();
    const lock = join(dir, "refs", "heads", "main.lock");
    await writeFile(lock, "");
    const merged = await mergePull(app, alice, repo, pull, "squash");
    expect(merged.ok).toBe(false);
    expect(!merged.ok && (merged.error as { kind: string }).kind).toBe("ref_locked");
    expect(getIssueById(app.db, pull.id)?.state).toBe("open");
    expect(app.db.query("SELECT 1 FROM pull_landing WHERE issue_id = ?1").get(pull.id)).toBeNull();
    await rm(lock);
    expect((await mergePull(app, alice, repo, pull, "squash")).ok).toBe(true);
  });
  test("sweepLocks leaves a young lock and removes an old one", async () => {
    const young = join(dir, "HEAD.lock");
    await writeFile(young, "");
    expect(await sweepLocks(dir, 5_000)).toEqual([]);
    expect(await sweepLocks(dir, 5_000, Date.now() + 10_000)).toEqual([young]);
  });
});
describe("runs a restart orphaned", () => {
  test("a run whose process of this host is gone is settled at once; a live or foreign one is not", async () => {
    const dataDir = await mkdtemp(join(tmpdir(), "forge-orphans-"));
    const app = createApp({ dataDir, port: 0 });
    try {
      const made = await signUp(app, { username: "bob", email: "[email protected]", password: "a-long-passphrase" });
      if (!made.ok) throw new Error("sign-up");
      promote(app.db, made.value);
      const r = await createRepo(app, made.value, { owner: "bob", name: "site", visibility: "public" });
      if (!r.ok) throw new Error(String(r.error));
      const mk = (holder: string) => {
        const run = createRun(app.db, { repoId: r.value.id, kind: "deploy" });
        app.db.run("UPDATE run SET state = 'running', holder = ?1, heartbeat_at = ?2 WHERE id = ?3", [
          holder,
          new Date().toISOString(),
          (run as { id: string }).id,
        ]);
        return (run as { id: string }).id;
      };
      const state = (id: string) => app.db.query<{ state: string }, [string]>("SELECT state FROM run WHERE id = ?1").get(id)!.state;
      const self = `pod:${process.pid}:5`;
      const gone = mk("pod:4242:1");
      const alive = mk("pod:4243:1");
      const elsewhere = mk("other:4244:1");
      const mine = mk(self);
      const samePid = mk(`pod:${process.pid}:1`); // our pid, an earlier life of it
      const live = (pid: number) => pid === 4243;
      expect(lapseOrphans(app.db, self, live)).toBe(2);
      failAbandonedRuns(app.db, 90_000);
      expect([gone, samePid].map(state)).toEqual(["failed", "failed"]);
      expect([alive, elsewhere, mine].map(state)).toEqual(["running", "running", "running"]);
    } finally {
      app.close();
      await rm(dataDir, { recursive: true, force: true });
    }
  });
});
		@@ -0,0 +1,6 @@
	1	# A forge killed in the middle of a merge
	2
	3	- A merge writes its intent (`pull_landing`: the commit, who lands it, on what) before it moves the base ref. At boot (`core/reconcile.ts`) and every 30 s (`main.ts`, lease tick; intents under a minute old are left to their own merge) `settleLandings` closes as merged any open pull whose landing commit is already in its base, with a `pull.merged` event; an intent whose ref never moved is dropped.
	4	- Lock files a kill left under `refs/` (and `HEAD.lock`, `packed-refs.lock`) older than 5 s are removed at boot and when a merge meets one (`git/locks.ts`). A lock still held answers `ref_locked`, not `base_moved`.
	5	- A run records its holder (`host:pid:start`) at each heartbeat. At boot a running run whose holder was a process of this host that is gone has its lease lapsed, so the existing sweeps settle it at once instead of after 90 s.
	6	- Not done: the slow tick does not walk every repository for locks; saying why an app did not answer within 90 s of a restart.
		@@ -31,6 +31,8 @@ export function takeBackWords(error: RevertPullError): string {
31	31	return "git refused it";
32	32	case "base_moved":
33	33	return "the base moved under it";
	34	case "ref_locked":
	35	return "the branch is locked by another git operation";
34	36	case "git":
35	37	return `git said ${error.message}`;
36	38	case "octopus":
		@@ -0,0 +1,130 @@
	1	/**
	2	* The intent to merge, and its settling (#12143). A merge moves the base ref
	3	* in git and then closes the pull in the database; a kill between the two
	4	* left main ahead of its own record. `recordLanding` writes what is about to
	5	* land before the ref moves; `clearLanding` drops it once the pull is closed
	6	* or the move failed; `settleLandings` closes, as merged, every open pull
	7	* whose recorded landing commit is already in its base — at boot and on a
	8	* slow tick — and forgets intents whose ref never moved.
	9	*/
	10
	11	import type { Database } from "bun:sqlite";
	12	import type { App } from "../app.ts";
	13	import { isAncestor } from "../git/builds.ts";
	14	import { repoDir } from "../git/repos.ts";
	15	import { resolveOid, writeRef } from "../git/merge.ts";
	16	import { closeAsMerged } from "./pulls.ts";
	17
	18	export interface Landing {
	19	readonly mergeOid: string;
	20	readonly actorId: string;
	21	readonly baseOid: string;
	22	readonly headOid: string;
	23	}
	24
	25	export function recordLanding(db: Database, pullId: string, landing: Landing): void {
	26	db.query(
	27	`INSERT OR REPLACE INTO pull_landing (issue_id, merge_oid, actor_id, base_oid, head_oid, created_at)
	28	VALUES (?1, ?2, ?3, ?4, ?5, ?6)`,
	29	).run(pullId, landing.mergeOid, landing.actorId, landing.baseOid, landing.headOid, new Date().toISOString());
	30	}
	31
	32	export function clearLanding(db: Database, pullId: string): void {
	33	db.query("DELETE FROM pull_landing WHERE issue_id = ?1").run(pullId);
	34	}
	35
	36	interface Row {
	37	issue_id: string;
	38	number: number;
	39	repo_id: string;
	40	owner_name: string;
	41	repo_name: string;
	42	base_branch: string;
	43	merged_at: string \| null;
	44	merge_oid: string;
	45	actor_id: string;
	46	base_oid: string;
	47	head_oid: string;
	48	created_at: string;
	49	}
	50
	51	/**
	52	* An intent younger than this may be a merge still in flight in this process: the slow tick
	53	* leaves it.
	54	*/
	55	const IN_FLIGHT_MS = 60_000;
	56
	57	/**
	58	* Settle every recorded landing. `all` (boot) treats each as no longer in
	59	* flight; otherwise intents younger than a minute are left to their merge.
	60	* Returns the numbers of pulls closed as merged.
	61	*/
	62	let settling = false;
	63
	64	/** One settle at a time: a tick that meets boot's, or its own predecessor, does nothing. */
	65	export async function settleLandings(app: App, all: boolean): Promise<number[]> {
	66	if (settling) return [];
	67	settling = true;
	68	try {
	69	return await settle(app, all);
	70	} finally {
	71	settling = false;
	72	}
	73	}
	74
	75	async function settle(app: App, all: boolean): Promise<number[]> {
	76	const rows = app.db
	77	.query<Row, []>(
	78	`SELECT pl.*, issue.number, issue.repo_id, user.username AS owner_name, repo.name AS repo_name,
	79	pull.base_branch, pull.merged_at
	80	FROM pull_landing pl
	81	JOIN pull ON pull.issue_id = pl.issue_id
	82	JOIN issue ON issue.id = pl.issue_id
	83	JOIN repo ON repo.id = issue.repo_id
	84	JOIN user ON user.id = repo.owner_id`,
	85	)
	86	.all();
	87	const closed: number[] = [];
	88	const now = Date.now();
	89	for (const row of rows) {
	90	if (!all && now - Date.parse(row.created_at) < IN_FLIGHT_MS) continue;
	91	if (row.merged_at !== null) {
	92	clearLanding(app.db, row.issue_id);
	93	continue;
	94	}
	95	const dir = repoDir(app.reposRoot, row.owner_name, row.repo_name);
	96	const tip = await resolveOid(dir, `refs/heads/${row.base_branch}`);
	97	if (tip === undefined) continue; // unreadable now: keep the intent for the next tick
	98	if (!(await isAncestor(dir, row.merge_oid, tip))) {
	99	// The ref never moved: nothing landed, nothing to settle.
	100	clearLanding(app.db, row.issue_id);
	101	continue;
	102	}
	103	// Re-read: only the call that closes the pull announces it.
	104	const fresh = app.db.query<{ merged_at: string \| null }, [string]>("SELECT merged_at FROM pull WHERE issue_id = ?1").get(row.issue_id);
	105	if (fresh?.merged_at != null) {
	106	clearLanding(app.db, row.issue_id);
	107	continue;
	108	}
	109	closeAsMerged(app.db, row.issue_id, {
	110	mergeOid: row.merge_oid,
	111	actorId: row.actor_id,
	112	baseOid: row.base_oid,
	113	headOid: row.head_oid,
	114	});
	115	clearLanding(app.db, row.issue_id);
	116	await writeRef(dir, `refs/pull/${row.number}/merged`, row.merge_oid);
	117	app.bus.append({
	118	kind: "pull.merged",
	119	repoId: row.repo_id,
	120	actorId: row.actor_id,
	121	payload: { number: row.number, strategy: "recovered", mergeOid: row.merge_oid },
	122	});
	123	app.log.warn("settled a pull whose merge was cut off: its commit is on the base", {
	124	repo: `${row.owner_name}/${row.repo_name}`,
	125	pull: row.number,
	126	});
	127	closed.push(row.number);
	128	}
	129	return closed;
	130	}
		@@ -0,0 +1,1 @@
	1	5e60c7d4d17e29f4a6d2293882fdc1207ccd48fd96621ccbc0e444f3fc60d827
		@@ -0,0 +1,17 @@
	1	// A merge moves the base ref in git and then closes the pull in this database; a kill between the
	2	// two left main ahead and the pull open for good (#12143). The intent is now written first:
	3	// `pull_landing` holds the commit about to land, who lands it and on what, until the pull is
	4	// closed; boot and a slow tick close any pull whose landing commit is already on its base.
	5	// `run.holder` names the process (host:pid:start) that last renewed a run's lease, so a boot can
	6	// settle at once a run whose process is gone. Additive.
	7	export default `
	8	CREATE TABLE IF NOT EXISTS pull_landing (
	9	issue_id TEXT PRIMARY KEY REFERENCES issue(id) ON DELETE CASCADE,
	10	merge_oid TEXT NOT NULL,
	11	actor_id TEXT NOT NULL,
	12	base_oid TEXT NOT NULL,
	13	head_oid TEXT NOT NULL,
	14	created_at TEXT NOT NULL
	15	);
	16	ALTER TABLE run ADD COLUMN holder TEXT;
	17	`;
		@@ -8,6 +8,7 @@
8	8
9	9	import { getRepoById } from "./repos.ts";
10	10	import { repoDir } from "../git/repos.ts";
	11	import { clearLanding, recordLanding } from "./landing.ts";
11	12	import { sealedByNumber } from "./sealed.ts";
12	13	import type { Database } from "bun:sqlite";
13	14	import { diffRange, type Diff, type DiffLaps } from "../git/diff.ts";
		@@ -1247,8 +1248,20 @@ export async function mergePull(
1247	1248	author: input.author,
1248	1249	...(input.committer === undefined ? {} : { committer: input.committer }),
1249	1250	...(input.expect === undefined ? {} : { treeOid: input.expect.treeOid }),
	1251	// The intent, before the ref moves: a kill between the move and the close
	1252	// is settled from it at boot (`settleLandings`, #12143).
	1253	beforeMove: (mergeOid) =>
	1254	recordLanding(db, pull.id, {
	1255	mergeOid,
	1256	actorId: input.actorId,
	1257	baseOid: status.baseOid,
	1258	headOid: status.headOid,
	1259	}),
1250	1260	});
1251		if (!landed.ok) return landed;
	1261	if (!landed.ok) {
	1262	clearLanding(db, pull.id);
	1263	return landed;
	1264	}
1252	1265
1253	1266	closeAsMerged(db, pull.id, {
1254	1267	mergeOid: landed.value.mergeOid,
		@@ -1256,6 +1269,7 @@ export async function mergePull(
1256	1269	baseOid: status.baseOid,
1257	1270	headOid: status.headOid,
1258	1271	});
	1272	clearLanding(db, pull.id);
1259	1273
1260	1274	// The merge commit is kept reachable forever, independent of the base branch.
1261	1275	await writeRef(dir, `refs/pull/${pull.number}/merged`, landed.value.mergeOid);
		@@ -1291,7 +1305,7 @@ async function carriedBy(
1291	1305
1292	1306	/** Record the landing on the row and close the pull — once: a pull already
1293	1307	* merged keeps the commit it first landed as. */
1294		function closeAsMerged(
	1308	export function closeAsMerged(
1295	1309	db: Database,
1296	1310	pullId: string,
1297	1311	landed: {
		@@ -20,10 +20,13 @@
20	20	*/
21	21
22	22	import type { App } from "../app.ts";
	23	import { sweepLocks } from "../git/locks.ts";
23	24	import { installHooks } from "../git/hooks.ts";
24	25	import { listRefs } from "../git/read.ts";
25	26	import { repoDir } from "../git/repos.ts";
26	27	import { repoExists } from "../git/repos.ts";
	28	import { settleLandings } from "./landing.ts";
	29	import { lapseOrphans } from "./runs.ts";
27	30	import { getPull, trackPullHead } from "./pulls.ts";
28	31
29	32	/** Repos reconciled at once. Filesystem work, so wider than the git limit. */
		@@ -66,6 +69,9 @@ export async function reconcile(app: App): Promise<void> {
66	69	if (await repoExists(dir)) {
67	70	await installHooks(dir);
68	71	refreshed++;
	72	// Locks a kill -9 left (#12143) would refuse every later move of that ref.
	73	const swept = await sweepLocks(dir);
	74	if (swept.length > 0) app.log.warn("removed stale git lock files", { repo: `${row.owner_name}/${row.name}`, files: swept.length });
69	75	} else {
70	76	app.log.warn("repo row without a git directory", {
71	77	repo: `${row.owner_name}/${row.name}`,
		@@ -73,12 +79,17 @@ export async function reconcile(app: App): Promise<void> {
73	79	}
74	80	});
75	81
	82	// A merge cut off between the ref and the row, and runs whose process is gone (#12143).
	83	const settled = await settleLandings(app, true);
	84	const orphaned = lapseOrphans(app.db);
76	85	const heads = await reconcileHeads(app);
77	86
78	87	app.log.info("reconciled", {
79	88	repos: rows.length,
80	89	hooks_refreshed: refreshed,
81	90	heads_repaired: heads,
	91	landings_settled: settled.length,
	92	orphaned_runs: orphaned,
82	93	});
83	94	}
84	95
		@@ -9,6 +9,7 @@
9	9	*/
10	10
11	11	import type { Database } from "bun:sqlite";
	12	import { hostname } from "node:os";
12	13	import { BUDGET_PREFIX } from "../kernel/budgets.ts";
13	14	import { id } from "../kernel/id.ts";
14	15
		@@ -490,13 +491,59 @@ export function lastAgentRun(
490	491	return row ? toRun(row) : undefined;
491	492	}
492	493
493		/** The process working these runs says it still is. */
	494	/**
	495	* This process, as a run's lease names it: host, pid and start, so a reused pid is not mistaken
	496	* for it.
	497	*/
	498	export const HOLDER = `${hostname()}:${process.pid}:${Date.now()}`;
	499
	500	/** The process working these runs says it still is, and who it is. */
494	501	export function heartbeatRuns(db: Database, runIds: readonly string[]): void {
495	502	if (runIds.length === 0) return;
496	503	db.query(
497		`UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
498		WHERE state = 'running' AND id IN (${runIds.map((_, n) => `?${n + 1}`).join(",")})`,
499		).run(...runIds);
	504	`UPDATE run SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'), holder = ?1
	505	WHERE state = 'running' AND id IN (${runIds.map((_, n) => `?${n + 2}`).join(",")})`,
	506	).run(HOLDER, ...runIds);
	507	}
	508
	509	/**
	510	* Lapse the lease of every running run whose holder was a process of this
	511	* host that is gone (#12143): the next sweep settles it at once, instead of
	512	* after the 90 s a silent lease waits. A holder on another host, or one
	513	* that cannot be told apart, is left to its lease. `alive` is a seam for tests.
	514	*/
	515	export function lapseOrphans(
	516	db: Database,
	517	self: string = HOLDER,
	518	alive: (pid: number) => boolean = pidAlive,
	519	): number {
	520	const host = self.split(":")[0]!;
	521	const rows = db
	522	.query<{ id: string; holder: string }, []>(
	523	"SELECT id, holder FROM run WHERE state = 'running' AND holder IS NOT NULL",
	524	)
	525	.all();
	526	let lapsed = 0;
	527	for (const row of rows) {
	528	if (row.holder === self) continue;
	529	const [h, pid] = row.holder.split(":");
	530	if (h !== host \|\| !/^\d+$/.test(pid ?? "")) continue;
		sentinelcrew 1d ago A same-hostname holder in another pid namespace, or another instance, reads as ESRCH and its live run is lapsed. Now that this runs every tick, restrict it to boot or add an instance id to the holder.
	531	// Our own pid under another start is a predecessor (a container's pid 1 again): gone.
	532	if (Number(pid) !== process.pid && alive(Number(pid))) continue;
	533	lapsed += db
	534	.query("UPDATE run SET heartbeat_at = '1970-01-01T00:00:00.000Z' WHERE id = ?1 AND state = 'running' AND holder = ?2")
	535	.run(row.id, row.holder).changes;
	536	}
	537	return lapsed;
	538	}
	539
	540	function pidAlive(pid: number): boolean {
	541	try {
	542	process.kill(pid, 0);
	543	return true;
	544	} catch (e) {
	545	return (e as NodeJS.ErrnoException).code === "EPERM";
	546	}
500	547	}
501	548
502	549	/**
		@@ -0,0 +1,84 @@
	1	/**
	2	* Lock files a killed git process left behind (#12143). git takes `<ref>.lock`
	3	* for the milliseconds a ref moves; a `kill -9` in that window leaves it, and
	4	* every later update of that ref answers "File exists" — through a reboot —
	5	* until someone deletes it. A lock older than any git operation could last is
	6	* nobody's: `sweepLocks` removes those, under `refs/` and the two at the top,
	7	* at boot only, when no git of ours is alive; a merge removes only its own
	8	* ref's (`sweepRefLock`).
	9	*/
	10
	11	import { readdir, rm, stat } from "node:fs/promises";
	12	import { join } from "node:path";
	13
	14	/**
	15	* What boot and a landing treat as older than any live ref update (git holds one for
	16	* milliseconds).
	17	*/
	18	export const STALE_LOCK_MS = 5_000;
	19	/** The message git gives when a ref's lock is held. */
	20	export const LOCK_HELD = /unable to create '[^']*\.lock'/i;
	21
	22	export function isLockHeld(stderr: string): boolean {
	23	return LOCK_HELD.test(stderr);
	24	}
	25
	26	async function walk(dir: string, out: string[], depth: number): Promise<void> {
	27	if (depth > 12) return;
	28	let names: import("node:fs").Dirent[];
	29	try {
	30	names = await readdir(dir, { withFileTypes: true });
	31	} catch {
	32	return;
	33	}
	34	for (const entry of names) {
	35	const path = join(dir, entry.name);
	36	if (entry.isDirectory()) await walk(path, out, depth + 1);
	37	else if (entry.isFile() && entry.name.endsWith(".lock")) out.push(path);
	38	}
	39	}
	40
	41	/** Remove the `.lock` files of a git directory older than `minAgeMs`; the paths removed. /
	42	export async function sweepLocks(
	43	gitDir: string,
	44	minAgeMs: number = STALE_LOCK_MS,
	45	now: number = Date.now(),
	46	): Promise<string[]> {
	47	const found: string[] = [join(gitDir, "HEAD.lock"), join(gitDir, "packed-refs.lock")];
	48	await walk(join(gitDir, "refs"), found, 0);
	49	const removed: string[] = [];
	50	for (const path of found) {
	51	try {
	52	const s = await stat(path);
		sentinelcrew 1d ago outdated Removes locks held by live git processes if they are older than 5 s (packed-refs.lock during pack-refs/gc, HEAD.lock). That breaks git's mutual exclusion. Scope it to the one ref at merge time and sweep the whole repository only at boot.
	53	if (!s.isFile() \|\| now - s.mtimeMs < minAgeMs) continue;
	54	await rm(path, { force: true });
	55	removed.push(path);
	56	} catch {
	57	// not there, or gone as we looked
	58	}
	59	}
	60	return removed;
	61	}
	62
	63	/**
	64	* The lock of one ref, when it is older than `minAgeMs`: what a merge may
	65	* remove while other git processes run. Never another file's lock: a live
	66	* `pack-refs` holds `packed-refs.lock` for longer than any ref update.
	67	*/
	68	export async function sweepRefLock(
	69	gitDir: string,
	70	ref: string,
	71	minAgeMs: number = STALE_LOCK_MS,
	72	now: number = Date.now(),
	73	): Promise<boolean> {
	74	if (!/^refs\/heads\/[A-Za-z0-9._\/-]+$/.test(ref) \|\| ref.includes("..")) return false;
	75	const path = join(gitDir, `${ref}.lock`);
	76	try {
	77	const s = await stat(path);
	78	if (!s.isFile() \|\| now - s.mtimeMs < minAgeMs) return false;
	79	await rm(path, { force: true });
	80	return true;
	81	} catch {
	82	return false;
	83	}
	84	}
		@@ -8,6 +8,7 @@
8	8	import { err, ok, type Result } from "../kernel/result.ts";
9	9	import { diffMemoryKey, recalledBase, rememberBase } from "./diff-memory.ts";
10	10	import { runGit, runGitCapture } from "./env.ts";
	11	import { isLockHeld, sweepRefLock } from "./locks.ts";
11	12	import { isSafeOid, isSafeRef, isSafeTreePath } from "./guard.ts";
12	13
13	14	export type MergeStrategy = "merge" \| "squash";
		@@ -21,6 +22,11 @@ export type MergeError =
21	22	\| { kind: "conflict"; files: string[] }
22	23	\| { kind: "unsafe" }
23	24	\| { kind: "base_moved" }
	25	/**
	26	* A ref's lock file is held (by a live git process, or a recent one's leftover): not the base
	27	* moving.
	28	*/
	29	\| { kind: "ref_locked" }
24	30	\| { kind: "git"; message: string };
25	31
26	32	export interface MergeResult {
		@@ -221,6 +227,11 @@ export async function merge(
221	227	* nothing is committed.
222	228	*/
223	229	treeOid?: string;
	230	/**
	231	* Called with the commit about to land, before the ref moves and in the
	232	* same tick: the intent to merge, recorded where a kill can find it (#12143).
	233	*/
	234	beforeMove?: (mergeOid: string) => void;
224	235	},
225	236	): Promise<Result<MergeResult, MergeError>> {
226	237	if (
		@@ -256,10 +267,20 @@ export async function merge(
256	267	const mergeOid = commit.value.trim();
257	268
258	269	// CAS the branch forward: fails if someone pushed between our read and now.
259		const update = await runGit(
260		["update-ref", `refs/heads/${input.baseBranch}`, mergeOid, input.baseOid],
261		{ cwd: dir },
262		);
	270	input.beforeMove?.(mergeOid);
	271	const move = () =>
	272	runGit(
	273	["update-ref", `refs/heads/${input.baseBranch}`, mergeOid, input.baseOid],
	274	{ cwd: dir },
	275	);
	276	let update = await move();
	277	// A lock a killed git left behind is not the base moving (#12143): one
	278	// of this ref older than any live update is removed and the move tried once more; a
	279	// lock still held is reported as held.
		sentinelcrew 1d ago This sweeps every old lock in the repo on any lock-held error, not just the lock of this ref. Remove only `refs/heads/<base>.lock`, and only when it is stale.
	280	if (!update.ok && isLockHeld(update.error.stderr)) {
	281	if (await sweepRefLock(dir, `refs/heads/${input.baseBranch}`)) update = await move();
	282	if (!update.ok && isLockHeld(update.error.stderr)) return err({ kind: "ref_locked" });
	283	}
263	284	if (!update.ok) return err({ kind: "base_moved" });
264	285
265	286	return ok({ mergeOid, previousOid: input.baseOid });