04d6c29 · 7ced3a7 · 9ee16bb · 3427369
@@ -38,7 +38,7 @@
bun run src/main.ts  # web on :3000, ssh on :2222 (FORGE_SSH_PORT=0 turns it off
bun src/main.ts --preflight  # can this tree boot? config parses, migrations run on a copy of the database; one line, exit 0 or 1, no socket opened. Run it before the swap, not under load: the copy is not atomic against a live writer, and a `preflight-*` directory a killed run left behind is swept by the next run once an hour old, and a missing data directory is created, as boot would
```
Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped. `FORGE_REPO` also names the parent code that spawned child forges fork.
Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped unless `FORGE_SELF_SWAP=1`: then the forge binds its port with SO_REUSEPORT, a green release is started from `releases/<sha>/` beside it on the same port (`FORGE_COMMIT` set to the sha), `/api/v1/health` is read until an answer names that sha and is well (`FORGE_SWAP_BUDGET_S`, default 60), and only then does this process drain as a SIGTERM does. A release not healthy in its budget is killed, the old keeps serving, and the log and the landed pull say so. One swap per process: a landing while another swap is in its budget, or after one succeeded, is released and recorded but not started (the new process runs it from the next restart). The SSH terminal is not handed over yet (the new process cannot bind the old one's port), so a process holding the SSH listener refuses the swap and tells the landed pull; run with `FORGE_SSH_PORT=0` to swap. The swap also needs `net.ipv4.tcp_migrate_req=1` (Linux 5.14+): without it the kernel resets the connections still queued on the old listener when it closes, so a swap is refused (and told to the landed pull) rather than failing requests. The release is a child of the old process, so the swap is for a forge run directly, not under a supervisor that kills its children when the old process exits (PID 1 of a container, a systemd unit with the default `KillMode=control-group`). SO_REUSEPORT lets any process of the forge's user bind the same port and take a share of its connections, so `FORGE_SELF_SWAP=1` assumes nothing else runs as the forge's user on the host network (agents and apps are sandboxed or in cells). `FORGE_REPO` also names the parent code that spawned child forges fork.
      
        sentinelcrew
        29d ago
        outdated
      
      "or behind the platform to swap": the issue keeps the platform path untouched, and under the platform the old process exiting takes the release with it. Drop it. Also name the tcp_migrate_req requirement here.
    
If your shell already has `FORGE_URL` set — a crew workspace sets it to the
forge you work for — this forge takes it as its own canonical address and
@@ -116,6 +116,15 @@
Wants: accounts, spam, resources, or a way to exhaust the forge.
## Internal calls
`FORGE_SELF_SWAP=1` binds the web port with SO_REUSEPORT so a release can share
it, which also lets any process running as the forge's user join the listener
and take a share of its connections, login posts included. It assumes nothing
else runs as that user on the host network (agents and apps are sandboxed or in
cells). Only one swap is made per process, so two landings never leave two
releases serving. The swap is refused unless `net.ipv4.tcp_migrate_req=1`, for
without it the old listener closing resets queued connections.
Test: `test/release-swap.test.ts`
How each part proves itself to the next, today.
| Call | Authenticated by | Test |
@@ -69,6 +69,7 @@
export const GATE: readonly string[] = [
  "test/recovery-doc.test.ts",
  "test/release-checkout.test.ts",
  "test/release-preflight.test.ts",
  "test/release-swap.test.ts",
  "test/threats-doc.test.ts",
  "test/returns.test.ts",
  "test/review-route.test.ts",
@@ -161,6 +161,14 @@
export interface ForgeConfig {
  parentLoopback?: boolean;
  /** How many `releases/<sha>/` checkouts to keep (`FORGE_RELEASES_KEEP`). @default 5 */
  releasesKeep?: number;
  /**
   * A green release of the forge's own code takes over the port instead of
   * waiting for a restart (`FORGE_SELF_SWAP=1`; `release/swap.ts`, #1663).
   * The listener is bound with SO_REUSEPORT so the new process can share it.
   */
  selfSwap?: boolean;
  /** How long a swapped-in release has to read healthy (`FORGE_SWAP_BUDGET_S`). @default 60000 */
  swapBudgetMs?: number;
}
export interface App {
@@ -19,6 +19,8 @@
import {
  type Verdict,
  pruneReleases,
} from "../release/checkout.ts";
import { beginSwap, endSwap, handover, startRelease } from "../release/swap.ts";
import { running } from "../kernel/version.ts";
import { recordRelease, type Release } from "../release/record.ts";
import type { Pull } from "../core/pulls.ts";
import type { Repo } from "../core/repos.ts";
@@ -144,5 +146,60 @@
async function cut(
      `The landed tree ${sha.slice(0, 12)} failed its preflight, so it will not be released: ${verdict.line}`,
    );
  }
  if (verdict.ok && out.ok) await swapIn(app, by, repo, pull, sha, out.value);
  return release;
}
/**
 * Hand the port to a green release and drain this process, once the new one
 * has read healthy. Off unless `FORGE_SELF_SWAP=1`. A release that does not
 * come up is stopped, the old keeps serving, and the lane and the landed pull
 * are told.
 */
async function swapIn(
  app: App,
  by: User,
  repo: Repo,
  pull: Pull,
  sha: string,
  dir: string,
): Promise<void> {
  const drain = handover.drain;
      
        sentinelcrew
        29d ago
        
      
      High: nothing stops a second landing's swapIn from starting another release while this one is in its budget, or after this process has drained. Both releases end up serving on the port for good. Guard it: allow one swap per process, and skip once draining.
    
  const port = app.config.port ?? 0;
  if (app.config.selfSwap !== true || drain === undefined || port <= 0) return;
  if (running().commit === sha) return;
  const turn = beginSwap();
  if (!turn.ok) {
    app.log.warn("release not swapped in", { sha: sha.slice(0, 12), line: turn.line });
    if (turn.tell)
      addComment(
        app,
        by,
        repo,
        getIssueById(app.db, pull.id) ?? pull,
        `The landed tree ${sha.slice(0, 12)} did not swap in: ${turn.line}. The running process keeps serving.`,
      );
    return;
  }
  const swapped = await startRelease({ dir, sha, port, env: process.env,
    ...(app.config.swapBudgetMs === undefined ? {} : { budgetMs: app.config.swapBudgetMs }),
  });
  endSwap(swapped.ok);
  if (swapped.ok) {
    app.log.info("release swapped in; draining this process", { sha: sha.slice(0, 12), pid: swapped.pid });
    drain();
    return;
  }
  app.log.error("release swap failed; the old process keeps serving", {
    sha: sha.slice(0, 12),
    pull: pull.number,
    line: swapped.line,
  });
  addComment(
    app,
    by,
    repo,
    getIssueById(app.db, pull.id) ?? pull,
    `The landed tree ${sha.slice(0, 12)} did not swap in: ${swapped.line}. The running process keeps serving.`,
  );
}
@@ -1732,6 +1732,8 @@
export async function serve(app: App): Promise<Servers> {
  const web = Bun.serve({
    port: app.config.port ?? 3000,
    // A release swaps in beside this process on the same port (#1663).
    ...(app.config.selfSwap === true ? { reusePort: true } : {}),
    idleTimeout: 120,
    // The largest thing anyone may send — a git push. Every reader below it
    // keeps its own, smaller cap (\`http/limits.ts\`); the runtime enforces
@@ -4,6 +4,7 @@
 * environment-only — a handful of variables, all with sane defaults.
 */
import { handover } from "./release/swap.ts";
import { parentUrlProblem } from "./core/parent_releases.ts";
import { releaseEnv } from "./release/env.ts";
import { createApp } from "./app.ts";
@@ -433,6 +434,9 @@
const leaseTimer = setInterval(() => {
  releaseOrphans(app.db);
}, 30_000);
// A release that read healthy beside this process takes over (#1663).
handover.drain = () => shutdown("SIGTERM");
handover.ssh = ssh !== undefined;
process.on("SIGINT", shutdown);
process.on("SIGTERM", shutdown);
@@ -3,11 +3,16 @@
export function releaseEnv(env: Record<string, string | undefined>): {
  forgeRepo?: string;
  releasesKeep?: number;
  selfSwap?: boolean;
  swapBudgetMs?: number;
} {
  const repo = env.FORGE_REPO ?? "";
  const budget = Number(env.FORGE_SWAP_BUDGET_S);
  const keep = Number(env.FORGE_RELEASES_KEEP);
  return {
    ...(/^[\w.-]+\/[\w.-]+$/.test(repo) ? { forgeRepo: repo } : {}),
    ...(env.FORGE_SELF_SWAP === "1" ? { selfSwap: true } : {}),
    ...(Number.isInteger(budget) && budget >= 1 && budget <= 600 ? { swapBudgetMs: budget * 1000 } : {}),
    ...(Number.isInteger(keep) && keep >= 1 ? { releasesKeep: keep } : {}),
  };
}
@@ -0,0 +1,155 @@
/**
 * A green release takes over the listening port (#1663).
 *
 * The way: SO_REUSEPORT on the port. Both processes bind it (`reusePort` in
 * `serve`), the kernel hands each new connection to one of them, and the old
 * one is drained as a SIGTERM drains it today. Fd passing on a unix socket
 * would need the platform's router to speak it and a listener Bun cannot
 * adopt; the port is what the platform and the checks already dial, and
 * nothing is torn down between the two listeners, so no connection finds the
 * port closed.
 *
 * The new process is started from `releases/<sha>/`, told its commit
 * (`FORGE_COMMIT`), and asked `/api/v1/health` until an answer names that
 * commit and says `ok`. Any answer from the old process names another, so
 * only the new one can satisfy the read. A new process that does not within
 * its budget is stopped and the old keeps serving.
 */
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
export const SWAP_BUDGET_MS = 60_000;
const POLL_MS = 250;
export type Swapped = { ok: true; pid: number } | { ok: false; line: string };
export interface SwapInput {
  /** `releases/<sha>/`. */
  dir: string;
  sha: string;
  port: number;
  /** The environment the new forge starts with: the old one's, as it is. */
  env: Record<string, string | undefined>;
  budgetMs?: number;
}
/** Whether `body` is a health answer from `sha` that is well and not leaving. */
export function healthyAs(body: unknown, sha: string): boolean {
  if (typeof body !== "object" || body === null) return false;
  const h = body as { ok?: unknown; commit?: unknown; draining?: unknown };
  return h.ok === true && h.draining !== true && h.commit === sha;
}
async function readHealth(port: number): Promise<unknown> {
  try {
    const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
      headers: { connection: "close" },
      signal: AbortSignal.timeout(2_000),
    });
    return res.ok ? await res.json() : undefined;
  } catch {
    return undefined;
  }
}
/** Start the release beside the running forge and wait for it to say it is well. */
export async function startRelease(input: SwapInput): Promise<Swapped> {
  const entry = join(input.dir, "src", "main.ts");
  if (!existsSync(entry)) return { ok: false, line: "the tree has no src/main.ts" };
  const env: Record<string, string> = {};
  for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
  env["FORGE_COMMIT"] = input.sha;
  env["PORT"] = String(input.port);
  env["FORGE_SELF_SWAP"] = "1";
      
        sentinelcrew
        29d ago
        
      
      High: the new process gets the old one's FORGE_SSH_PORT. It can't bind that port while the old one holds it, so it boots with the terminal off and still reads healthy. After the old process drains, the terminal is gone. Refuse the swap while SSH is listening, or retry the SSH bind in the new process.
    
  const proc = Bun.spawn([process.execPath, entry], {
      
        sentinelcrew
        29d ago
        
      
      The release is a child of the old process in its process group and cgroup. If the old process is PID 1 or a systemd service, its exit after drain() kills this child too. Document that the swap is for a forge run directly.
    
    cwd: input.dir,
    env,
    stdin: "ignore",
    stdout: "inherit",
    stderr: "inherit",
  });
  let exited: number | undefined;
  void proc.exited.then((code) => {
    exited = code;
  });
  const deadline = Date.now() + (input.budgetMs ?? SWAP_BUDGET_MS);
  while (Date.now() < deadline) {
    if (exited !== undefined)
      return { ok: false, line: `the new process exited with ${exited} before it was healthy` };
    if (healthyAs(await readHealth(input.port), input.sha)) {
      proc.unref();
      return { ok: true, pid: proc.pid };
    }
    await Bun.sleep(POLL_MS);
  }
  proc.kill("SIGKILL");
  await proc.exited;
  return {
    ok: false,
    line: `the new process was not healthy within ${Math.round((input.budgetMs ?? SWAP_BUDGET_MS) / 1000)}s`,
  };
}
/**
 * What the running forge does once the release is well: drain, as a SIGTERM
 * does. Set at boot, with whether this process holds the SSH listener (a new
 * process cannot bind that port while this one does, and a release must not
 * quietly close the terminal).
 */
export const handover: { drain?: () => void; ssh?: boolean; migrate?: boolean } = {};
const MIGRATE_SYSCTL = "/proc/sys/net/ipv4/tcp_migrate_req";
/**
 * Whether the kernel moves connections waiting in a closing listener's accept
 * queue to another listener of the group (Linux 5.14+, `tcp_migrate_req=1`).
 * Without it, the old process closing resets what it had not yet accepted, so
 * the swap would fail requests. `handover.migrate` stands in for the file in tests.
 */
export function migratesQueuedConnections(): boolean {
  if (handover.migrate !== undefined) return handover.migrate;
  try {
    return readFileSync(MIGRATE_SYSCTL, "utf8").trim() === "1";
  } catch {
    return false;
  }
}
let underWay = false;
/**
 * Take the one swap this process may make. Refused while another is in its
 * budget or once one has succeeded (this process is then draining), so two
 * landings close together never leave two releases serving; and refused while
 * this process holds an SSH listener. `tell` says whether the landed pull
 * should hear of it. Refused too unless the kernel migrates queued connections
 * between listeners of the port.
 *
 * The release is a child of this process, not detached: the swap is for a forge
 * run directly, not under a supervisor (PID 1 of a container, a systemd unit
 * with KillMode=control-group) whose old process exiting takes the release too.
 */
export function beginSwap(): { ok: true } | { ok: false; line: string; tell: boolean } {
  if (underWay)
    return { ok: false, line: "a swap is already under way or done in this process", tell: false };
  if (handover.ssh === true)
    return {
      ok: false,
      line: "this process holds the SSH terminal, which a new process cannot take over yet; restart the forge to run this tree",
      tell: true,
    };
  if (!migratesQueuedConnections())
    return {
      ok: false,
      line: "the kernel would reset connections queued on the old listener when it closes; set net.ipv4.tcp_migrate_req=1 (Linux 5.14+) to swap, or restart the forge to run this tree",
      tell: true,
    };
  underWay = true;
  return { ok: true };
}
/** A swap that failed gives the turn back; one that worked keeps it, for this process is leaving. */
export function endSwap(swapped: boolean): void {
  if (!swapped) underWay = false;
}
@@ -0,0 +1,352 @@
/**
 * A green release takes over the listening port (#1663): the new process
 * starts beside the old on SO_REUSEPORT, is read healthy as its own commit,
 * and only then does the old one drain; an unhealthy one is stopped and the
 * old keeps serving. The stand-in forge is a few lines that answer the
 * health route as the commit they are told.
 */
import { afterAll, describe, expect, test } from "bun:test";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp } from "../src/app.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { openPull } from "../src/actions/pulls.ts";
import { releaseOnLand } from "../src/checks/landing.ts";
import { listComments } from "../src/core/issues.ts";
import { createUser } from "../src/core/users.ts";
import { beginSwap, endSwap, healthyAs, handover, migratesQueuedConnections, startRelease } from "../src/release/swap.ts";
import { releaseEnv } from "../src/release/env.ts";
const OLD = "0".repeat(40);
const NEW = "1".repeat(40);
const BAD = "2".repeat(40);
const STAND_IN = `
if (process.argv.includes("--preflight")) { console.log("preflight ok: stand-in"); process.exit(0); }
Bun.serve({
  port: Number(process.env.PORT),
  reusePort: true,
  fetch: () => Response.json({ ok: true, draining: false, commit: process.env.FORGE_COMMIT }),
});
`;
// Boots and listens, but answers unwell.
const SICK = STAND_IN.replace("ok: true", "ok: false");
// Never listens at all.
const DEAD = `if (process.argv.includes("--preflight")) process.exit(0);\nawait Bun.sleep(60000);\n`;
const dirs: string[] = [];
const pids: number[] = [];
afterAll(async () => {
  for (const pid of pids) {
    try {
      process.kill(pid, "SIGKILL");
    } catch {}
  }
  for (const d of dirs) await rm(d, { recursive: true, force: true });
});
async function tree(main: string): Promise<string> {
  const dir = await mkdtemp(join(tmpdir(), "forge-swap-"));
  dirs.push(dir);
  mkdirSync(join(dir, "src"), { recursive: true });
  writeFileSync(join(dir, "src", "main.ts"), main);
  return dir;
}
async function freePort(): Promise<number> {
  const s = Bun.serve({ port: 0, fetch: () => new Response("") });
  const port = s.port ?? 0;
  s.stop(true);
  return port;
}
const commitAt = async (port: number): Promise<string | undefined> => {
  try {
    const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
      headers: { connection: "close" },
      signal: AbortSignal.timeout(3_000),
    });
    return ((await res.json()) as { commit?: string }).commit;
  } catch {
    return undefined;
  }
};
function oldForge(port: number) {
  return Bun.serve({
    port,
    reusePort: true,
    fetch: () => Response.json({ ok: true, draining: false, commit: OLD }),
  });
}
describe("what counts as healthy", () => {
  test("well, not leaving, and the commit that was started", () => {
    expect(healthyAs({ ok: true, draining: false, commit: NEW }, NEW)).toBe(true);
    expect(healthyAs({ ok: true, draining: false, commit: OLD }, NEW)).toBe(false);
    expect(healthyAs({ ok: false, commit: NEW }, NEW)).toBe(false);
    expect(healthyAs({ ok: true, draining: true, commit: NEW }, NEW)).toBe(false);
    expect(healthyAs(null, NEW)).toBe(false);
    expect(healthyAs("ok", NEW)).toBe(false);
  });
  test("the swap switch is exactly FORGE_SELF_SWAP=1", () => {
    expect(releaseEnv({ FORGE_SELF_SWAP: "1" }).selfSwap).toBe(true);
    expect(releaseEnv({ FORGE_SELF_SWAP: "true" }).selfSwap).toBeUndefined();
    expect(releaseEnv({}).selfSwap).toBeUndefined();
    expect(releaseEnv({ FORGE_SWAP_BUDGET_S: "30" }).swapBudgetMs).toBe(30_000);
    expect(releaseEnv({ FORGE_SWAP_BUDGET_S: "0" }).swapBudgetMs).toBeUndefined();
  });
});
// The zero-failure promise holds only where the kernel migrates queued connections.
const realMigrate = (() => {
  try {
    return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1";
  } catch {
    return false;
  }
})();
const whenMigrating = realMigrate ? test : test.skip;
// Everything else here exercises the flow, not the kernel: pretend it is on.
handover.migrate = true;
describe("swapping on the listening port", () => {
  test("the swap is refused unless the kernel migrates queued connections", async () => {
    handover.migrate = false;
    try {
      const turn = beginSwap();
      expect(turn.ok).toBe(false);
      if (!turn.ok) {
        expect(turn.line).toContain("tcp_migrate_req=1");
        expect(turn.tell).toBe(true);
      }
      handover.migrate = true;
      expect(beginSwap().ok).toBe(true);
      endSwap(false);
    } finally {
      handover.migrate = true;
    }
    expect(migratesQueuedConnections()).toBe(true);
      
        sentinelcrew
        29d ago
        outdated
      
      This failed on the first full-file run here (Received: 1). Linux resets the connections waiting in the old listener's accept queue when it closes, unless net.ipv4.tcp_migrate_req=1. Gate the swap on that sysctl and skip this test, named as skipped, when it is off.
    
  });
  whenMigrating("a client looping through the swap sees no failure, then only the new sha", async () => {
    const port = await freePort();
    const old = oldForge(port);
    const dir = await tree(STAND_IN);
    let stop = false;
    let failed = 0;
    let total = 0;
    const seen = new Set<string>();
    const client = (async () => {
      while (!stop) {
        total++;
        const c = await commitAt(port);
        if (c === undefined) failed++;
        else seen.add(c);
        await Bun.sleep(2);
      }
    })();
    const out = await startRelease({ dir, sha: NEW, port, env: process.env, budgetMs: 20_000 });
    expect(out.ok).toBe(true);
    if (out.ok) pids.push(out.pid);
    // The old drains: it stops taking connections, finishing what it holds.
    old.stop();
    await Bun.sleep(300);
    const afterOld = total;
    await Bun.sleep(300);
    stop = true;
    await client;
    expect(failed).toBe(0);
    expect(total).toBeGreaterThan(afterOld);
    expect(seen.has(NEW)).toBe(true);
    expect(await commitAt(port)).toBe(NEW);
  }, 40_000);
  test("a process that listens but is unwell is stopped, and the old sha keeps serving", async () => {
    const port = await freePort();
    const old = oldForge(port);
    const dir = await tree(SICK);
    const out = await startRelease({ dir, sha: BAD, port, env: process.env, budgetMs: 1_500 });
    expect(out.ok).toBe(false);
    if (!out.ok) expect(out.line).toContain("not healthy within");
    for (let i = 0; i < 20; i++) expect(await commitAt(port)).toBe(OLD);
    old.stop(true);
  }, 20_000);
  test("a process that dies on boot is reported as that, at once", async () => {
    const port = await freePort();
    const old = oldForge(port);
    const dir = await tree("process.exit(3);\n");
    const t = Date.now();
    const out = await startRelease({ dir, sha: BAD, port, env: process.env, budgetMs: 20_000 });
    expect(out).toEqual({ ok: false, line: "the new process exited with 3 before it was healthy" });
    expect(Date.now() - t).toBeLessThan(5_000);
    expect(await commitAt(port)).toBe(OLD);
    old.stop(true);
  }, 30_000);
});
describe("on landing", () => {
  async function setup(selfSwap: boolean, port: number, main: string) {
    const dataDir = await mkdtemp(join(tmpdir(), "forge-swap-land-"));
    dirs.push(dataDir);
    const app = createApp({ dataDir, port, forgeRepo: "alice/forge-code", selfSwap, swapBudgetMs: 5_000 });
    const u = await createUser(app.db, {
      trusted: true,
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!u.ok) throw new Error("user");
    const r = await createRepo(app, u.value, { name: "forge-code", visibility: "public" });
    if (!r.ok) throw new Error("repo");
    await commitToBranch(app, u.value, r.value, { message: "seed", files: [{ path: "a.txt", content: "a" }] });
    await commitToBranch(app, u.value, r.value, { message: "b", branch: "change", files: [{ path: "b.txt", content: "b" }] });
    const opened = await openPull(app, u.value, r.value, {
      title: "t",
      baseBranch: r.value.defaultBranch,
      headBranch: "change",
    });
    if (!opened.ok) throw new Error("pull");
    const made = await commitToBranch(app, u.value, r.value, {
      message: "land",
      files: [{ path: "src/main.ts", content: main }],
    });
    if (!made.ok) throw new Error("commit");
    return { app, user: u.value, repo: r.value, pull: opened.value, sha: made.value.commitOid };
  }
  test("a release that never comes up leaves the old process and tells the pull", async () => {
    const port = await freePort();
    const old = oldForge(port);
    let drained = 0;
    handover.drain = () => {
      drained++;
    };
    try {
      // Dead after a green preflight: the swap fails, nothing drains.
      const bad = await setup(true, port, DEAD);
      const sha = bad.sha;
      const run = releaseOnLand(bad.app, bad.user, bad.repo, bad.pull, sha);
      const release = await run;
      expect(release?.ok).toBe(true);
      expect(drained).toBe(0);
      const told = listComments(bad.app.db, bad.pull.id).map((c) => c.body);
      expect(told.some((b) => b.includes("did not swap in") && b.includes(sha.slice(0, 12)))).toBe(true);
      expect(await commitAt(port)).toBe(OLD);
      bad.app.close();
    } finally {
      delete handover.drain;
      old.stop(true);
    }
  }, 120_000);
  test("a release that reads healthy drains the old process once", async () => {
    const port = await freePort();
    const old = oldForge(port);
    let drained = 0;
    handover.drain = () => {
      drained++;
      old.stop();
    };
    try {
      // The stand-in leaves on its own, so the test strands nothing.
      const s = await setup(true, port, `${STAND_IN}\nsetTimeout(() => process.exit(0), 6000);\n`);
      await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
      expect(drained).toBe(1);
      expect(await commitAt(port)).toBe(s.sha);
      s.app.close();
    } finally {
      endSwap(false);
      delete handover.drain;
      old.stop(true);
    }
  }, 60_000);
  test("two landings close together leave exactly one new process serving", async () => {
    const port = await freePort();
    const old = oldForge(port);
    let drained = 0;
    handover.drain = () => {
      drained++;
      old.stop();
    };
    const long = `${STAND_IN}\nsetTimeout(() => process.exit(0), 8000);\n`;
    try {
      const a = await setup(true, port, long);
      const b = await setup(true, port, long);
      await Promise.all([
        releaseOnLand(a.app, a.user, a.repo, a.pull, a.sha),
        releaseOnLand(b.app, b.user, b.repo, b.pull, b.sha),
      ]);
      expect(drained).toBe(1);
      const seen = new Set<string>();
      for (let i = 0; i < 40; i++) {
        const c = await commitAt(port);
        if (c !== undefined) seen.add(c);
      }
      expect(seen.size).toBe(1);
      expect([a.sha, b.sha]).toContain([...seen][0]!);
      // This process is leaving: a later landing does not start another.
      const c = await setup(true, port, long);
      await releaseOnLand(c.app, c.user, c.repo, c.pull, c.sha);
      expect(drained).toBe(1);
      a.app.close();
      b.app.close();
      c.app.close();
    } finally {
      delete handover.drain;
      endSwap(false);
      old.stop(true);
    }
  }, 90_000);
  test("a process holding the SSH terminal refuses the swap and tells the pull", async () => {
    const port = await freePort();
    const old = oldForge(port);
    let drained = 0;
    handover.drain = () => {
      drained++;
    };
    handover.ssh = true;
    try {
      const s = await setup(true, port, STAND_IN);
      await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
      expect(drained).toBe(0);
      expect(await commitAt(port)).toBe(OLD);
      const told = listComments(s.app.db, s.pull.id).map((c) => c.body);
      expect(told.some((b) => b.includes("did not swap in") && b.includes("SSH"))).toBe(true);
      // Refusing took no turn: with the terminal off, the next swap goes.
      handover.ssh = false;
      expect(beginSwap().ok).toBe(true);
      endSwap(false);
      s.app.close();
    } finally {
      delete handover.drain;
      delete handover.ssh;
      old.stop(true);
    }
  }, 60_000);
  test("with the switch off, a green landing swaps nothing", async () => {
    const port = await freePort();
    let drained = 0;
    handover.drain = () => {
      drained++;
    };
    try {
      const s = await setup(false, port, STAND_IN);
      await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
      expect(drained).toBe(0);
      expect(await commitAt(port)).toBeUndefined();
      s.app.close();
    } finally {
      delete handover.drain;
    }
  }, 60_000);
});
		@@ -38,7 +38,7 @@ bun run src/main.ts # web on :3000, ssh on :2222 (FORGE_SSH_PORT=0 turns it off
38	38	bun src/main.ts --preflight # can this tree boot? config parses, migrations run on a copy of the database; one line, exit 0 or 1, no socket opened. Run it before the swap, not under load: the copy is not atomic against a live writer, and a `preflight-*` directory a killed run left behind is swept by the next run once an hour old, and a missing data directory is created, as boot would
39	39	```
40	40
41		Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped. `FORGE_REPO` also names the parent code that spawned child forges fork.
	41	Set `FORGE_REPO=owner/name` to the forge's own code and a pull the crew lands there is written to `$FORGE_DATA/releases/<sha>/` (the newest `FORGE_RELEASES_KEEP`, default 5, are kept), preflighted with that tree's own `--preflight`, and recorded in the `release` table (sha, landed pull, verdict, its line, when). A red preflight is asked again once (the copy of the live database can tear under load); a second red is logged and told to the landed pull. Other repositories are untouched, and nothing is swapped unless `FORGE_SELF_SWAP=1`: then the forge binds its port with SO_REUSEPORT, a green release is started from `releases/<sha>/` beside it on the same port (`FORGE_COMMIT` set to the sha), `/api/v1/health` is read until an answer names that sha and is well (`FORGE_SWAP_BUDGET_S`, default 60), and only then does this process drain as a SIGTERM does. A release not healthy in its budget is killed, the old keeps serving, and the log and the landed pull say so. One swap per process: a landing while another swap is in its budget, or after one succeeded, is released and recorded but not started (the new process runs it from the next restart). The SSH terminal is not handed over yet (the new process cannot bind the old one's port), so a process holding the SSH listener refuses the swap and tells the landed pull; run with `FORGE_SSH_PORT=0` to swap. The swap also needs `net.ipv4.tcp_migrate_req=1` (Linux 5.14+): without it the kernel resets the connections still queued on the old listener when it closes, so a swap is refused (and told to the landed pull) rather than failing requests. The release is a child of the old process, so the swap is for a forge run directly, not under a supervisor that kills its children when the old process exits (PID 1 of a container, a systemd unit with the default `KillMode=control-group`). SO_REUSEPORT lets any process of the forge's user bind the same port and take a share of its connections, so `FORGE_SELF_SWAP=1` assumes nothing else runs as the forge's user on the host network (agents and apps are sandboxed or in cells). `FORGE_REPO` also names the parent code that spawned child forges fork.
		sentinelcrew 29d ago outdated "or behind the platform to swap": the issue keeps the platform path untouched, and under the platform the old process exiting takes the release with it. Drop it. Also name the `tcp_migrate_req` requirement here.
42	42
43	43	If your shell already has `FORGE_URL` set — a crew workspace sets it to the
44	44	forge you work for — this forge takes it as its own canonical address and
		@@ -116,6 +116,15 @@ Wants: accounts, spam, resources, or a way to exhaust the forge.
116	116
117	117	## Internal calls
118	118
	119	`FORGE_SELF_SWAP=1` binds the web port with SO_REUSEPORT so a release can share
	120	it, which also lets any process running as the forge's user join the listener
	121	and take a share of its connections, login posts included. It assumes nothing
	122	else runs as that user on the host network (agents and apps are sandboxed or in
	123	cells). Only one swap is made per process, so two landings never leave two
	124	releases serving. The swap is refused unless `net.ipv4.tcp_migrate_req=1`, for
	125	without it the old listener closing resets queued connections.
	126	Test: `test/release-swap.test.ts`
	127
119	128	How each part proves itself to the next, today.
120	129
121	130	\| Call \| Authenticated by \| Test \|
		@@ -69,6 +69,7 @@ export const GATE: readonly string[] = [
69	69	"test/recovery-doc.test.ts",
70	70	"test/release-checkout.test.ts",
71	71	"test/release-preflight.test.ts",
	72	"test/release-swap.test.ts",
72	73	"test/threats-doc.test.ts",
73	74	"test/returns.test.ts",
74	75	"test/review-route.test.ts",
		@@ -161,6 +161,14 @@ export interface ForgeConfig {
161	161	parentLoopback?: boolean;
162	162	/** How many `releases/<sha>/` checkouts to keep (`FORGE_RELEASES_KEEP`). @default 5 */
163	163	releasesKeep?: number;
	164	/**
	165	* A green release of the forge's own code takes over the port instead of
	166	* waiting for a restart (`FORGE_SELF_SWAP=1`; `release/swap.ts`, #1663).
	167	* The listener is bound with SO_REUSEPORT so the new process can share it.
	168	*/
	169	selfSwap?: boolean;
	170	/** How long a swapped-in release has to read healthy (`FORGE_SWAP_BUDGET_S`). @default 60000 */
	171	swapBudgetMs?: number;
164	172	}
165	173
166	174	export interface App {
		@@ -19,6 +19,8 @@ import {
19	19	type Verdict,
20	20	pruneReleases,
21	21	} from "../release/checkout.ts";
	22	import { beginSwap, endSwap, handover, startRelease } from "../release/swap.ts";
	23	import { running } from "../kernel/version.ts";
22	24	import { recordRelease, type Release } from "../release/record.ts";
23	25	import type { Pull } from "../core/pulls.ts";
24	26	import type { Repo } from "../core/repos.ts";
		@@ -144,5 +146,60 @@ async function cut(
144	146	`The landed tree ${sha.slice(0, 12)} failed its preflight, so it will not be released: ${verdict.line}`,
145	147	);
146	148	}
	149	if (verdict.ok && out.ok) await swapIn(app, by, repo, pull, sha, out.value);
147	150	return release;
148	151	}
	152
	153	/**
	154	* Hand the port to a green release and drain this process, once the new one
	155	* has read healthy. Off unless `FORGE_SELF_SWAP=1`. A release that does not
	156	* come up is stopped, the old keeps serving, and the lane and the landed pull
	157	* are told.
	158	*/
	159	async function swapIn(
	160	app: App,
	161	by: User,
	162	repo: Repo,
	163	pull: Pull,
	164	sha: string,
	165	dir: string,
	166	): Promise<void> {
	167	const drain = handover.drain;
		sentinelcrew 29d ago High: nothing stops a second landing's `swapIn` from starting another release while this one is in its budget, or after this process has drained. Both releases end up serving on the port for good. Guard it: allow one swap per process, and skip once draining.
	168	const port = app.config.port ?? 0;
	169	if (app.config.selfSwap !== true \|\| drain === undefined \|\| port <= 0) return;
	170	if (running().commit === sha) return;
	171	const turn = beginSwap();
	172	if (!turn.ok) {
	173	app.log.warn("release not swapped in", { sha: sha.slice(0, 12), line: turn.line });
	174	if (turn.tell)
	175	addComment(
	176	app,
	177	by,
	178	repo,
	179	getIssueById(app.db, pull.id) ?? pull,
	180	`The landed tree ${sha.slice(0, 12)} did not swap in: ${turn.line}. The running process keeps serving.`,
	181	);
	182	return;
	183	}
	184	const swapped = await startRelease({ dir, sha, port, env: process.env,
	185	...(app.config.swapBudgetMs === undefined ? {} : { budgetMs: app.config.swapBudgetMs }),
	186	});
	187	endSwap(swapped.ok);
	188	if (swapped.ok) {
	189	app.log.info("release swapped in; draining this process", { sha: sha.slice(0, 12), pid: swapped.pid });
	190	drain();
	191	return;
	192	}
	193	app.log.error("release swap failed; the old process keeps serving", {
	194	sha: sha.slice(0, 12),
	195	pull: pull.number,
	196	line: swapped.line,
	197	});
	198	addComment(
	199	app,
	200	by,
	201	repo,
	202	getIssueById(app.db, pull.id) ?? pull,
	203	`The landed tree ${sha.slice(0, 12)} did not swap in: ${swapped.line}. The running process keeps serving.`,
	204	);
	205	}
		@@ -1732,6 +1732,8 @@ export async function serve(app: App): Promise<Servers> {
1732	1732
1733	1733	const web = Bun.serve({
1734	1734	port: app.config.port ?? 3000,
	1735	// A release swaps in beside this process on the same port (#1663).
	1736	...(app.config.selfSwap === true ? { reusePort: true } : {}),
1735	1737	idleTimeout: 120,
1736	1738	// The largest thing anyone may send — a git push. Every reader below it
1737	1739	// keeps its own, smaller cap (\`http/limits.ts\`); the runtime enforces
		@@ -4,6 +4,7 @@
4	4	* environment-only — a handful of variables, all with sane defaults.
5	5	*/
6	6
	7	import { handover } from "./release/swap.ts";
7	8	import { parentUrlProblem } from "./core/parent_releases.ts";
8	9	import { releaseEnv } from "./release/env.ts";
9	10	import { createApp } from "./app.ts";
		@@ -433,6 +434,9 @@ const leaseTimer = setInterval(() => {
433	434	releaseOrphans(app.db);
434	435	}, 30_000);
435	436
	437	// A release that read healthy beside this process takes over (#1663).
	438	handover.drain = () => shutdown("SIGTERM");
	439	handover.ssh = ssh !== undefined;
436	440	process.on("SIGINT", shutdown);
437	441	process.on("SIGTERM", shutdown);
438	442
		@@ -3,11 +3,16 @@
3	3	export function releaseEnv(env: Record<string, string \| undefined>): {
4	4	forgeRepo?: string;
5	5	releasesKeep?: number;
	6	selfSwap?: boolean;
	7	swapBudgetMs?: number;
6	8	} {
7	9	const repo = env.FORGE_REPO ?? "";
	10	const budget = Number(env.FORGE_SWAP_BUDGET_S);
8	11	const keep = Number(env.FORGE_RELEASES_KEEP);
9	12	return {
10	13	...(/^[\w.-]+\/[\w.-]+$/.test(repo) ? { forgeRepo: repo } : {}),
	14	...(env.FORGE_SELF_SWAP === "1" ? { selfSwap: true } : {}),
	15	...(Number.isInteger(budget) && budget >= 1 && budget <= 600 ? { swapBudgetMs: budget * 1000 } : {}),
11	16	...(Number.isInteger(keep) && keep >= 1 ? { releasesKeep: keep } : {}),
12	17	};
13	18	}
		@@ -0,0 +1,155 @@
	1	/**
	2	* A green release takes over the listening port (#1663).
	3	*
	4	* The way: SO_REUSEPORT on the port. Both processes bind it (`reusePort` in
	5	* `serve`), the kernel hands each new connection to one of them, and the old
	6	* one is drained as a SIGTERM drains it today. Fd passing on a unix socket
	7	* would need the platform's router to speak it and a listener Bun cannot
	8	* adopt; the port is what the platform and the checks already dial, and
	9	* nothing is torn down between the two listeners, so no connection finds the
	10	* port closed.
	11	*
	12	* The new process is started from `releases/<sha>/`, told its commit
	13	* (`FORGE_COMMIT`), and asked `/api/v1/health` until an answer names that
	14	* commit and says `ok`. Any answer from the old process names another, so
	15	* only the new one can satisfy the read. A new process that does not within
	16	* its budget is stopped and the old keeps serving.
	17	*/
	18
	19	import { existsSync, readFileSync } from "node:fs";
	20	import { join } from "node:path";
	21
	22	export const SWAP_BUDGET_MS = 60_000;
	23	const POLL_MS = 250;
	24
	25	export type Swapped = { ok: true; pid: number } \| { ok: false; line: string };
	26
	27	export interface SwapInput {
	28	/** `releases/<sha>/`. */
	29	dir: string;
	30	sha: string;
	31	port: number;
	32	/** The environment the new forge starts with: the old one's, as it is. */
	33	env: Record<string, string \| undefined>;
	34	budgetMs?: number;
	35	}
	36
	37	/** Whether `body` is a health answer from `sha` that is well and not leaving. */
	38	export function healthyAs(body: unknown, sha: string): boolean {
	39	if (typeof body !== "object" \|\| body === null) return false;
	40	const h = body as { ok?: unknown; commit?: unknown; draining?: unknown };
	41	return h.ok === true && h.draining !== true && h.commit === sha;
	42	}
	43
	44	async function readHealth(port: number): Promise<unknown> {
	45	try {
	46	const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
	47	headers: { connection: "close" },
	48	signal: AbortSignal.timeout(2_000),
	49	});
	50	return res.ok ? await res.json() : undefined;
	51	} catch {
	52	return undefined;
	53	}
	54	}
	55
	56	/** Start the release beside the running forge and wait for it to say it is well. */
	57	export async function startRelease(input: SwapInput): Promise<Swapped> {
	58	const entry = join(input.dir, "src", "main.ts");
	59	if (!existsSync(entry)) return { ok: false, line: "the tree has no src/main.ts" };
	60	const env: Record<string, string> = {};
	61	for (const [k, v] of Object.entries(input.env)) if (v !== undefined) env[k] = v;
	62	env["FORGE_COMMIT"] = input.sha;
	63	env["PORT"] = String(input.port);
	64	env["FORGE_SELF_SWAP"] = "1";
		sentinelcrew 29d ago High: the new process gets the old one's `FORGE_SSH_PORT`. It can't bind that port while the old one holds it, so it boots with the terminal off and still reads healthy. After the old process drains, the terminal is gone. Refuse the swap while SSH is listening, or retry the SSH bind in the new process.
	65	const proc = Bun.spawn([process.execPath, entry], {
		sentinelcrew 29d ago The release is a child of the old process in its process group and cgroup. If the old process is PID 1 or a systemd service, its exit after `drain()` kills this child too. Document that the swap is for a forge run directly.
	66	cwd: input.dir,
	67	env,
	68	stdin: "ignore",
	69	stdout: "inherit",
	70	stderr: "inherit",
	71	});
	72	let exited: number \| undefined;
	73	void proc.exited.then((code) => {
	74	exited = code;
	75	});
	76	const deadline = Date.now() + (input.budgetMs ?? SWAP_BUDGET_MS);
	77	while (Date.now() < deadline) {
	78	if (exited !== undefined)
	79	return { ok: false, line: `the new process exited with ${exited} before it was healthy` };
	80	if (healthyAs(await readHealth(input.port), input.sha)) {
	81	proc.unref();
	82	return { ok: true, pid: proc.pid };
	83	}
	84	await Bun.sleep(POLL_MS);
	85	}
	86	proc.kill("SIGKILL");
	87	await proc.exited;
	88	return {
	89	ok: false,
	90	line: `the new process was not healthy within ${Math.round((input.budgetMs ?? SWAP_BUDGET_MS) / 1000)}s`,
	91	};
	92	}
	93
	94	/**
	95	* What the running forge does once the release is well: drain, as a SIGTERM
	96	* does. Set at boot, with whether this process holds the SSH listener (a new
	97	* process cannot bind that port while this one does, and a release must not
	98	* quietly close the terminal).
	99	*/
	100	export const handover: { drain?: () => void; ssh?: boolean; migrate?: boolean } = {};
	101
	102	const MIGRATE_SYSCTL = "/proc/sys/net/ipv4/tcp_migrate_req";
	103
	104	/**
	105	* Whether the kernel moves connections waiting in a closing listener's accept
	106	* queue to another listener of the group (Linux 5.14+, `tcp_migrate_req=1`).
	107	* Without it, the old process closing resets what it had not yet accepted, so
	108	* the swap would fail requests. `handover.migrate` stands in for the file in tests.
	109	*/
	110	export function migratesQueuedConnections(): boolean {
	111	if (handover.migrate !== undefined) return handover.migrate;
	112	try {
	113	return readFileSync(MIGRATE_SYSCTL, "utf8").trim() === "1";
	114	} catch {
	115	return false;
	116	}
	117	}
	118
	119	let underWay = false;
	120
	121	/**
	122	* Take the one swap this process may make. Refused while another is in its
	123	* budget or once one has succeeded (this process is then draining), so two
	124	* landings close together never leave two releases serving; and refused while
	125	* this process holds an SSH listener. `tell` says whether the landed pull
	126	* should hear of it. Refused too unless the kernel migrates queued connections
	127	* between listeners of the port.
	128	*
	129	* The release is a child of this process, not detached: the swap is for a forge
	130	* run directly, not under a supervisor (PID 1 of a container, a systemd unit
	131	* with KillMode=control-group) whose old process exiting takes the release too.
	132	*/
	133	export function beginSwap(): { ok: true } \| { ok: false; line: string; tell: boolean } {
	134	if (underWay)
	135	return { ok: false, line: "a swap is already under way or done in this process", tell: false };
	136	if (handover.ssh === true)
	137	return {
	138	ok: false,
	139	line: "this process holds the SSH terminal, which a new process cannot take over yet; restart the forge to run this tree",
	140	tell: true,
	141	};
	142	if (!migratesQueuedConnections())
	143	return {
	144	ok: false,
	145	line: "the kernel would reset connections queued on the old listener when it closes; set net.ipv4.tcp_migrate_req=1 (Linux 5.14+) to swap, or restart the forge to run this tree",
	146	tell: true,
	147	};
	148	underWay = true;
	149	return { ok: true };
	150	}
	151
	152	/** A swap that failed gives the turn back; one that worked keeps it, for this process is leaving. */
	153	export function endSwap(swapped: boolean): void {
	154	if (!swapped) underWay = false;
	155	}
		@@ -0,0 +1,352 @@
	1	/**
	2	* A green release takes over the listening port (#1663): the new process
	3	* starts beside the old on SO_REUSEPORT, is read healthy as its own commit,
	4	* and only then does the old one drain; an unhealthy one is stopped and the
	5	* old keeps serving. The stand-in forge is a few lines that answer the
	6	* health route as the commit they are told.
	7	*/
	8
	9	import { afterAll, describe, expect, test } from "bun:test";
	10	import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
	11	import { mkdtemp, rm } from "node:fs/promises";
	12	import { tmpdir } from "node:os";
	13	import { join } from "node:path";
	14	import { createApp } from "../src/app.ts";
	15	import { commitToBranch, createRepo } from "../src/actions/repos.ts";
	16	import { openPull } from "../src/actions/pulls.ts";
	17	import { releaseOnLand } from "../src/checks/landing.ts";
	18	import { listComments } from "../src/core/issues.ts";
	19	import { createUser } from "../src/core/users.ts";
	20	import { beginSwap, endSwap, healthyAs, handover, migratesQueuedConnections, startRelease } from "../src/release/swap.ts";
	21	import { releaseEnv } from "../src/release/env.ts";
	22
	23	const OLD = "0".repeat(40);
	24	const NEW = "1".repeat(40);
	25	const BAD = "2".repeat(40);
	26
	27	const STAND_IN = `
	28	if (process.argv.includes("--preflight")) { console.log("preflight ok: stand-in"); process.exit(0); }
	29	Bun.serve({
	30	port: Number(process.env.PORT),
	31	reusePort: true,
	32	fetch: () => Response.json({ ok: true, draining: false, commit: process.env.FORGE_COMMIT }),
	33	});
	34	`;
	35	// Boots and listens, but answers unwell.
	36	const SICK = STAND_IN.replace("ok: true", "ok: false");
	37	// Never listens at all.
	38	const DEAD = `if (process.argv.includes("--preflight")) process.exit(0);\nawait Bun.sleep(60000);\n`;
	39
	40	const dirs: string[] = [];
	41	const pids: number[] = [];
	42	afterAll(async () => {
	43	for (const pid of pids) {
	44	try {
	45	process.kill(pid, "SIGKILL");
	46	} catch {}
	47	}
	48	for (const d of dirs) await rm(d, { recursive: true, force: true });
	49	});
	50
	51	async function tree(main: string): Promise<string> {
	52	const dir = await mkdtemp(join(tmpdir(), "forge-swap-"));
	53	dirs.push(dir);
	54	mkdirSync(join(dir, "src"), { recursive: true });
	55	writeFileSync(join(dir, "src", "main.ts"), main);
	56	return dir;
	57	}
	58
	59	async function freePort(): Promise<number> {
	60	const s = Bun.serve({ port: 0, fetch: () => new Response("") });
	61	const port = s.port ?? 0;
	62	s.stop(true);
	63	return port;
	64	}
	65
	66	const commitAt = async (port: number): Promise<string \| undefined> => {
	67	try {
	68	const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, {
	69	headers: { connection: "close" },
	70	signal: AbortSignal.timeout(3_000),
	71	});
	72	return ((await res.json()) as { commit?: string }).commit;
	73	} catch {
	74	return undefined;
	75	}
	76	};
	77
	78	function oldForge(port: number) {
	79	return Bun.serve({
	80	port,
	81	reusePort: true,
	82	fetch: () => Response.json({ ok: true, draining: false, commit: OLD }),
	83	});
	84	}
	85
	86	describe("what counts as healthy", () => {
	87	test("well, not leaving, and the commit that was started", () => {
	88	expect(healthyAs({ ok: true, draining: false, commit: NEW }, NEW)).toBe(true);
	89	expect(healthyAs({ ok: true, draining: false, commit: OLD }, NEW)).toBe(false);
	90	expect(healthyAs({ ok: false, commit: NEW }, NEW)).toBe(false);
	91	expect(healthyAs({ ok: true, draining: true, commit: NEW }, NEW)).toBe(false);
	92	expect(healthyAs(null, NEW)).toBe(false);
	93	expect(healthyAs("ok", NEW)).toBe(false);
	94	});
	95
	96	test("the swap switch is exactly FORGE_SELF_SWAP=1", () => {
	97	expect(releaseEnv({ FORGE_SELF_SWAP: "1" }).selfSwap).toBe(true);
	98	expect(releaseEnv({ FORGE_SELF_SWAP: "true" }).selfSwap).toBeUndefined();
	99	expect(releaseEnv({}).selfSwap).toBeUndefined();
	100	expect(releaseEnv({ FORGE_SWAP_BUDGET_S: "30" }).swapBudgetMs).toBe(30_000);
	101	expect(releaseEnv({ FORGE_SWAP_BUDGET_S: "0" }).swapBudgetMs).toBeUndefined();
	102	});
	103	});
	104
	105	// The zero-failure promise holds only where the kernel migrates queued connections.
	106	const realMigrate = (() => {
	107	try {
	108	return readFileSync("/proc/sys/net/ipv4/tcp_migrate_req", "utf8").trim() === "1";
	109	} catch {
	110	return false;
	111	}
	112	})();
	113	const whenMigrating = realMigrate ? test : test.skip;
	114	// Everything else here exercises the flow, not the kernel: pretend it is on.
	115	handover.migrate = true;
	116
	117	describe("swapping on the listening port", () => {
	118	test("the swap is refused unless the kernel migrates queued connections", async () => {
	119	handover.migrate = false;
	120	try {
	121	const turn = beginSwap();
	122	expect(turn.ok).toBe(false);
	123	if (!turn.ok) {
	124	expect(turn.line).toContain("tcp_migrate_req=1");
	125	expect(turn.tell).toBe(true);
	126	}
	127	handover.migrate = true;
	128	expect(beginSwap().ok).toBe(true);
	129	endSwap(false);
	130	} finally {
	131	handover.migrate = true;
	132	}
	133	expect(migratesQueuedConnections()).toBe(true);
		sentinelcrew 29d ago outdated This failed on the first full-file run here (`Received: 1`). Linux resets the connections waiting in the old listener's accept queue when it closes, unless `net.ipv4.tcp_migrate_req=1`. Gate the swap on that sysctl and skip this test, named as skipped, when it is off.
	134	});
	135
	136	whenMigrating("a client looping through the swap sees no failure, then only the new sha", async () => {
	137	const port = await freePort();
	138	const old = oldForge(port);
	139	const dir = await tree(STAND_IN);
	140	let stop = false;
	141	let failed = 0;
	142	let total = 0;
	143	const seen = new Set<string>();
	144	const client = (async () => {
	145	while (!stop) {
	146	total++;
	147	const c = await commitAt(port);
	148	if (c === undefined) failed++;
	149	else seen.add(c);
	150	await Bun.sleep(2);
	151	}
	152	})();
	153	const out = await startRelease({ dir, sha: NEW, port, env: process.env, budgetMs: 20_000 });
	154	expect(out.ok).toBe(true);
	155	if (out.ok) pids.push(out.pid);
	156	// The old drains: it stops taking connections, finishing what it holds.
	157	old.stop();
	158	await Bun.sleep(300);
	159	const afterOld = total;
	160	await Bun.sleep(300);
	161	stop = true;
	162	await client;
	163	expect(failed).toBe(0);
	164	expect(total).toBeGreaterThan(afterOld);
	165	expect(seen.has(NEW)).toBe(true);
	166	expect(await commitAt(port)).toBe(NEW);
	167	}, 40_000);
	168
	169	test("a process that listens but is unwell is stopped, and the old sha keeps serving", async () => {
	170	const port = await freePort();
	171	const old = oldForge(port);
	172	const dir = await tree(SICK);
	173	const out = await startRelease({ dir, sha: BAD, port, env: process.env, budgetMs: 1_500 });
	174	expect(out.ok).toBe(false);
	175	if (!out.ok) expect(out.line).toContain("not healthy within");
	176	for (let i = 0; i < 20; i++) expect(await commitAt(port)).toBe(OLD);
	177	old.stop(true);
	178	}, 20_000);
	179
	180	test("a process that dies on boot is reported as that, at once", async () => {
	181	const port = await freePort();
	182	const old = oldForge(port);
	183	const dir = await tree("process.exit(3);\n");
	184	const t = Date.now();
	185	const out = await startRelease({ dir, sha: BAD, port, env: process.env, budgetMs: 20_000 });
	186	expect(out).toEqual({ ok: false, line: "the new process exited with 3 before it was healthy" });
	187	expect(Date.now() - t).toBeLessThan(5_000);
	188	expect(await commitAt(port)).toBe(OLD);
	189	old.stop(true);
	190	}, 30_000);
	191	});
	192
	193	describe("on landing", () => {
	194	async function setup(selfSwap: boolean, port: number, main: string) {
	195	const dataDir = await mkdtemp(join(tmpdir(), "forge-swap-land-"));
	196	dirs.push(dataDir);
	197	const app = createApp({ dataDir, port, forgeRepo: "alice/forge-code", selfSwap, swapBudgetMs: 5_000 });
	198	const u = await createUser(app.db, {
	199	trusted: true,
	200	username: "alice",
	201	email: "[email protected]",
	202	password: "a-good-passphrase",
	203	adminIfFirst: true,
	204	});
	205	if (!u.ok) throw new Error("user");
	206	const r = await createRepo(app, u.value, { name: "forge-code", visibility: "public" });
	207	if (!r.ok) throw new Error("repo");
	208	await commitToBranch(app, u.value, r.value, { message: "seed", files: [{ path: "a.txt", content: "a" }] });
	209	await commitToBranch(app, u.value, r.value, { message: "b", branch: "change", files: [{ path: "b.txt", content: "b" }] });
	210	const opened = await openPull(app, u.value, r.value, {
	211	title: "t",
	212	baseBranch: r.value.defaultBranch,
	213	headBranch: "change",
	214	});
	215	if (!opened.ok) throw new Error("pull");
	216	const made = await commitToBranch(app, u.value, r.value, {
	217	message: "land",
	218	files: [{ path: "src/main.ts", content: main }],
	219	});
	220	if (!made.ok) throw new Error("commit");
	221	return { app, user: u.value, repo: r.value, pull: opened.value, sha: made.value.commitOid };
	222	}
	223
	224	test("a release that never comes up leaves the old process and tells the pull", async () => {
	225	const port = await freePort();
	226	const old = oldForge(port);
	227	let drained = 0;
	228	handover.drain = () => {
	229	drained++;
	230	};
	231	try {
	232	// Dead after a green preflight: the swap fails, nothing drains.
	233	const bad = await setup(true, port, DEAD);
	234	const sha = bad.sha;
	235	const run = releaseOnLand(bad.app, bad.user, bad.repo, bad.pull, sha);
	236	const release = await run;
	237	expect(release?.ok).toBe(true);
	238	expect(drained).toBe(0);
	239	const told = listComments(bad.app.db, bad.pull.id).map((c) => c.body);
	240	expect(told.some((b) => b.includes("did not swap in") && b.includes(sha.slice(0, 12)))).toBe(true);
	241	expect(await commitAt(port)).toBe(OLD);
	242	bad.app.close();
	243	} finally {
	244	delete handover.drain;
	245	old.stop(true);
	246	}
	247	}, 120_000);
	248
	249	test("a release that reads healthy drains the old process once", async () => {
	250	const port = await freePort();
	251	const old = oldForge(port);
	252	let drained = 0;
	253	handover.drain = () => {
	254	drained++;
	255	old.stop();
	256	};
	257	try {
	258	// The stand-in leaves on its own, so the test strands nothing.
	259	const s = await setup(true, port, `${STAND_IN}\nsetTimeout(() => process.exit(0), 6000);\n`);
	260	await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
	261	expect(drained).toBe(1);
	262	expect(await commitAt(port)).toBe(s.sha);
	263	s.app.close();
	264	} finally {
	265	endSwap(false);
	266	delete handover.drain;
	267	old.stop(true);
	268	}
	269	}, 60_000);
	270
	271	test("two landings close together leave exactly one new process serving", async () => {
	272	const port = await freePort();
	273	const old = oldForge(port);
	274	let drained = 0;
	275	handover.drain = () => {
	276	drained++;
	277	old.stop();
	278	};
	279	const long = `${STAND_IN}\nsetTimeout(() => process.exit(0), 8000);\n`;
	280	try {
	281	const a = await setup(true, port, long);
	282	const b = await setup(true, port, long);
	283	await Promise.all([
	284	releaseOnLand(a.app, a.user, a.repo, a.pull, a.sha),
	285	releaseOnLand(b.app, b.user, b.repo, b.pull, b.sha),
	286	]);
	287	expect(drained).toBe(1);
	288	const seen = new Set<string>();
	289	for (let i = 0; i < 40; i++) {
	290	const c = await commitAt(port);
	291	if (c !== undefined) seen.add(c);
	292	}
	293	expect(seen.size).toBe(1);
	294	expect([a.sha, b.sha]).toContain([...seen][0]!);
	295	// This process is leaving: a later landing does not start another.
	296	const c = await setup(true, port, long);
	297	await releaseOnLand(c.app, c.user, c.repo, c.pull, c.sha);
	298	expect(drained).toBe(1);
	299	a.app.close();
	300	b.app.close();
	301	c.app.close();
	302	} finally {
	303	delete handover.drain;
	304	endSwap(false);
	305	old.stop(true);
	306	}
	307	}, 90_000);
	308
	309	test("a process holding the SSH terminal refuses the swap and tells the pull", async () => {
	310	const port = await freePort();
	311	const old = oldForge(port);
	312	let drained = 0;
	313	handover.drain = () => {
	314	drained++;
	315	};
	316	handover.ssh = true;
	317	try {
	318	const s = await setup(true, port, STAND_IN);
	319	await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
	320	expect(drained).toBe(0);
	321	expect(await commitAt(port)).toBe(OLD);
	322	const told = listComments(s.app.db, s.pull.id).map((c) => c.body);
	323	expect(told.some((b) => b.includes("did not swap in") && b.includes("SSH"))).toBe(true);
	324	// Refusing took no turn: with the terminal off, the next swap goes.
	325	handover.ssh = false;
	326	expect(beginSwap().ok).toBe(true);
	327	endSwap(false);
	328	s.app.close();
	329	} finally {
	330	delete handover.drain;
	331	delete handover.ssh;
	332	old.stop(true);
	333	}
	334	}, 60_000);
	335
	336	test("with the switch off, a green landing swaps nothing", async () => {
	337	const port = await freePort();
	338	let drained = 0;
	339	handover.drain = () => {
	340	drained++;
	341	};
	342	try {
	343	const s = await setup(false, port, STAND_IN);
	344	await releaseOnLand(s.app, s.user, s.repo, s.pull, s.sha);
	345	expect(drained).toBe(0);
	346	expect(await commitAt(port)).toBeUndefined();
	347	s.app.close();
	348	} finally {
	349	delete handover.drain;
	350	}
	351	}, 60_000);
	352	});