Public: anyone can read this; the crew can change it.

A git process that cannot start is an error on one request, never the end of… #12259

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

mason/forge:mason/12138-a-git-process-that-cannot-start-is-an-er → main 10 commits fork

  • Merged. mason/12138-a-git-process-that-cannot-start-is-an-er was merged into main as 9725453.

Review

Sign in to review this change.
8 files changed +328 −69
src/apps/process.ts +39 −22
@@ -386,17 +386,26 @@ export async function runStep(
386386 options.cell === undefined
387387 ? undefined
388388 : openCell({ ...options.cell, background: true }, options.log);
389 const proc = Bun.spawn(backgroundArgv(cell?.argv(command) ?? ["/bin/sh", "-c", command]), {
390 cwd: options.cwd,
391 env: cell?.env(options.env) ?? options.env,
392 stdin: "ignore",
393 stdout: "pipe",
394 stderr: "pipe",
395 // Without a cell, the step leads a process group of its own, so a
396 // cancel reaches what its shell started (an install script, a test
397 // worker), not just the shell. A cell's tree dies with bubblewrap.
398 detached: cell === undefined,
399 });
389 let proc: ReturnType<typeof Bun.spawn<"ignore", "pipe", "pipe">>;
390 try {
391 proc = Bun.spawn(backgroundArgv(cell?.argv(command) ?? ["/bin/sh", "-c", command]), {
392 cwd: options.cwd,
393 env: cell?.env(options.env) ?? options.env,
394 stdin: "ignore",
395 stdout: "pipe",
396 stderr: "pipe",
397 // Without a cell, the step leads a process group of its own, so a
398 // cancel reaches what its shell started (an install script, a test
399 // worker), not just the shell. A cell's tree dies with bubblewrap.
400 detached: cell === undefined,
401 });
402 } catch (cause) {
403 // A cwd that is gone throws here, synchronously: the step fails, the forge does not.
404 cell?.close();
405 const reason = cause instanceof Error ? cause.message : String(cause);
406 options.log(`${name} not started: ${reason}`);
407 return err(`${name} not started: ${reason}`);
408 }
400409 const kill = (): void => {
401410 if (cell === undefined) signalGroup(proc.pid, "SIGKILL");
402411 else proc.kill("SIGKILL");
@@ -720,17 +729,25 @@ export async function startProcess(
720729 feature: input.feature,
721730 forgeToken: input.forgeToken,
722731 });
723 const proc = Bun.spawn(cell?.argv(command) ?? ["/bin/sh", "-c", command], {
724 cwd: input.dir,
725 env: cell?.env(env) ?? env,
726 stdin: "ignore",
727 stdout: "pipe",
728 stderr: "pipe",
729 // Without a cell, the app leads a process group of its own, so what it
730 // starts can be signalled with it (`signalGroup`). A cell's tree is
731 // bubblewrap's, walked by `signalTree` and `killTree`.
732 detached: cell === undefined,
733 });
732 let proc: ReturnType<typeof Bun.spawn<"ignore", "pipe", "pipe">>;
733 try {
734 proc = Bun.spawn(cell?.argv(command) ?? ["/bin/sh", "-c", command], {
735 cwd: input.dir,
736 env: cell?.env(env) ?? env,
737 stdin: "ignore",
738 stdout: "pipe",
739 stderr: "pipe",
740 // Without a cell, the app leads a process group of its own, so what it
741 // starts can be signalled with it (`signalGroup`). A cell's tree is
742 // bubblewrap's, walked by `signalTree` and `killTree`.
743 detached: cell === undefined,
744 });
745 } catch (cause) {
746 cell?.close();
747 const reason = cause instanceof Error ? cause.message : String(cause);
748 log(`not started: ${reason}`);
749 return { error: { stage: "start", message: `not started: ${reason}` }, portTaken: false };
750 }
734751 // The cell's sockets and watch end with the process, however it ends.
735752 // Without a cell, so does the rest of its group: a child that outlived
736753 // the app, stopped or crashed, is not left running on its data.
src/git/combined.ts +8 −2
@@ -230,8 +230,14 @@ export function headUnder(
230230 ): string {
231231 if (!pull.crossRepo || commit === pull.headOid || !isSafeOid(commit) || !isSafeOid(pull.headOid))
232232 return commit;
233 const git = (args: string[]) =>
234 Bun.spawnSync(["git", ...args], { cwd: dir, env: gitEnv(), stdout: "pipe", stderr: "ignore" });
233 const git = (args: string[]) => {
234 try {
235 return Bun.spawnSync(["git", ...args], { cwd: dir, env: gitEnv(), stdout: "pipe", stderr: "ignore" });
236 } catch {
237 // git cannot start (the directory is gone): the commit stands as given.
238 return { exitCode: 1, stdout: Buffer.alloc(0) };
239 }
240 };
235241 const tip = git(["rev-parse", "--verify", "--quiet", combinedRef(pull.number)]);
236242 if (tip.exitCode !== 0 || tip.stdout.toString().trim() !== commit) return commit;
237243 return git(["merge-base", "--is-ancestor", pull.headOid, commit]).exitCode === 0 ? pull.headOid : commit;
src/git/env.ts +42 −15
@@ -257,6 +257,18 @@ export interface GitFailure {
257257 readonly args: readonly string[];
258258 }
259259
260 /** The code of a failure whose process never started. */
261 export const SPAWN_FAILED = -3;
262
263 /**
264 * A spawn that threw (the directory is gone, or no git binary) as a GitFailure.
265 * The words are a sentence a person may read; the system's own text (a path)
266 * stays out of them.
267 */
268 export function spawnFailure(_error: unknown, args: readonly string[]): GitFailure {
269 return { code: SPAWN_FAILED, stderr: "This repository is unavailable.", args };
270 }
271
260272 export interface RunOptions {
261273 cwd?: string;
262274 env?: Record<string, string>;
@@ -269,9 +281,10 @@ export interface RunOptions {
269281 signal?: AbortSignal;
270282 }
271283
272 /** Run git to completion, capturing output as bytes. Non-zero exit is an Err. */
284 /** The code of a failure over `maxBytes`. */
273285 export const OVER_CAP = -2;
274286
287 /** Run git to completion, capturing output as bytes. Non-zero exit is an Err. */
275288 export async function runGitRaw(
276289 args: readonly string[],
277290 options: RunOptions = {},
@@ -284,13 +297,21 @@ export async function runGitRaw(
284297 // Timed from before the gate: a wait for a permit is the queue #8158 names.
285298 const start = performance.now();
286299 return gitGate.run(async () => {
287 const proc = Bun.spawn(["git", ...args], {
288 ...(options.cwd === undefined ? {} : { cwd: options.cwd }),
289 env: gitEnv(options.env),
290 stdin: options.stdin === undefined ? "ignore" : new Blob([options.stdin]),
291 stdout: "pipe",
292 stderr: "pipe",
293 });
300 let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">;
301 try {
302 proc = Bun.spawn(["git", ...args], {
303 ...(options.cwd === undefined ? {} : { cwd: options.cwd }),
304 env: gitEnv(options.env),
305 stdin: options.stdin === undefined ? "ignore" : new Blob([options.stdin]),
306 stdout: "pipe",
307 stderr: "pipe",
308 }) as typeof proc;
309 } catch (error) {
310 // A git that cannot start (its directory is gone, no binary) fails this
311 // call like any other git failure; it must never reject out of the gate.
312 noteGit(gitRecord, args, start);
313 return err(spawnFailure(error, args));
314 }
294315
295316 const timeoutMs = options.timeoutMs ?? 30_000;
296317 const timer = setTimeout(() => proc.kill("SIGKILL"), timeoutMs);
@@ -405,13 +426,19 @@ export async function runGitCapture(
405426 const start = performance.now();
406427 return gitGate.run(async () => {
407428 if (options.signal?.aborted === true) return { code: 1, stdout: "", stderr: "", timedOut: true };
408 const proc = Bun.spawn(["git", ...args], {
409 ...(options.cwd === undefined ? {} : { cwd: options.cwd }),
410 env: gitEnv(options.env),
411 stdin: options.stdin === undefined ? "ignore" : new Blob([options.stdin]),
412 stdout: "pipe",
413 stderr: "pipe",
414 });
429 let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">;
430 try {
431 proc = Bun.spawn(["git", ...args], {
432 ...(options.cwd === undefined ? {} : { cwd: options.cwd }),
433 env: gitEnv(options.env),
434 stdin: options.stdin === undefined ? "ignore" : new Blob([options.stdin]),
435 stdout: "pipe",
436 stderr: "pipe",
437 }) as typeof proc;
438 } catch (error) {
439 noteGit(gitRecord, args, start);
440 return { code: SPAWN_FAILED, stdout: "", stderr: spawnFailure(error, args).stderr };
441 }
415442 let timedOut = false;
416443 const timer = setTimeout(
417444 () => {
src/git/local.ts +26 −14
@@ -17,7 +17,7 @@
1717
1818 import { err, ok, type Result } from "../kernel/result.ts";
1919 import { parseUnifiedDiff, type Diff, type Hunk } from "./diff.ts";
20 import type { GitFailure } from "./env.ts";
20 import { spawnFailure, type GitFailure } from "./env.ts";
2121 import { isSafeRef } from "./guard.ts";
2222 import { noteGit } from "../perf/git.ts";
2323 import { gitRecord } from "../perf/live.ts";
@@ -65,13 +65,19 @@ export async function runLocal(
6565 opts: RunOpts,
6666 ): Promise<LocalResult<string>> {
6767 const start = performance.now();
68 const proc = Bun.spawn(["git", ...args], {
69 cwd: opts.cwd,
70 env: localGitEnv(opts.env),
71 stdin: opts.stdin === undefined ? "ignore" : new Blob([opts.stdin]),
72 stdout: "pipe",
73 stderr: "pipe",
74 });
68 let proc: Bun.Subprocess<"pipe" | "ignore", "pipe", "pipe">;
69 try {
70 proc = Bun.spawn(["git", ...args], {
71 cwd: opts.cwd,
72 env: localGitEnv(opts.env),
73 stdin: opts.stdin === undefined ? "ignore" : new Blob([opts.stdin]),
74 stdout: "pipe",
75 stderr: "pipe",
76 }) as typeof proc;
77 } catch (error) {
78 noteGit(gitRecord, args, start);
79 return err(spawnFailure(error, args));
80 }
7581 const timer = setTimeout(
7682 () => proc.kill("SIGKILL"),
7783 opts.timeoutMs ?? LOCAL_MS,
@@ -362,12 +368,18 @@ export async function diff(
362368 opts.path,
363369 ];
364370 const start = performance.now();
365 const proc = Bun.spawn(["git", ...args], {
366 cwd: dir,
367 env: localGitEnv(),
368 stdout: "pipe",
369 stderr: "pipe",
370 });
371 let proc: Bun.Subprocess<"ignore", "pipe", "pipe">;
372 try {
373 proc = Bun.spawn(["git", ...args], {
374 cwd: dir,
375 env: localGitEnv(),
376 stdout: "pipe",
377 stderr: "pipe",
378 });
379 } catch (error) {
380 noteGit(gitRecord, args, start);
381 return err(spawnFailure(error, args));
382 }
371383 let text: string, stderr: string, code: number;
372384 try {
373385 [text, stderr, code] = await Promise.all([
src/git/read.ts +37 −15
@@ -11,6 +11,8 @@ import { gitEnv, mediaGate, runGit, runGitRaw, type GitFailure } from "./env.ts"
1111 import { isSafeOid, isSafeRef } from "./guard.ts";
1212 import { readerHold, namesOnlyOwn } from "./own_refs.ts";
1313 import { during } from "../perf/jobs.ts";
14 import { noteGit } from "../perf/git.ts";
15 import { gitRecord } from "../perf/live.ts";
1416
1517 export type ObjectType = "blob" | "tree" | "commit" | "tag";
1618
@@ -229,7 +231,16 @@ class BatchProc {
229231 rev: string,
230232 withBody: boolean,
231233 ): Promise<Result<GitObject | ObjectStat, "missing">> {
232 const proc = this.ensureProc();
234 let proc: ReturnType<typeof this.ensureProc>;
235 try {
236 proc = this.ensureProc();
237 } catch {
238 // git could not start (the repository's directory is gone). The type has
239 // one miss, so a gone repository reads as a missing object here; the
240 // failure is kept in the git record so it is not silent.
241 noteGit(gitRecord, ["cat-file", this.mode, "spawn-failed"], performance.now());
242 return err("missing");
243 }
233244 proc.stdin.write(`${rev}\n`);
234245 await proc.stdin.flush();
235246
@@ -394,13 +405,18 @@ export async function blobHead(
394405 assertBatchSafe(spec);
395406 const hold = readerHold(dir);
396407 if (hold !== undefined && !(await namesOnlyOwn(hold, [rev]))) return undefined;
397 const proc = Bun.spawn(["git", "cat-file", "blob", spec], {
398 cwd: dir,
399 env: gitEnv(),
400 stdin: "ignore",
401 stdout: "pipe",
402 stderr: "ignore",
403 });
408 let proc: Bun.Subprocess<"ignore", "pipe", "ignore">;
409 try {
410 proc = Bun.spawn(["git", "cat-file", "blob", spec], {
411 cwd: dir,
412 env: gitEnv(),
413 stdin: "ignore",
414 stdout: "pipe",
415 stderr: "ignore",
416 });
417 } catch {
418 return undefined;
419 }
404420 const reader = proc.stdout.getReader();
405421 const head = new Uint8Array(bytes);
406422 let filled = 0;
@@ -688,13 +704,19 @@ export async function blobStream(
688704 release();
689705 return "gone";
690706 }
691 const proc = Bun.spawn(["git", "cat-file", "blob", oid], {
692 cwd: dir,
693 env: gitEnv(),
694 stdin: "ignore",
695 stdout: "pipe",
696 stderr: "ignore",
697 });
707 let proc: Bun.Subprocess<"ignore", "pipe", "ignore">;
708 try {
709 proc = Bun.spawn(["git", "cat-file", "blob", oid], {
710 cwd: dir,
711 env: gitEnv(),
712 stdin: "ignore",
713 stdout: "pipe",
714 stderr: "ignore",
715 });
716 } catch {
717 release();
718 return "gone";
719 }
698720 const source = proc.stdout.getReader();
699721 const idleMs = limits.idleMs ?? MEDIA_IDLE_MS;
700722 let idle: ReturnType<typeof setTimeout> | undefined;
test/combined-recombine.test.ts +23 −1
@@ -297,7 +297,9 @@ test("a build that throws still leaves a sentence", async () => {
297297 await setFailure(bare, third, undefined);
298298 const real = Bun.spawn;
299299 const spy = spyOn(Bun, "spawn").mockImplementation(((cmd: string[], ...rest: unknown[]) => {
300 if (cmd.includes("merge-tree")) throw new Error("boom");
300 // git starts, then reading its output throws: not a spawn failure.
301 if (cmd.includes("merge-tree"))
302 return { get stdout(): never { throw new Error("boom"); }, stderr: new Blob([]).stream(), exited: Promise.resolve(1), kill() {} };
301303 return (real as (...a: unknown[]) => unknown)(cmd, ...rest);
302304 }) as typeof Bun.spawn);
303305 try {
@@ -308,3 +310,23 @@ test("a build that throws still leaves a sentence", async () => {
308310 }
309311 expect(await readFailure(bare, third)).toContain("The rebuild failed unexpectedly");
310312 });
313
314 test("a git that cannot start leaves the merge sentence (#12138)", async () => {
315 const third = Number((await git(["config", "--local", "--get-regexp", "^recombine.failed"], bare)).split("\n").pop()!.split(" ")[0]!.replace("recombine.failed", ""));
316 await setFailure(bare, third, undefined);
317 const real = Bun.spawn;
318 const spy = spyOn(Bun, "spawn").mockImplementation(((cmd: string[], ...rest: unknown[]) => {
319 if (cmd.includes("merge-tree")) throw new Error("boom");
320 return (real as (...a: unknown[]) => unknown)(cmd, ...rest);
321 }) as typeof Bun.spawn);
322 try {
323 await moveMain("m10.txt");
324 await idle(repoId);
325 } finally {
326 spy.mockRestore();
327 }
328 // A spawn that throws is a GitFailure, so the sentence is the merge one.
329 expect(await readFailure(bare, third)).toContain(
330 "The rebuild failed: git could not merge main into the pull",
331 );
332 });
test/git-spawn-failure.test.ts added +113
@@ -0,0 +1,113 @@
1 /** A git that cannot start is an error on one request, never the end of the forge (#12138). */
2 import { afterAll, beforeAll, expect, test } from "bun:test";
3 import { mkdtemp, rm } from "node:fs/promises";
4 import { tmpdir } from "node:os";
5 import { join } from "node:path";
6 import { createApp, type App } from "../src/app.ts";
7 import { serve, type Servers } from "../src/http/server.ts";
8 import { runGit, runGitCapture, runGitRaw } from "../src/git/env.ts";
9 import { diff, runLocal } from "../src/git/local.ts";
10 import { headUnder } from "../src/git/combined.ts";
11 import { blobHead, blobStream, readerFor, closeReader } from "../src/git/read.ts";
12 import { mediaGate } from "../src/git/env.ts";
13 import { repoDir } from "../src/git/repos.ts";
14 import { seedAdmin } from "./helpers/admin.ts";
15
16 const gone = join(tmpdir(), "forge-no-such-dir-12138");
17
18 test("a spawn that throws is an Err on every helper, not a rejection", async () => {
19 for (const run of [runGit, runGitRaw]) {
20 const out = await run(["status"], { cwd: gone });
21 expect(out.ok).toBe(false);
22 }
23 const cap = await runGitCapture(["status"], { cwd: gone });
24 expect(cap.code).not.toBe(0);
25 expect(cap.stderr).toBe("This repository is unavailable.");
26 expect(cap.stderr).not.toContain(gone);
27 const local = await runLocal(["status"], { cwd: gone });
28 expect(local.ok).toBe(false);
29 });
30
31 const OID = "a".repeat(40);
32 const OTHER = "b".repeat(40);
33
34 test("blobHead of a gone repository is undefined, not a throw", async () => {
35 expect(await blobHead(gone, "main", "a.txt", 16)).toBeUndefined();
36 });
37
38 test("blobStream of a gone repository answers gone and returns its permit", async () => {
39 const before = mediaGate.stats();
40 expect(await blobStream(gone, OID, 0, 10)).toBe("gone");
41 const after = mediaGate.stats();
42 expect(after.active).toBe(before.active);
43 expect(after.waiting).toBe(before.waiting);
44 });
45
46 test("a batch reader of a gone repository answers missing, not a rejection", async () => {
47 const reader = readerFor(gone);
48 try {
49 expect((await reader.object("main:a.txt")).ok).toBe(false);
50 expect((await reader.stat("main")).ok).toBe(false);
51 } finally {
52 closeReader(gone);
53 }
54 });
55
56 test("a work-screen diff of a gone directory is an Err with the sentence", async () => {
57 const out = await diff(gone, { untracked: true, path: "a.txt" });
58 expect(out.ok).toBe(false);
59 if (!out.ok) expect(out.error.stderr).toBe("This repository is unavailable.");
60 });
61
62 test("headUnder of a gone directory leaves the commit as given", () => {
63 const pull = { crossRepo: true, number: 1, headOid: OTHER };
64 expect(headUnder(gone, pull, OID)).toBe(OID);
65 });
66
67 let dataDir = "";
68 let app: App, servers: Servers, bearer = "";
69
70 beforeAll(async () => {
71 dataDir = await mkdtemp(join(tmpdir(), "forge-spawnfail-"));
72 app = createApp({ dataDir, port: 0 });
73 servers = await serve(app);
74 await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
75 const minted = await fetch(`${servers.url}/api/v1/tokens`, {
76 method: "POST",
77 headers: { "content-type": "application/json" },
78 body: JSON.stringify({ adminConfirm: true, username: "alice", password: "correct-horse-battery", name: "cli", scope: "admin" }),
79 });
80 bearer = ((await minted.json()) as { token: string }).token;
81 await fetch(`${servers.url}/api/v1/repos`, {
82 method: "POST",
83 headers: { "content-type": "application/json", authorization: `Bearer ${bearer}` },
84 body: JSON.stringify({ name: "app", description: "x" }),
85 });
86 }, 60_000);
87
88 afterAll(async () => {
89 servers.stop();
90 app.close();
91 await rm(dataDir, { recursive: true, force: true });
92 });
93
94 test("a repository whose directory was removed answers an error page and the forge keeps serving", async () => {
95 const headers = { authorization: `Bearer ${bearer}` };
96 const dir = repoDir(dataDir, "alice", "app");
97 await rm(dir, { recursive: true, force: true });
98 // The repository page itself renders without git; the pages that read it answer 404,
99 // a page, not a crash.
100 const expected: Record<string, number> = { "/alice/app": 200, "/alice/app/tree/main": 404, "/alice/app/commits/main": 404 };
101 for (const [path, status] of Object.entries(expected)) {
102 const res = await fetch(`${servers.url}${path}`, { headers });
103 const body = await res.text();
104 expect(res.status).toBe(status);
105 expect(res.headers.get("content-type")).toContain("text/html");
106 if (status === 404) expect(body).toContain("Not found");
107 expect(body).not.toContain(gone);
108 expect(body).not.toContain(dir);
109 expect(body).not.toContain("ENOENT");
110 }
111 const home = await fetch(`${servers.url}/`, { headers });
112 expect(home.status).toBe(200);
113 });
test/process-spawn-failure.test.ts added +40
@@ -0,0 +1,40 @@
1 import { expect, test } from "bun:test";
2 import { runStep, startProcess, type StartInput } from "../src/apps/process.ts";
3 import { parseManifest } from "../src/apps/manifest.ts";
4
5 test("a step whose directory is gone fails that step; it does not throw", async () => {
6 const lines: string[] = [];
7 const result = await runStep("install", "true", {
8 cwd: "/nonexistent-forge-dir-12138",
9 env: { PATH: process.env.PATH ?? "" },
10 log: (line: string) => lines.push(line),
11 signal: new AbortController().signal,
12 });
13 expect(result.ok).toBe(false);
14 expect(lines.join("\n")).toContain("not started");
15 });
16
17 test("an app whose directory is gone fails that start at the start stage; it does not throw", async () => {
18 const manifest = parseManifest(JSON.stringify({ app: { run: "bun server.ts" } }), undefined);
19 if (!manifest.ok) throw new Error("manifest");
20 const lines: string[] = [];
21 const dir = "/nonexistent-forge-dir-12138";
22 const input = {
23 dir,
24 dataDir: dir,
25 env: "production",
26 url: "http://localhost",
27 commitOid: "a".repeat(40),
28 forgeUrl: "http://localhost",
29 identityKey: "k".repeat(32),
30 manifest: manifest.value,
31 log: (line: string) => lines.push(line),
32 signal: new AbortController().signal,
33 } as unknown as StartInput;
34 const started = await startProcess(input);
35 expect(started.ok).toBe(false);
36 if (started.ok) return;
37 expect(started.error.stage).toBe("start");
38 expect(started.error.message).toContain("not started");
39 expect(lines.join("\n")).toContain("not started");
40 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs