b716947
@@ -65,7 +65,11 @@
around it. None is advisory.
   where the filesystem offers it — APFS, XFS, btrfs — a plain copy
   otherwise, and `empty` is a per-app choice). A preview process is handed
   its own `FORGE_APP_DATA` and nothing that names production's. Merging a
   pull moves code; it never touches data.
   pull moves code; it never touches data. A copy the build cannot read —
   its process exits before it ever answers, most often a migration on
   `main` the pull's commit predates — is tried exactly once more on an
   empty data directory before the deploy is failed; production is never
   seeded from a copy, so this never applies to it.
4. **A deployment is a check.** Every deploy is a run; the run's state is
   what the pull shows. With `require_checks` on, `mergePull` refuses
   `checks_required` until the head's deploy run is `ok`, read live inside
@@ -99,6 +99,18 @@
export function ensureEnvData(
  return cloneDir(source, target);
}
/**
 * Discard a data directory that turned out to be unusable and recreate it
 * empty. Used only after a copy has already been tried and its process
 * would not come up: the copy is not kept beside the fresh directory, and
 * nothing outside this one environment's data directory is touched.
 */
export function reseedEmpty(dataDir: string, appId: string, env: EnvName): void {
  const target = envDataDir(dataDir, appId, env);
  rmSync(target, { recursive: true, force: true });
  mkdirSync(target, { recursive: true });
}
/** Remove an environment entirely: checkouts, data, everything. */
export function removeEnv(dataDir: string, appId: string, env: EnvName): void {
  rmSync(envDir(dataDir, appId, env), { recursive: true, force: true });
@@ -52,11 +52,17 @@
import {
  previewEnv,
  removeApp,
  removeEnv,
  reseedEmpty,
  type EnvName,
} from "./data.ts";
import { parseManifest, type AppManifest } from "./manifest.ts";
import { identityKey } from "./proxy.ts";
import { startProcess, type ProcessHandle } from "./process.ts";
import {
  startProcess,
  type ProcessHandle,
  type StartError,
  type StartInput,
} from "./process.ts";
const log = createLog("apps");
@@ -124,6 +130,19 @@
function elapsedSeconds(fromIso: string, toIso: string): string {
  return `${(ms / 1000).toFixed(1)}s`;
}
/**
 * Whether a start failure is the process itself exiting before it ever
 * answered its health path — as opposed to install failing, or the process
 * staying up but never answering. Only this shape of failure is worth
 * retrying on empty data: it is what a build too old for the data it was
 * given looks like.
 */
function diedBeforeAnswering(error: StartError): boolean {
  return error.stage === "start" && /^exited with code \d+ before answering/.test(
    error.message,
  );
}
/** A running environment, as a terminal session needs it. */
export interface LiveEnv {
  readonly env: EnvName;
@@ -729,7 +748,7 @@
export class AppsRuntime {
    else if (seeded === "empty" && state.env !== "production")
      write("data: starting empty");
    const started = await startProcess({
    const startInput: Omit<StartInput, "signal"> = {
      dir,
      dataDir,
      manifest: manifest.value,
@@ -743,9 +762,33 @@
export class AppsRuntime {
      // process that comes up is the one that sees it.
      vars: resolveAppEnv(this.app.db, this.app.config.dataDir, state.app.id),
      log: write,
      signal,
    });
    if (!started.ok) {
    };
    let started = await startProcess({ ...startInput, signal });
    // The one retry this deploy gets: a preview seeded from production whose
    // process died before it ever answered — schema ahead of what this build
    // knows, most often — is tried again on empty data, once. A production
    // deploy is never seeded from a copy, so it never takes this path.
    if (
      !started.ok &&
      (seeded === "cloned" || seeded === "copied") &&
      diedBeforeAnswering(started.error)
    ) {
      const first = started.error;
      write(`failed: ${first.stage}: ${first.message}`);
      write(
        "the copied data was not usable by this build; the preview is starting on empty data",
      );
      reseedEmpty(this.app.config.dataDir, state.app.id, state.env);
      started = await startProcess({ ...startInput, signal });
      if (!started.ok) {
        rmSync(dir, { recursive: true, force: true });
        return {
          ok: false,
          error: `${started.error.stage}: ${started.error.message} (retried on empty data after the copy failed with ${first.stage}: ${first.message})`,
        };
      }
    } else if (!started.ok) {
      rmSync(dir, { recursive: true, force: true });
      return {
        ok: false,
@@ -14,8 +14,13 @@
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { AppsRuntime } from "../src/apps/runtime.ts";
import { envDir } from "../src/apps/data.ts";
import { newApp, openEnvUrl, updateAppSettings } from "../src/actions/apps.ts";
import { envDataDir, envDir, previewEnv } from "../src/apps/data.ts";
import {
  enableApp,
  newApp,
  openEnvUrl,
  updateAppSettings,
} from "../src/actions/apps.ts";
import { addToCrew, createAgent, listCrew } from "../src/actions/crew.ts";
import {
  assignIssue,
@@ -38,6 +43,7 @@
import { listRuns, type Run } from "../src/core/runs.ts";
import { createToken } from "../src/core/tokens.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { readRunLog } from "../src/runs/log.ts";
import { grantedCookie } from "./helpers/identity.ts";
const clientEnv = {
@@ -830,6 +836,228 @@
describe("apps: production, previews, data, gates, crew", () => {
  }, 90_000);
});
describe("a preview whose copied data the build cannot read starts fresh (issue #36)", () => {
  let dataDir: string;
  let app: App;
  let servers: Servers;
  let runtime: AppsRuntime;
  let alice: User;
  const DOMAIN = "retry.test";
  async function settled(
    where: (run: Run) => boolean,
    repoId: string,
    timeoutMs = 30_000,
  ): Promise<Run> {
    const deadline = Date.now() + timeoutMs;
    while (Date.now() < deadline) {
      const run = listRuns(app.db, repoId, { kind: "deploy", limit: 50 }).find(
        where,
      );
      if (
        run &&
        (run.state === "ok" ||
          run.state === "failed" ||
          run.state === "cancelled")
      )
        return run;
      await Bun.sleep(50);
    }
    throw new Error("run did not settle in time");
  }
  // Never checks in, so `install` has nothing to do: the deploy starts the
  // process directly, no `bun install` on the critical path.
  const FORGE_JSON = JSON.stringify({
    app: { run: "sh run.sh", install: "", health: "/", ready: 5 },
  });
  beforeAll(async () => {
    dataDir = await mkdtemp(join(tmpdir(), "forge-retry-"));
    app = createApp({ dataDir, port: 0 });
    servers = await serve(app);
    runtime = new AppsRuntime(app, {
      domain: DOMAIN,
      scheme: "http",
      publicPort: servers.port,
      forgeUrl: servers.url,
      concurrency: 2,
    });
    await runtime.start();
    const created = await createUser(app.db, {
      username: "alice",
      email: "[email protected]",
      password: "a-good-passphrase",
      adminIfFirst: true,
    });
    if (!created.ok) throw new Error(created.error);
    alice = created.value;
  }, 30_000);
  afterAll(async () => {
    await runtime.stop();
    servers.stop();
    app.close();
    await rm(dataDir, { recursive: true, force: true });
  });
  test("a build too old for a migration on main comes up on the retry", async () => {
    const made = await createRepo(app, alice, {
      name: "schema-ahead",
      visibility: "public",
    });
    expect(made.ok).toBe(true);
    if (!made.ok) return;
    const repo = made.value;
    const committed = await commitToBranch(app, alice, repo, {
      message: "Refuse data from a schema this build does not know",
      files: [
        { path: "forge.json", content: FORGE_JSON },
        {
          path: "run.sh",
          content: `if [ -f "$FORGE_APP_DATA/schema-22" ]; then
  echo "database is at schema 22, this build only knows 21 - refusing to run" 1>&2
  exit 1
fi
exec bun -e "Bun.serve({port:Number(process.env.PORT),hostname:'127.0.0.1',fetch:()=>new Response('ok')})"
`,
        },
      ],
    });
    expect(committed.ok).toBe(true);
    const enabled = enableApp(app, alice, repo, {});
    expect(enabled.ok).toBe(true);
    if (!enabled.ok) return;
    const record = enabled.value;
    const production = await settled(
      (r) => r.env === "production",
      repo.id,
    );
    expect(production.state).toBe("ok");
    // A migration lands on `main` — simulated here the way the issue
    // describes it, a schema file production now has that an older build
    // does not know how to read.
    writeFileSync(
      join(envDataDir(dataDir, record.id, "production"), "schema-22"),
      "22",
    );
    const branch = await commitToBranch(app, alice, repo, {
      branch: "note",
      base: repo.defaultBranch,
      message: "A trivial, unrelated change",
      files: [{ path: "NOTE.md", content: "hi\n" }],
    });
    expect(branch.ok).toBe(true);
    const opened = await openPull(app, alice, repo, {
      title: "Unrelated change",
      baseBranch: repo.defaultBranch,
      headBranch: "note",
    });
    expect(opened.ok).toBe(true);
    if (!opened.ok) return;
    const run = await settled(
      (r) => r.env === `pull-${opened.value.number}`,
      repo.id,
    );
    expect(run.state).toBe("ok");
    const { lines } = readRunLog(dataDir, run.id);
    const text = lines.join("\n");
    expect(text).toContain("data: copied from production");
    expect(text).toContain("exited with code 1 before answering");
    expect(text).toContain(
      "the copied data was not usable by this build; the preview is starting on empty data",
    );
    expect(text).toContain(`serving http://`);
    // The retry's data directory does not carry the discarded copy's file.
    const previewData = envDataDir(
      dataDir,
      record.id,
      previewEnv(opened.value.number),
    );
    expect(existsSync(join(previewData, "schema-22"))).toBe(false);
    const page = await fetch(`${servers.url}/`, {
      headers: { host: `schema-ahead--${opened.value.number}.${DOMAIN}` },
    });
    expect(page.status).toBe(200);
  }, 60_000);
  test("an app that always exits retries exactly once in a preview, and never in production", async () => {
    const made = await createRepo(app, alice, {
      name: "always-fails",
      visibility: "public",
    });
    expect(made.ok).toBe(true);
    if (!made.ok) return;
    const repo = made.value;
    const committed = await commitToBranch(app, alice, repo, {
      message: "A run that never comes up",
      files: [
        { path: "forge.json", content: FORGE_JSON },
        {
          path: "run.sh",
          content: `echo "boom" 1>&2\nexit 1\n`,
        },
      ],
    });
    expect(committed.ok).toBe(true);
    const enabled = enableApp(app, alice, repo, {});
    expect(enabled.ok).toBe(true);
    if (!enabled.ok) return;
    // Production: seeded empty, never a copy, so it gets exactly one try.
    const production = await settled((r) => r.env === "production", repo.id);
    expect(production.state).toBe("failed");
    const prodLog = readRunLog(dataDir, production.id).lines;
    expect(prodLog.filter((l) => l.startsWith("$ sh run.sh")).length).toBe(1);
    expect(
      prodLog.some((l) => l.includes("starting on empty data")),
    ).toBe(false);
    const branch = await commitToBranch(app, alice, repo, {
      branch: "note",
      base: repo.defaultBranch,
      message: "A trivial change",
      files: [{ path: "NOTE.md", content: "hi\n" }],
    });
    expect(branch.ok).toBe(true);
    const opened = await openPull(app, alice, repo, {
      title: "Trivial",
      baseBranch: repo.defaultBranch,
      headBranch: "note",
    });
    expect(opened.ok).toBe(true);
    if (!opened.ok) return;
    const run = await settled(
      (r) => r.env === `pull-${opened.value.number}`,
      repo.id,
    );
    expect(run.state).toBe("failed");
    const previewLog = readRunLog(dataDir, run.id).lines;
    // Exactly one retry: the process is started twice, never more.
    expect(
      previewLog.filter((l) => l.startsWith("$ sh run.sh")).length,
    ).toBe(2);
    expect(
      previewLog.some((l) =>
        l.includes(
          "the copied data was not usable by this build; the preview is starting on empty data",
        ),
      ),
    ).toBe(true);
    // The run's summary names the first failure's reason too.
    expect(run.summary ?? "").toContain("retried on empty data");
    expect(run.summary ?? "").toContain("exited with code 1 before answering");
  }, 60_000);
});
describe("New app: turn the sentence into a plan and pick the crew", () => {
  let dataDir: string;
  let app: App;
		@@ -65,7 +65,11 @@ around it. None is advisory.
65	65	where the filesystem offers it — APFS, XFS, btrfs — a plain copy
66	66	otherwise, and `empty` is a per-app choice). A preview process is handed
67	67	its own `FORGE_APP_DATA` and nothing that names production's. Merging a
68		pull moves code; it never touches data.
	68	pull moves code; it never touches data. A copy the build cannot read —
	69	its process exits before it ever answers, most often a migration on
	70	`main` the pull's commit predates — is tried exactly once more on an
	71	empty data directory before the deploy is failed; production is never
	72	seeded from a copy, so this never applies to it.
69	73	4. A deployment is a check. Every deploy is a run; the run's state is
70	74	what the pull shows. With `require_checks` on, `mergePull` refuses
71	75	`checks_required` until the head's deploy run is `ok`, read live inside
		@@ -99,6 +99,18 @@ export function ensureEnvData(
99	99	return cloneDir(source, target);
100	100	}
101	101
	102	/**
	103	* Discard a data directory that turned out to be unusable and recreate it
	104	* empty. Used only after a copy has already been tried and its process
	105	* would not come up: the copy is not kept beside the fresh directory, and
	106	* nothing outside this one environment's data directory is touched.
	107	*/
	108	export function reseedEmpty(dataDir: string, appId: string, env: EnvName): void {
	109	const target = envDataDir(dataDir, appId, env);
	110	rmSync(target, { recursive: true, force: true });
	111	mkdirSync(target, { recursive: true });
	112	}
	113
102	114	/** Remove an environment entirely: checkouts, data, everything. */
103	115	export function removeEnv(dataDir: string, appId: string, env: EnvName): void {
104	116	rmSync(envDir(dataDir, appId, env), { recursive: true, force: true });
		@@ -52,11 +52,17 @@ import {
52	52	previewEnv,
53	53	removeApp,
54	54	removeEnv,
	55	reseedEmpty,
55	56	type EnvName,
56	57	} from "./data.ts";
57	58	import { parseManifest, type AppManifest } from "./manifest.ts";
58	59	import { identityKey } from "./proxy.ts";
59		import { startProcess, type ProcessHandle } from "./process.ts";
	60	import {
	61	startProcess,
	62	type ProcessHandle,
	63	type StartError,
	64	type StartInput,
	65	} from "./process.ts";
60	66
61	67	const log = createLog("apps");
62	68
		@@ -124,6 +130,19 @@ function elapsedSeconds(fromIso: string, toIso: string): string {
124	130	return `${(ms / 1000).toFixed(1)}s`;
125	131	}
126	132
	133	/**
	134	* Whether a start failure is the process itself exiting before it ever
	135	* answered its health path — as opposed to install failing, or the process
	136	* staying up but never answering. Only this shape of failure is worth
	137	* retrying on empty data: it is what a build too old for the data it was
	138	* given looks like.
	139	*/
	140	function diedBeforeAnswering(error: StartError): boolean {
	141	return error.stage === "start" && /^exited with code \d+ before answering/.test(
	142	error.message,
	143	);
	144	}
	145
127	146	/** A running environment, as a terminal session needs it. */
128	147	export interface LiveEnv {
129	148	readonly env: EnvName;
		@@ -729,7 +748,7 @@ export class AppsRuntime {
729	748	else if (seeded === "empty" && state.env !== "production")
730	749	write("data: starting empty");
731	750
732		const started = await startProcess({
	751	const startInput: Omit<StartInput, "signal"> = {
733	752	dir,
734	753	dataDir,
735	754	manifest: manifest.value,
		@@ -743,9 +762,33 @@ export class AppsRuntime {
743	762	// process that comes up is the one that sees it.
744	763	vars: resolveAppEnv(this.app.db, this.app.config.dataDir, state.app.id),
745	764	log: write,
746		signal,
747		});
748		if (!started.ok) {
	765	};
	766	let started = await startProcess({ ...startInput, signal });
	767
	768	// The one retry this deploy gets: a preview seeded from production whose
	769	// process died before it ever answered — schema ahead of what this build
	770	// knows, most often — is tried again on empty data, once. A production
	771	// deploy is never seeded from a copy, so it never takes this path.
	772	if (
	773	!started.ok &&
	774	(seeded === "cloned" \|\| seeded === "copied") &&
	775	diedBeforeAnswering(started.error)
	776	) {
	777	const first = started.error;
	778	write(`failed: ${first.stage}: ${first.message}`);
	779	write(
	780	"the copied data was not usable by this build; the preview is starting on empty data",
	781	);
	782	reseedEmpty(this.app.config.dataDir, state.app.id, state.env);
	783	started = await startProcess({ ...startInput, signal });
	784	if (!started.ok) {
	785	rmSync(dir, { recursive: true, force: true });
	786	return {
	787	ok: false,
	788	error: `${started.error.stage}: ${started.error.message} (retried on empty data after the copy failed with ${first.stage}: ${first.message})`,
	789	};
	790	}
	791	} else if (!started.ok) {
749	792	rmSync(dir, { recursive: true, force: true });
750	793	return {
751	794	ok: false,
		@@ -14,8 +14,13 @@ import { tmpdir } from "node:os";
14	14	import { join } from "node:path";
15	15	import { createApp, type App } from "../src/app.ts";
16	16	import { AppsRuntime } from "../src/apps/runtime.ts";
17		import { envDir } from "../src/apps/data.ts";
18		import { newApp, openEnvUrl, updateAppSettings } from "../src/actions/apps.ts";
	17	import { envDataDir, envDir, previewEnv } from "../src/apps/data.ts";
	18	import {
	19	enableApp,
	20	newApp,
	21	openEnvUrl,
	22	updateAppSettings,
	23	} from "../src/actions/apps.ts";
19	24	import { addToCrew, createAgent, listCrew } from "../src/actions/crew.ts";
20	25	import {
21	26	assignIssue,
		@@ -38,6 +43,7 @@ import { listRuns, type Run } from "../src/core/runs.ts";
38	43	import { createToken } from "../src/core/tokens.ts";
39	44	import { createUser, type User } from "../src/core/users.ts";
40	45	import { serve, type Servers } from "../src/http/server.ts";
	46	import { readRunLog } from "../src/runs/log.ts";
41	47	import { grantedCookie } from "./helpers/identity.ts";
42	48
43	49	const clientEnv = {
		@@ -830,6 +836,228 @@ describe("apps: production, previews, data, gates, crew", () => {
830	836	}, 90_000);
831	837	});
832	838
	839	describe("a preview whose copied data the build cannot read starts fresh (issue #36)", () => {
	840	let dataDir: string;
	841	let app: App;
	842	let servers: Servers;
	843	let runtime: AppsRuntime;
	844	let alice: User;
	845	const DOMAIN = "retry.test";
	846
	847	async function settled(
	848	where: (run: Run) => boolean,
	849	repoId: string,
	850	timeoutMs = 30_000,
	851	): Promise<Run> {
	852	const deadline = Date.now() + timeoutMs;
	853	while (Date.now() < deadline) {
	854	const run = listRuns(app.db, repoId, { kind: "deploy", limit: 50 }).find(
	855	where,
	856	);
	857	if (
	858	run &&
	859	(run.state === "ok" \|\|
	860	run.state === "failed" \|\|
	861	run.state === "cancelled")
	862	)
	863	return run;
	864	await Bun.sleep(50);
	865	}
	866	throw new Error("run did not settle in time");
	867	}
	868
	869	// Never checks in, so `install` has nothing to do: the deploy starts the
	870	// process directly, no `bun install` on the critical path.
	871	const FORGE_JSON = JSON.stringify({
	872	app: { run: "sh run.sh", install: "", health: "/", ready: 5 },
	873	});
	874
	875	beforeAll(async () => {
	876	dataDir = await mkdtemp(join(tmpdir(), "forge-retry-"));
	877	app = createApp({ dataDir, port: 0 });
	878	servers = await serve(app);
	879	runtime = new AppsRuntime(app, {
	880	domain: DOMAIN,
	881	scheme: "http",
	882	publicPort: servers.port,
	883	forgeUrl: servers.url,
	884	concurrency: 2,
	885	});
	886	await runtime.start();
	887	const created = await createUser(app.db, {
	888	username: "alice",
	889	email: "[email protected]",
	890	password: "a-good-passphrase",
	891	adminIfFirst: true,
	892	});
	893	if (!created.ok) throw new Error(created.error);
	894	alice = created.value;
	895	}, 30_000);
	896
	897	afterAll(async () => {
	898	await runtime.stop();
	899	servers.stop();
	900	app.close();
	901	await rm(dataDir, { recursive: true, force: true });
	902	});
	903
	904	test("a build too old for a migration on main comes up on the retry", async () => {
	905	const made = await createRepo(app, alice, {
	906	name: "schema-ahead",
	907	visibility: "public",
	908	});
	909	expect(made.ok).toBe(true);
	910	if (!made.ok) return;
	911	const repo = made.value;
	912	const committed = await commitToBranch(app, alice, repo, {
	913	message: "Refuse data from a schema this build does not know",
	914	files: [
	915	{ path: "forge.json", content: FORGE_JSON },
	916	{
	917	path: "run.sh",
	918	content: `if [ -f "$FORGE_APP_DATA/schema-22" ]; then
	919	echo "database is at schema 22, this build only knows 21 - refusing to run" 1>&2
	920	exit 1
	921	fi
	922	exec bun -e "Bun.serve({port:Number(process.env.PORT),hostname:'127.0.0.1',fetch:()=>new Response('ok')})"
	923	`,
	924	},
	925	],
	926	});
	927	expect(committed.ok).toBe(true);
	928	const enabled = enableApp(app, alice, repo, {});
	929	expect(enabled.ok).toBe(true);
	930	if (!enabled.ok) return;
	931	const record = enabled.value;
	932
	933	const production = await settled(
	934	(r) => r.env === "production",
	935	repo.id,
	936	);
	937	expect(production.state).toBe("ok");
	938
	939	// A migration lands on `main` — simulated here the way the issue
	940	// describes it, a schema file production now has that an older build
	941	// does not know how to read.
	942	writeFileSync(
	943	join(envDataDir(dataDir, record.id, "production"), "schema-22"),
	944	"22",
	945	);
	946
	947	const branch = await commitToBranch(app, alice, repo, {
	948	branch: "note",
	949	base: repo.defaultBranch,
	950	message: "A trivial, unrelated change",
	951	files: [{ path: "NOTE.md", content: "hi\n" }],
	952	});
	953	expect(branch.ok).toBe(true);
	954	const opened = await openPull(app, alice, repo, {
	955	title: "Unrelated change",
	956	baseBranch: repo.defaultBranch,
	957	headBranch: "note",
	958	});
	959	expect(opened.ok).toBe(true);
	960	if (!opened.ok) return;
	961
	962	const run = await settled(
	963	(r) => r.env === `pull-${opened.value.number}`,
	964	repo.id,
	965	);
	966	expect(run.state).toBe("ok");
	967
	968	const { lines } = readRunLog(dataDir, run.id);
	969	const text = lines.join("\n");
	970	expect(text).toContain("data: copied from production");
	971	expect(text).toContain("exited with code 1 before answering");
	972	expect(text).toContain(
	973	"the copied data was not usable by this build; the preview is starting on empty data",
	974	);
	975	expect(text).toContain(`serving http://`);
	976
	977	// The retry's data directory does not carry the discarded copy's file.
	978	const previewData = envDataDir(
	979	dataDir,
	980	record.id,
	981	previewEnv(opened.value.number),
	982	);
	983	expect(existsSync(join(previewData, "schema-22"))).toBe(false);
	984
	985	const page = await fetch(`${servers.url}/`, {
	986	headers: { host: `schema-ahead--${opened.value.number}.${DOMAIN}` },
	987	});
	988	expect(page.status).toBe(200);
	989	}, 60_000);
	990
	991	test("an app that always exits retries exactly once in a preview, and never in production", async () => {
	992	const made = await createRepo(app, alice, {
	993	name: "always-fails",
	994	visibility: "public",
	995	});
	996	expect(made.ok).toBe(true);
	997	if (!made.ok) return;
	998	const repo = made.value;
	999	const committed = await commitToBranch(app, alice, repo, {
	1000	message: "A run that never comes up",
	1001	files: [
	1002	{ path: "forge.json", content: FORGE_JSON },
	1003	{
	1004	path: "run.sh",
	1005	content: `echo "boom" 1>&2\nexit 1\n`,
	1006	},
	1007	],
	1008	});
	1009	expect(committed.ok).toBe(true);
	1010	const enabled = enableApp(app, alice, repo, {});
	1011	expect(enabled.ok).toBe(true);
	1012	if (!enabled.ok) return;
	1013
	1014	// Production: seeded empty, never a copy, so it gets exactly one try.
	1015	const production = await settled((r) => r.env === "production", repo.id);
	1016	expect(production.state).toBe("failed");
	1017	const prodLog = readRunLog(dataDir, production.id).lines;
	1018	expect(prodLog.filter((l) => l.startsWith("$ sh run.sh")).length).toBe(1);
	1019	expect(
	1020	prodLog.some((l) => l.includes("starting on empty data")),
	1021	).toBe(false);
	1022
	1023	const branch = await commitToBranch(app, alice, repo, {
	1024	branch: "note",
	1025	base: repo.defaultBranch,
	1026	message: "A trivial change",
	1027	files: [{ path: "NOTE.md", content: "hi\n" }],
	1028	});
	1029	expect(branch.ok).toBe(true);
	1030	const opened = await openPull(app, alice, repo, {
	1031	title: "Trivial",
	1032	baseBranch: repo.defaultBranch,
	1033	headBranch: "note",
	1034	});
	1035	expect(opened.ok).toBe(true);
	1036	if (!opened.ok) return;
	1037
	1038	const run = await settled(
	1039	(r) => r.env === `pull-${opened.value.number}`,
	1040	repo.id,
	1041	);
	1042	expect(run.state).toBe("failed");
	1043	const previewLog = readRunLog(dataDir, run.id).lines;
	1044	// Exactly one retry: the process is started twice, never more.
	1045	expect(
	1046	previewLog.filter((l) => l.startsWith("$ sh run.sh")).length,
	1047	).toBe(2);
	1048	expect(
	1049	previewLog.some((l) =>
	1050	l.includes(
	1051	"the copied data was not usable by this build; the preview is starting on empty data",
	1052	),
	1053	),
	1054	).toBe(true);
	1055	// The run's summary names the first failure's reason too.
	1056	expect(run.summary ?? "").toContain("retried on empty data");
	1057	expect(run.summary ?? "").toContain("exited with code 1 before answering");
	1058	}, 60_000);
	1059	});
	1060
833	1061	describe("New app: turn the sentence into a plan and pick the crew", () => {
834	1062	let dataDir: string;
835	1063	let app: App;