4b90a17
@@ -22,26 +22,46 @@
import { serve, type Servers } from "../src/http/server.ts";
import { readRunLog } from "../src/runs/log.ts";
/**
 * The app under test: `/work?s=N` starts a child that works for N seconds
 * and answers at once; `/stream` is an event stream that stays open until
 * the browser goes away.
 * The app under test: `/work` starts a child that works until `/done` ends
 * it; `/slow` answers only once `/answer` is asked (`/waiting` counts
 * those held); `/stream` is an event
 * stream that stays open until the browser goes away. Nothing in it waits
 * on the wall clock, so the suite never does either.
 */
const BUSY_SERVER = `const port = Number(process.env.PORT ?? "0");
const children = new Set<ReturnType<typeof Bun.spawn>>();
const waiting: Array<() => void> = [];
export const server = Bun.serve({
  port,
  fetch(request) {
  async fetch(request) {
    const url = new URL(request.url);
    if (url.pathname === "/work") {
      Bun.spawn(["sleep", String(Number(url.searchParams.get("s") ?? "1"))], {
      const child = Bun.spawn(["sleep", "300"], {
        stdio: ["ignore", "ignore", "ignore"],
      });
      children.add(child);
      return new Response("working");
    }
    if (url.pathname === "/done") {
      for (const child of children) {
        child.kill();
        await child.exited;
      }
      children.clear();
      return new Response("done");
    }
    if (url.pathname === "/slow") {
      return new Promise((resolve) =>
        setTimeout(() => resolve(new Response("late")), 2000),
        waiting.push(() => resolve(new Response("late"))),
      );
    }
    if (url.pathname === "/waiting") {
      return new Response(String(waiting.length));
    }
    if (url.pathname === "/answer") {
      for (const answer of waiting.splice(0)) answer();
      return new Response("answered");
    }
    if (url.pathname === "/stream") {
      let timer: ReturnType<typeof setInterval> | undefined;
      const body = new ReadableStream({
@@ -86,8 +106,14 @@
describe("apps that are busy", () => {
  let record: AppRecord;
  const DOMAIN = "apps.test";
  const HOST = `busy.${DOMAIN}`;
  const WINDOW = 700;
  const GRACE = 4_000;
  // Windows of the sweep's own clock, which the suite runs ahead by hand:
  // no real wait ever reaches them, so nothing sleeps unless a test says so.
  const WINDOW = 60 * 60_000;
  const GRACE = 10 * 60_000;
  /** How far the sweep's clock runs ahead of the wall's. */
  let ahead = 0;
  /** How many sweeps have read the clock. */
  let sweeps = 0;
  const viaHost = (path = "/", init: RequestInit = {}) =>
    fetch(`${servers.url}${path}`, {
@@ -104,7 +130,7 @@
describe("apps that are busy", () => {
    while (Date.now() < deadline) {
      const value = read();
      if (value !== undefined) return value;
      await Bun.sleep(50);
      await Bun.sleep(5);
    }
    throw new Error("did not happen in time");
  }
@@ -114,14 +140,32 @@
describe("apps that are busy", () => {
        (r) => where(r) && (r.state === "ok" || r.state === "failed"),
      ),
    );
  /**
   * Run the sweep's clock to `ms` past the wall and wait for two sweeps to
   * read it: the first decides synchronously after its read, so by the
   * second every environment has been judged at that hour.
   */
  async function sweepAt(ms: number): Promise<void> {
    ahead = ms;
    const from = sweeps;
    await until(() => (sweeps >= from + 2 ? true : undefined), 10_000);
  }
  const asleep = () =>
    until(() => (production().state === "asleep" ? true : undefined), 20_000);
    until(() => (production().state === "asleep" ? true : undefined), 10_000);
  const log = (): string =>
    readRunLog(dataDir, production().runId!, 64 * 1024).lines.join("\n");
  /** What the run log gained after `before`. */
  const since = (before: string): string => log().slice(before.length);
  /** Wake production and wait for it to fall asleep, so each test starts rested. */
  /** Put production to sleep and wake it, so each test starts rested. */
  async function freshlyAwake(): Promise<void> {
    if (production().state !== "asleep") await asleep();
    if (production().state !== "asleep") {
      await viaHost("/done");
      await viaHost("/answer");
      await sweepAt(WINDOW + GRACE + 60_000);
      await asleep();
    }
    ahead = 0;
    expect((await viaHost("/")).status).toBe(200);
    expect(production().state).toBe("running");
  }
@@ -135,9 +179,12 @@
describe("apps that are busy", () => {
      scheme: "http",
      publicPort: servers.port,
      forgeUrl: servers.url,
      // Long enough for the scaffold and the busy app to deploy and settle.
      sleepAfterMs: { production: 3_600_000, preview: 3_600_000 },
      sweepEveryMs: 100,
      sleepAfterMs: { production: WINDOW, preview: WINDOW },
      clock: () => {
        sweeps++;
        return Date.now() + ahead;
      },
      sweepEveryMs: 10,
      busyGraceMs: GRACE,
    });
    await runtime.start();
@@ -176,18 +223,6 @@
describe("apps that are busy", () => {
        r.env === "production" && r.commitOid === committed.value.commitOid,
    );
    if (run.state !== "ok") throw new Error(`deploy failed: ${run.summary}`);
    // The runtime the rest of the suite judges: the same data, a short window.
    await runtime.stop();
    runtime = new AppsRuntime(app, {
      domain: DOMAIN,
      scheme: "http",
      publicPort: servers.port,
      forgeUrl: servers.url,
      sleepAfterMs: { production: WINDOW, preview: WINDOW },
      sweepEveryMs: 100,
      busyGraceMs: GRACE,
    });
    await runtime.start();
    await until(() => (production().state === "running" ? true : undefined));
  }, 180_000);
@@ -199,24 +234,29 @@
describe("apps that are busy", () => {
  });
  test("an idle app still sleeps, and says nothing about being busy", async () => {
    // Invariant: idle past its window, an app with no work sleeps, and no busy is said.
    await freshlyAwake();
    const before = log();
    await sweepAt(WINDOW - 60_000);
    expect(production().state).toBe("running");
    await sweepAt(WINDOW);
    await asleep();
    expect(production().busy).toBeUndefined();
    expect(log()).toContain("asleep: nothing asked for it");
  }, 30_000);
    expect(since(before)).toContain("asleep: nothing asked for it");
  });
  test.skipIf(noProc)(
    "an app whose child is still working survives the sweep, then sleeps once it is done",
    async () => {
      // Invariant: a working child keeps it awake past the window, the pages say so, and it sleeps once the child is gone.
      await freshlyAwake();
      // Five seconds of work: long enough that a loaded machine still checks
      // while the child is busy (two seconds raced the check under load).
      expect(await (await viaHost("/work?s=5")).text()).toBe("working");
      const before = log();
      expect(await (await viaHost("/work")).text()).toBe("working");
      // Well past the idle window, and still up: it was asked, and it is busy.
      await Bun.sleep(WINDOW + 700);
      await sweepAt(WINDOW + 60_000);
      expect(production().state).toBe("running");
      expect(production().busy).toContain("child processes");
      expect(log()).toContain("kept awake: busy");
      expect(since(before)).toContain("kept awake: busy");
      // The app page and the Now card both say why it is up.
      const page = await (await fetch(`${servers.url}/alice/busy/app`)).text();
@@ -228,22 +268,30 @@
describe("apps that are busy", () => {
      expect(now).toContain("its child processes are still working");
      // The child finishes; the next sweep finds nothing to wait for.
      expect(await (await viaHost("/done")).text()).toBe("done");
      
        junipercrew
        2d ago
        
      
      Unbounded poll: use until(...) with a deadline so a miss gives a clear failure.
    
      await sweepAt(WINDOW + 60_000);
      await asleep();
      expect(production().busy).toBeUndefined();
    },
    30_000,
  );
  test("a request still waiting on its answer keeps the app awake", async () => {
    // Invariant: a request in flight keeps it awake past the window; once answered it sleeps.
    await freshlyAwake();
    // The answer takes three windows; the sweep must not stop the app first.
    const response = await viaHost("/slow");
    expect(await response.text()).toBe("late");
    const response = viaHost("/slow");
    // The app holds it, so the proxy is holding it too.
    while ((await (await viaHost("/waiting")).text()) !== "1") await Bun.sleep(5);
    await sweepAt(WINDOW + 60_000);
    expect(production().state).toBe("running");
    expect(production().busy).toContain("a request is still waiting");
    await viaHost("/answer");
    expect(await (await response).text()).toBe("late");
    await sweepAt(WINDOW + 60_000);
    await asleep();
  }, 30_000);
  });
  test("an open stream keeps the app awake until it closes", async () => {
    // Invariant: an open stream keeps it awake past the window; once closed it sleeps.
    await freshlyAwake();
    const abort = new AbortController();
    const response = await viaHost("/stream", { signal: abort.signal });
@@ -251,29 +299,28 @@
describe("apps that are busy", () => {
    const reader = response.body!.getReader();
    const first = await reader.read();
    expect(new TextDecoder().decode(first.value)).toContain("data: hello");
    await Bun.sleep(WINDOW + 700);
    await sweepAt(WINDOW + 60_000);
    expect(production().state).toBe("running");
    expect(production().busy).toContain("a stream to it is still open");
    abort.abort();
    await reader.cancel().catch(() => {});
    await asleep();
  }, 30_000);
  });
  test.skipIf(noProc)(
    "an app that stays busy sleeps anyway once the grace is spent",
    async () => {
      // Invariant: busy spares an app for one grace past its window, never longer.
      await freshlyAwake();
      const started = Date.now();
      // Works far longer than the window and the grace together.
      await viaHost("/work?s=12");
      await Bun.sleep(WINDOW + 700);
      const before = log();
      await viaHost("/work");
      await sweepAt(WINDOW + GRACE - 60_000);
      expect(production().state).toBe("running");
      expect(production().busy).toContain("child processes");
      await sweepAt(WINDOW + GRACE);
      await asleep();
      expect(Date.now() - started).toBeGreaterThanOrEqual(WINDOW + GRACE);
      expect(Date.now() - started).toBeLessThan(10_000);
      expect(log()).toContain("asleep: still busy");
      expect(since(before)).toContain("asleep: still busy");
    },
    30_000,
  );
});
		@@ -22,26 +22,46 @@ import { serve, type Servers } from "../src/http/server.ts";
22	22	import { readRunLog } from "../src/runs/log.ts";
23	23
24	24	/**
25		* The app under test: `/work?s=N` starts a child that works for N seconds
26		* and answers at once; `/stream` is an event stream that stays open until
27		* the browser goes away.
	25	* The app under test: `/work` starts a child that works until `/done` ends
	26	* it; `/slow` answers only once `/answer` is asked (`/waiting` counts
	27	* those held); `/stream` is an event
	28	* stream that stays open until the browser goes away. Nothing in it waits
	29	* on the wall clock, so the suite never does either.
28	30	*/
29	31	const BUSY_SERVER = `const port = Number(process.env.PORT ?? "0");
	32	const children = new Set<ReturnType<typeof Bun.spawn>>();
	33	const waiting: Array<() => void> = [];
30	34	export const server = Bun.serve({
31	35	port,
32		fetch(request) {
	36	async fetch(request) {
33	37	const url = new URL(request.url);
34	38	if (url.pathname === "/work") {
35		Bun.spawn(["sleep", String(Number(url.searchParams.get("s") ?? "1"))], {
	39	const child = Bun.spawn(["sleep", "300"], {
36	40	stdio: ["ignore", "ignore", "ignore"],
37	41	});
	42	children.add(child);
38	43	return new Response("working");
39	44	}
	45	if (url.pathname === "/done") {
	46	for (const child of children) {
	47	child.kill();
	48	await child.exited;
	49	}
	50	children.clear();
	51	return new Response("done");
	52	}
40	53	if (url.pathname === "/slow") {
41	54	return new Promise((resolve) =>
42		setTimeout(() => resolve(new Response("late")), 2000),
	55	waiting.push(() => resolve(new Response("late"))),
43	56	);
44	57	}
	58	if (url.pathname === "/waiting") {
	59	return new Response(String(waiting.length));
	60	}
	61	if (url.pathname === "/answer") {
	62	for (const answer of waiting.splice(0)) answer();
	63	return new Response("answered");
	64	}
45	65	if (url.pathname === "/stream") {
46	66	let timer: ReturnType<typeof setInterval> \| undefined;
47	67	const body = new ReadableStream({
		@@ -86,8 +106,14 @@ describe("apps that are busy", () => {
86	106	let record: AppRecord;
87	107	const DOMAIN = "apps.test";
88	108	const HOST = `busy.${DOMAIN}`;
89		const WINDOW = 700;
90		const GRACE = 4_000;
	109	// Windows of the sweep's own clock, which the suite runs ahead by hand:
	110	// no real wait ever reaches them, so nothing sleeps unless a test says so.
	111	const WINDOW = 60 * 60_000;
	112	const GRACE = 10 * 60_000;
	113	/** How far the sweep's clock runs ahead of the wall's. */
	114	let ahead = 0;
	115	/** How many sweeps have read the clock. */
	116	let sweeps = 0;
91	117
92	118	const viaHost = (path = "/", init: RequestInit = {}) =>
93	119	fetch(`${servers.url}${path}`, {
		@@ -104,7 +130,7 @@ describe("apps that are busy", () => {
104	130	while (Date.now() < deadline) {
105	131	const value = read();
106	132	if (value !== undefined) return value;
107		await Bun.sleep(50);
	133	await Bun.sleep(5);
108	134	}
109	135	throw new Error("did not happen in time");
110	136	}
		@@ -114,14 +140,32 @@ describe("apps that are busy", () => {
114	140	(r) => where(r) && (r.state === "ok" \|\| r.state === "failed"),
115	141	),
116	142	);
	143	/**
	144	* Run the sweep's clock to `ms` past the wall and wait for two sweeps to
	145	* read it: the first decides synchronously after its read, so by the
	146	* second every environment has been judged at that hour.
	147	*/
	148	async function sweepAt(ms: number): Promise<void> {
	149	ahead = ms;
	150	const from = sweeps;
	151	await until(() => (sweeps >= from + 2 ? true : undefined), 10_000);
	152	}
117	153	const asleep = () =>
118		until(() => (production().state === "asleep" ? true : undefined), 20_000);
	154	until(() => (production().state === "asleep" ? true : undefined), 10_000);
119	155	const log = (): string =>
120	156	readRunLog(dataDir, production().runId!, 64 * 1024).lines.join("\n");
	157	/** What the run log gained after `before`. */
	158	const since = (before: string): string => log().slice(before.length);
121	159
122		/** Wake production and wait for it to fall asleep, so each test starts rested. */
	160	/** Put production to sleep and wake it, so each test starts rested. */
123	161	async function freshlyAwake(): Promise<void> {
124		if (production().state !== "asleep") await asleep();
	162	if (production().state !== "asleep") {
	163	await viaHost("/done");
	164	await viaHost("/answer");
	165	await sweepAt(WINDOW + GRACE + 60_000);
	166	await asleep();
	167	}
	168	ahead = 0;
125	169	expect((await viaHost("/")).status).toBe(200);
126	170	expect(production().state).toBe("running");
127	171	}
		@@ -135,9 +179,12 @@ describe("apps that are busy", () => {
135	179	scheme: "http",
136	180	publicPort: servers.port,
137	181	forgeUrl: servers.url,
138		// Long enough for the scaffold and the busy app to deploy and settle.
139		sleepAfterMs: { production: 3_600_000, preview: 3_600_000 },
140		sweepEveryMs: 100,
	182	sleepAfterMs: { production: WINDOW, preview: WINDOW },
	183	clock: () => {
	184	sweeps++;
	185	return Date.now() + ahead;
	186	},
	187	sweepEveryMs: 10,
141	188	busyGraceMs: GRACE,
142	189	});
143	190	await runtime.start();
		@@ -176,18 +223,6 @@ describe("apps that are busy", () => {
176	223	r.env === "production" && r.commitOid === committed.value.commitOid,
177	224	);
178	225	if (run.state !== "ok") throw new Error(`deploy failed: ${run.summary}`);
179		// The runtime the rest of the suite judges: the same data, a short window.
180		await runtime.stop();
181		runtime = new AppsRuntime(app, {
182		domain: DOMAIN,
183		scheme: "http",
184		publicPort: servers.port,
185		forgeUrl: servers.url,
186		sleepAfterMs: { production: WINDOW, preview: WINDOW },
187		sweepEveryMs: 100,
188		busyGraceMs: GRACE,
189		});
190		await runtime.start();
191	226	await until(() => (production().state === "running" ? true : undefined));
192	227	}, 180_000);
193	228
		@@ -199,24 +234,29 @@ describe("apps that are busy", () => {
199	234	});
200	235
201	236	test("an idle app still sleeps, and says nothing about being busy", async () => {
	237	// Invariant: idle past its window, an app with no work sleeps, and no busy is said.
202	238	await freshlyAwake();
	239	const before = log();
	240	await sweepAt(WINDOW - 60_000);
	241	expect(production().state).toBe("running");
	242	await sweepAt(WINDOW);
203	243	await asleep();
204	244	expect(production().busy).toBeUndefined();
205		expect(log()).toContain("asleep: nothing asked for it");
206		}, 30_000);
	245	expect(since(before)).toContain("asleep: nothing asked for it");
	246	});
207	247
208	248	test.skipIf(noProc)(
209	249	"an app whose child is still working survives the sweep, then sleeps once it is done",
210	250	async () => {
	251	// Invariant: a working child keeps it awake past the window, the pages say so, and it sleeps once the child is gone.
211	252	await freshlyAwake();
212		// Five seconds of work: long enough that a loaded machine still checks
213		// while the child is busy (two seconds raced the check under load).
214		expect(await (await viaHost("/work?s=5")).text()).toBe("working");
	253	const before = log();
	254	expect(await (await viaHost("/work")).text()).toBe("working");
215	255	// Well past the idle window, and still up: it was asked, and it is busy.
216		await Bun.sleep(WINDOW + 700);
	256	await sweepAt(WINDOW + 60_000);
217	257	expect(production().state).toBe("running");
218	258	expect(production().busy).toContain("child processes");
219		expect(log()).toContain("kept awake: busy");
	259	expect(since(before)).toContain("kept awake: busy");
220	260
221	261	// The app page and the Now card both say why it is up.
222	262	const page = await (await fetch(`${servers.url}/alice/busy/app`)).text();
		@@ -228,22 +268,30 @@ describe("apps that are busy", () => {
228	268	expect(now).toContain("its child processes are still working");
229	269
230	270	// The child finishes; the next sweep finds nothing to wait for.
	271	expect(await (await viaHost("/done")).text()).toBe("done");
		junipercrew 2d ago Unbounded poll: use `until(...)` with a deadline so a miss gives a clear failure.
	272	await sweepAt(WINDOW + 60_000);
231	273	await asleep();
232	274	expect(production().busy).toBeUndefined();
233	275	},
234		30_000,
235	276	);
236	277
237	278	test("a request still waiting on its answer keeps the app awake", async () => {
	279	// Invariant: a request in flight keeps it awake past the window; once answered it sleeps.
238	280	await freshlyAwake();
239		// The answer takes three windows; the sweep must not stop the app first.
240		const response = await viaHost("/slow");
241		expect(await response.text()).toBe("late");
	281	const response = viaHost("/slow");
	282	// The app holds it, so the proxy is holding it too.
	283	while ((await (await viaHost("/waiting")).text()) !== "1") await Bun.sleep(5);
	284	await sweepAt(WINDOW + 60_000);
242	285	expect(production().state).toBe("running");
	286	expect(production().busy).toContain("a request is still waiting");
	287	await viaHost("/answer");
	288	expect(await (await response).text()).toBe("late");
	289	await sweepAt(WINDOW + 60_000);
243	290	await asleep();
244		}, 30_000);
	291	});
245	292
246	293	test("an open stream keeps the app awake until it closes", async () => {
	294	// Invariant: an open stream keeps it awake past the window; once closed it sleeps.
247	295	await freshlyAwake();
248	296	const abort = new AbortController();
249	297	const response = await viaHost("/stream", { signal: abort.signal });
		@@ -251,29 +299,28 @@ describe("apps that are busy", () => {
251	299	const reader = response.body!.getReader();
252	300	const first = await reader.read();
253	301	expect(new TextDecoder().decode(first.value)).toContain("data: hello");
254		await Bun.sleep(WINDOW + 700);
	302	await sweepAt(WINDOW + 60_000);
255	303	expect(production().state).toBe("running");
256	304	expect(production().busy).toContain("a stream to it is still open");
257	305
258	306	abort.abort();
259	307	await reader.cancel().catch(() => {});
260	308	await asleep();
261		}, 30_000);
	309	});
262	310
263	311	test.skipIf(noProc)(
264	312	"an app that stays busy sleeps anyway once the grace is spent",
265	313	async () => {
	314	// Invariant: busy spares an app for one grace past its window, never longer.
266	315	await freshlyAwake();
267		const started = Date.now();
268		// Works far longer than the window and the grace together.
269		await viaHost("/work?s=12");
270		await Bun.sleep(WINDOW + 700);
	316	const before = log();
	317	await viaHost("/work");
	318	await sweepAt(WINDOW + GRACE - 60_000);
271	319	expect(production().state).toBe("running");
	320	expect(production().busy).toContain("child processes");
	321	await sweepAt(WINDOW + GRACE);
272	322	await asleep();
273		expect(Date.now() - started).toBeGreaterThanOrEqual(WINDOW + GRACE);
274		expect(Date.now() - started).toBeLessThan(10_000);
275		expect(log()).toContain("asleep: still busy");
	323	expect(since(before)).toContain("asleep: still busy");
276	324	},
277		30_000,
278	325	);
279	326	});