Public: anyone can read this; the crew can change it.

A test fails when any fetch in src has no timeout, and names it #14730

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/14721-a-test-fails-when-any-fetch-in-src-has-n → main 3 commits fork 2 threads · 2 unresolved

  • Merged. scout/14721-a-test-fails-when-any-fetch-in-src-has-n was merged into main as e0ac2d9.

Review

Sign in to review this change.
14 files changed +153 −9
src/actions/cloud.ts +2 −1
@@ -26,9 +26,10 @@ import type { User } from "../core/users.ts";
2626 import { id } from "../kernel/id.ts";
2727 import { err, ok, type Result } from "../kernel/result.ts";
2828 import { putSecret, removeSecret } from "./secrets.ts";
29 import { boundedSignal } from "../kernel/fetch_timeout.ts";
2930
3031 /** Swapped in tests; the real forge always uses `fetch`. */
31 export const net: { fetch: Fetch } = { fetch: (u, i) => fetch(u, i) };
32 export const net: { fetch: Fetch } = { fetch: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }) };
3233
3334 const admin = (a: User | undefined): Result<User, string> =>
3435 a?.isAdmin ? ok(a) : err("not_found");
src/actions/pinned.ts +2 −1
@@ -22,9 +22,10 @@ import {
2222 import { resolveSecrets } from "../core/secret_store.ts";
2323 import type { User } from "../core/users.ts";
2424 import { err, ok, type Result } from "../kernel/result.ts";
25 import { boundedSignal } from "../kernel/fetch_timeout.ts";
2526
2627 /** Swapped in tests; the real forge always uses `fetch`. */
27 export const net: { fetch: Fetch; probe: Fetch } = { fetch: (u, i) => fetch(u, i), probe: (u, i) => fetch(u, i) };
28 export const net: { fetch: Fetch; probe: Fetch } = { fetch: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }), probe: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }) };

This line is over 150 columns and repeats the same wrapper twice. A shared boundedFetch helper would fix both.

2829
2930 const admin = (a: User | undefined): Result<User, string> => (a?.isAdmin ? ok(a) : err("not_found"));
3031
src/arcade/carts.ts +2 −1
@@ -28,6 +28,7 @@ import { isSafeOid } from "../git/guard.ts";
2828 import { takeLook, type Capture } from "../looks/capture.ts";
2929 import { createLog } from "../kernel/log.ts";
3030 import { err, ok, type Result } from "../kernel/result.ts";
31 import { boundedSignal } from "../kernel/fetch_timeout.ts";
3132
3233 const log = createLog("carts");
3334
@@ -230,7 +231,7 @@ export async function keepPage(
230231 ): Promise<Result<void, "page_missing">> {
231232 let bytes: Uint8Array | undefined;
232233 try {
233 const answer = await fetch(url, { redirect: "manual" });
234 const answer = await fetch(url, { redirect: "manual", signal: boundedSignal() });
234235 if (!answer.ok) return err("page_missing");
235236 bytes = await readCapped(answer, max);
236237 } catch {
src/chaos/report.ts +2
@@ -7,6 +7,7 @@
77 */
88
99 import type { Night } from "./run.ts";
10 import { boundedSignal } from "../kernel/fetch_timeout.ts";
1011
1112 export interface Finding {
1213 readonly fault: string;
@@ -49,6 +50,7 @@ async function call(path: string, body: unknown): Promise<Response> {
4950 method: "POST",
5051 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
5152 body: JSON.stringify(body),
53 signal: boundedSignal(),
5254 });
5355 }
5456
src/cluster/jobs.ts +2 −1
@@ -6,6 +6,7 @@
66 * injectable. Plain data in (a `JobSpec`), plain data out (a `JobResult`).
77 */
88 import { readFileSync } from "node:fs";
9 import { boundedSignal } from "../kernel/fetch_timeout.ts";
910
1011 const SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount";
1112 const NAME = /^[a-z0-9]([-a-z0-9]{0,51}[a-z0-9])?$/;
@@ -161,7 +162,7 @@ type PodList = { items?: Pod[] };
161162 export async function runJob(spec: JobSpec, opts: JobOptions = {}): Promise<JobResult> {
162163 const body = JSON.stringify(manifest(spec));
163164 const cred = opts.credentials ?? serviceAccount();
164 const doFetch = opts.fetch ?? ((u, i) => fetch(u, i as RequestInit));
165 const doFetch = opts.fetch ?? ((u, i) => fetch(u, { ...(i as RequestInit), signal: boundedSignal(i?.signal) }));
165166 const now = opts.now ?? Date.now;
166167 const sleep = opts.sleep ?? ((ms) => new Promise<void>((r) => setTimeout(r, ms)));
167168 const timeoutMs = opts.timeoutMs ?? 600_000;
src/git/hook-address.ts +3 −1
@@ -6,6 +6,8 @@
66 * so no unix socket is ever exposed to the sandbox to get there.
77 */
88
9 import { boundedSignal } from "../kernel/fetch_timeout.ts";
10
911 const TCP = /^tcp:(127\.0\.0\.1):([0-9]{1,5})$/;
1012
1113 /** The address of a loopback listener. */
@@ -38,5 +40,5 @@ export function fetchInternal(
3840 init: RequestInit = {},
3941 ): Promise<Response> {
4042 const target = internalRequest(address, path);
41 return fetch(target.url, { ...init, ...target.init } as RequestInit);
43 return fetch(target.url, { ...init, ...target.init, signal: boundedSignal(init.signal) } as RequestInit);
4244 }
src/kernel/fetch_timeout.ts added +13
@@ -0,0 +1,13 @@
1 /**
2 * The one limit on an outbound call that names none of its own (#14721): no
3 * fetch can hang a run, a deploy or a page forever. A caller's own signal
4 * always wins; this is only the floor under a call that brought none.
5 * `test/fetch-timeout.test.ts` fails on any `fetch(` in src without a signal.
6 */
7
8 /** Longer than any answer the forge waits on, short enough that nothing waits forever. */
9 export const FETCH_TIMEOUT_MS = 30_000;
10
11 /** The caller's signal, or one that aborts after `FETCH_TIMEOUT_MS`. */
12 export const boundedSignal = (signal?: AbortSignal | null): AbortSignal =>
13 signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS);
src/kernel/outbound.ts +3
@@ -17,6 +17,7 @@ import { isIP } from "node:net";
1717 import { err, ok, type Result } from "./result.ts";
1818 import { addressBytes, inCidr, inCidrs, parseCidr, type Cidr } from "./cidr.ts";
1919 import { cellProxy, UNRESOLVED_WORDS } from "../apps/egress.ts";
20 import { boundedSignal } from "./fetch_timeout.ts";
2021
2122 const REFUSED = [
2223 "0.0.0.0/8",
@@ -188,6 +189,7 @@ export async function outboundFetch(
188189 try {
189190 answer = await fetch(target, {
190191 ...init,
192 signal: boundedSignal(init.signal),
191193 redirect: "manual",
192194 proxy: vetted.value.proxy,
193195 ...(target.protocol === "https:" ? { tls } : {}),
@@ -213,6 +215,7 @@ export async function outboundFetch(
213215 headers.set("host", target.host);
214216 return fetch(pinned, {
215217 ...init,
218 signal: boundedSignal(init.signal),
216219 headers,
217220 redirect: "manual",
218221 ...(target.protocol === "https:" ? { tls: { serverName: name, ...tls } } : {}),
src/origins/origin_fetch.ts +2 −1
@@ -9,6 +9,7 @@
99
1010 import { outboundFetch } from "../kernel/outbound.ts";
1111 import { trustedHosts } from "./trusted-hosts.ts";
12 import { boundedSignal } from "../kernel/fetch_timeout.ts";
1213
1314 /**
1415 * Hosts an operator has explicitly vouched for, comma-separated. This is the
@@ -80,6 +81,6 @@ export function originFetch(
8081 init: RequestInit = {},
8182 env: Record<string, string | undefined> = process.env,
8283 ): Promise<Response> {
83 if (exemptOrigin(url, env)) return fetch(url, init);
84 if (exemptOrigin(url, env)) return fetch(url, { ...init, signal: boundedSignal(init.signal) });
8485 return outboundFetch(url, init, { trusted: trustedHosts() });
8586 }
src/perf/github_traffic.ts +2 −1
@@ -12,6 +12,7 @@
1212 import type { Database } from "bun:sqlite";
1313 import { quietly } from "../core/write_clock.ts";
1414 import { during } from "./jobs.ts";
15 import { boundedSignal } from "../kernel/fetch_timeout.ts";
1516
1617 const API = "https://api.github.com";
1718 const REPO = /^(?!\.+\/)[A-Za-z0-9_.-]{1,100}\/(?!\.+$)[A-Za-z0-9_.-]{1,100}$/;
@@ -74,7 +75,7 @@ export function entriesOf(
7475 export async function pullGithub(
7576 db: Database,
7677 config: GithubConfig,
77 fetcher: Fetch = (url, init) => fetch(url, init),
78 fetcher: Fetch = (url, init) => fetch(url, { ...init, signal: boundedSignal(init?.signal) }),
7879 now: number = Date.now(),
7980 ): Promise<number> {
8081 const today = new Date(now).toISOString().slice(0, 10);
src/snapshot/probe.ts +2 −1
@@ -10,6 +10,7 @@
1010
1111 import { listSnapshots } from "./snapshot.ts";
1212 import type { Store } from "./store.ts";
13 import { boundedSignal } from "../kernel/fetch_timeout.ts";
1314
1415 /** One nightly plus a day-and-a-half's grace: older than this is a missed night. */
1516 export const MAX_AGE_MS = 36 * 3_600_000;
@@ -90,7 +91,7 @@ export async function probeForge(
9091 export async function probe(deps: ProbeDeps): Promise<ProbeResult> {
9192 const [bucket, forge] = await Promise.all([
9293 probeBucket(deps.store, deps.now ?? Date.now(), deps.maxAgeMs ?? MAX_AGE_MS),
93 probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, i)), deps.timeoutMs ?? FORGE_TIMEOUT_MS),
94 probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) })), deps.timeoutMs ?? FORGE_TIMEOUT_MS),
9495 ]);
9596 return { ok: bucket.ok && forge.ok, findings: [forge, bucket] };
9697 }
src/spawn/cell.ts +2
@@ -830,6 +830,8 @@ export async function forward(
830830 headers,
831831 body: hasBody ? request.body : null,
832832 redirect: "manual",
833 // Bound by the visitor: a stream lasts as long as they stay, and ends when they leave.
834 signal: request.signal,

This is bound by the visitor only. If the app never sends headers, the forward waits indefinitely. Add a named header-wait limit.

833835 decompress: false,
834836 } as RequestInit);
835837 const out = new Headers();
src/system/footing.ts +2 −1
@@ -27,6 +27,7 @@
2727 * those were true on one day and are not re-read here.
2828 */
2929 import { serviceAccount, type Credentials } from "../cluster/jobs.ts";
30 import { boundedSignal } from "../kernel/fetch_timeout.ts";
3031
3132 export type Need = "answer" | "sign-in" | "build" | "release" | "backup";
3233
@@ -353,7 +354,7 @@ export async function probeFooting(opts: ProbeOptions = {}): Promise<Footing> {
353354 } catch (e) {
354355 return declaredFooting(`not read: ${e instanceof Error ? e.message : String(e)}`, now());
355356 }
356 const doFetch: KubeFetch = opts.fetch ?? ((u, i) => fetch(u, i as RequestInit));
357 const doFetch: KubeFetch = opts.fetch ?? ((u, i) => fetch(u, { ...(i as RequestInit), signal: boundedSignal(i?.signal) }));
357358 const get = async <T>(path: string): Promise<{ ok: true; value: T } | { ok: false; status: number | string }> => {
358359 try {
359360 const res = await doFetch(`${cred.server}${path}`, {
test/fetch-timeout.test.ts added +114
@@ -0,0 +1,114 @@
1 /**
2 * No outbound call can hang a run, a deploy or a page forever (#14721):
3 * every `fetch(` in src passes a `signal`, or sits on the short list below
4 * with the reason it may go without. A text scan, not a syntax tree: from
5 * each `fetch(` it reads to the matching `)` and looks for `signal`.
6 */
7
8 import { describe, expect, test } from "bun:test";
9 import { readFileSync } from "node:fs";
10 import { join, relative } from "node:path";
11 import { Glob } from "bun";
12 import { boundedSignal, FETCH_TIMEOUT_MS } from "../src/kernel/fetch_timeout.ts";
13
14 const ROOT = join(import.meta.dir, "..");
15
16 /** Directories whose `fetch(` runs in the reader's browser, not in the forge. */
17 const BROWSER = [
18 // Client scripts shipped as strings to the page: the reader's tab waits, and the reader can leave
19 // it.
20 "src/web/",
21 // The toolbar script the proxy injects into an app's pages: it runs in the visitor's browser.
22 "src/apps/toolbar.ts",
23 ];
24
25 /**
26 * `file: call` → why that call may go without a signal in its own text. Keyed
27 * by the call's line as written, not its number, so an edit above it does
28 * not move it off the list.
29 */
30 const ALLOWED: Record<string, string> = {
31 "src/origins/discover.ts: ? await fetch(target, init)": "its init, built just above, carries AbortSignal.timeout(TIMEOUT_MS)",
32 };
33
34 /**
35 * `fetch(` that is the global: not a method, a longer name like `doFetch(`, or
36 * a word in a regular expression like `|fetch(?:`.
37 */
38 const CALL = /(^|[^\w.$|])fetch\((?!\?)/g;
39
40 /** From the `(` at `open`, the text up to its matching `)`. */
41 const argsOf = (text: string, open: number): string => {
42 let depth = 0;
43 for (let i = open; i < text.length; i++) {
44 const c = text[i];
45 if (c === "(") depth++;
46 else if (c === ")" && --depth === 0) return text.slice(open, i + 1);
47 }
48 return text.slice(open);
49 };
50
51 /** Each unbounded `fetch(` in one file's text: its line number and that line, trimmed. */
52 const calls = (text: string): { line: number; code: string }[] => {
53 const found: { line: number; code: string }[] = [];
54 for (const m of text.matchAll(CALL)) {
55 const at = m.index + (m[1] ?? "").length;
56 const lineStart = text.lastIndexOf("\n", at) + 1;
57 const before = text.slice(lineStart, at).trim();
58 if (/^(\*|\/\*)/.test(before) || /(^|\s)\/\//.test(before)) continue;
59 if (/\bfunction\s*$/.test(before)) continue;
60 if (/\bsignal\b/.test(argsOf(text, at + "fetch".length))) continue;
61 const lineEnd = text.indexOf("\n", at);
62 found.push({ line: text.slice(0, at).split("\n").length, code: text.slice(lineStart, lineEnd < 0 ? undefined : lineEnd).trim() });
63 }
64 return found;
65 };
66
67 /** The line numbers of each unbounded `fetch(` in one file's text. */
68 export const unbounded = (text: string): number[] => calls(text).map((c) => c.line);
69
70 const offenders = (): string[] => {
71 const found: string[] = [];
72 for (const path of new Glob("src/**/*.ts").scanSync({ cwd: ROOT })) {
73 const rel = relative(ROOT, join(ROOT, path));
74 if (BROWSER.some((dir) => rel.startsWith(dir))) continue;
75 for (const { line, code } of calls(readFileSync(join(ROOT, path), "utf8"))) {
76 if (!(`${rel}: ${code}` in ALLOWED)) found.push(`${rel}:${line} ${code}`);
77 }
78 }
79 return found.sort();
80 };
81
82 describe("fetch timeouts", () => {
83 test("the scan names a fetch with no signal, across lines, and passes one that has it", () => {
84 expect(unbounded(`await fetch(url, {\n method: "POST",\n});`)).toEqual([1]);
85 expect(unbounded(`x;\nawait fetch(url, {\n signal: AbortSignal.timeout(5),\n});`)).toEqual([]);
86 expect(unbounded(`net.fetch(url); doFetch(url); // fetch(url)\n * fetch(url)\n/a|fetch(?:x)/`)).toEqual([]);
87 expect(unbounded(`export async function fetch(dir: string) {}`)).toEqual([]);
88 expect(unbounded(`const f = (u, i) => fetch(u, i);`)).toEqual([1]);
89 });
90
91 test("the floor keeps a caller's own signal and gives one to a call that has none", () => {
92 const own = new AbortController().signal;
93 expect(boundedSignal(own)).toBe(own);
94 const floor = boundedSignal(undefined);
95 expect(floor).toBeInstanceOf(AbortSignal);
96 expect(floor.aborted).toBe(false);
97 expect(FETCH_TIMEOUT_MS).toBeGreaterThan(0);
98 });
99
100 test("every fetch in src carries a signal or a reason", () => {
101 const found = offenders();
102 if (found.length) throw new Error(`fetch with no signal; add AbortSignal.timeout(<named limit>):\n ${found.join("\n ")}`);
103 expect(found).toEqual([]);
104 });
105
106 test("every allowed entry still names an unbounded fetch", () => {
107 const stale = Object.keys(ALLOWED).filter((key) => {
108 const cut = key.indexOf(": ");
109 const file = key.slice(0, cut);
110 return !calls(readFileSync(join(ROOT, file), "utf8")).some((c) => c.code === key.slice(cut + 2));
111 });
112 expect(stale).toEqual([]);
113 });
114 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs