Public: anyone can read this; the crew can change it.
A test fails when any fetch in src has no timeout, and names it #14730
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/14721-a-test-fails-when-any-fetch-in-src-has-n → main 3 commits fork 2 threads · 2 unresolved
14 files
changed +153
−9
src/actions/cloud.ts +2 −1
| @@ -26,9 +26,10 @@ import type { User } from "../core/users.ts"; | ||
| 26 | 26 | import { id } from "../kernel/id.ts"; |
| 27 | 27 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 28 | 28 | import { putSecret, removeSecret } from "./secrets.ts"; |
| 29 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 29 | 30 | |
| 30 | 31 | /** Swapped in tests; the real forge always uses `fetch`. */ |
| 31 | export const net: { fetch: Fetch } = { fetch: (u, i) => fetch(u, i) }; | |
| 32 | export const net: { fetch: Fetch } = { fetch: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }) }; | |
| 32 | 33 | |
| 33 | 34 | const admin = (a: User | undefined): Result<User, string> => |
| 34 | 35 | a?.isAdmin ? ok(a) : err("not_found"); |
src/actions/pinned.ts +2 −1
| @@ -22,9 +22,10 @@ import { | ||
| 22 | 22 | import { resolveSecrets } from "../core/secret_store.ts"; |
| 23 | 23 | import type { User } from "../core/users.ts"; |
| 24 | 24 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 25 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 25 | 26 | |
| 26 | 27 | /** Swapped in tests; the real forge always uses `fetch`. */ |
| 27 | export const net: { fetch: Fetch; probe: Fetch } = { fetch: (u, i) => fetch(u, i), probe: (u, i) => fetch(u, i) }; | |
| 28 | export const net: { fetch: Fetch; probe: Fetch } = { fetch: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }), probe: (u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) }) }; | |
| 28 | 29 | |
| 29 | 30 | const admin = (a: User | undefined): Result<User, string> => (a?.isAdmin ? ok(a) : err("not_found")); |
| 30 | 31 |
src/arcade/carts.ts +2 −1
| @@ -28,6 +28,7 @@ import { isSafeOid } from "../git/guard.ts"; | ||
| 28 | 28 | import { takeLook, type Capture } from "../looks/capture.ts"; |
| 29 | 29 | import { createLog } from "../kernel/log.ts"; |
| 30 | 30 | import { err, ok, type Result } from "../kernel/result.ts"; |
| 31 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 31 | 32 | |
| 32 | 33 | const log = createLog("carts"); |
| 33 | 34 | |
| @@ -230,7 +231,7 @@ export async function keepPage( | ||
| 230 | 231 | ): Promise<Result<void, "page_missing">> { |
| 231 | 232 | let bytes: Uint8Array | undefined; |
| 232 | 233 | try { |
| 233 | const answer = await fetch(url, { redirect: "manual" }); | |
| 234 | const answer = await fetch(url, { redirect: "manual", signal: boundedSignal() }); | |
| 234 | 235 | if (!answer.ok) return err("page_missing"); |
| 235 | 236 | bytes = await readCapped(answer, max); |
| 236 | 237 | } catch { |
src/chaos/report.ts +2
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | 9 | import type { Night } from "./run.ts"; |
| 10 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 10 | 11 | |
| 11 | 12 | export interface Finding { |
| 12 | 13 | readonly fault: string; |
| @@ -49,6 +50,7 @@ async function call(path: string, body: unknown): Promise<Response> { | ||
| 49 | 50 | method: "POST", |
| 50 | 51 | headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, |
| 51 | 52 | body: JSON.stringify(body), |
| 53 | signal: boundedSignal(), | |
| 52 | 54 | }); |
| 53 | 55 | } |
| 54 | 56 |
src/cluster/jobs.ts +2 −1
| @@ -6,6 +6,7 @@ | ||
| 6 | 6 | * injectable. Plain data in (a `JobSpec`), plain data out (a `JobResult`). |
| 7 | 7 | */ |
| 8 | 8 | import { readFileSync } from "node:fs"; |
| 9 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 9 | 10 | |
| 10 | 11 | const SA_DIR = "/var/run/secrets/kubernetes.io/serviceaccount"; |
| 11 | 12 | const NAME = /^[a-z0-9]([-a-z0-9]{0,51}[a-z0-9])?$/; |
| @@ -161,7 +162,7 @@ type PodList = { items?: Pod[] }; | ||
| 161 | 162 | export async function runJob(spec: JobSpec, opts: JobOptions = {}): Promise<JobResult> { |
| 162 | 163 | const body = JSON.stringify(manifest(spec)); |
| 163 | 164 | const cred = opts.credentials ?? serviceAccount(); |
| 164 | const doFetch = opts.fetch ?? ((u, i) => fetch(u, i as RequestInit)); | |
| 165 | const doFetch = opts.fetch ?? ((u, i) => fetch(u, { ...(i as RequestInit), signal: boundedSignal(i?.signal) })); | |
| 165 | 166 | const now = opts.now ?? Date.now; |
| 166 | 167 | const sleep = opts.sleep ?? ((ms) => new Promise<void>((r) => setTimeout(r, ms))); |
| 167 | 168 | const timeoutMs = opts.timeoutMs ?? 600_000; |
src/git/hook-address.ts +3 −1
| @@ -6,6 +6,8 @@ | ||
| 6 | 6 | * so no unix socket is ever exposed to the sandbox to get there. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 10 | ||
| 9 | 11 | const TCP = /^tcp:(127\.0\.0\.1):([0-9]{1,5})$/; |
| 10 | 12 | |
| 11 | 13 | /** The address of a loopback listener. */ |
| @@ -38,5 +40,5 @@ export function fetchInternal( | ||
| 38 | 40 | init: RequestInit = {}, |
| 39 | 41 | ): Promise<Response> { |
| 40 | 42 | const target = internalRequest(address, path); |
| 41 | return fetch(target.url, { ...init, ...target.init } as RequestInit); | |
| 43 | return fetch(target.url, { ...init, ...target.init, signal: boundedSignal(init.signal) } as RequestInit); | |
| 42 | 44 | } |
src/kernel/fetch_timeout.ts added +13
| @@ -0,0 +1,13 @@ | ||
| 1 | /** | |
| 2 | * The one limit on an outbound call that names none of its own (#14721): no | |
| 3 | * fetch can hang a run, a deploy or a page forever. A caller's own signal | |
| 4 | * always wins; this is only the floor under a call that brought none. | |
| 5 | * `test/fetch-timeout.test.ts` fails on any `fetch(` in src without a signal. | |
| 6 | */ | |
| 7 | ||
| 8 | /** Longer than any answer the forge waits on, short enough that nothing waits forever. */ | |
| 9 | export const FETCH_TIMEOUT_MS = 30_000; | |
| 10 | ||
| 11 | /** The caller's signal, or one that aborts after `FETCH_TIMEOUT_MS`. */ | |
| 12 | export const boundedSignal = (signal?: AbortSignal | null): AbortSignal => | |
| 13 | signal ?? AbortSignal.timeout(FETCH_TIMEOUT_MS); |
src/kernel/outbound.ts +3
| @@ -17,6 +17,7 @@ import { isIP } from "node:net"; | ||
| 17 | 17 | import { err, ok, type Result } from "./result.ts"; |
| 18 | 18 | import { addressBytes, inCidr, inCidrs, parseCidr, type Cidr } from "./cidr.ts"; |
| 19 | 19 | import { cellProxy, UNRESOLVED_WORDS } from "../apps/egress.ts"; |
| 20 | import { boundedSignal } from "./fetch_timeout.ts"; | |
| 20 | 21 | |
| 21 | 22 | const REFUSED = [ |
| 22 | 23 | "0.0.0.0/8", |
| @@ -188,6 +189,7 @@ export async function outboundFetch( | ||
| 188 | 189 | try { |
| 189 | 190 | answer = await fetch(target, { |
| 190 | 191 | ...init, |
| 192 | signal: boundedSignal(init.signal), | |
| 191 | 193 | redirect: "manual", |
| 192 | 194 | proxy: vetted.value.proxy, |
| 193 | 195 | ...(target.protocol === "https:" ? { tls } : {}), |
| @@ -213,6 +215,7 @@ export async function outboundFetch( | ||
| 213 | 215 | headers.set("host", target.host); |
| 214 | 216 | return fetch(pinned, { |
| 215 | 217 | ...init, |
| 218 | signal: boundedSignal(init.signal), | |
| 216 | 219 | headers, |
| 217 | 220 | redirect: "manual", |
| 218 | 221 | ...(target.protocol === "https:" ? { tls: { serverName: name, ...tls } } : {}), |
src/origins/origin_fetch.ts +2 −1
| @@ -9,6 +9,7 @@ | ||
| 9 | 9 | |
| 10 | 10 | import { outboundFetch } from "../kernel/outbound.ts"; |
| 11 | 11 | import { trustedHosts } from "./trusted-hosts.ts"; |
| 12 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 12 | 13 | |
| 13 | 14 | /** |
| 14 | 15 | * Hosts an operator has explicitly vouched for, comma-separated. This is the |
| @@ -80,6 +81,6 @@ export function originFetch( | ||
| 80 | 81 | init: RequestInit = {}, |
| 81 | 82 | env: Record<string, string | undefined> = process.env, |
| 82 | 83 | ): Promise<Response> { |
| 83 | if (exemptOrigin(url, env)) return fetch(url, init); | |
| 84 | if (exemptOrigin(url, env)) return fetch(url, { ...init, signal: boundedSignal(init.signal) }); | |
| 84 | 85 | return outboundFetch(url, init, { trusted: trustedHosts() }); |
| 85 | 86 | } |
src/perf/github_traffic.ts +2 −1
| @@ -12,6 +12,7 @@ | ||
| 12 | 12 | import type { Database } from "bun:sqlite"; |
| 13 | 13 | import { quietly } from "../core/write_clock.ts"; |
| 14 | 14 | import { during } from "./jobs.ts"; |
| 15 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 15 | 16 | |
| 16 | 17 | const API = "https://api.github.com"; |
| 17 | 18 | const REPO = /^(?!\.+\/)[A-Za-z0-9_.-]{1,100}\/(?!\.+$)[A-Za-z0-9_.-]{1,100}$/; |
| @@ -74,7 +75,7 @@ export function entriesOf( | ||
| 74 | 75 | export async function pullGithub( |
| 75 | 76 | db: Database, |
| 76 | 77 | config: GithubConfig, |
| 77 | fetcher: Fetch = (url, init) => fetch(url, init), | |
| 78 | fetcher: Fetch = (url, init) => fetch(url, { ...init, signal: boundedSignal(init?.signal) }), | |
| 78 | 79 | now: number = Date.now(), |
| 79 | 80 | ): Promise<number> { |
| 80 | 81 | const today = new Date(now).toISOString().slice(0, 10); |
src/snapshot/probe.ts +2 −1
| @@ -10,6 +10,7 @@ | ||
| 10 | 10 | |
| 11 | 11 | import { listSnapshots } from "./snapshot.ts"; |
| 12 | 12 | import type { Store } from "./store.ts"; |
| 13 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 13 | 14 | |
| 14 | 15 | /** One nightly plus a day-and-a-half's grace: older than this is a missed night. */ |
| 15 | 16 | export const MAX_AGE_MS = 36 * 3_600_000; |
| @@ -90,7 +91,7 @@ export async function probeForge( | ||
| 90 | 91 | export async function probe(deps: ProbeDeps): Promise<ProbeResult> { |
| 91 | 92 | const [bucket, forge] = await Promise.all([ |
| 92 | 93 | probeBucket(deps.store, deps.now ?? Date.now(), deps.maxAgeMs ?? MAX_AGE_MS), |
| 93 | probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, i)), deps.timeoutMs ?? FORGE_TIMEOUT_MS), | |
| 94 | probeForge(deps.forgeUrl, deps.fetch ?? ((u, i) => fetch(u, { ...i, signal: boundedSignal(i?.signal) })), deps.timeoutMs ?? FORGE_TIMEOUT_MS), | |
| 94 | 95 | ]); |
| 95 | 96 | return { ok: bucket.ok && forge.ok, findings: [forge, bucket] }; |
| 96 | 97 | } |
src/spawn/cell.ts +2
| @@ -830,6 +830,8 @@ export async function forward( | ||
| 830 | 830 | headers, |
| 831 | 831 | body: hasBody ? request.body : null, |
| 832 | 832 | redirect: "manual", |
| 833 | // Bound by the visitor: a stream lasts as long as they stay, and ends when they leave. | |
| 834 | signal: request.signal, | |
This is bound by the visitor only. If the app never sends headers, the forward waits indefinitely. Add a named header-wait limit. |
||
| 833 | 835 | decompress: false, |
| 834 | 836 | } as RequestInit); |
| 835 | 837 | const out = new Headers(); |
src/system/footing.ts +2 −1
| @@ -27,6 +27,7 @@ | ||
| 27 | 27 | * those were true on one day and are not re-read here. |
| 28 | 28 | */ |
| 29 | 29 | import { serviceAccount, type Credentials } from "../cluster/jobs.ts"; |
| 30 | import { boundedSignal } from "../kernel/fetch_timeout.ts"; | |
| 30 | 31 | |
| 31 | 32 | export type Need = "answer" | "sign-in" | "build" | "release" | "backup"; |
| 32 | 33 | |
| @@ -353,7 +354,7 @@ export async function probeFooting(opts: ProbeOptions = {}): Promise<Footing> { | ||
| 353 | 354 | } catch (e) { |
| 354 | 355 | return declaredFooting(`not read: ${e instanceof Error ? e.message : String(e)}`, now()); |
| 355 | 356 | } |
| 356 | const doFetch: KubeFetch = opts.fetch ?? ((u, i) => fetch(u, i as RequestInit)); | |
| 357 | const doFetch: KubeFetch = opts.fetch ?? ((u, i) => fetch(u, { ...(i as RequestInit), signal: boundedSignal(i?.signal) })); | |
| 357 | 358 | const get = async <T>(path: string): Promise<{ ok: true; value: T } | { ok: false; status: number | string }> => { |
| 358 | 359 | try { |
| 359 | 360 | const res = await doFetch(`${cred.server}${path}`, { |
test/fetch-timeout.test.ts added +114
| @@ -0,0 +1,114 @@ | ||
| 1 | /** | |
| 2 | * No outbound call can hang a run, a deploy or a page forever (#14721): | |
| 3 | * every `fetch(` in src passes a `signal`, or sits on the short list below | |
| 4 | * with the reason it may go without. A text scan, not a syntax tree: from | |
| 5 | * each `fetch(` it reads to the matching `)` and looks for `signal`. | |
| 6 | */ | |
| 7 | ||
| 8 | import { describe, expect, test } from "bun:test"; | |
| 9 | import { readFileSync } from "node:fs"; | |
| 10 | import { join, relative } from "node:path"; | |
| 11 | import { Glob } from "bun"; | |
| 12 | import { boundedSignal, FETCH_TIMEOUT_MS } from "../src/kernel/fetch_timeout.ts"; | |
| 13 | ||
| 14 | const ROOT = join(import.meta.dir, ".."); | |
| 15 | ||
| 16 | /** Directories whose `fetch(` runs in the reader's browser, not in the forge. */ | |
| 17 | const BROWSER = [ | |
| 18 | // Client scripts shipped as strings to the page: the reader's tab waits, and the reader can leave | |
| 19 | // it. | |
| 20 | "src/web/", | |
| 21 | // The toolbar script the proxy injects into an app's pages: it runs in the visitor's browser. | |
| 22 | "src/apps/toolbar.ts", | |
| 23 | ]; | |
| 24 | ||
| 25 | /** | |
| 26 | * `file: call` → why that call may go without a signal in its own text. Keyed | |
| 27 | * by the call's line as written, not its number, so an edit above it does | |
| 28 | * not move it off the list. | |
| 29 | */ | |
| 30 | const ALLOWED: Record<string, string> = { | |
| 31 | "src/origins/discover.ts: ? await fetch(target, init)": "its init, built just above, carries AbortSignal.timeout(TIMEOUT_MS)", | |
| 32 | }; | |
| 33 | ||
| 34 | /** | |
| 35 | * `fetch(` that is the global: not a method, a longer name like `doFetch(`, or | |
| 36 | * a word in a regular expression like `|fetch(?:`. | |
| 37 | */ | |
| 38 | const CALL = /(^|[^\w.$|])fetch\((?!\?)/g; | |
| 39 | ||
| 40 | /** From the `(` at `open`, the text up to its matching `)`. */ | |
| 41 | const argsOf = (text: string, open: number): string => { | |
| 42 | let depth = 0; | |
| 43 | for (let i = open; i < text.length; i++) { | |
| 44 | const c = text[i]; | |
| 45 | if (c === "(") depth++; | |
| 46 | else if (c === ")" && --depth === 0) return text.slice(open, i + 1); | |
| 47 | } | |
| 48 | return text.slice(open); | |
| 49 | }; | |
| 50 | ||
| 51 | /** Each unbounded `fetch(` in one file's text: its line number and that line, trimmed. */ | |
| 52 | const calls = (text: string): { line: number; code: string }[] => { | |
| 53 | const found: { line: number; code: string }[] = []; | |
| 54 | for (const m of text.matchAll(CALL)) { | |
| 55 | const at = m.index + (m[1] ?? "").length; | |
| 56 | const lineStart = text.lastIndexOf("\n", at) + 1; | |
| 57 | const before = text.slice(lineStart, at).trim(); | |
| 58 | if (/^(\*|\/\*)/.test(before) || /(^|\s)\/\//.test(before)) continue; | |
| 59 | if (/\bfunction\s*$/.test(before)) continue; | |
| 60 | if (/\bsignal\b/.test(argsOf(text, at + "fetch".length))) continue; | |
| 61 | const lineEnd = text.indexOf("\n", at); | |
| 62 | found.push({ line: text.slice(0, at).split("\n").length, code: text.slice(lineStart, lineEnd < 0 ? undefined : lineEnd).trim() }); | |
| 63 | } | |
| 64 | return found; | |
| 65 | }; | |
| 66 | ||
| 67 | /** The line numbers of each unbounded `fetch(` in one file's text. */ | |
| 68 | export const unbounded = (text: string): number[] => calls(text).map((c) => c.line); | |
| 69 | ||
| 70 | const offenders = (): string[] => { | |
| 71 | const found: string[] = []; | |
| 72 | for (const path of new Glob("src/**/*.ts").scanSync({ cwd: ROOT })) { | |
| 73 | const rel = relative(ROOT, join(ROOT, path)); | |
| 74 | if (BROWSER.some((dir) => rel.startsWith(dir))) continue; | |
| 75 | for (const { line, code } of calls(readFileSync(join(ROOT, path), "utf8"))) { | |
| 76 | if (!(`${rel}: ${code}` in ALLOWED)) found.push(`${rel}:${line} ${code}`); | |
| 77 | } | |
| 78 | } | |
| 79 | return found.sort(); | |
| 80 | }; | |
| 81 | ||
| 82 | describe("fetch timeouts", () => { | |
| 83 | test("the scan names a fetch with no signal, across lines, and passes one that has it", () => { | |
| 84 | expect(unbounded(`await fetch(url, {\n method: "POST",\n});`)).toEqual([1]); | |
| 85 | expect(unbounded(`x;\nawait fetch(url, {\n signal: AbortSignal.timeout(5),\n});`)).toEqual([]); | |
| 86 | expect(unbounded(`net.fetch(url); doFetch(url); // fetch(url)\n * fetch(url)\n/a|fetch(?:x)/`)).toEqual([]); | |
| 87 | expect(unbounded(`export async function fetch(dir: string) {}`)).toEqual([]); | |
| 88 | expect(unbounded(`const f = (u, i) => fetch(u, i);`)).toEqual([1]); | |
| 89 | }); | |
| 90 | ||
| 91 | test("the floor keeps a caller's own signal and gives one to a call that has none", () => { | |
| 92 | const own = new AbortController().signal; | |
| 93 | expect(boundedSignal(own)).toBe(own); | |
| 94 | const floor = boundedSignal(undefined); | |
| 95 | expect(floor).toBeInstanceOf(AbortSignal); | |
| 96 | expect(floor.aborted).toBe(false); | |
| 97 | expect(FETCH_TIMEOUT_MS).toBeGreaterThan(0); | |
| 98 | }); | |
| 99 | ||
| 100 | test("every fetch in src carries a signal or a reason", () => { | |
| 101 | const found = offenders(); | |
| 102 | if (found.length) throw new Error(`fetch with no signal; add AbortSignal.timeout(<named limit>):\n ${found.join("\n ")}`); | |
| 103 | expect(found).toEqual([]); | |
| 104 | }); | |
| 105 | ||
| 106 | test("every allowed entry still names an unbounded fetch", () => { | |
| 107 | const stale = Object.keys(ALLOWED).filter((key) => { | |
| 108 | const cut = key.indexOf(": "); | |
| 109 | const file = key.slice(0, cut); | |
| 110 | return !calls(readFileSync(join(ROOT, file), "utf8")).some((c) => c.code === key.slice(cut + 2)); | |
| 111 | }); | |
| 112 | expect(stale).toEqual([]); | |
| 113 | }); | |
| 114 | }); |
new above
offline
This line is over 150 columns and repeats the same wrapper twice. A shared boundedFetch helper would fix both.