Public: anyone can read this; the crew can change it.
From outside: App proxy returns a bare 400 for a few seconds during app restarts #13724
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/13717-from-outside-app-proxy-returns-a-bare-40 → main 3 commits fork 5 threads · 5 unresolved
src/apps/proxy.ts +36
| @@ -1371,6 +1371,22 @@ export async function proxyApp( | ||
| 1371 | 1371 | relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who); |
| 1372 | 1372 | return relayAnswer(request, unavailable(runtime, target, status, "the app stopped answering")); |
| 1373 | 1373 | } |
| 1374 | // A bare 400 from a deployment the environment has moved off (being replaced, or no | |
| 1375 | // longer the one on this port) is taken as the restart window, not the app: say so | |
| 1376 | // with a 503 and a retry, never a "your request is wrong" the app did not send. | |
| 1377 | // A mitigation on an unproven cause (#13724): the 400 has not been reproduced. | |
outdated
The check runs after |
||
| 1378 | // When #13738 captures its headers and they show another source, remove this. | |
| 1379 | if ( | |
| 1380 | bareRefusal(response, runtime.envStatus(target.app, target.env), port) | |
| 1381 | ) { | |
| 1382 | void response.body?.cancel(); | |
| 1383 | answered(); | |
| 1384 | relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who); | |
| 1385 | return relayAnswer( | |
| 1386 | request, | |
| 1387 | unavailable(runtime, target, { ...status, state: "deploying" }, "it is restarting."), | |
| 1388 | ); | |
| 1389 | } | |
| 1374 | 1390 | relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who); |
| 1375 | 1391 | const out = new Headers(response.headers); |
| 1376 | 1392 | for (const name of HOP_BY_HOP) out.delete(name); |
| @@ -1858,6 +1874,26 @@ function cookieValue(request: Request, name: string): string | undefined { | ||
| 1858 | 1874 | return undefined; |
| 1859 | 1875 | } |
| 1860 | 1876 | |
| 1877 | /** | |
| 1878 | * Whether `response` is a bare 400 (no body, no type, no `server` mark: | |
| 1879 | * nothing an app wrote) from a deployment the environment has moved off: | |
| 1880 | * it is being replaced (`deploying`) or another port serves now. An app's | |
| 1881 | * own bare 400 from the deployment that serves passes through. Accepted: an | |
| 1882 | * app's own bare 400 while its environment is `deploying` is called the | |
| 1883 | * restart window too, and answers 503; the two cannot be told apart here. | |
| 1884 | */ | |
| 1885 | export function bareRefusal( | |
| 1886 | response: Response, | |
| 1887 | now: EnvStatus | undefined, | |
| 1888 | port: number, | |
| 1889 | ): boolean { | |
| 1890 | if (response.status !== 400) return false; | |
| 1891 | if (response.headers.has("content-type") || response.headers.has("server")) return false; | |
| 1892 | const length = response.headers.get("content-length"); | |
| 1893 | if (length !== null && length !== "0") return false; | |
outdated
|
||
| 1894 | return now !== undefined && (now.state === "deploying" || (now.port !== undefined && now.port !== port)); | |
| 1895 | } | |
| 1896 | ||
| 1861 | 1897 | /** |
| 1862 | 1898 | * The page an environment that is not serving shows: what it is doing, in |
| 1863 | 1899 | * words, with the run to look at. Never a bare gateway error. |
test/apps-proxy-restart-window.test.ts added +157
| @@ -0,0 +1,157 @@ | ||
| 1 | /** | |
outdated
Add a test through |
||
| 2 | * A bare 400 from a deployment the environment has moved off is the restart | |
| 3 | * window: the proxy answers 503 with a retry and the holding page, while the | |
| 4 | * app's own 400 (and a bare one from the deployment that serves) passes | |
| 5 | * through (#13724). A mitigation, not a confirmed fix: #13738 captures the | |
outdated
Wrong issue number (#13717). Use #13724. |
||
| 6 | * headers of the real 400. | |
| 7 | * | |
| 8 | * The upstream is a stub and envStatus is spied, so nothing deploys and | |
| 9 | * nothing waits on the clock: the stub holds each answer until the test, | |
| 10 | * having moved the environment, releases it. | |
| 11 | */ | |
| 12 | import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; | |
| 13 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 14 | import { tmpdir } from "node:os"; | |
| 15 | import { join } from "node:path"; | |
| 16 | import { createApp, type App } from "../src/app.ts"; | |
| 17 | import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts"; | |
| 18 | import { bareRefusal } from "../src/apps/proxy.ts"; | |
| 19 | import { newApp } from "../src/actions/apps.ts"; | |
| 20 | import { createUser } from "../src/core/users.ts"; | |
| 21 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 22 | import { quietLogs } from "./helpers/globals.ts"; | |
| 23 | ||
| 24 | quietLogs(); | |
| 25 | ||
| 26 | const DOMAIN = "apps.test"; | |
| 27 | ||
| 28 | const bare = () => new Response(null, { status: 400 }); | |
| 29 | const running = (port: number) => ({ state: "running", port }) as EnvStatus; | |
| 30 | ||
| 31 | describe("bareRefusal", () => { | |
| 32 | test("only a bare 400 from a deployment the environment moved off", () => { | |
| 33 | expect(bareRefusal(bare(), running(8), 7)).toBe(true); | |
| 34 | expect(bareRefusal(bare(), running(7), 7)).toBe(false); | |
| 35 | expect(bareRefusal(bare(), undefined, 7)).toBe(false); | |
| 36 | expect(bareRefusal(bare(), { state: "stopped" } as EnvStatus, 7)).toBe(false); | |
| 37 | }); | |
| 38 | test("accepted: an app's own bare 400 while its environment is deploying is a 503", () => { | |
| 39 | // The same port and the app's own answer, but a redeploy is under way: | |
| 40 | // indistinguishable from the restart window, so it is called one. | |
| 41 | expect(bareRefusal(bare(), { state: "deploying", port: 7 } as EnvStatus, 7)).toBe(true); | |
| 42 | }); | |
| 43 | test("a 400 with a type, a body or a server mark is the app's", () => { | |
| 44 | const moved = running(8); | |
| 45 | const own = (init: ResponseInit) => new Response(init.status === 400 ? "x" : null, init); | |
| 46 | expect(bareRefusal(own({ status: 400, headers: { "content-type": "text/plain" } }), moved, 7)).toBe(false); | |
| 47 | expect(bareRefusal(own({ status: 400, headers: { "content-length": "1" } }), moved, 7)).toBe(false); | |
| 48 | expect(bareRefusal(new Response(null, { status: 400, headers: { server: "app" } }), moved, 7)).toBe(false); | |
| 49 | expect(bareRefusal(new Response("ok"), moved, 7)).toBe(false); | |
| 50 | }); | |
| 51 | }); | |
| 52 | ||
| 53 | describe("through the proxy", () => { | |
| 54 | let dataDir: string; | |
| 55 | let app: App; | |
| 56 | let servers: Servers; | |
| 57 | let runtime: AppsRuntime; | |
| 58 | let backend: ReturnType<typeof Bun.serve>; | |
| 59 | let status: ReturnType<typeof spyOn>; | |
| 60 | /** What envStatus reports; the test changes it while an answer is held. */ | |
| 61 | let now: Partial<EnvStatus> = {}; | |
| 62 | /** Resolved by the test to let the held answer go. */ | |
| 63 | let release: () => void = () => {}; | |
| 64 | /** Resolved by the stub once the request has reached it. */ | |
| 65 | let arrived: Promise<void> = Promise.resolve(); | |
| 66 | ||
| 67 | const hold = () => { | |
| 68 | let reached!: () => void; | |
| 69 | arrived = new Promise((r) => (reached = r)); | |
| 70 | const gate = new Promise<void>((r) => (release = r)); | |
| 71 | return { reached: () => reached(), gate }; | |
| 72 | }; | |
| 73 | ||
| 74 | const get = (path: string) => | |
| 75 | fetch(`${servers.url}${path}`, { headers: { host: `hello.${DOMAIN}`, accept: "text/plain" }, redirect: "manual" }); | |
| 76 | ||
| 77 | /** Ask for `path`, and once the stub holds it, move the environment to `then` and let it go. */ | |
| 78 | async function across(path: string, then: Partial<EnvStatus>): Promise<Response> { | |
| 79 | const pending = get(path); | |
| 80 | await arrived; | |
| 81 | now = then; | |
| 82 | release(); | |
| 83 | return pending; | |
| 84 | } | |
| 85 | ||
| 86 | let held: ReturnType<typeof hold> | undefined; | |
| 87 | ||
| 88 | beforeAll(async () => { | |
| 89 | backend = Bun.serve({ | |
| 90 | port: 0, | |
| 91 | async fetch(request) { | |
| 92 | const path = new URL(request.url).pathname; | |
| 93 | if (held !== undefined) { | |
| 94 | held.reached(); | |
| 95 | await held.gate; | |
| 96 | } | |
| 97 | if (path === "/bare") return new Response(null, { status: 400 }); | |
| 98 | if (path === "/own") | |
| 99 | return new Response("name is required", { status: 400, headers: { "content-type": "text/plain" } }); | |
| 100 | return new Response("ok", { headers: { "content-type": "text/plain" } }); | |
| 101 | }, | |
| 102 | }); | |
| 103 | dataDir = await mkdtemp(join(tmpdir(), "forge-restart-")); | |
| 104 | app = createApp({ dataDir, port: 0 }); | |
| 105 | servers = await serve(app); | |
| 106 | runtime = new AppsRuntime(app, { domain: DOMAIN, scheme: "http", publicPort: servers.port, forgeUrl: servers.url, concurrency: 1 }); | |
| 107 | await runtime.start(); | |
| 108 | status = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation( | |
| 109 | () => ({ env: "production", state: "running", port: backend.port, commitOid: "a".repeat(40), ...now }) as EnvStatus, | |
| 110 | ); | |
| 111 | const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice", | |
| 112 | email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true }); | |
| 113 | if (!made.ok) throw new Error(made.error); | |
| 114 | const created = await newApp(app, made.value, { name: "hello", description: "d", visibility: "public" }); | |
| 115 | if (!created.ok) throw new Error(String(created.error)); | |
| 116 | }, 30_000); | |
| 117 | ||
| 118 | afterAll(async () => { | |
| 119 | status.mockRestore(); | |
| 120 | await runtime.stop(); | |
| 121 | servers.stop(); | |
| 122 | backend.stop(true); | |
| 123 | app.close(); | |
| 124 | await rm(dataDir, { recursive: true, force: true }); | |
| 125 | }); | |
| 126 | ||
| 127 | test("a settled app's own bare 400 passes through", async () => { | |
| 128 | now = {}; | |
| 129 | held = undefined; | |
| 130 | const res = await get("/bare"); | |
| 131 | expect(res.status).toBe(400); | |
| 132 | }); | |
| 133 | ||
| 134 | test("an app's 400 with a body passes through even as the environment moves", async () => { | |
| 135 | now = {}; | |
| 136 | held = hold(); | |
| 137 | const res = await across("/own", { port: backend.port! + 1 }); | |
| 138 | expect(res.status).toBe(400); | |
| 139 | expect(await res.text()).toBe("name is required"); | |
| 140 | }); | |
| 141 | ||
| 142 | test("a bare 400 as the deployment is replaced is a 503 with a retry and the holding page", async () => { | |
| 143 | now = {}; | |
| 144 | held = hold(); | |
| 145 | const res = await across("/bare", { port: backend.port! + 1 }); | |
| 146 | expect(res.status).toBe(503); | |
| 147 | expect(res.headers.get("retry-after")).toBe("3"); | |
| 148 | expect(await res.text()).toContain("is starting"); | |
| 149 | }); | |
| 150 | ||
| 151 | test("accepted: an app's own bare 400 while it is deploying is a 503", async () => { | |
| 152 | now = {}; | |
| 153 | held = hold(); | |
| 154 | const res = await across("/bare", { state: "deploying" }); | |
| 155 | expect(res.status).toBe(503); | |
| 156 | }); | |
| 157 | }); |
This masks a symptom on an unproven cause. Say so in the comment, and link #13738 so the next person knows to remove it if the headers show another source.