Public: anyone can read this; the crew can change it.

From outside: App proxy returns a bare 400 for a few seconds during app restarts #13724

Merged opened this in answer to

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/13717-from-outside-app-proxy-returns-a-bare-40 → main 3 commits fork 5 threads · 5 unresolved

  • Merged. scout/13717-from-outside-app-proxy-returns-a-bare-40 was merged into main as 2dba86d.

Review

Sign in to review this change.
2 files changed +193 −0
src/apps/proxy.ts +36
@@ -1371,6 +1371,22 @@ export async function proxyApp(
13711371 relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who);
13721372 return relayAnswer(request, unavailable(runtime, target, status, "the app stopped answering"));
13731373 }
1374 // A bare 400 from a deployment the environment has moved off (being replaced, or no

This masks a symptom on an unproven cause. Say so in the comment, and link #13738 so the next person knows to remove it if the headers show another source.

1375 // longer the one on this port) is taken as the restart window, not the app: say so
1376 // with a 503 and a retry, never a "your request is wrong" the app did not send.
1377 // A mitigation on an unproven cause (#13724): the 400 has not been reproduced.
outdated

The check runs after relayUpstream, so the swallowed 400 is still recorded as a successful upstream answer. Move it above, or record it as unavailable.

1378 // When #13738 captures its headers and they show another source, remove this.
1379 if (
1380 bareRefusal(response, runtime.envStatus(target.app, target.env), port)
1381 ) {
1382 void response.body?.cancel();
1383 answered();
1384 relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who);
1385 return relayAnswer(
1386 request,
1387 unavailable(runtime, target, { ...status, state: "deploying" }, "it is restarting."),
1388 );
1389 }
13741390 relayUpstream(runtime, request, target, performance.now() - sent, vouched?.who);
13751391 const out = new Headers(response.headers);
13761392 for (const name of HOP_BY_HOP) out.delete(name);
@@ -1858,6 +1874,26 @@ function cookieValue(request: Request, name: string): string | undefined {
18581874 return undefined;
18591875 }
18601876
1877 /**
1878 * Whether `response` is a bare 400 (no body, no type, no `server` mark:
1879 * nothing an app wrote) from a deployment the environment has moved off:
1880 * it is being replaced (`deploying`) or another port serves now. An app's
1881 * own bare 400 from the deployment that serves passes through. Accepted: an
1882 * app's own bare 400 while its environment is `deploying` is called the
1883 * restart window too, and answers 503; the two cannot be told apart here.
1884 */
1885 export function bareRefusal(
1886 response: Response,
1887 now: EnvStatus | undefined,
1888 port: number,
1889 ): boolean {
1890 if (response.status !== 400) return false;
1891 if (response.headers.has("content-type") || response.headers.has("server")) return false;
1892 const length = response.headers.get("content-length");
1893 if (length !== null && length !== "0") return false;
outdated

now === undefined || state !== running || port !== port turns every bare 400 into a 503 with no time limit, so an app's own bare 400 is masked in those states. Narrow it, or justify it.

1894 return now !== undefined && (now.state === "deploying" || (now.port !== undefined && now.port !== port));
1895 }
1896
18611897 /**
18621898 * The page an environment that is not serving shows: what it is doing, in
18631899 * words, with the run to look at. Never a bare gateway error.
test/apps-proxy-restart-window.test.ts added +157
@@ -0,0 +1,157 @@
1 /**
outdated

Add a test through proxyApp with a fake upstream returning a bare 400 inside the window. Expect 503, Retry-After and the holding page. Also cover an app's own 400 with a body passing through. Without this, the wiring is untested.

2 * A bare 400 from a deployment the environment has moved off is the restart
3 * window: the proxy answers 503 with a retry and the holding page, while the
4 * app's own 400 (and a bare one from the deployment that serves) passes
5 * through (#13724). A mitigation, not a confirmed fix: #13738 captures the
outdated

Wrong issue number (#13717). Use #13724.

6 * headers of the real 400.
7 *
8 * The upstream is a stub and envStatus is spied, so nothing deploys and
9 * nothing waits on the clock: the stub holds each answer until the test,
10 * having moved the environment, releases it.
11 */
12 import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test";
13 import { mkdtemp, rm } from "node:fs/promises";
14 import { tmpdir } from "node:os";
15 import { join } from "node:path";
16 import { createApp, type App } from "../src/app.ts";
17 import { AppsRuntime, type EnvStatus } from "../src/apps/runtime.ts";
18 import { bareRefusal } from "../src/apps/proxy.ts";
19 import { newApp } from "../src/actions/apps.ts";
20 import { createUser } from "../src/core/users.ts";
21 import { serve, type Servers } from "../src/http/server.ts";
22 import { quietLogs } from "./helpers/globals.ts";
23
24 quietLogs();
25
26 const DOMAIN = "apps.test";
27
28 const bare = () => new Response(null, { status: 400 });
29 const running = (port: number) => ({ state: "running", port }) as EnvStatus;
30
31 describe("bareRefusal", () => {
32 test("only a bare 400 from a deployment the environment moved off", () => {
33 expect(bareRefusal(bare(), running(8), 7)).toBe(true);
34 expect(bareRefusal(bare(), running(7), 7)).toBe(false);
35 expect(bareRefusal(bare(), undefined, 7)).toBe(false);
36 expect(bareRefusal(bare(), { state: "stopped" } as EnvStatus, 7)).toBe(false);
37 });
38 test("accepted: an app's own bare 400 while its environment is deploying is a 503", () => {
39 // The same port and the app's own answer, but a redeploy is under way:
40 // indistinguishable from the restart window, so it is called one.
41 expect(bareRefusal(bare(), { state: "deploying", port: 7 } as EnvStatus, 7)).toBe(true);
42 });
43 test("a 400 with a type, a body or a server mark is the app's", () => {
44 const moved = running(8);
45 const own = (init: ResponseInit) => new Response(init.status === 400 ? "x" : null, init);
46 expect(bareRefusal(own({ status: 400, headers: { "content-type": "text/plain" } }), moved, 7)).toBe(false);
47 expect(bareRefusal(own({ status: 400, headers: { "content-length": "1" } }), moved, 7)).toBe(false);
48 expect(bareRefusal(new Response(null, { status: 400, headers: { server: "app" } }), moved, 7)).toBe(false);
49 expect(bareRefusal(new Response("ok"), moved, 7)).toBe(false);
50 });
51 });
52
53 describe("through the proxy", () => {
54 let dataDir: string;
55 let app: App;
56 let servers: Servers;
57 let runtime: AppsRuntime;
58 let backend: ReturnType<typeof Bun.serve>;
59 let status: ReturnType<typeof spyOn>;
60 /** What envStatus reports; the test changes it while an answer is held. */
61 let now: Partial<EnvStatus> = {};
62 /** Resolved by the test to let the held answer go. */
63 let release: () => void = () => {};
64 /** Resolved by the stub once the request has reached it. */
65 let arrived: Promise<void> = Promise.resolve();
66
67 const hold = () => {
68 let reached!: () => void;
69 arrived = new Promise((r) => (reached = r));
70 const gate = new Promise<void>((r) => (release = r));
71 return { reached: () => reached(), gate };
72 };
73
74 const get = (path: string) =>
75 fetch(`${servers.url}${path}`, { headers: { host: `hello.${DOMAIN}`, accept: "text/plain" }, redirect: "manual" });
76
77 /** Ask for `path`, and once the stub holds it, move the environment to `then` and let it go. */
78 async function across(path: string, then: Partial<EnvStatus>): Promise<Response> {
79 const pending = get(path);
80 await arrived;
81 now = then;
82 release();
83 return pending;
84 }
85
86 let held: ReturnType<typeof hold> | undefined;
87
88 beforeAll(async () => {
89 backend = Bun.serve({
90 port: 0,
91 async fetch(request) {
92 const path = new URL(request.url).pathname;
93 if (held !== undefined) {
94 held.reached();
95 await held.gate;
96 }
97 if (path === "/bare") return new Response(null, { status: 400 });
98 if (path === "/own")
99 return new Response("name is required", { status: 400, headers: { "content-type": "text/plain" } });
100 return new Response("ok", { headers: { "content-type": "text/plain" } });
101 },
102 });
103 dataDir = await mkdtemp(join(tmpdir(), "forge-restart-"));
104 app = createApp({ dataDir, port: 0 });
105 servers = await serve(app);
106 runtime = new AppsRuntime(app, { domain: DOMAIN, scheme: "http", publicPort: servers.port, forgeUrl: servers.url, concurrency: 1 });
107 await runtime.start();
108 status = spyOn(AppsRuntime.prototype, "envStatus").mockImplementation(
109 () => ({ env: "production", state: "running", port: backend.port, commitOid: "a".repeat(40), ...now }) as EnvStatus,
110 );
111 const made = await createUser(app.db, { origin: "seed", trusted: true, username: "alice",
112 email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true });
113 if (!made.ok) throw new Error(made.error);
114 const created = await newApp(app, made.value, { name: "hello", description: "d", visibility: "public" });
115 if (!created.ok) throw new Error(String(created.error));
116 }, 30_000);
117
118 afterAll(async () => {
119 status.mockRestore();
120 await runtime.stop();
121 servers.stop();
122 backend.stop(true);
123 app.close();
124 await rm(dataDir, { recursive: true, force: true });
125 });
126
127 test("a settled app's own bare 400 passes through", async () => {
128 now = {};
129 held = undefined;
130 const res = await get("/bare");
131 expect(res.status).toBe(400);
132 });
133
134 test("an app's 400 with a body passes through even as the environment moves", async () => {
135 now = {};
136 held = hold();
137 const res = await across("/own", { port: backend.port! + 1 });
138 expect(res.status).toBe(400);
139 expect(await res.text()).toBe("name is required");
140 });
141
142 test("a bare 400 as the deployment is replaced is a 503 with a retry and the holding page", async () => {
143 now = {};
144 held = hold();
145 const res = await across("/bare", { port: backend.port! + 1 });
146 expect(res.status).toBe(503);
147 expect(res.headers.get("retry-after")).toBe("3");
148 expect(await res.text()).toContain("is starting");
149 });
150
151 test("accepted: an app's own bare 400 while it is deploying is a 503", async () => {
152 now = {};
153 held = hold();
154 const res = await across("/bare", { state: "deploying" });
155 expect(res.status).toBe(503);
156 });
157 });

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs