Public: anyone can read this; the crew can change it.
No page refreshes itself on a timer: a test walks the rendered pages and fails on meta refresh or a reload interval #10962
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/10947-no-page-refreshes-itself-on-a-timer-a-te → main 2 commits fork 1 thread · 1 unresolved
1 file
changed +238
−3
test/no-polling.test.ts +238 −3
| @@ -9,9 +9,19 @@ import { Glob } from "bun"; | ||
| 9 | 9 | import { mkdtemp, rm } from "node:fs/promises"; |
| 10 | 10 | import { tmpdir } from "node:os"; |
| 11 | 11 | import { join } from "node:path"; |
| 12 | import { createApp } from "../src/app.ts"; | |
| 13 | import { serve } from "../src/http/server.ts"; | |
| 14 | import { adminCookie } from "./helpers/admin.ts"; | |
| 12 | import { afterAll, beforeAll } from "bun:test"; | |
| 13 | import { createApp, type App } from "../src/app.ts"; | |
| 14 | import { commitToBranch } from "../src/actions/repos.ts"; | |
| 15 | import { createIssue } from "../src/core/issues.ts"; | |
| 16 | import { createRepo } from "../src/core/repos.ts"; | |
| 17 | import { createSession } from "../src/core/sessions.ts"; | |
| 18 | import { buildRouter, serve, type Servers } from "../src/http/server.ts"; | |
| 19 | import { pollJs } from "../src/web/client.ts"; | |
| 20 | import { adminCookie, seedAdmin } from "./helpers/admin.ts"; | |
| 21 | import { quietLogs } from "./helpers/globals.ts"; | |
| 22 | import { pool } from "./helpers/sweep-clock.ts"; | |
| 23 | ||
| 24 | quietLogs(); | |
| 15 | 25 | |
| 16 | 26 | /** |
| 17 | 27 | * The owner's main pages. /crew is not here: it answers 404 for the owner (the |
| @@ -212,3 +222,228 @@ describe("no rendered page polls", () => { | ||
| 212 | 222 | } |
| 213 | 223 | }); |
| 214 | 224 | }); |
| 225 | ||
| 226 | /** | |
| 227 | * Every page the router lists (#10947). The owner's pages above are held to | |
| 228 | * more; this walks the whole GET table, signed out and signed in as admin, and | |
| 229 | * reads each page's markup and the scripts it loads. | |
| 230 | */ | |
| 231 | ||
| 232 | /** A script that legitimately waits on something outside the page: marker in its text → why. */ | |
| 233 | const POLL_ALLOWED: Record<string, string> = { | |
| 234 | "form[data-poll]": | |
| 235 | "the device-code connect page (pollJs, src/web/client.ts) waits on a provider that sets the cadence (its `interval`), so there is no event of ours to stream; it posts the form the page already offers, stops after 200 tries, and reloads only once the flow leaves pending", | |
| 236 | }; | |
| 237 | ||
| 238 | /** The text of the balanced `(…)` or `{…}` whose opener is at `open`. Strings and comments are not understood: a bracket inside one skews it. */ | |
| 239 | function balanced(src: string, open: number): string { | |
| 240 | const pairs: Record<string, string> = { "(": ")", "{": "}" }; | |
| 241 | const o = src[open]!, c = pairs[o]!; | |
| 242 | let depth = 0; | |
| 243 | for (let i = open; i < src.length; i++) { | |
| 244 | if (src[i] === o) depth++; | |
| 245 | else if (src[i] === c && --depth === 0) return src.slice(open + 1, i); | |
| 246 | } | |
| 247 | return src.slice(open + 1); | |
| 248 | } | |
| 249 | ||
| 250 | /** The body of a function a script names: `function name(…) {…}` or `name = (…) => {…}` / `name = function (…) {…}`. */ | |
| 251 | function bodyOf(src: string, name: string): string | undefined { | |
| 252 | const m = new RegExp(`(?:function\\s+${name}\\s*\\(|\\b${name}\\s*=\\s*(?:async\\s*)?(?:function\\s*\\(|\\([^)]*\\)\\s*=>|\\w+\\s*=>))`).exec(src); | |
| 253 | if (!m) return undefined; | |
| 254 | const brace = src.indexOf("{", m.index + m[0].length - 1); | |
| 255 | return brace < 0 ? undefined : balanced(src, brace); | |
| 256 | } | |
| 257 | ||
| 258 | /** | |
| 259 | * What a script or page does that polls. Each timer's callback is read whole | |
| 260 | * (balanced to its closing bracket); a named callback is resolved to its body | |
| 261 | * in the same text. A callback built elsewhere (an imported or computed one) | |
| 262 | * cannot be read: that is the detector's limit, pinned in a test below. | |
| 263 | */ | |
| 264 | export function findPolling(source: string): string[] { | |
| 265 | const found: string[] = []; | |
| 266 | if (/<meta[^>]+http-equiv=["']?refresh/i.test(source)) found.push("meta refresh"); | |
| 267 | if (/data-live-every/.test(source)) found.push("data-live-every"); | |
| 268 | const re = /\b(setInterval|setTimeout)\s*\(/g; | |
| 269 | for (let m = re.exec(source); m; m = re.exec(source)) { | |
| 270 | const args = balanced(source, m.index + m[0].length - 1); | |
| 271 | const first = /^\s*([A-Za-z_$][\w$]*)\s*(?:,|$)/.exec(args); | |
| 272 | const callback = first ? (bodyOf(source, first[1]!) ?? "") : args; | |
| 273 | if (/location\.reload|location\.replace\(\s*location|history\.go\(\s*0/.test(callback)) found.push(`${m[1]} reloads the page`); | |
| 274 | else if (m[1] === "setInterval" && /\bfetch\s*\(|XMLHttpRequest/.test(callback)) found.push("setInterval fetches"); | |
| 275 | // A timeout that asks again is an interval by another name. | |
| 276 | else if (first && /\bfetch\s*\(/.test(callback) && new RegExp(`setTimeout\\(\\s*${first[1]}\\b`).test(callback)) found.push("a fetch re-arms itself on a timeout"); | |
| 277 | } | |
| 278 | for (const m of source.matchAll(/function\s+([A-Za-z_$][\w$]*)\s*\(/g)) { | |
| 279 | const body = bodyOf(source, m[1]!) ?? ""; | |
| 280 | if (/\bfetch\s*\(/.test(body) && new RegExp(`setTimeout\\(\\s*${m[1]}\\b`).test(body)) found.push("a fetch re-arms itself on a timeout"); | |
| 281 | } | |
| 282 | return [...new Set(found)]; | |
| 283 | } | |
| 284 | ||
| 285 | const pollReason = (source: string): string | undefined => | |
| 286 | Object.entries(POLL_ALLOWED).find(([marker]) => source.includes(marker))?.[1]; | |
| 287 | ||
| 288 | /** | |
| 289 | * GET patterns the walk cannot fill from its seed, each with why: the walk is | |
| 290 | * every pattern minus these, and the list is checked both ways, so a new page | |
| 291 | * is walked unless someone names it here. | |
| 292 | */ | |
| 293 | const ORIGIN = "needs a connected origin account; the seed connects none"; | |
| 294 | const NOT_WALKED: Record<string, string> = { | |
| 295 | "/:owner/-/teams/:team": "needs an organization and a team", | |
| 296 | "/:owner/:name/app/:env/open": "needs a running app environment to open", | |
| 297 | "/:owner/:name/bets/:id": "needs a proposed bet", | |
| 298 | "/:owner/:name/builds/:label/cart": "needs a pinned build with a cart", | |
| 299 | "/:owner/:name/builds/:label/cart.png": "needs a pinned build with a cart", | |
| 300 | "/:owner/:name/builds/:label/shots": "needs a pinned build with pictures", | |
| 301 | "/:owner/:name/builds/:label/shots/:file": "needs a pinned build with pictures", | |
| 302 | "/:owner/:name/commit/:oid": "the seed does not keep the commit id; the commit page is a plain diff with the shell script only", | |
| 303 | "/:owner/:name/looks/:env/:commit/:kind.png": "an image of a captured look, not a page", | |
| 304 | "/:owner/:name/machines/:machine/shell": "needs a running machine to attach a terminal to", | |
| 305 | "/:owner/:name/milestones/:number/limb/:op": "needs a milestone with a limb", | |
| 306 | "/:owner/:name/runs/:id": "the seed creates no run", | |
| 307 | "/:owner/:name/variants/:group": "needs a variant group of agent tries", | |
| 308 | "/:owner/:name/waves/:version": "needs a wave of a milestone", | |
| 309 | "/boards/:slug": "needs a board", | |
| 310 | "/boards/:slug/live": "needs a board; a stream, not a page", | |
| 311 | "/bring/:id/:job": "needs a bring job", | |
| 312 | "/fleet/ops/:id": "needs a fleet operation", | |
| 313 | "/go/:code": "needs a short code minted by a share", | |
| 314 | "/home/archived/:op": "needs an archived-home operation", | |
| 315 | "/icons/:spec.png": "an image, not a page", | |
| 316 | "/migrate/jobs/:job": "needs a migration job", | |
| 317 | "/now/:stage": "the stage word is one the seed cannot guess", | |
| 318 | "/resilience/nightly/:job": "needs a nightly job name", | |
| 319 | "/runs/:id": "the seed creates no run", | |
| 320 | "/settings/features/:slug": "needs a feature slug", | |
| 321 | "/settings/origins/:id/archived": ORIGIN, | |
| 322 | "/settings/origins/:id/mirror/:job": ORIGIN, | |
| 323 | "/settings/origins/:id/repos": ORIGIN, | |
| 324 | "/settings/origins/flows/:id": ORIGIN, | |
| 325 | "/settings/teams/:slug": "needs a team", | |
| 326 | "/talk/:id": "needs a conversation of the signed-in person", | |
| 327 | }; | |
| 328 | /** The whole browse-an-origin family is one reason. */ | |
| 329 | const whyNot = (p: string): string | undefined => (p.startsWith("/origins/:id/r/") ? ORIGIN : NOT_WALKED[p]); | |
| 330 | ||
| 331 | describe("every page the router lists", () => { | |
| 332 | let dataDir = ""; | |
| 333 | let app: App, servers: Servers; | |
| 334 | let cookie = ""; | |
| 335 | const fill: Record<string, string> = {}; | |
| 336 | ||
| 337 | beforeAll(async () => { | |
| 338 | dataDir = await mkdtemp(join(tmpdir(), "forge-nopoll-walk-")); | |
| 339 | app = createApp({ dataDir, port: 0 }); | |
| 340 | servers = await serve(app); | |
| 341 | const admin = await seedAdmin(app, { username: "ada", email: "[email protected]", password: "correct-horse-battery" }); | |
| 342 | if (!admin.ok) throw new Error(admin.error); | |
| 343 | const made = await createRepo(app.db, app.reposRoot, { ownerId: admin.value.id, name: "notes", visibility: "public" }); | |
| 344 | if (!made.ok) throw new Error(String(made.error)); | |
| 345 | const seed = await commitToBranch(app, admin.value, made.value, { message: "seed", files: [{ path: "a.txt", content: "a\n" }] }); | |
| 346 | if (!seed.ok) throw new Error(String(seed.error)); | |
| 347 | const issue = createIssue(app.db, { repoId: made.value.id, title: "an issue", body: "b", authorId: admin.value.id }); | |
| 348 | if (!issue.ok) throw new Error(String(issue.error)); | |
| 349 | Object.assign(fill, { owner: "ada", name: "notes", username: "ada", ref: "main", path: "a.txt", number: String(issue.value.number) }); | |
| 350 | cookie = `forge_session=${createSession(app.db, admin.value.id).token}`; | |
| 351 | }, 30_000); | |
| 352 | ||
| 353 | afterAll(async () => { | |
| 354 | servers?.stop(); | |
| 355 | app?.close(); | |
| 356 | await rm(dataDir, { recursive: true, force: true }); | |
| 357 | }); | |
| 358 | ||
| 359 | const patterns = (): string[] => [...new Set(buildRouter().routes().filter((r) => r.method === "GET").map((r) => r.pattern))]; | |
| 360 | const keysOf = (p: string): string[] => [...p.matchAll(/[:*]([a-z_]+)/g)].map((m) => m[1]!); | |
| 361 | const fillable = (p: string): boolean => keysOf(p).every((k) => k in fill); | |
| 362 | const pathOf = (p: string): string => p.replace(/[:*]([a-z_]+)/g, (_, k: string) => fill[k]!); | |
| 363 | ||
| 364 | async function walk(headers: Record<string, string>) { | |
| 365 | const scripts = new Map<string, string>(); | |
| 366 | const offenders: string[] = []; | |
| 367 | const allowed: string[] = []; | |
| 368 | const htmlPages: string[] = []; | |
| 369 | await pool(patterns().filter(fillable).map(pathOf), async (path) => { | |
| 370 | const res = await fetch(`${servers.url}${path}`, { headers: { "accept-encoding": "identity", ...headers }, redirect: "manual", signal: AbortSignal.timeout(8_000) }); | |
| 371 | if (res.headers.get("refresh")) offenders.push(`${path}: Refresh header`); | |
| 372 | if (res.status !== 200 || !(res.headers.get("content-type") ?? "").includes("text/html")) { await res.body?.cancel(); return; } | |
| 373 | htmlPages.push(path); | |
| 374 | const html = await res.text(); | |
| 375 | for (const found of findPolling(html.replace(/<script\b[^>]*>[^]*?<\/script\s*>/gi, ""))) offenders.push(`${path}: ${found}`); | |
| 376 | const { srcs, inline } = scriptTags(html); | |
| 377 | inline.forEach((code, i) => scripts.set(`${path} inline#${i}`, code)); | |
| 378 | for (const src of srcs) if (src.startsWith("/") && !src.startsWith("//") && !scripts.has(src)) scripts.set(src, await (await fetch(`${servers.url}${src}`)).text()); | |
| 379 | }); | |
| 380 | for (const [name, source] of scripts) { | |
| 381 | const found = findPolling(source); | |
| 382 | if (!found.length) continue; | |
| 383 | if (pollReason(source)) allowed.push(name); | |
| 384 | else offenders.push(`${name}: ${found.join("; ")}`); | |
| 385 | } | |
| 386 | return { htmlPages: htmlPages.sort(), offenders, allowed }; | |
| 387 | } | |
| 388 | ||
| 389 | test("signed out, no page the walk reaches refreshes itself on a timer", async () => { | |
| 390 | const out = await walk({}); | |
| 391 | expect(out.offenders).toEqual([]); | |
| 392 | expect(out.htmlPages.length).toBeGreaterThanOrEqual(ANON_FLOOR); | |
| 393 | expect(out.htmlPages).toContain("/"); | |
| 394 | }, 30_000); | |
| 395 | ||
| 396 | test("signed in as admin, no page the walk reaches refreshes itself on a timer", async () => { | |
| 397 | const out = await walk({ cookie }); | |
| 398 | expect(out.offenders).toEqual([]); | |
| 399 | expect(out.htmlPages).toContain("/system"); | |
| 400 | expect(out.htmlPages).toContain("/flow"); | |
| 401 | expect(out.htmlPages.length).toBeGreaterThanOrEqual(WALKED_FLOOR); | |
| 402 | }, 30_000); | |
| 403 | ||
| 404 | test("the patterns the walk skips are exactly the ones named, each with a reason", () => { | |
| 405 | const skipped = patterns().filter((p) => !fillable(p)).sort(); | |
| 406 | expect(skipped.filter((p) => !whyNot(p))).toEqual([]); | |
| 407 | // Every named entry is still skipped: a pattern the seed can now fill must leave the list. | |
| 408 | expect(Object.keys(NOT_WALKED).filter((p) => !skipped.includes(p))).toEqual([]); | |
| 409 | for (const p of skipped) expect(whyNot(p)!.length).toBeGreaterThan(10); | |
| 410 | }); | |
| 411 | }); | |
| 412 | ||
| 413 | /** Fewest HTML pages the admin walk reaches; observed when written. A page lost from the walk drops under it. */ | |
| 414 | const WALKED_FLOOR = 130; | |
| 415 | /** The same, signed out (observed 44). */ | |
| 416 | const ANON_FLOOR = 40; | |
| 417 | ||
| 418 | describe("the polling detector", () => { | |
| 419 | test("names each kind, including a named callback and a re-arming fetch", () => { | |
| 420 | expect(findPolling(`setInterval(function(){ location.reload(); }, 1000)`)).toContain("setInterval reloads the page"); | |
| 421 | expect(findPolling(`setTimeout(function(){ location.reload() }, 5000)`)).toContain("setTimeout reloads the page"); | |
| 422 | expect(findPolling(`setInterval(() => fetch("/x").then(r => r.text()), 2000)`)).toContain("setInterval fetches"); | |
| 423 | expect(findPolling(`<meta http-equiv="refresh" content="5">`)).toContain("meta refresh"); | |
| 424 | expect(findPolling(`<p data-live-every="5">`)).toContain("data-live-every"); | |
| 425 | expect(findPolling(`function poll(){ fetch("/x").then(draw); }\nsetInterval(poll, 1000)`)).toContain("setInterval fetches"); | |
| 426 | expect(findPolling(`const poll = async () => { await fetch("/x"); };\nsetInterval(poll, 1000)`)).toContain("setInterval fetches"); | |
| 427 | expect(findPolling(`function reloadSoon(){ location.reload(); }\nsetTimeout(reloadSoon, 10)`)).toContain("setTimeout reloads the page"); | |
| 428 | expect(findPolling(`function ask(){ fetch("/x").then(function(){ setTimeout(ask, 1000); }); }`)).toContain("a fetch re-arms itself on a timeout"); | |
| 429 | }); | |
| 430 | ||
| 431 | test("a clean script passes, and a long unrelated callback does not leak into the next timer", () => { | |
| 432 | expect(findPolling(`setInterval(tick, 100); setTimeout(function(){ s.textContent = ''; }, 4000)`)).toEqual([]); | |
| 433 | const filler = "x();".repeat(400); | |
| 434 | expect(findPolling(`setTimeout(function(){ ${filler} }, 1); function later(){ location.reload(); }`)).toEqual([]); | |
| 435 | // a named clock that only redraws text is no poll, even beside a fetch elsewhere | |
| 436 | expect(findPolling(`function tick(){ el.textContent = now(); } setInterval(tick, 1000); function go(){ fetch("/x"); }`)).toEqual([]); | |
| 437 | }); | |
| 438 | ||
| 439 | test("its limit: a callback it cannot read (imported or computed) is not seen", () => { | |
| 440 | expect(findPolling(`import { poll } from "./poll.js"; setInterval(poll, 1000)`)).toEqual([]); | |
| 441 | expect(findPolling(`setInterval(pick("fast"), 1000)`)).toEqual([]); | |
| 442 | }); | |
| 443 | ||
| 444 | test("the device-code poll is flagged, and the one allowlist entry matches it and says why", () => { | |
| 445 | expect(findPolling(pollJs).length).toBeGreaterThan(0); | |
| 446 | expect(pollReason(pollJs)).toBeDefined(); | |
| 447 | for (const why of Object.values(POLL_ALLOWED)) expect(why.length).toBeGreaterThan(30); | |
| 448 | }); | |
| 449 | }); |
Non-blocking: the one allowlist entry has its reason and a test that it still matches. It should go once #10964 lands.