Public: anyone can read this; the crew can change it.
A large clone or fetch does not starve the pages: git pack responses yield to page responses when the line is contended #13779
Check · Review · Land · Live done — Landed and live in production.
torque/forge:torque/13778-a-large-clone-or-fetch-does-not-starve-t → main 9 commits fork 3 threads · 3 unresolved
10 files
changed +394
−1
docs/features/README.md +1
| @@ -88,6 +88,7 @@ Read with the guide page [Your first app in five minutes](../guide-first-app.md) | ||
| 88 | 88 | - [](make-it-yours.md) |
| 89 | 89 | - [](new-app.md) |
| 90 | 90 | - [](plain-git.md) |
| 91 | - [](clone-pacing.md) | |
| 91 | 92 | - [](send-a-change.md) |
| 92 | 93 | - [](start.md) |
| 93 | 94 | - [](welcome.md) |
docs/features/clone-pacing.md added +9
| @@ -0,0 +1,9 @@ | ||
| 1 | # Clones yield to pages | |
| 2 | ||
| 3 | Pages and git packs leave through the same uplink, so a large clone could starve every page (#13778). | |
| 4 | ||
| 5 | - **Pacing** (`src/git/pace.ts`): while a page request is open, or one began or ended in the last 3 seconds, a `git-upload-pack` response is sent at `FORGE_PACK_PACED_BPS` bytes a second (default 131072, about 1 Mbit/s). A value that is not a finite number above zero is ignored and the default is used. With no page traffic a clone gets the whole line: there is no fixed cap. An open request older than 20 seconds (a held stream) is not a page, and a request to `/:owner/:name.git/…` is a transfer, not a page. Pushes are not paced. | |
| 6 | - **Stalls and slots**: a paced pack moves far faster than the stall floor (16 KiB in 30 s, `src/git/progress.ts`), so pacing alone never ends a transfer as a stall. It does hold its git process and transfer slot longer while pages are open, and the two-hour cap still ends a pack that would need longer than that at the paced rate. The slot limit is unchanged. | |
| 7 | - **The line** is said on /flow and in the terminal's Flow screen: "Git packs are being slowed so pages go first." for a minute after a pack was paced. | |
| 8 | - **Smaller clones**: upload-pack already honours partial clone (`git clone --filter=blob:none`) and shallow fetches (`--depth`); use them. | |
| 9 | - Not done: SSH transfers are not paced; the rate is a setting, not measured from the uplink (#13712). Crawler and media pacing, `robots.txt` and `/stream` for crawlers are separate changes. |
src/git/pace.ts added +136
| @@ -0,0 +1,136 @@ | ||
| 1 | /** | |
| 2 | * Pages first, packs with what is left (#13778). | |
| 3 | * | |
| 4 | * Pages and git packs leave through the same uplink, and a shaper cannot tell | |
| 5 | * them apart: two clones of a large repository starved the home page to 11 s. | |
| 6 | * So a pack response yields to page responses. While a page request is open | |
| 7 | * (younger than `OPEN_FRESH_MS`: a stream held for minutes is not a page being | |
| 8 | * answered) or one began or ended in the last `QUIET_MS`, a pack is sent at | |
| 9 | * `pagedBps`; with no page traffic it is not slowed at all, so there is no | |
| 10 | * fixed cap. The rate is the pack's share of the line while pages want it: | |
| 11 | * `FORGE_PACK_PACED_BPS` bytes a second, 128 KiB by default (about 1 Mbit/s of a | |
| 12 | * 3.5 Mbit/s line). A paced pack still moves far faster than `progress.ts`'s stall | |
| 13 | * floor (16 KiB in 30 s), so it is not killed as a stall; the two-hour cap and the | |
| 14 | * transfer slot it holds are unchanged. Pacing delays reads of the response, so git blocks on a | |
| 15 | * full pipe (`pull-pipe.ts`); nothing is buffered. State: one number and one Map of the open page | |
| 16 | * requests, at most as many as are in flight; gone on restart. | |
| 17 | */ | |
| 18 | ||
| 19 | import { during } from "../perf/jobs.ts"; | |
| 20 | ||
| 21 | /** A page began or ended this recently: the line is contended. */ | |
| 22 | export const QUIET_MS = 3_000; | |
| 23 | /** An open request older than this is a held stream, not a page waiting for the line. */ | |
| 24 | export const OPEN_FRESH_MS = 20_000; | |
| 25 | /** The most sent at once while paced, so a pace is smooth and not a burst per second. */ | |
| 26 | const PIECE = 16 * 1024; | |
| 27 | export const DEFAULT_BPS = 128 * 1024; | |
| 28 | ||
| 29 | /** | |
| 30 | * The paced rate from `FORGE_PACK_PACED_BPS`: a finite number above zero, else the default. A | |
| 31 | * negative, zero or non-numeric value would otherwise switch pacing off without a word. | |
| 32 | */ | |
| 33 | export function pacedRate(raw: string | undefined): number { | |
| 34 | const n = Number(raw); | |
| 35 | return raw !== undefined && Number.isFinite(n) && n > 0 ? n : DEFAULT_BPS; | |
| 36 | } | |
| 37 | ||
| 38 | export interface Pacer { | |
| 39 | /** A page request opened; hand back the call that says it is answered. */ | |
| 40 | page(): () => void; | |
| 41 | /** Is a page being served now or was one in the last QUIET_MS? */ | |
| 42 | contended(): boolean; | |
| 43 | /** Page requests open now. */ | |
| 44 | inFlight(): number; | |
| 45 | /** `stream`, held to the paced rate while the line is contended. */ | |
| 46 | pace(stream: ReadableStream<Uint8Array>): ReadableStream<Uint8Array>; | |
| 47 | /** Pack bytes sent, bytes sent while paced, and pages opened, since start. */ | |
| 48 | readonly stats: { sent: number; paced: number; lastPacedAt: number; opened: number }; | |
| 49 | } | |
| 50 | ||
| 51 | export function createPacer( | |
| 52 | now: () => number = Date.now, | |
| 53 | pagedBps: number = pacedRate(process.env.FORGE_PACK_PACED_BPS), | |
| 54 | sleep: (ms: number) => Promise<void> = (ms) => new Promise((done) => setTimeout(() => during("git-pace", () => done(undefined)), ms)), | |
| 55 | ): Pacer { | |
| 56 | const open = new Map<object, number>(); | |
| 57 | let lastEdge = -Infinity; | |
| 58 | const stats = { sent: 0, paced: 0, lastPacedAt: 0, opened: 0 }; | |
| 59 | const contended = (): boolean => { | |
| 60 | const t = now(); | |
| 61 | if (t - lastEdge < QUIET_MS) return true; | |
| 62 | for (const at of open.values()) if (t - at < OPEN_FRESH_MS) return true; | |
| 63 | return false; | |
| 64 | }; | |
| 65 | return { | |
| 66 | stats, | |
| 67 | contended, | |
| 68 | inFlight: () => open.size, | |
| 69 | page() { | |
| 70 | const token = {}; | |
| 71 | stats.opened += 1; | |
| 72 | lastEdge = now(); | |
| 73 | open.set(token, lastEdge); | |
| 74 | return () => { | |
| 75 | open.delete(token); | |
| 76 | lastEdge = now(); | |
| 77 | }; | |
| 78 | }, | |
| 79 | pace(stream) { | |
| 80 | const reader = stream.getReader(); | |
| 81 | let held: Uint8Array = new Uint8Array(0); | |
| 82 | let next = 0; | |
| 83 | return new ReadableStream<Uint8Array>( | |
| 84 | { | |
| 85 | async pull(controller) { | |
| 86 | if (held.length === 0) { | |
| 87 | const r = await reader.read(); | |
| 88 | if (r.done) return controller.close(); | |
| 89 | held = r.value; | |
| 90 | } | |
| 91 | const busy = contended(); | |
| 92 | const piece = busy ? held.subarray(0, PIECE) : held; | |
| 93 | held = held.subarray(piece.length); | |
| 94 | if (busy) { | |
| 95 | // Each piece owes the line piece/rate seconds; wait out what the last one owed. | |
| 96 | const wait = next - now(); | |
| 97 | if (wait > 0) await sleep(wait); | |
| 98 | next = Math.max(next, now()) + (piece.length * 1000) / pagedBps; | |
| 99 | stats.paced += piece.length; | |
| 100 | stats.lastPacedAt = now(); | |
| 101 | } else next = 0; | |
| 102 | stats.sent += piece.length; | |
| 103 | controller.enqueue(piece); | |
| 104 | }, | |
| 105 | cancel: (reason) => reader.cancel(reason), | |
| 106 | }, | |
| 107 | { highWaterMark: 0 }, | |
| 108 | ); | |
| 109 | }, | |
| 110 | }; | |
| 111 | } | |
| 112 | ||
| 113 | /** The process's pacer: the server opens pages on it, the pack responses are paced by it. */ | |
| 114 | export const pacer = createPacer(); | |
outdated
|
||
| 115 | ||
| 116 | /** `/:owner/:name.git/…`: the prefix of a git transfer, not a `.git/` anywhere in a page's path. */ | |
| 117 | const PACK_PATH = /^\/[^/]+\/[^/]+\.git\//; | |
| 118 | ||
| 119 | /** Is this URL a git transfer (`/owner/name.git/…`) and not a page? */ | |
| 120 | export function isPackRequest(url: string): boolean { | |
| 121 | let path: string; | |
| 122 | try { | |
| 123 | path = new URL(url).pathname; | |
outdated
Principles 1 and 8: this is one long sentence with a parenthetical, and it says 'a clone' when crawler media also sets |
||
| 124 | } catch { | |
| 125 | path = url.indexOf("?") < 0 ? url : url.slice(0, url.indexOf("?")); | |
| 126 | } | |
| 127 | return PACK_PATH.test(path); | |
| 128 | } | |
| 129 | ||
| 130 | /** | |
| 131 | * What /flow and the terminal say for a minute after a pack was held back for pages; null | |
| 132 | * otherwise. | |
| 133 | */ | |
| 134 | export function lineSaid(now: number = Date.now(), p: Pacer = pacer): string | null { | |
| 135 | return now - p.stats.lastPacedAt > 60_000 ? null : "Git packs are being slowed so pages go first."; | |
| 136 | } |
src/git/smart-http.ts +3 −1
| @@ -18,6 +18,7 @@ import { | ||
| 18 | 18 | import { gitEnv, transferGate } from "./env.ts"; |
| 19 | 19 | import { pullPipe, type PullPipe } from "./pull-pipe.ts"; |
| 20 | 20 | import { transferDeadline, type Stall } from "./progress.ts"; |
| 21 | import { pacer } from "./pace.ts"; | |
| 21 | 22 | import { OWN_REFS_ONLY, reachableFromOwnRefs, refusalLine, wantsOf } from "./own_refs.ts"; |
| 22 | 23 | |
| 23 | 24 | export type GitService = "git-upload-pack" | "git-receive-pack"; |
| @@ -168,7 +169,8 @@ export async function serviceRpc( | ||
| 168 | 169 | proc.kill("SIGKILL"), |
| 169 | 170 | ); |
| 170 | 171 | |
| 171 | return new Response(stdout, { | |
| 172 | // A fetch yields to pages while the line is contended (`pace.ts`). | |
| 173 | return new Response(service === "git-upload-pack" ? pacer.pace(stdout) : stdout, { | |
| 172 | 174 | headers: { |
| 173 | 175 | "content-type": `application/x-${service}-result`, |
| 174 | 176 | "cache-control": "no-cache", |
src/http/server.ts +4
| @@ -578,6 +578,7 @@ import { | ||
| 578 | 578 | takeDbTime, |
| 579 | 579 | type RouteTiming, |
| 580 | 580 | } from "../core/route_timing.ts"; |
| 581 | import { isPackRequest, pacer } from "../git/pace.ts"; | |
| 581 | 582 | import { captureStore, inFlight, liveOf, readLedger } from "../perf/live.ts"; |
| 582 | 583 | import { heldOf } from "../perf/held.ts"; |
| 583 | 584 | import { childLabel, markChildRequest, noteChildRequest } from "../perf/child_time.ts"; |
| @@ -3114,6 +3115,8 @@ export async function serve(app: App): Promise<Servers> { | ||
| 3114 | 3115 | fetch: (raw, server) => { |
| 3115 | 3116 | const request = parseableRequest(raw); |
| 3116 | 3117 | const bound = request === raw ? server : answeringFor(server, request, raw); |
| 3118 | // A page is open until answered, and git packs yield to it (`git/pace.ts`). | |
| 3119 | const pageDone = isPackRequest(request.url) ? undefined : pacer.page(); | |
| 3117 | 3120 | return inFlight.during(request, performance.now(), async () => { |
| 3118 | 3121 | try { |
| 3119 | 3122 | const response = await handle(request, bound); |
| @@ -3121,6 +3124,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 3121 | 3124 | } finally { |
| 3122 | 3125 | // Whatever door answered: the gate holds no request past its answer (#12901). |
| 3123 | 3126 | turnOf(app).leave(request); |
| 3127 | pageDone?.(); | |
| 3124 | 3128 | } |
| 3125 | 3129 | }); |
| 3126 | 3130 | }, |
src/tui/screens/flow.ts +3
| @@ -22,6 +22,7 @@ | ||
| 22 | 22 | * Read-only; Enter, Esc or q closes it. |
| 23 | 23 | */ |
| 24 | 24 | |
| 25 | import { lineSaid } from "../../git/pace.ts"; | |
| 25 | 26 | import type { Database } from "bun:sqlite"; |
| 26 | 27 | import { doctorRuns, type DoctorRun } from "../../checks/doctor.ts"; |
| 27 | 28 | import { recentGates, type KeptResult } from "../../checks/gate_result.ts"; |
| @@ -137,8 +138,10 @@ export function flowLines( | ||
| 137 | 138 | const warming = warmingLine(db, now.getTime(), pullCounts, agent !== undefined); |
| 138 | 139 | const reviewing = reviewingSays(fig.reviewing); |
| 139 | 140 | const ids = repoIds(db, queues.map((q) => q.repo)); |
| 141 | const line = lineSaid(now.getTime()); | |
| 140 | 142 | return [ |
| 141 | 143 | ...wrap(verdict(stages, now.getTime()), inner), |
| 144 | ...(line === null ? [] : wrap(line, inner)), | |
| 142 | 145 | ...(keeper === undefined || keeperLine(keeper) === "" ? [] : ["", ...wrap(keeperLine(keeper), inner)]), |
| 143 | 146 | ...measureLines(measuresCached(db, now, agent !== undefined), inner), |
| 144 | 147 | ...(heartbeat === undefined ? [] : ["", ...wrap(heartbeat, inner)]), |
src/web/pages/flow-panel.ts +3
| @@ -7,6 +7,7 @@ | ||
| 7 | 7 | * count links back to the page that drew it with `?out=<what>[&hour=<0–23>]`. |
| 8 | 8 | */ |
| 9 | 9 | |
| 10 | import { lineSaid } from "../../git/pace.ts"; | |
| 10 | 11 | import { MERGE_QUEUE_HELP } from "../../flow/help.ts"; |
| 11 | 12 | import { helpAsked, pageHelp } from "../page_help.ts"; |
| 12 | 13 | import { HELD_EXPLAINED } from "./boards-held.ts"; |
| @@ -119,6 +120,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string, slot | ||
| 119 | 120 | const now = Date.now(); |
| 120 | 121 | const stages = readPipeline(app.db, new Date(now)); |
| 121 | 122 | const worst = bottleneck(stages); |
| 123 | const line = lineSaid(now); | |
| 122 | 124 | const owner = reader.kind === "owner"; |
| 123 | 125 | // A page hands in the picture a worker read (`flow/pages.ts`); only a direct caller (a test) reads it here. |
| 124 | 126 | const pic: PanelPicture = slots.picture ?? { |
| @@ -162,6 +164,7 @@ export function flowPanel(app: App, reader: Reader, url: URL, base: string, slot | ||
| 162 | 164 | return html`<div class="bd-live" id="flow-live" data-live="${FLOW_LIVE}"> |
| 163 | 165 | <section class="bd-now" id="flow-summary" aria-label="How the pipeline stands"> |
| 164 | 166 | <p class="bd-lead" id="bottleneck" data-stage="${worst?.key ?? ""}"><strong>${verdict(stages, now)}</strong></p> |
| 167 | ${line === null ? "" : html`<p id="line-busy">${line}</p>`} | |
| 165 | 168 | ${slots.now ?? null} |
| 166 | 169 | ${keeperRow(app.db, owner)} |
| 167 | 170 | <p id="pile">${pile}</p> |
test/git-pace-wiring.test.ts added +130
| @@ -0,0 +1,130 @@ | ||
| 1 | /** | |
| 2 | * #13778: the wiring that makes pacing work — pages open and close on the pacer, only fetches are | |
| 3 | * paced, /flow and the terminal say it. | |
| 4 | */ | |
| 5 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 6 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 7 | import { tmpdir } from "node:os"; | |
| 8 | import { join } from "node:path"; | |
| 9 | import { createApp, type App } from "../src/app.ts"; | |
| 10 | import { pacer } from "../src/git/pace.ts"; | |
| 11 | import { initBareRepo, repoDir } from "../src/git/repos.ts"; | |
| 12 | import { advertiseRefs, isGitService, serviceRpc } from "../src/git/smart-http.ts"; | |
| 13 | import { serve, type Servers } from "../src/http/server.ts"; | |
| 14 | import { createUser } from "../src/core/users.ts"; | |
| 15 | import { unwrap } from "../src/kernel/result.ts"; | |
| 16 | import { flowLines } from "../src/tui/screens/flow.ts"; | |
| 17 | import { gitClientEnv } from "./helpers/git-client.ts"; | |
| 18 | ||
| 19 | let root: string; | |
| 20 | let work: string; | |
| 21 | let app: App; | |
| 22 | let servers: Servers; | |
| 23 | let gitServer: ReturnType<typeof Bun.serve>; | |
| 24 | let base: string; | |
| 25 | let cookie = ""; | |
| 26 | const must = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => { | |
| 27 | if (!r.ok) throw new Error(JSON.stringify(r.error)); | |
| 28 | return r.value; | |
| 29 | }; | |
| 30 | /** | |
| 31 | * The bottleneck panel is drawn on /boards, from a picture a worker reads: ask until it is drawn. | |
| 32 | */ | |
| 33 | const flow = async (): Promise<string> => { | |
| 34 | for (let i = 0; i < 50; i++) { | |
| 35 | const text = await (await fetch(`${servers.url}/boards`, { headers: { cookie } })).text(); | |
| 36 | if (text.includes('id="bottleneck"')) return text; | |
| 37 | await Bun.sleep(100); // real-timer: polls a real server until its worker has drawn the panel | |
| 38 | } | |
| 39 | throw new Error("the bottleneck panel was never drawn"); | |
| 40 | }; | |
| 41 | ||
| 42 | async function git(args: string[], cwd: string): Promise<void> { | |
| 43 | const proc = Bun.spawn(["git", ...args], { cwd, env: gitClientEnv(), stdout: "pipe", stderr: "pipe" }); | |
| 44 | const [err, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); | |
| 45 | if (code !== 0) throw new Error(`git ${args.join(" ")}: ${err}`); | |
| 46 | } | |
| 47 | ||
| 48 | beforeAll(async () => { | |
| 49 | root = await mkdtemp(join(tmpdir(), "forge-pacewire-")); | |
| 50 | work = await mkdtemp(join(tmpdir(), "forge-pacewire-work-")); | |
| 51 | app = createApp({ dataDir: root, port: 0 }); | |
| 52 | servers = await serve(app); | |
| 53 | must(await createUser(app.db, { origin: "seed", trusted: true, username: "alice", email: "[email protected]", password: "a-good-passphrase", adminIfFirst: true })); | |
| 54 | const res = await fetch(`${servers.url}/signin`, { | |
| 55 | method: "POST", | |
| 56 | redirect: "manual", | |
| 57 | headers: { "content-type": "application/x-www-form-urlencoded", origin: servers.url }, | |
| 58 | body: new URLSearchParams({ username: "alice", password: "a-good-passphrase" }), | |
| 59 | }); | |
| 60 | cookie = (res.headers.get("set-cookie") ?? "").split(";")[0] ?? ""; | |
| 61 | const bare = repoDir(root, "ada", "arm"); | |
| 62 | unwrap(await initBareRepo(bare)); | |
| 63 | const seed = join(work, "seed"); | |
| 64 | await git(["init", "-q", "-b", "main", seed], work); | |
| 65 | await Bun.write(join(seed, "a.txt"), "a\n"); | |
| 66 | await git(["add", "."], seed); | |
| 67 | await git(["commit", "-q", "-m", "one"], seed); | |
| 68 | await git(["push", "-q", bare, "main"], seed); | |
| 69 | const open = { stall: { windowMs: 120_000, minBytes: 1 }, timeoutMs: 300_000, idleMs: 120_000 }; | |
| 70 | gitServer = Bun.serve({ | |
| 71 | port: 0, | |
| 72 | idleTimeout: 0, | |
| 73 | async fetch(request) { | |
| 74 | const url = new URL(request.url); | |
| 75 | if (url.pathname === "/ada/arm.git/info/refs") { | |
| 76 | const service = url.searchParams.get("service"); | |
| 77 | return isGitService(service) ? advertiseRefs(bare, service, request) : new Response("no", { status: 400 }); | |
| 78 | } | |
| 79 | for (const service of ["git-upload-pack", "git-receive-pack"] as const) | |
| 80 | if (url.pathname === `/ada/arm.git/${service}`) return serviceRpc(bare, service, request, open); | |
| 81 | return new Response("no", { status: 404 }); | |
| 82 | }, | |
| 83 | }); | |
| 84 | base = `http://127.0.0.1:${gitServer.port}/ada/arm.git`; | |
| 85 | }, 60_000); | |
| 86 | ||
| 87 | afterAll(async () => { | |
| 88 | gitServer.stop(true); | |
| 89 | servers.stop(); | |
| 90 | app.close(); | |
| 91 | pacer.stats.lastPacedAt = 0; | |
| 92 | await rm(root, { recursive: true, force: true }); | |
| 93 | await rm(work, { recursive: true, force: true }); | |
| 94 | }); | |
| 95 | ||
| 96 | test("serviceRpc sends a fetch's pack through the pacer and a push's report not", async () => { | |
| 97 | const before = pacer.stats.sent; | |
| 98 | await git(["clone", "-q", base, join(work, "c")], work); | |
| 99 | const afterClone = pacer.stats.sent; | |
| 100 | expect(afterClone).toBeGreaterThan(before); | |
| 101 | await Bun.write(join(work, "c", "b.txt"), "b\n"); | |
| 102 | await git(["add", "."], join(work, "c")); | |
| 103 | await git(["commit", "-q", "-m", "two"], join(work, "c")); | |
| 104 | await git(["push", "-q", "origin", "main"], join(work, "c")); | |
| 105 | expect(pacer.stats.sent).toBe(afterClone); | |
| 106 | }); | |
| 107 | ||
| 108 | test("serve() opens a page on the pacer and closes it when answered; a git transfer is not a page", async () => { | |
| 109 | const opened = pacer.stats.opened; | |
| 110 | expect((await fetch(`${servers.url}/`)).status).toBe(200); | |
| 111 | expect(pacer.stats.opened).toBe(opened + 1); | |
| 112 | await Bun.sleep(20); // real-timer: lets the real server finish closing the page | |
| 113 | expect(pacer.inFlight()).toBe(0); | |
| 114 | await (await fetch(`${servers.url}/ada/arm.git/info/refs?service=git-upload-pack`)).text(); | |
| 115 | expect(pacer.stats.opened).toBe(opened + 1); | |
| 116 | await (await fetch(`${servers.url}/a/b/raw/main/x.git/y`)).text(); | |
| 117 | expect(pacer.stats.opened).toBe(opened + 2); | |
| 118 | }); | |
| 119 | ||
| 120 | test("the bottleneck panel and the terminal's Flow screen say it once while packs are paced, and not when quiet", async () => { | |
| 121 | const said = "Git packs are being slowed so pages go first."; | |
| 122 | pacer.stats.lastPacedAt = 0; | |
| 123 | expect(await flow()).not.toContain('id="line-busy"'); | |
| 124 | expect(flowLines(app.db).join("\n")).not.toContain("Git packs are being slowed"); | |
| 125 | pacer.stats.lastPacedAt = Date.now(); | |
| 126 | const html = await flow(); | |
| 127 | expect(html.match(/id="line-busy"/g)?.length).toBe(1); | |
| 128 | expect(html).toContain(said); | |
| 129 | expect(flowLines(app.db).join(" ").replace(/\s+/g, " ")).toContain(said); | |
| 130 | }); |
test/git-pace.test.ts added +104
| @@ -0,0 +1,104 @@ | ||
| 1 | import { expect, test } from "bun:test"; | |
| 2 | import { createPacer, DEFAULT_BPS, isPackRequest, lineSaid, pacedRate, QUIET_MS } from "../src/git/pace.ts"; | |
| 3 | import { transferDeadline } from "../src/git/progress.ts"; | |
| 4 | ||
| 5 | function source(total: number, chunk: number): ReadableStream<Uint8Array> { | |
| 6 | let left = total; | |
| 7 | return new ReadableStream({ | |
| 8 | pull(c) { | |
| 9 | if (left <= 0) return c.close(); | |
| 10 | const n = Math.min(chunk, left); | |
| 11 | left -= n; | |
| 12 | c.enqueue(new Uint8Array(n)); | |
| 13 | }, | |
| 14 | }); | |
| 15 | } | |
| 16 | async function drain(s: ReadableStream<Uint8Array>): Promise<number> { | |
| 17 | let n = 0; | |
| 18 | for await (const c of s) n += c.length; | |
| 19 | return n; | |
| 20 | } | |
| 21 | ||
| 22 | test("with no page traffic a pack is not slowed", async () => { | |
| 23 | let t = 0; | |
| 24 | const slept: number[] = []; | |
| 25 | const p = createPacer(() => t, 1000, async (ms) => void slept.push(ms)); | |
| 26 | expect(await drain(p.pace(source(1_000_000, 100_000)))).toBe(1_000_000); | |
| 27 | expect(slept).toEqual([]); | |
| 28 | expect(p.stats.paced).toBe(0); | |
| 29 | }); | |
| 30 | ||
| 31 | test("while a page is open a pack is held to the paced rate", async () => { | |
| 32 | let t = 0; | |
| 33 | let slept = 0; | |
| 34 | const p = createPacer(() => t, 32 * 1024, async (ms) => { | |
| 35 | slept += ms; | |
| 36 | t += ms; | |
| 37 | }); | |
| 38 | const done = p.page(); | |
| 39 | // 256 KiB at 32 KiB/s is 8 s, less the first piece which goes at once. | |
| 40 | expect(await drain(p.pace(source(256 * 1024, 64 * 1024)))).toBe(256 * 1024); | |
| 41 | expect(slept).toBeGreaterThan(7_000); | |
| 42 | expect(slept).toBeLessThanOrEqual(8_000); | |
| 43 | expect(p.stats.paced).toBe(256 * 1024); | |
| 44 | done(); | |
| 45 | }); | |
| 46 | ||
| 47 | test("a page that ended is remembered for a few seconds, then the line is free", () => { | |
| 48 | let t = 1_000_000; | |
| 49 | const p = createPacer(() => t); | |
| 50 | expect(p.contended()).toBe(false); | |
| 51 | p.page()(); | |
| 52 | expect(p.contended()).toBe(true); | |
| 53 | t += QUIET_MS + 1; | |
| 54 | expect(p.contended()).toBe(false); | |
| 55 | }); | |
| 56 | ||
| 57 | test("a request held open for minutes is a stream, not a page", () => { | |
| 58 | let t = 1_000_000; | |
| 59 | const p = createPacer(() => t); | |
| 60 | p.page(); | |
| 61 | t += 60_000; | |
| 62 | expect(p.contended()).toBe(false); | |
| 63 | }); | |
| 64 | ||
| 65 | test("git transfer URLs are not pages, and a page path that contains .git/ is", () => { | |
| 66 | expect(isPackRequest("https://x/a/b.git/git-upload-pack")).toBe(true); | |
| 67 | expect(isPackRequest("https://x/a/b.git/info/refs?service=git-upload-pack")).toBe(true); | |
| 68 | expect(isPackRequest("https://x/a/b")).toBe(false); | |
| 69 | expect(isPackRequest("https://x/?q=.git/")).toBe(false); | |
| 70 | expect(isPackRequest("https://x/a/b/raw/main/x.git/y")).toBe(false); | |
| 71 | expect(isPackRequest("https://x/a.git/y")).toBe(false); | |
| 72 | }); | |
| 73 | ||
| 74 | test("the paced rate is a finite number above zero, else the default", () => { | |
| 75 | expect(pacedRate(undefined)).toBe(DEFAULT_BPS); | |
| 76 | for (const bad of ["", "-5", "0", "abc", "Infinity", "NaN"]) expect(pacedRate(bad)).toBe(DEFAULT_BPS); | |
| 77 | expect(pacedRate("4096")).toBe(4096); | |
| 78 | }); | |
| 79 | ||
| 80 | test("lineSaid speaks for a minute after a pack was paced, once, in one sentence", () => { | |
| 81 | let t = 1_000_000; | |
| 82 | const p = createPacer(() => t); | |
| 83 | expect(lineSaid(t, p)).toBeNull(); | |
| 84 | p.stats.lastPacedAt = t; | |
| 85 | const said = lineSaid(t + 59_000, p); | |
| 86 | expect(said).toBe("Git packs are being slowed so pages go first."); | |
| 87 | expect(lineSaid(t + 61_000, p)).toBeNull(); | |
| 88 | }); | |
| 89 | ||
| 90 | test("a paced pack moving over the stall floor is not killed as a stall (#4482)", async () => { | |
| 91 | let killed = false; | |
| 92 | const stall = { windowMs: 1_200, minBytes: 100 }; | |
| 93 | const deadline = transferDeadline(true, source(96 * 1024, 4 * 1024), { stall, timeoutMs: 60_000 }, () => (killed = true)); | |
| 94 | const p = createPacer(Date.now, 48 * 1024); | |
| 95 | const done = p.page(); | |
| 96 | const start = Date.now(); | |
| 97 | // 96 KiB at 48 KiB/s: about two seconds, more than a window, paced the whole way. | |
| 98 | expect(await drain(p.pace(deadline.stdout))).toBe(96 * 1024); | |
| 99 | deadline.stop(); | |
| 100 | done(); | |
| 101 | expect(Date.now() - start).toBeGreaterThan(1_200); | |
| 102 | expect(p.stats.paced).toBe(96 * 1024); | |
| 103 | expect(killed).toBe(false); | |
| 104 | }); |
test/job-slices.test.ts +1
| @@ -261,6 +261,7 @@ function jobs(): Record<string, () => unknown> { | ||
| 261 | 261 | * on. |
| 262 | 262 | */ |
| 263 | 263 | const UNTIMED: Record<string, string> = { |
| 264 | "git-pace": "a one-shot timer in git/pace.ts that only resolves the promise a paced pack write waits on: no work of its own, armed per pack chunk while a page response is in flight; git-pace.test.ts drives the pacer with a fake clock", | |
| 264 | 265 | "drill-watch": "a /proc read of the drill child's memory and a state write every WATCH_MS, only while a restore drill runs in its own process; drill-job tests drive runDrillInChild directly", |
| 265 | 266 | desk: "expires notes past their time: one indexed delete on the desk table every 30 s (actions/desk.ts); the fixture has no notes", |
| 266 | 267 | ledger: "walks the cost ledger's hash chain at boot and once a day (#12273): file reads proportional to the ledger, off the request path; the fixture has no ledger", |
new above
offline
Unvalidated env:
Number('-5')is truthy, so a negative rate makesnextgo backwards and pacing silently stops. Accept only a finite number above 0, else the default.