5c40013 · 04dd74d
@@ -39,7 +39,8 @@
 */
import { readFileSync, writeFileSync } from "node:fs";
import { fell, missed, summarize, type LoadResult } from "../src/perf/load_verdict.ts";
import { fell, missed, summarize, TARGET, type LoadResult } from "../src/perf/load_verdict.ts";
import { refuseProduction, systemResolve } from "../src/perf/production_guard.ts";
const arg = (name: string, fallback?: string): string => {
  const at = process.argv.indexOf(`--${name}`);
@@ -49,6 +50,12 @@
const arg = (name: string, fallback?: string): string => {
};
const base = arg("url").replace(/\/$/, "");
// Never at production (#9596): decided from the URL, before the first request.
const notHere = await refuseProduction(base, systemResolve);
if (notHere !== undefined) {
  console.error(`load: refused: ${notHere}`);
  process.exit(2);
}
const repo = arg("repo");
const rate = Number(arg("rate", "100"));
const seconds = Number(arg("seconds", "10"));
@@ -180,11 +187,17 @@
counting = true;
await flood(seconds);
await Promise.all(inflight);
const result: LoadResult = { ...summarize(version, rate, seconds, latencies, refused), cache: bypass ? "bypass" : "edge", refusedBy };
const warmTotals = warmPerSecond.reduce(([s, r], [a, b]) => [s + a, r + b], [0, 0]);
const result: LoadResult = {
  ...summarize(version, rate, seconds, latencies, refused),
  cache: bypass ? "bypass" : "edge",
  refusedBy,
  warm: { seconds: warm, served: warmTotals[0]!, refused: warmTotals[1]! },
};
console.log(JSON.stringify(result, null, 2));
console.log(`per second, served/refused: ${perSecond.map(([a, b]) => `${a}/${b}`).join(" ")}`);
const short = missed(result);
console.log(short === undefined ? "target met: none refused, p95 under 100 ms" : `target missed: ${short}`);
console.log(short === undefined ? `target met: ${TARGET.rate}/s, none refused, p95 under ${TARGET.p95UnderMs} ms` : `target missed: ${short}`);
if (cookie !== undefined) {
  const s = [...signedIn].sort((a, b) => a - b);
  const at = (p: number) => (s.length === 0 ? null : Math.round(s[Math.min(s.length - 1, Math.floor(p * s.length))]! * 10) / 10);
@@ -192,7 +205,7 @@
if (cookie !== undefined) {
}
const out = process.argv.includes("--out") ? arg("out") : undefined;
// The file says itself whether the run met the goal, so a kept result can be read without rerunning it.
if (out !== undefined) writeFileSync(out, JSON.stringify({ ...result, verdict: short ?? "target met" }, null, 2) + "\n");
if (out !== undefined) writeFileSync(out, JSON.stringify({ ...result, target: TARGET, verdict: short ?? "target met" }, null, 2) + "\n");
const last = process.argv.includes("--compare") ? (JSON.parse(readFileSync(arg("compare"), "utf8")) as LoadResult) : undefined;
const why = last === undefined ? undefined : fell(last, result);
if (why !== undefined) {
@@ -15,9 +15,9 @@
Media (`/media/…`, the route a PDF or image is drawn from) is **not covered**:
The bypasses are tested (`test/anonymous-deep.test.ts`): a signed-in reader, a repository made private, `?fragment=1` and a branch-name ref never get a page kept for a stranger. `--compare` exiting 1 is tested (`test/load-bench.test.ts`).
**Release step:** `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name>` runs the load twice, through the edge and with it bypassed, at 500/s for 20 s after a 20 s warm-up, writes `bench/results/<version>.json` and `<version>.bypass.json`, and compares each with the newest older kept file of its kind (`src/perf/baseline.ts`); exit 1 from either is the gate. With no older file it only writes the first; commit it. `scripts/cut-release.ts` prints this command as its last step; it does not run it (it needs a copy with data), and no baseline is kept yet — the first release run writes it.
**Release step:** `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name>` runs the load twice, through the edge and with it bypassed, at 500/s for 20 s after a 20 s warm-up, writes `bench/results/<version>.json` and `<version>.bypass.json` (`--results DIR` writes them elsewhere), and compares each with the newest older kept file of its kind (`src/perf/baseline.ts`); exit 1 from either is the gate. With no older file it only writes the first; commit it. `scripts/cut-release.ts` prints this command as its last step; it does not run it (it needs a copy with data). See [the person's run](#the-persons-release-run-9596) below for the steps.
No release baseline (`<version>.json`) is kept in `bench/results/` yet: the pull's preview holds no repository to read, and the load must not be aimed at production. The first files are written by the first run against a copy with data. The two `9514-*.json` files there are the in-process measurements below; `lastBaseline` ignores them.
No release baseline (`<version>.json`) is kept in `bench/results/` yet: the pull's preview holds no repository to read, and the load must not be aimed at production. The first files are written by the first run against a copy with data. The two `9514-*.json` files there are the in-process measurements below; `lastBaseline` ignores them. The crew writes no result file it did not measure.
Follow-ups: profile #9503, baseline #9504 (the file itself, still to be written from a copy with data), media #9505.
@@ -96,12 +96,28 @@
What the caches hold (entries, bytes beside the cap, hits and misses) is on `/sy
A bigger git pool (`FORGE_GIT_CONCURRENCY=32`) changed nothing before item 5: the main thread was the limit, not git.
### Not done: the run through the edge, on a copy with production's data (#9565)
### The run through the edge, on a copy with production's data (#9565, #9596)
This sandbox reaches no copy holding production's repositories and no edge in front of one. The load must not be aimed at production, and a pull's preview holds no repository. The numbers above are the best case on a copy of one repository: no edge, the client on the same machine, and a warm start. They are not the `Done` of #9514. To take them, run from a machine that is not the server:
A crew sandbox reaches no copy holding production's repositories and no edge in front of one, so the numbers above are the best case: one repository, no edge, the client on the same machine, a warm start. They are not the `Done` of #9514. The run below is a person's, from a machine that is not the server.
```sh
bun scripts/release-load.ts <version> --url <copy behind the edge> --repo trevato/forge
```
## The person's release run (#9596)
**Never at production.** `scripts/release-load.ts` and `bench/load.ts` both refuse, with exit 2 and before any request goes out, a `--url` whose host is production's (`forge.open-platform.sh`), any name under it, or an address production's name resolves to (`src/perf/production_guard.ts`). A different name on the same edge (a hosted child, `<name>--forge.open-platform.sh`) is a different forge and is not refused. Load it only if it is yours.
1. **Restore a copy of production's data** on a box that is not production: `bun scripts/restore-drill.ts /srv/forge-copy` restores the newest snapshot into that empty directory (the `S3_*` variables, or `FORGE_SNAPSHOT_DIR`, and `FORGE_SNAPSHOT_PASSPHRASE`, as on the restore card), boots it once on a side port and checks it against production's public counts. The directory stays.
2. **Boot the copy as a forge**: `FORGE_DATA=/srv/forge-copy PORT=3100 bun run src/main.ts` from a clone at the release's tag. It runs the code you are releasing on production's repositories.
3. **Put the edge in front of it**, the same way production has it: a Cloudflare tunnel (or the cluster ingress) to `localhost:3100` under a name of your own, with the cache rule production has: cache HTML that is `public` and `s-maxage`, **bypass any request with a Cookie or Authorization header** (`src/http/addressed.ts`). Without that rule, the bypass pass reads cached pages and the two files show the same thing. Behind the tunnel, set `FORGE_TRUSTED_PROXIES` and `FORGE_CLIENT_IP_HEADER=cf-connecting-ip` ([OPERATING](../OPERATING.md)) so strangers are counted per address, not as one.
4. **Check the copy answers** at its name: `curl -sI https://<copy>/trevato/forge` is 200.
5. **Run it from a third machine** (not the copy's box, not production):
   ```sh
   bun install
   bun scripts/release-load.ts <version> --url https://<copy> --repo trevato/forge
   ```
   It takes about 80 s: per pass, a 20 s warm-up that is not counted (its per-second `served/refused` line is printed), then 20 s at 500/s. Add `--crawl 3000` to spread the load over the repository rather than its top folder.
6. **Read the verdicts.** It ends with one line per file. Each of `bench/results/<version>.json` (through the edge) and `bench/results/<version>.bypass.json` (past it) holds `asked` (the rate), `servedPerSecond`, `refused` and `refusedBy`, `p50` and `p95` in ms, `warm` (`seconds`, `served`, `refused`), `target` (`{"rate": 500, "refused": 0, "p95UnderMs": 100}`) and `verdict`: `"target met"`, or how far it missed, e.g. `"12 refused of 10000; p95 131.4 ms, 31.4 ms over the 100 ms target"` or `"served 400/s, 100/s under the 500/s target"`. A missed target is written, not an error; exit 1 means the served rate fell more than 20% from the last kept file of the same kind.
7. **Post both files** as a comment on #9514, with the version, the machine the load ran from and the copy's box, and commit them in a pull to `bench/results/`. They become the first baseline the next release compares with.
Tests: `test/release-load-run.test.ts` runs the whole command against a local fake forge, one that keeps up (both files `"target met"`) and one that answers in 150 ms (both files say how far p95 missed), and checks that production is refused by name and by address before any request.
This writes both kinds and compares each with its last kept result.
@@ -1,22 +1,41 @@
/**
 * `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name> [--rate N] [--seconds N]`:
 * the release step's load check (#9504, #9514). Runs bench/load.ts against a
 * copy with data twice, through the edge cache and with it bypassed, writes
 * bench/results/<version>.json and <version>.bypass.json, and compares each
 * with the newest kept older result of its kind; exit 1 is a fall of more
 * than 20% in either. With no older result of a kind it only writes the
 * first. Commit the files it writes.
 * `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name> [--rate N] [--seconds N] [--warm N] [--results DIR]`:
 * the release step's load check (#9504, #9514, #9596). Runs bench/load.ts
 * against a copy with data twice, through the edge cache and with it
 * bypassed, writes `<version>.json` and `<version>.bypass.json` into
 * bench/results (or `--results DIR`), each with its warm-up, the target and
 * `"verdict"` ("target met" or how far it missed), and compares each with
 * the newest kept older result of its kind; exit 1 is a fall of more than
 * 20% in either. With no older result of a kind it only writes the first.
 * A `--url` that is production is refused (exit 2) before any request goes
 * out (`src/perf/production_guard.ts`). Commit the files it writes.
 */
import { existsSync, mkdirSync, readdirSync } from "node:fs";
import { CACHE_MODES, lastBaseline, loadArgs } from "../src/perf/baseline.ts";
import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { CACHE_MODES, lastBaseline, loadArgs, resultFile } from "../src/perf/baseline.ts";
import { refuseProduction, systemResolve } from "../src/perf/production_guard.ts";
const [version, ...rest] = process.argv.slice(2);
if (version === undefined || !/^\d+\.\d+\.\d+$/.test(version) || !rest.includes("--url") || !rest.includes("--repo")) {
  console.error("usage: bun scripts/release-load.ts <major.minor.patch> --url <copy> --repo <owner/name> [--rate N] [--seconds N]");
const USAGE = "usage: bun scripts/release-load.ts <major.minor.patch> --url <copy> --repo <owner/name> [--rate N] [--seconds N] [--warm N] [--results DIR]";
const [version, ...args] = process.argv.slice(2);
const valueOf = (name: string): string | undefined => {
  const at = args.indexOf(`--${name}`);
  return at >= 0 ? args[at + 1] : undefined;
};
const url = valueOf("url");
if (version === undefined || !/^\d+\.\d+\.\d+$/.test(version) || url === undefined || valueOf("repo") === undefined) {
  console.error(USAGE);
  process.exit(2);
}
const dir = `${import.meta.dir}/../bench/results`;
const notHere = await refuseProduction(url, systemResolve);
if (notHere !== undefined) {
  console.error(`release-load: refused: ${notHere}`);
  process.exit(2);
}
// `--results` is this script's own; bench/load.ts is handed the rest.
const resultsAt = args.indexOf("--results");
const rest = resultsAt >= 0 ? [...args.slice(0, resultsAt), ...args.slice(resultsAt + 2)] : args;
const dir = resolve(valueOf("results") ?? `${import.meta.dir}/../bench/results`);
mkdirSync(dir, { recursive: true });
// Through the edge, then with it bypassed (#9514): both run, and either falling is the gate.
let failed = 0;
@@ -27,4 +46,10 @@
for (const cache of CACHE_MODES) {
  const code = await Bun.spawn(cmd, { stdout: "inherit", stderr: "inherit", env: { BUN_CONFIG_MAX_HTTP_REQUESTS: "65000", ...process.env } }).exited;
  if (code !== 0) failed = code;
}
// Both verdicts together at the end, read back from the files that will be committed.
for (const cache of CACHE_MODES) {
  const file = `${dir}/${resultFile(version, cache)}`;
  const verdict = existsSync(file) ? (JSON.parse(readFileSync(file, "utf8")) as { verdict?: string }).verdict : "not written";
  console.error(`release-load (${cache}): ${file}: ${verdict}`);
}
process.exit(failed);
@@ -227,8 +227,7 @@
export function parseSentence(text: string, now: number): Record<string, string>
    else if (/\byesterday\b/.test(t)) {
      out.from = since(Date.parse(startOfToday(now)), 1);
      out.to = startOfToday(now);
    }
    else if (/\b(today|so far today)\b/.test(t)) out.from = startOfToday(now);
    } else if (/\b(today|so far today)\b/.test(t)) out.from = startOfToday(now);
    else if (/\b(a week|weekly)\b/.test(t)) days = 7;
  }
  if (days !== undefined) out.from = since(now, days);
@@ -26,17 +26,38 @@
export interface LoadResult {
  readonly refusedBy?: Readonly<Record<string, number>>;
  /** `"target met"`, or `missed()`'s words, as the run that wrote it judged (#9548). Absent in results kept before. */
  readonly verdict?: string;
  /** The uncounted warm-up before the measured window (#9596): its seconds and what it got. Absent in results kept before. */
  readonly warm?: { readonly seconds: number; readonly served: number; readonly refused: number };
}
/** The goal a release is held to (#9514): at the asked rate, no refusal and p95 under this, ms. */
export const TARGET_P95_MS = 100;
/** The rate the goal is held at, requests per second (#9514). */
export const TARGET_RATE = 500;
/**
 * The share of the asked rate a run must serve. The open loop sends
 * floor(elapsed × rate), so a full run lands a request or two under; 95% is
 * the slack for that, not for a forge that cannot keep up (refusals count apart).
 */
export const SERVED_SHARE = 0.95;
/** Words for how a run fell short of the goal, or undefined when it refused none and its p95 was under `TARGET_P95_MS`. */
/** What the goal is, written into each result file so it reads on its own (#9596). */
export const TARGET = { rate: TARGET_RATE, refused: 0, p95UnderMs: TARGET_P95_MS } as const;
const tenth = (n: number) => Math.round(n * 10) / 10;
/**
 * Words for how far a run fell short of the goal (500/s asked and served, none
 * refused, p95 under `TARGET_P95_MS`), or undefined when it met it.
 */
export function missed(r: LoadResult): string | undefined {
  const why: string[] = [];
  if (r.refused > 0) why.push(`${r.refused} refused`);
  if (r.asked < TARGET_RATE) why.push(`asked ${r.asked}/s, ${tenth(TARGET_RATE - r.asked)}/s under the ${TARGET_RATE}/s target`);
  else if (r.served > 0 && r.servedPerSecond < TARGET_RATE * SERVED_SHARE)
    why.push(`served ${r.servedPerSecond}/s, ${tenth(TARGET_RATE - r.servedPerSecond)}/s under the ${TARGET_RATE}/s target`);
  if (r.refused > 0) why.push(`${r.refused} refused of ${r.served + r.refused}`);
  if (r.served === 0) why.push("none served");
  else if (r.p95 >= TARGET_P95_MS) why.push(`p95 ${r.p95} ms, not under ${TARGET_P95_MS} ms`);
  else if (r.p95 >= TARGET_P95_MS) why.push(`p95 ${r.p95} ms, ${tenth(r.p95 - TARGET_P95_MS)} ms over the ${TARGET_P95_MS} ms target`);
  return why.length === 0 ? undefined : why.join("; ");
}
@@ -0,0 +1,49 @@
/**
 * The load is never aimed at production (#9596). Decides from the URL alone,
 * before any request goes out, whether a `--url` would land on the forge's
 * own public host: its name, any name under it, or an address literal that
 * production's name resolves to. A different name on the same edge is a
 * different forge (the edge routes by Host), so it is not refused here.
 *
 * Plain data in, words out: undefined when the URL may be loaded, else why not.
 */
import { isIP } from "node:net";
/** The forge's own public host: production (as in `scripts/restore-drill.ts`). */
export const PRODUCTION_HOST = "forge.open-platform.sh";
/** Addresses a name resolves to; injected so tests ask no resolver. */
export type Resolve = (host: string) => Promise<readonly string[]>;
const bare = (host: string): string => host.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
/** A loopback or unspecified address: production is never one. */
const local = (ip: string): boolean => /^127\./.test(ip) || ip === "::1" || ip === "0.0.0.0" || ip === "::";
export async function refuseProduction(url: string, resolve: Resolve, production: string = PRODUCTION_HOST): Promise<string | undefined> {
  let parsed: URL;
  try {
    parsed = new URL(url);
  } catch {
    return `--url ${JSON.stringify(url)} is not a URL`;
  }
  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return `--url must be http or https, not ${parsed.protocol}`;
  const host = bare(parsed.hostname);
  const prod = bare(production);
  if (host === prod || host.endsWith(`.${prod}`)) return `--url ${parsed.origin} is production (${prod}); run the load only at a copy you stand up`;
  if (isIP(host) === 0 || local(host)) return undefined;
  let addresses: readonly string[];
  try {
    addresses = (await resolve(prod)).map(bare);
  } catch {
    return `--url ${parsed.origin} is an address and ${prod} could not be resolved to rule out that it is production; use the copy's name`;
  }
  return addresses.includes(host) ? `--url ${parsed.origin} is an address of production (${prod}); run the load only at a copy you stand up` : undefined;
}
/** The system resolver, every address of a name. */
export const systemResolve: Resolve = async (host) => {
  const { lookup } = await import("node:dns/promises");
  return (await lookup(host, { all: true })).map((a) => a.address);
};
@@ -67,7 +67,8 @@
test("--bypass sends every request with a cookie the edge must pass, --warm is n
    expect(withCookie).toBeGreaterThan(result.served);
    expect(result.served).toBeLessThanOrEqual(41);
    expect(out).toContain("warm-up, per second");
    expect(out).toContain("target met");
    // 40/s is under the 500/s the goal is held at, so the verdict says by how much (#9596).
    expect(out).toContain("target missed: asked 40/s, 460/s under the 500/s target");
  } finally {
    server.stop(true);
  }
@@ -0,0 +1,118 @@
import { expect, test } from "bun:test";
import { existsSync } from "node:fs";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { missed } from "../src/perf/load_verdict.ts";
import { PRODUCTION_HOST, refuseProduction } from "../src/perf/production_guard.ts";
// The person's release run (#9596): one command, both passes, both files, each with its verdict.
const oid = "b".repeat(40);
const page = `<a href="/o/r/blob/${oid}/f.txt">f</a><a href="/o/r/tree/${oid}/d">d</a>`;
/** Runs scripts/release-load.ts against a fake forge answering after `delayMs`; returns both files and what each pass's request carried. */
const release = async (delayMs: number) => {
  const dir = await mkdtemp(join(tmpdir(), "forge-release-load-"));
  const cookies = { bypass: 0, none: 0 };
  const server = Bun.serve({
    port: 0,
    fetch: async (req) => {
      if (/\/(blob|tree|raw)\/[0-9a-f]{40}/.test(new URL(req.url).pathname)) {
        req.headers.get("cookie") === "forge_load=bypass" ? cookies.bypass++ : cookies.none++;
        if (delayMs > 0) await Bun.sleep(delayMs);
      }
      return new Response(page, { headers: { "content-type": "text/html" } });
    },
  });
  try {
    const p = Bun.spawn(
      ["bun", "scripts/release-load.ts", "9.9.9", "--url", `http://127.0.0.1:${server.port}`, "--repo", "o/r", "--seconds", "1", "--warm", "1", "--results", dir],
      { stdout: "pipe", stderr: "pipe" },
    );
    const [code, , err] = await Promise.all([p.exited, new Response(p.stdout).text(), new Response(p.stderr).text()]);
    const read = async (f: string) => JSON.parse(await readFile(join(dir, f), "utf8"));
    return { code, err, cookies, edge: await read("9.9.9.json"), bypass: await read("9.9.9.bypass.json") };
  } finally {
    server.stop(true);
    await rm(dir, { recursive: true, force: true });
  }
};
test("a forge that keeps up gets both files at 500/s, each with its warm-up and \"target met\"", async () => {
  const r = await release(0);
  expect(r.code).toBe(0);
  for (const [file, cache] of [[r.edge, "edge"], [r.bypass, "bypass"]] as const) {
    expect(file.version).toBe("9.9.9");
    expect(file.cache).toBe(cache);
    expect(file.asked).toBe(500);
    expect(file.refused).toBe(0);
    expect(file.servedPerSecond).toBeGreaterThanOrEqual(475);
    expect(file.p95).toBeLessThan(100);
    expect(typeof file.p50).toBe("number");
    expect(file.warm.seconds).toBe(1);
    expect(file.warm.served).toBeGreaterThan(0);
    expect(file.target).toEqual({ rate: 500, refused: 0, p95UnderMs: 100 });
    expect(file.verdict).toBe("target met");
  }
  // The edge pass carries no cookie; every request of the bypass pass carries the one the edge must pass on.
  expect(r.cookies.none).toBeGreaterThan(0);
  expect(r.cookies.bypass).toBeGreaterThan(0);
  expect(r.err).toContain("9.9.9.json: target met");
  expect(r.err).toContain("9.9.9.bypass.json: target met");
}, 30_000);
test("a forge answering in 150 ms gets both files saying how far p95 missed", async () => {
  const r = await release(150);
  // Missing the target is written down, not a failed run: only a fall from a kept result exits 1.
  expect(r.code).toBe(0);
  for (const file of [r.edge, r.bypass]) {
    expect(file.p95).toBeGreaterThanOrEqual(150);
    expect(file.verdict).toBe(`p95 ${file.p95} ms, ${Math.round((file.p95 - 100) * 10) / 10} ms over the 100 ms target`);
  }
  expect(r.err).toContain("9.9.9.bypass.json: p95 ");
}, 30_000);
test("the verdict says how far each part of the goal was missed", () => {
  const r = (o: object) => ({ version: "1", asked: 500, seconds: 20, served: 10_000, refused: 0, servedPerSecond: 500, p50: 3, p95: 40, ...o });
  expect(missed(r({}))).toBeUndefined();
  expect(missed(r({ asked: 100 }))).toBe("asked 100/s, 400/s under the 500/s target");
  expect(missed(r({ servedPerSecond: 400 }))).toBe("served 400/s, 100/s under the 500/s target");
  expect(missed(r({ servedPerSecond: 476 }))).toBeUndefined();
  expect(missed(r({ refused: 12, served: 9_988 }))).toBe("12 refused of 10000");
  expect(missed(r({ p95: 131.4 }))).toBe("p95 131.4 ms, 31.4 ms over the 100 ms target");
});
test("production is refused by name, by any name under it and by its address; a copy is not", async () => {
  let asked = 0;
  const resolve = async () => (asked++, ["203.0.113.7", "2001:db8::7"]);
  for (const url of [`https://${PRODUCTION_HOST}`, `https://FORGE.open-platform.sh./`, `http://${PRODUCTION_HOST}:8080/x`, `https://edge.${PRODUCTION_HOST}`, "https://203.0.113.7", "http://[2001:db8::7]:80"])
    expect(await refuseProduction(url, resolve)).toContain("production");
  expect(await refuseProduction("https://copy.example", resolve)).toBeUndefined();
  expect(await refuseProduction("https://copy--forge.open-platform.sh", resolve)).toBeUndefined();
  expect(await refuseProduction("http://127.0.0.1:9999", resolve)).toBeUndefined();
  expect(await refuseProduction("https://198.51.100.1", resolve)).toBeUndefined();
  // A name is decided without asking a resolver; only an address literal needs production's addresses.
  expect(asked).toBe(3);
  expect(await refuseProduction("https://198.51.100.1", async () => Promise.reject(new Error("no dns")))).toContain("could not be resolved");
  expect(await refuseProduction("ftp://copy.example", resolve)).toContain("http or https");
  expect(await refuseProduction("not a url", resolve)).toContain("not a URL");
});
test("release-load and bench/load.ts refuse a production --url before any request, writing nothing", async () => {
  const dir = await mkdtemp(join(tmpdir(), "forge-release-load-"));
  try {
    for (const cmd of [
      ["bun", "scripts/release-load.ts", "9.9.9", "--url", `https://${PRODUCTION_HOST}`, "--repo", "trevato/forge", "--results", dir],
      ["bun", "bench/load.ts", "--url", `https://${PRODUCTION_HOST}/`, "--repo", "trevato/forge", "--out", join(dir, "x.json")],
    ]) {
      // A proxy that is not there: were a request sent, it would fail as a connection error, not this refusal.
      const p = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe", env: { ...process.env, HTTPS_PROXY: "http://127.0.0.1:9", HTTP_PROXY: "http://127.0.0.1:9" } });
      const [code, err] = await Promise.all([p.exited, new Response(p.stderr).text()]);
      expect(code).toBe(2);
      expect(err).toContain(`refused: --url https://${PRODUCTION_HOST} is production`);
    }
    expect(existsSync(join(dir, "9.9.9.json")) || existsSync(join(dir, "9.9.9.bypass.json")) || existsSync(join(dir, "x.json"))).toBe(false);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});
		@@ -39,7 +39,8 @@
39	39	*/
40	40
41	41	import { readFileSync, writeFileSync } from "node:fs";
42		import { fell, missed, summarize, type LoadResult } from "../src/perf/load_verdict.ts";
	42	import { fell, missed, summarize, TARGET, type LoadResult } from "../src/perf/load_verdict.ts";
	43	import { refuseProduction, systemResolve } from "../src/perf/production_guard.ts";
43	44
44	45	const arg = (name: string, fallback?: string): string => {
45	46	const at = process.argv.indexOf(`--${name}`);
		@@ -49,6 +50,12 @@ const arg = (name: string, fallback?: string): string => {
49	50	};
50	51
51	52	const base = arg("url").replace(/\/$/, "");
	53	// Never at production (#9596): decided from the URL, before the first request.
	54	const notHere = await refuseProduction(base, systemResolve);
	55	if (notHere !== undefined) {
	56	console.error(`load: refused: ${notHere}`);
	57	process.exit(2);
	58	}
52	59	const repo = arg("repo");
53	60	const rate = Number(arg("rate", "100"));
54	61	const seconds = Number(arg("seconds", "10"));
		@@ -180,11 +187,17 @@ counting = true;
180	187	await flood(seconds);
181	188	await Promise.all(inflight);
182	189
183		const result: LoadResult = { ...summarize(version, rate, seconds, latencies, refused), cache: bypass ? "bypass" : "edge", refusedBy };
	190	const warmTotals = warmPerSecond.reduce(([s, r], [a, b]) => [s + a, r + b], [0, 0]);
	191	const result: LoadResult = {
	192	...summarize(version, rate, seconds, latencies, refused),
	193	cache: bypass ? "bypass" : "edge",
	194	refusedBy,
	195	warm: { seconds: warm, served: warmTotals[0]!, refused: warmTotals[1]! },
	196	};
184	197	console.log(JSON.stringify(result, null, 2));
185	198	console.log(`per second, served/refused: ${perSecond.map(([a, b]) => `${a}/${b}`).join(" ")}`);
186	199	const short = missed(result);
187		console.log(short === undefined ? "target met: none refused, p95 under 100 ms" : `target missed: ${short}`);
	200	console.log(short === undefined ? `target met: ${TARGET.rate}/s, none refused, p95 under ${TARGET.p95UnderMs} ms` : `target missed: ${short}`);
188	201	if (cookie !== undefined) {
189	202	const s = [...signedIn].sort((a, b) => a - b);
190	203	const at = (p: number) => (s.length === 0 ? null : Math.round(s[Math.min(s.length - 1, Math.floor(p * s.length))]! * 10) / 10);
		@@ -192,7 +205,7 @@ if (cookie !== undefined) {
192	205	}
193	206	const out = process.argv.includes("--out") ? arg("out") : undefined;
194	207	// The file says itself whether the run met the goal, so a kept result can be read without rerunning it.
195		if (out !== undefined) writeFileSync(out, JSON.stringify({ ...result, verdict: short ?? "target met" }, null, 2) + "\n");
	208	if (out !== undefined) writeFileSync(out, JSON.stringify({ ...result, target: TARGET, verdict: short ?? "target met" }, null, 2) + "\n");
196	209	const last = process.argv.includes("--compare") ? (JSON.parse(readFileSync(arg("compare"), "utf8")) as LoadResult) : undefined;
197	210	const why = last === undefined ? undefined : fell(last, result);
198	211	if (why !== undefined) {
		@@ -15,9 +15,9 @@ Media (`/media/…`, the route a PDF or image is drawn from) is not covered:
15	15
16	16	The bypasses are tested (`test/anonymous-deep.test.ts`): a signed-in reader, a repository made private, `?fragment=1` and a branch-name ref never get a page kept for a stranger. `--compare` exiting 1 is tested (`test/load-bench.test.ts`).
17	17
18		Release step: `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name>` runs the load twice, through the edge and with it bypassed, at 500/s for 20 s after a 20 s warm-up, writes `bench/results/<version>.json` and `<version>.bypass.json`, and compares each with the newest older kept file of its kind (`src/perf/baseline.ts`); exit 1 from either is the gate. With no older file it only writes the first; commit it. `scripts/cut-release.ts` prints this command as its last step; it does not run it (it needs a copy with data), and no baseline is kept yet — the first release run writes it.
	18	Release step: `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name>` runs the load twice, through the edge and with it bypassed, at 500/s for 20 s after a 20 s warm-up, writes `bench/results/<version>.json` and `<version>.bypass.json` (`--results DIR` writes them elsewhere), and compares each with the newest older kept file of its kind (`src/perf/baseline.ts`); exit 1 from either is the gate. With no older file it only writes the first; commit it. `scripts/cut-release.ts` prints this command as its last step; it does not run it (it needs a copy with data). See [the person's run](#the-persons-release-run-9596) below for the steps.
19	19
20		No release baseline (`<version>.json`) is kept in `bench/results/` yet: the pull's preview holds no repository to read, and the load must not be aimed at production. The first files are written by the first run against a copy with data. The two `9514-*.json` files there are the in-process measurements below; `lastBaseline` ignores them.
	20	No release baseline (`<version>.json`) is kept in `bench/results/` yet: the pull's preview holds no repository to read, and the load must not be aimed at production. The first files are written by the first run against a copy with data. The two `9514-*.json` files there are the in-process measurements below; `lastBaseline` ignores them. The crew writes no result file it did not measure.
21	21
22	22	Follow-ups: profile #9503, baseline #9504 (the file itself, still to be written from a copy with data), media #9505.
23	23
		@@ -96,12 +96,28 @@ What the caches hold (entries, bytes beside the cap, hits and misses) is on `/sy
96	96
97	97	A bigger git pool (`FORGE_GIT_CONCURRENCY=32`) changed nothing before item 5: the main thread was the limit, not git.
98	98
99		### Not done: the run through the edge, on a copy with production's data (#9565)
	99	### The run through the edge, on a copy with production's data (#9565, #9596)
100	100
101		This sandbox reaches no copy holding production's repositories and no edge in front of one. The load must not be aimed at production, and a pull's preview holds no repository. The numbers above are the best case on a copy of one repository: no edge, the client on the same machine, and a warm start. They are not the `Done` of #9514. To take them, run from a machine that is not the server:
	101	A crew sandbox reaches no copy holding production's repositories and no edge in front of one, so the numbers above are the best case: one repository, no edge, the client on the same machine, a warm start. They are not the `Done` of #9514. The run below is a person's, from a machine that is not the server.
102	102
103		```sh
104		bun scripts/release-load.ts <version> --url <copy behind the edge> --repo trevato/forge
105		```
	103	## The person's release run (#9596)
	104
	105	Never at production. `scripts/release-load.ts` and `bench/load.ts` both refuse, with exit 2 and before any request goes out, a `--url` whose host is production's (`forge.open-platform.sh`), any name under it, or an address production's name resolves to (`src/perf/production_guard.ts`). A different name on the same edge (a hosted child, `<name>--forge.open-platform.sh`) is a different forge and is not refused. Load it only if it is yours.
	106
	107	1. Restore a copy of production's data on a box that is not production: `bun scripts/restore-drill.ts /srv/forge-copy` restores the newest snapshot into that empty directory (the `S3_*` variables, or `FORGE_SNAPSHOT_DIR`, and `FORGE_SNAPSHOT_PASSPHRASE`, as on the restore card), boots it once on a side port and checks it against production's public counts. The directory stays.
	108	2. Boot the copy as a forge: `FORGE_DATA=/srv/forge-copy PORT=3100 bun run src/main.ts` from a clone at the release's tag. It runs the code you are releasing on production's repositories.
	109	3. Put the edge in front of it, the same way production has it: a Cloudflare tunnel (or the cluster ingress) to `localhost:3100` under a name of your own, with the cache rule production has: cache HTML that is `public` and `s-maxage`, bypass any request with a Cookie or Authorization header (`src/http/addressed.ts`). Without that rule, the bypass pass reads cached pages and the two files show the same thing. Behind the tunnel, set `FORGE_TRUSTED_PROXIES` and `FORGE_CLIENT_IP_HEADER=cf-connecting-ip` ([OPERATING](../OPERATING.md)) so strangers are counted per address, not as one.
	110	4. Check the copy answers at its name: `curl -sI https://<copy>/trevato/forge` is 200.
	111	5. Run it from a third machine (not the copy's box, not production):
	112
	113	```sh
	114	bun install
	115	bun scripts/release-load.ts <version> --url https://<copy> --repo trevato/forge
	116	```
	117
	118	It takes about 80 s: per pass, a 20 s warm-up that is not counted (its per-second `served/refused` line is printed), then 20 s at 500/s. Add `--crawl 3000` to spread the load over the repository rather than its top folder.
	119	6. Read the verdicts. It ends with one line per file. Each of `bench/results/<version>.json` (through the edge) and `bench/results/<version>.bypass.json` (past it) holds `asked` (the rate), `servedPerSecond`, `refused` and `refusedBy`, `p50` and `p95` in ms, `warm` (`seconds`, `served`, `refused`), `target` (`{"rate": 500, "refused": 0, "p95UnderMs": 100}`) and `verdict`: `"target met"`, or how far it missed, e.g. `"12 refused of 10000; p95 131.4 ms, 31.4 ms over the 100 ms target"` or `"served 400/s, 100/s under the 500/s target"`. A missed target is written, not an error; exit 1 means the served rate fell more than 20% from the last kept file of the same kind.
	120	7. Post both files as a comment on #9514, with the version, the machine the load ran from and the copy's box, and commit them in a pull to `bench/results/`. They become the first baseline the next release compares with.
	121
	122	Tests: `test/release-load-run.test.ts` runs the whole command against a local fake forge, one that keeps up (both files `"target met"`) and one that answers in 150 ms (both files say how far p95 missed), and checks that production is refused by name and by address before any request.
106	123
107		This writes both kinds and compares each with its last kept result.
		@@ -1,22 +1,41 @@
1	1	/**
2		* `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name> [--rate N] [--seconds N]`:
3		* the release step's load check (#9504, #9514). Runs bench/load.ts against a
4		* copy with data twice, through the edge cache and with it bypassed, writes
5		* bench/results/<version>.json and <version>.bypass.json, and compares each
6		* with the newest kept older result of its kind; exit 1 is a fall of more
7		* than 20% in either. With no older result of a kind it only writes the
8		* first. Commit the files it writes.
	2	* `bun scripts/release-load.ts <version> --url <copy> --repo <owner/name> [--rate N] [--seconds N] [--warm N] [--results DIR]`:
	3	* the release step's load check (#9504, #9514, #9596). Runs bench/load.ts
	4	* against a copy with data twice, through the edge cache and with it
	5	* bypassed, writes `<version>.json` and `<version>.bypass.json` into
	6	* bench/results (or `--results DIR`), each with its warm-up, the target and
	7	* `"verdict"` ("target met" or how far it missed), and compares each with
	8	* the newest kept older result of its kind; exit 1 is a fall of more than
	9	* 20% in either. With no older result of a kind it only writes the first.
	10	* A `--url` that is production is refused (exit 2) before any request goes
	11	* out (`src/perf/production_guard.ts`). Commit the files it writes.
9	12	*/
10	13
11		import { existsSync, mkdirSync, readdirSync } from "node:fs";
12		import { CACHE_MODES, lastBaseline, loadArgs } from "../src/perf/baseline.ts";
	14	import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
	15	import { resolve } from "node:path";
	16	import { CACHE_MODES, lastBaseline, loadArgs, resultFile } from "../src/perf/baseline.ts";
	17	import { refuseProduction, systemResolve } from "../src/perf/production_guard.ts";
13	18
14		const [version, ...rest] = process.argv.slice(2);
15		if (version === undefined \|\| !/^\d+\.\d+\.\d+$/.test(version) \|\| !rest.includes("--url") \|\| !rest.includes("--repo")) {
16		console.error("usage: bun scripts/release-load.ts <major.minor.patch> --url <copy> --repo <owner/name> [--rate N] [--seconds N]");
	19	const USAGE = "usage: bun scripts/release-load.ts <major.minor.patch> --url <copy> --repo <owner/name> [--rate N] [--seconds N] [--warm N] [--results DIR]";
	20	const [version, ...args] = process.argv.slice(2);
	21	const valueOf = (name: string): string \| undefined => {
	22	const at = args.indexOf(`--${name}`);
	23	return at >= 0 ? args[at + 1] : undefined;
	24	};
	25	const url = valueOf("url");
	26	if (version === undefined \|\| !/^\d+\.\d+\.\d+$/.test(version) \|\| url === undefined \|\| valueOf("repo") === undefined) {
	27	console.error(USAGE);
17	28	process.exit(2);
18	29	}
19		const dir = `${import.meta.dir}/../bench/results`;
	30	const notHere = await refuseProduction(url, systemResolve);
	31	if (notHere !== undefined) {
	32	console.error(`release-load: refused: ${notHere}`);
	33	process.exit(2);
	34	}
	35	// `--results` is this script's own; bench/load.ts is handed the rest.
	36	const resultsAt = args.indexOf("--results");
	37	const rest = resultsAt >= 0 ? [...args.slice(0, resultsAt), ...args.slice(resultsAt + 2)] : args;
	38	const dir = resolve(valueOf("results") ?? `${import.meta.dir}/../bench/results`);
20	39	mkdirSync(dir, { recursive: true });
21	40	// Through the edge, then with it bypassed (#9514): both run, and either falling is the gate.
22	41	let failed = 0;
		@@ -27,4 +46,10 @@ for (const cache of CACHE_MODES) {
27	46	const code = await Bun.spawn(cmd, { stdout: "inherit", stderr: "inherit", env: { BUN_CONFIG_MAX_HTTP_REQUESTS: "65000", ...process.env } }).exited;
28	47	if (code !== 0) failed = code;
29	48	}
	49	// Both verdicts together at the end, read back from the files that will be committed.
	50	for (const cache of CACHE_MODES) {
	51	const file = `${dir}/${resultFile(version, cache)}`;
	52	const verdict = existsSync(file) ? (JSON.parse(readFileSync(file, "utf8")) as { verdict?: string }).verdict : "not written";
	53	console.error(`release-load (${cache}): ${file}: ${verdict}`);
	54	}
30	55	process.exit(failed);
		@@ -227,8 +227,7 @@ export function parseSentence(text: string, now: number): Record<string, string>
227	227	else if (/\byesterday\b/.test(t)) {
228	228	out.from = since(Date.parse(startOfToday(now)), 1);
229	229	out.to = startOfToday(now);
230		}
231		else if (/\b(today\|so far today)\b/.test(t)) out.from = startOfToday(now);
	230	} else if (/\b(today\|so far today)\b/.test(t)) out.from = startOfToday(now);
232	231	else if (/\b(a week\|weekly)\b/.test(t)) days = 7;
233	232	}
234	233	if (days !== undefined) out.from = since(now, days);
		@@ -26,17 +26,38 @@ export interface LoadResult {
26	26	readonly refusedBy?: Readonly<Record<string, number>>;
27	27	/** `"target met"`, or `missed()`'s words, as the run that wrote it judged (#9548). Absent in results kept before. */
28	28	readonly verdict?: string;
	29	/** The uncounted warm-up before the measured window (#9596): its seconds and what it got. Absent in results kept before. */
	30	readonly warm?: { readonly seconds: number; readonly served: number; readonly refused: number };
29	31	}
30	32
31	33	/** The goal a release is held to (#9514): at the asked rate, no refusal and p95 under this, ms. */
32	34	export const TARGET_P95_MS = 100;
	35	/** The rate the goal is held at, requests per second (#9514). */
	36	export const TARGET_RATE = 500;
	37	/**
	38	* The share of the asked rate a run must serve. The open loop sends
	39	* floor(elapsed × rate), so a full run lands a request or two under; 95% is
	40	* the slack for that, not for a forge that cannot keep up (refusals count apart).
	41	*/
	42	export const SERVED_SHARE = 0.95;
33	43
34		/** Words for how a run fell short of the goal, or undefined when it refused none and its p95 was under `TARGET_P95_MS`. */
	44	/** What the goal is, written into each result file so it reads on its own (#9596). */
	45	export const TARGET = { rate: TARGET_RATE, refused: 0, p95UnderMs: TARGET_P95_MS } as const;
	46
	47	const tenth = (n: number) => Math.round(n * 10) / 10;
	48
	49	/**
	50	* Words for how far a run fell short of the goal (500/s asked and served, none
	51	* refused, p95 under `TARGET_P95_MS`), or undefined when it met it.
	52	*/
35	53	export function missed(r: LoadResult): string \| undefined {
36	54	const why: string[] = [];
37		if (r.refused > 0) why.push(`${r.refused} refused`);
	55	if (r.asked < TARGET_RATE) why.push(`asked ${r.asked}/s, ${tenth(TARGET_RATE - r.asked)}/s under the ${TARGET_RATE}/s target`);
	56	else if (r.served > 0 && r.servedPerSecond < TARGET_RATE * SERVED_SHARE)
	57	why.push(`served ${r.servedPerSecond}/s, ${tenth(TARGET_RATE - r.servedPerSecond)}/s under the ${TARGET_RATE}/s target`);
	58	if (r.refused > 0) why.push(`${r.refused} refused of ${r.served + r.refused}`);
38	59	if (r.served === 0) why.push("none served");
39		else if (r.p95 >= TARGET_P95_MS) why.push(`p95 ${r.p95} ms, not under ${TARGET_P95_MS} ms`);
	60	else if (r.p95 >= TARGET_P95_MS) why.push(`p95 ${r.p95} ms, ${tenth(r.p95 - TARGET_P95_MS)} ms over the ${TARGET_P95_MS} ms target`);
40	61	return why.length === 0 ? undefined : why.join("; ");
41	62	}
42	63
		@@ -0,0 +1,49 @@
	1	/**
	2	* The load is never aimed at production (#9596). Decides from the URL alone,
	3	* before any request goes out, whether a `--url` would land on the forge's
	4	* own public host: its name, any name under it, or an address literal that
	5	* production's name resolves to. A different name on the same edge is a
	6	* different forge (the edge routes by Host), so it is not refused here.
	7	*
	8	* Plain data in, words out: undefined when the URL may be loaded, else why not.
	9	*/
	10
	11	import { isIP } from "node:net";
	12
	13	/** The forge's own public host: production (as in `scripts/restore-drill.ts`). */
	14	export const PRODUCTION_HOST = "forge.open-platform.sh";
	15
	16	/** Addresses a name resolves to; injected so tests ask no resolver. */
	17	export type Resolve = (host: string) => Promise<readonly string[]>;
	18
	19	const bare = (host: string): string => host.toLowerCase().replace(/^\[\|\]$/g, "").replace(/\.$/, "");
	20
	21	/** A loopback or unspecified address: production is never one. */
	22	const local = (ip: string): boolean => /^127\./.test(ip) \|\| ip === "::1" \|\| ip === "0.0.0.0" \|\| ip === "::";
	23
	24	export async function refuseProduction(url: string, resolve: Resolve, production: string = PRODUCTION_HOST): Promise<string \| undefined> {
	25	let parsed: URL;
	26	try {
	27	parsed = new URL(url);
	28	} catch {
	29	return `--url ${JSON.stringify(url)} is not a URL`;
	30	}
	31	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return `--url must be http or https, not ${parsed.protocol}`;
	32	const host = bare(parsed.hostname);
	33	const prod = bare(production);
	34	if (host === prod \|\| host.endsWith(`.${prod}`)) return `--url ${parsed.origin} is production (${prod}); run the load only at a copy you stand up`;
	35	if (isIP(host) === 0 \|\| local(host)) return undefined;
	36	let addresses: readonly string[];
	37	try {
	38	addresses = (await resolve(prod)).map(bare);
	39	} catch {
	40	return `--url ${parsed.origin} is an address and ${prod} could not be resolved to rule out that it is production; use the copy's name`;
	41	}
	42	return addresses.includes(host) ? `--url ${parsed.origin} is an address of production (${prod}); run the load only at a copy you stand up` : undefined;
	43	}
	44
	45	/** The system resolver, every address of a name. */
	46	export const systemResolve: Resolve = async (host) => {
	47	const { lookup } = await import("node:dns/promises");
	48	return (await lookup(host, { all: true })).map((a) => a.address);
	49	};
		@@ -67,7 +67,8 @@ test("--bypass sends every request with a cookie the edge must pass, --warm is n
67	67	expect(withCookie).toBeGreaterThan(result.served);
68	68	expect(result.served).toBeLessThanOrEqual(41);
69	69	expect(out).toContain("warm-up, per second");
70		expect(out).toContain("target met");
	70	// 40/s is under the 500/s the goal is held at, so the verdict says by how much (#9596).
	71	expect(out).toContain("target missed: asked 40/s, 460/s under the 500/s target");
71	72	} finally {
72	73	server.stop(true);
73	74	}
		@@ -0,0 +1,118 @@
	1	import { expect, test } from "bun:test";
	2	import { existsSync } from "node:fs";
	3	import { mkdtemp, readFile, rm } from "node:fs/promises";
	4	import { tmpdir } from "node:os";
	5	import { join } from "node:path";
	6	import { missed } from "../src/perf/load_verdict.ts";
	7	import { PRODUCTION_HOST, refuseProduction } from "../src/perf/production_guard.ts";
	8
	9	// The person's release run (#9596): one command, both passes, both files, each with its verdict.
	10	const oid = "b".repeat(40);
	11	const page = `<a href="/o/r/blob/${oid}/f.txt">f</a><a href="/o/r/tree/${oid}/d">d</a>`;
	12
	13	/** Runs scripts/release-load.ts against a fake forge answering after `delayMs`; returns both files and what each pass's request carried. */
	14	const release = async (delayMs: number) => {
	15	const dir = await mkdtemp(join(tmpdir(), "forge-release-load-"));
	16	const cookies = { bypass: 0, none: 0 };
	17	const server = Bun.serve({
	18	port: 0,
	19	fetch: async (req) => {
	20	if (/\/(blob\|tree\|raw)\/[0-9a-f]{40}/.test(new URL(req.url).pathname)) {
	21	req.headers.get("cookie") === "forge_load=bypass" ? cookies.bypass++ : cookies.none++;
	22	if (delayMs > 0) await Bun.sleep(delayMs);
	23	}
	24	return new Response(page, { headers: { "content-type": "text/html" } });
	25	},
	26	});
	27	try {
	28	const p = Bun.spawn(
	29	["bun", "scripts/release-load.ts", "9.9.9", "--url", `http://127.0.0.1:${server.port}`, "--repo", "o/r", "--seconds", "1", "--warm", "1", "--results", dir],
	30	{ stdout: "pipe", stderr: "pipe" },
	31	);
	32	const [code, , err] = await Promise.all([p.exited, new Response(p.stdout).text(), new Response(p.stderr).text()]);
	33	const read = async (f: string) => JSON.parse(await readFile(join(dir, f), "utf8"));
	34	return { code, err, cookies, edge: await read("9.9.9.json"), bypass: await read("9.9.9.bypass.json") };
	35	} finally {
	36	server.stop(true);
	37	await rm(dir, { recursive: true, force: true });
	38	}
	39	};
	40
	41	test("a forge that keeps up gets both files at 500/s, each with its warm-up and \"target met\"", async () => {
	42	const r = await release(0);
	43	expect(r.code).toBe(0);
	44	for (const [file, cache] of [[r.edge, "edge"], [r.bypass, "bypass"]] as const) {
	45	expect(file.version).toBe("9.9.9");
	46	expect(file.cache).toBe(cache);
	47	expect(file.asked).toBe(500);
	48	expect(file.refused).toBe(0);
	49	expect(file.servedPerSecond).toBeGreaterThanOrEqual(475);
	50	expect(file.p95).toBeLessThan(100);
	51	expect(typeof file.p50).toBe("number");
	52	expect(file.warm.seconds).toBe(1);
	53	expect(file.warm.served).toBeGreaterThan(0);
	54	expect(file.target).toEqual({ rate: 500, refused: 0, p95UnderMs: 100 });
	55	expect(file.verdict).toBe("target met");
	56	}
	57	// The edge pass carries no cookie; every request of the bypass pass carries the one the edge must pass on.
	58	expect(r.cookies.none).toBeGreaterThan(0);
	59	expect(r.cookies.bypass).toBeGreaterThan(0);
	60	expect(r.err).toContain("9.9.9.json: target met");
	61	expect(r.err).toContain("9.9.9.bypass.json: target met");
	62	}, 30_000);
	63
	64	test("a forge answering in 150 ms gets both files saying how far p95 missed", async () => {
	65	const r = await release(150);
	66	// Missing the target is written down, not a failed run: only a fall from a kept result exits 1.
	67	expect(r.code).toBe(0);
	68	for (const file of [r.edge, r.bypass]) {
	69	expect(file.p95).toBeGreaterThanOrEqual(150);
	70	expect(file.verdict).toBe(`p95 ${file.p95} ms, ${Math.round((file.p95 - 100) * 10) / 10} ms over the 100 ms target`);
	71	}
	72	expect(r.err).toContain("9.9.9.bypass.json: p95 ");
	73	}, 30_000);
	74
	75	test("the verdict says how far each part of the goal was missed", () => {
	76	const r = (o: object) => ({ version: "1", asked: 500, seconds: 20, served: 10_000, refused: 0, servedPerSecond: 500, p50: 3, p95: 40, ...o });
	77	expect(missed(r({}))).toBeUndefined();
	78	expect(missed(r({ asked: 100 }))).toBe("asked 100/s, 400/s under the 500/s target");
	79	expect(missed(r({ servedPerSecond: 400 }))).toBe("served 400/s, 100/s under the 500/s target");
	80	expect(missed(r({ servedPerSecond: 476 }))).toBeUndefined();
	81	expect(missed(r({ refused: 12, served: 9_988 }))).toBe("12 refused of 10000");
	82	expect(missed(r({ p95: 131.4 }))).toBe("p95 131.4 ms, 31.4 ms over the 100 ms target");
	83	});
	84
	85	test("production is refused by name, by any name under it and by its address; a copy is not", async () => {
	86	let asked = 0;
	87	const resolve = async () => (asked++, ["203.0.113.7", "2001:db8::7"]);
	88	for (const url of [`https://${PRODUCTION_HOST}`, `https://FORGE.open-platform.sh./`, `http://${PRODUCTION_HOST}:8080/x`, `https://edge.${PRODUCTION_HOST}`, "https://203.0.113.7", "http://[2001:db8::7]:80"])
	89	expect(await refuseProduction(url, resolve)).toContain("production");
	90	expect(await refuseProduction("https://copy.example", resolve)).toBeUndefined();
	91	expect(await refuseProduction("https://copy--forge.open-platform.sh", resolve)).toBeUndefined();
	92	expect(await refuseProduction("http://127.0.0.1:9999", resolve)).toBeUndefined();
	93	expect(await refuseProduction("https://198.51.100.1", resolve)).toBeUndefined();
	94	// A name is decided without asking a resolver; only an address literal needs production's addresses.
	95	expect(asked).toBe(3);
	96	expect(await refuseProduction("https://198.51.100.1", async () => Promise.reject(new Error("no dns")))).toContain("could not be resolved");
	97	expect(await refuseProduction("ftp://copy.example", resolve)).toContain("http or https");
	98	expect(await refuseProduction("not a url", resolve)).toContain("not a URL");
	99	});
	100
	101	test("release-load and bench/load.ts refuse a production --url before any request, writing nothing", async () => {
	102	const dir = await mkdtemp(join(tmpdir(), "forge-release-load-"));
	103	try {
	104	for (const cmd of [
	105	["bun", "scripts/release-load.ts", "9.9.9", "--url", `https://${PRODUCTION_HOST}`, "--repo", "trevato/forge", "--results", dir],
	106	["bun", "bench/load.ts", "--url", `https://${PRODUCTION_HOST}/`, "--repo", "trevato/forge", "--out", join(dir, "x.json")],
	107	]) {
	108	// A proxy that is not there: were a request sent, it would fail as a connection error, not this refusal.
	109	const p = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe", env: { ...process.env, HTTPS_PROXY: "http://127.0.0.1:9", HTTP_PROXY: "http://127.0.0.1:9" } });
	110	const [code, err] = await Promise.all([p.exited, new Response(p.stderr).text()]);
	111	expect(code).toBe(2);
	112	expect(err).toContain(`refused: --url https://${PRODUCTION_HOST} is production`);
	113	}
	114	expect(existsSync(join(dir, "9.9.9.json")) \|\| existsSync(join(dir, "9.9.9.bypass.json")) \|\| existsSync(join(dir, "x.json"))).toBe(false);
	115	} finally {
	116	await rm(dir, { recursive: true, force: true });
	117	}
	118	});