8e9453c · b93ae66 · eb3858c · 72db91f · 4aec7ed · 5b060d8
@@ -77,6 +77,7 @@
export const GATE: readonly string[] = [
  "test/fleet-ops.test.ts",
  "test/meter.test.ts",
  "test/told-by.test.ts",
  "test/highlight.test.ts",
  "test/check-list.test.ts",
];
@@ -145,11 +146,6 @@
export const EXCLUDED: readonly Exclusion[] = [
  { file: "test/git-ssh.test.ts", reason: "not yet admitted", issue: 261 },
  { file: "test/git.test.ts", reason: "not yet admitted", issue: 261 },
  { file: "test/hardening.test.ts", reason: "not yet admitted", issue: 261 },
  {
    file: "test/highlight.test.ts",
    reason: "red: a 10ms highlighter bound fails on a slower machine",
    issue: 294,
  },
  { file: "test/image-gif.test.ts", reason: "not yet admitted", issue: 261 },
  {
    file: "test/image-graphics.test.ts",
@@ -14,8 +14,9 @@
 *   step, so no input can wedge it.
 *
 *   **Linear.** No backtracking regex ever touches a whole line; the hot loops
 *   compare character codes. A 10,000-character line scans in under a
 *   millisecond, which is what makes the editor's per-keystroke re-scan free.
 *   compare character codes. A line sixteen times longer costs about sixteen
 *   times as much, never two hundred and fifty-six, which is what keeps the
 *   editor's per-keystroke re-scan cheap however long the line.
 *
 * ## Two levels of token
 *
@@ -7,8 +7,9 @@
 *
 *   **lossless** — the token texts of a line concatenate back to the line, so
 *   nothing a file contains can be dropped or duplicated on the way to a page;
 *   **linear** — a 10,000-character line scans in well under a frame, which is
 *   what lets the editor re-highlight on every keystroke;
 *   **linear** — scanning a line eight times longer takes about eight times
 *   as long, never sixty-four, which is what lets the editor re-highlight a
 *   long line on every keystroke;
 *   **total** — random bytes in any language never throw.
 *
 * On top of that, one fixture per language, snapshotted, so a change to a rule
@@ -31,6 +32,46 @@
import { renderTokens, tokenClasses } from "../src/web/highlight-html.ts";
import { THEMES, THEME_NAMES } from "../src/design/themes.ts";
import { css } from "../src/web/style.ts";
/**
 * How `work`'s cost grows with its input, as the exponent k in time ∝ size^k:
 * timed at `small` and at `GROWTH` times that, in this run on this machine,
 * small and large interleaved so load lands on both, best of three each.
 *
 * Linearity is asserted as this exponent, never as milliseconds. A wall-clock
 * bound is a claim about the machine (10ms passed on one box and took 25ms on
 * a crew machine, #294); a ratio cancels the machine out. A linear scanner
 * measures k ≈ 1 (a little above under GC and allocation as the token array
 * grows), a quadratic one k ≈ 2 — the regression these tests exist for — so
 * `LINEAR_BOUND` sits halfway. Growth is 16× rather than 2× so the two are
 * 64× against 256× in time, far apart enough for a loaded machine's noise
 * not to bridge them. A spike can still land on one measurement, so an
 * exponent over the bound is measured again, up to three times, and the best
 * is returned: noise does not repeat, quadratic does.
 */
const GROWTH = 16;
const LINEAR_BOUND = 1.5;
function growth(work: (size: number) => void, small: number): number {
  const time = (size: number) => {
    const t0 = process.cpuUsage();
    work(size);
    const { user, system } = process.cpuUsage(t0);
    return user + system;
  };
  work(small); // warm the JIT so no sample pays for compiling
  let k = Infinity;
  for (let attempt = 0; attempt < 3 && k >= LINEAR_BOUND; attempt++) {
    let lo = Infinity;
    let hi = Infinity;
    for (let i = 0; i < 3; i++) {
      lo = Math.min(lo, time(small));
      hi = Math.min(hi, time(small * GROWTH));
    }
    k = Math.min(k, Math.log(Math.max(hi, 1) / Math.max(lo, 1)) / Math.log(GROWTH));
  }
  return k;
}
function join(code: string, lang: Language): string {
  return highlightLines(code, lang)
    .map((line) => line.map((t) => t.text).join(""))
@@ -467,34 +508,50 @@
describe("every language snapshots", () => {
});
describe("resilience", () => {
  test("a 10,000-character line scans in under 10ms in every language", () => {
  test("the growth measure tells linear from quadratic", () => {
    // The yardstick, checked against itself: a scanner that re-reads the
    // line for every character must land above the bound, or the two tests
    // below could never fail.
    const quadratic = (size: number) => {
      const line = "x".repeat(size);
      let sum = 0;
      for (let i = 0; i < size; i++)
        for (let j = i; j < size; j++) sum += line.charCodeAt(j);
      return sum;
    };
    expect(growth(quadratic, 250)).toBeGreaterThan(LINEAR_BOUND);
  }, 30_000);
  test("a long line scans in linear time in every language", () => {
    const unit =
      "const x = { a: 1, b: `t${y}`, c: /re/g, d: 'str', e: [1,2,3] }; // n ";
    const line = unit.repeat(200).slice(0, 10_000);
    const line = (size: number) => unit.repeat(Math.ceil(size / unit.length)).slice(0, size);
    for (const lang of LANGUAGES) {
      const t0 = Bun.nanoseconds();
      const rows = highlightLines(line, lang);
      const ms = (Bun.nanoseconds() - t0) / 1e6;
      expect(rows.length).toBe(1);
      expect(ms).toBeLessThan(10);
      expect(highlightLines(line(10_000), lang).length).toBe(1);
      const k = growth((size) => highlightLines(line(size), lang), 500);
      expect({ lang, linear: k < LINEAR_BOUND }).toEqual({ lang, linear: true });
    }
  });
  }, 30_000);
  test("pathological delimiter runs stay linear", () => {
    for (const junk of [
      "`".repeat(4000),
      "/".repeat(4000),
      "${".repeat(2000),
      "<".repeat(4000),
      '"'.repeat(4000),
      "#[".repeat(2000),
      "[[".repeat(2000),
    ]) {
      const t0 = Bun.nanoseconds();
      for (const lang of LANGUAGES) expect(join(junk, lang)).toBe(junk);
      expect((Bun.nanoseconds() - t0) / 1e6).toBeLessThan(120);
    for (const [unit, label] of [
      ["`", "backticks"],
      ["/", "slashes"],
      ["${", "interpolations"],
      ["<", "angles"],
      ['"', "quotes"],
      ["#[", "attributes"],
      ["[[", "long brackets"],
    ] as const) {
      const junk = (size: number) => unit.repeat(size / unit.length);
      for (const lang of LANGUAGES) expect(join(junk(4_000), lang)).toBe(junk(4_000));
      const k = growth((size) => {
        const text = junk(size);
        for (const lang of LANGUAGES) highlightLines(text, lang);
      }, 250);
      expect({ label, linear: k < LINEAR_BOUND }).toEqual({ label, linear: true });
    }
  });
  }, 30_000);
  test("random bytes never throw, in any language", () => {
    // Deterministic: a seeded generator, so a failure is reproducible.
		@@ -77,6 +77,7 @@ export const GATE: readonly string[] = [
77	77	"test/fleet-ops.test.ts",
78	78	"test/meter.test.ts",
79	79	"test/told-by.test.ts",
	80	"test/highlight.test.ts",
80	81	"test/check-list.test.ts",
81	82	];
82	83
		@@ -145,11 +146,6 @@ export const EXCLUDED: readonly Exclusion[] = [
145	146	{ file: "test/git-ssh.test.ts", reason: "not yet admitted", issue: 261 },
146	147	{ file: "test/git.test.ts", reason: "not yet admitted", issue: 261 },
147	148	{ file: "test/hardening.test.ts", reason: "not yet admitted", issue: 261 },
148		{
149		file: "test/highlight.test.ts",
150		reason: "red: a 10ms highlighter bound fails on a slower machine",
151		issue: 294,
152		},
153	149	{ file: "test/image-gif.test.ts", reason: "not yet admitted", issue: 261 },
154	150	{
155	151	file: "test/image-graphics.test.ts",
		@@ -14,8 +14,9 @@
14	14	* step, so no input can wedge it.
15	15	*
16	16	* Linear. No backtracking regex ever touches a whole line; the hot loops
17		* compare character codes. A 10,000-character line scans in under a
18		* millisecond, which is what makes the editor's per-keystroke re-scan free.
	17	* compare character codes. A line sixteen times longer costs about sixteen
	18	* times as much, never two hundred and fifty-six, which is what keeps the
	19	* editor's per-keystroke re-scan cheap however long the line.
19	20	*
20	21	* ## Two levels of token
21	22	*
		@@ -7,8 +7,9 @@
7	7	*
8	8	* lossless — the token texts of a line concatenate back to the line, so
9	9	* nothing a file contains can be dropped or duplicated on the way to a page;
10		* linear — a 10,000-character line scans in well under a frame, which is
11		* what lets the editor re-highlight on every keystroke;
	10	* linear — scanning a line eight times longer takes about eight times
	11	* as long, never sixty-four, which is what lets the editor re-highlight a
	12	* long line on every keystroke;
12	13	* total — random bytes in any language never throw.
13	14	*
14	15	* On top of that, one fixture per language, snapshotted, so a change to a rule
		@@ -31,6 +32,46 @@ import { renderTokens, tokenClasses } from "../src/web/highlight-html.ts";
31	32	import { THEMES, THEME_NAMES } from "../src/design/themes.ts";
32	33	import { css } from "../src/web/style.ts";
33	34
	35	/**
	36	* How `work`'s cost grows with its input, as the exponent k in time ∝ size^k:
	37	* timed at `small` and at `GROWTH` times that, in this run on this machine,
	38	* small and large interleaved so load lands on both, best of three each.
	39	*
	40	* Linearity is asserted as this exponent, never as milliseconds. A wall-clock
	41	* bound is a claim about the machine (10ms passed on one box and took 25ms on
	42	* a crew machine, #294); a ratio cancels the machine out. A linear scanner
	43	* measures k ≈ 1 (a little above under GC and allocation as the token array
	44	* grows), a quadratic one k ≈ 2 — the regression these tests exist for — so
	45	* `LINEAR_BOUND` sits halfway. Growth is 16× rather than 2× so the two are
	46	* 64× against 256× in time, far apart enough for a loaded machine's noise
	47	* not to bridge them. A spike can still land on one measurement, so an
	48	* exponent over the bound is measured again, up to three times, and the best
	49	* is returned: noise does not repeat, quadratic does.
	50	*/
	51	const GROWTH = 16;
	52	const LINEAR_BOUND = 1.5;
	53
	54	function growth(work: (size: number) => void, small: number): number {
	55	const time = (size: number) => {
	56	const t0 = process.cpuUsage();
	57	work(size);
	58	const { user, system } = process.cpuUsage(t0);
	59	return user + system;
	60	};
	61	work(small); // warm the JIT so no sample pays for compiling
	62	let k = Infinity;
	63	for (let attempt = 0; attempt < 3 && k >= LINEAR_BOUND; attempt++) {
	64	let lo = Infinity;
	65	let hi = Infinity;
	66	for (let i = 0; i < 3; i++) {
	67	lo = Math.min(lo, time(small));
	68	hi = Math.min(hi, time(small * GROWTH));
	69	}
	70	k = Math.min(k, Math.log(Math.max(hi, 1) / Math.max(lo, 1)) / Math.log(GROWTH));
	71	}
	72	return k;
	73	}
	74
34	75	function join(code: string, lang: Language): string {
35	76	return highlightLines(code, lang)
36	77	.map((line) => line.map((t) => t.text).join(""))
		@@ -467,34 +508,50 @@ describe("every language snapshots", () => {
467	508	});
468	509
469	510	describe("resilience", () => {
470		test("a 10,000-character line scans in under 10ms in every language", () => {
	511	test("the growth measure tells linear from quadratic", () => {
	512	// The yardstick, checked against itself: a scanner that re-reads the
	513	// line for every character must land above the bound, or the two tests
	514	// below could never fail.
	515	const quadratic = (size: number) => {
	516	const line = "x".repeat(size);
	517	let sum = 0;
	518	for (let i = 0; i < size; i++)
	519	for (let j = i; j < size; j++) sum += line.charCodeAt(j);
	520	return sum;
	521	};
	522	expect(growth(quadratic, 250)).toBeGreaterThan(LINEAR_BOUND);
	523	}, 30_000);
	524
	525	test("a long line scans in linear time in every language", () => {
471	526	const unit =
472	527	"const x = { a: 1, b: `t${y}`, c: /re/g, d: 'str', e: [1,2,3] }; // n ";
473		const line = unit.repeat(200).slice(0, 10_000);
	528	const line = (size: number) => unit.repeat(Math.ceil(size / unit.length)).slice(0, size);
474	529	for (const lang of LANGUAGES) {
475		const t0 = Bun.nanoseconds();
476		const rows = highlightLines(line, lang);
477		const ms = (Bun.nanoseconds() - t0) / 1e6;
478		expect(rows.length).toBe(1);
479		expect(ms).toBeLessThan(10);
	530	expect(highlightLines(line(10_000), lang).length).toBe(1);
	531	const k = growth((size) => highlightLines(line(size), lang), 500);
	532	expect({ lang, linear: k < LINEAR_BOUND }).toEqual({ lang, linear: true });
480	533	}
481		});
	534	}, 30_000);
482	535
483	536	test("pathological delimiter runs stay linear", () => {
484		for (const junk of [
485		"`".repeat(4000),
486		"/".repeat(4000),
487		"${".repeat(2000),
488		"<".repeat(4000),
489		'"'.repeat(4000),
490		"#[".repeat(2000),
491		"[[".repeat(2000),
492		]) {
493		const t0 = Bun.nanoseconds();
494		for (const lang of LANGUAGES) expect(join(junk, lang)).toBe(junk);
495		expect((Bun.nanoseconds() - t0) / 1e6).toBeLessThan(120);
	537	for (const [unit, label] of [
	538	["`", "backticks"],
	539	["/", "slashes"],
	540	["${", "interpolations"],
	541	["<", "angles"],
	542	['"', "quotes"],
	543	["#[", "attributes"],
	544	["[[", "long brackets"],
	545	] as const) {
	546	const junk = (size: number) => unit.repeat(size / unit.length);
	547	for (const lang of LANGUAGES) expect(join(junk(4_000), lang)).toBe(junk(4_000));
	548	const k = growth((size) => {
	549	const text = junk(size);
	550	for (const lang of LANGUAGES) highlightLines(text, lang);
	551	}, 250);
	552	expect({ label, linear: k < LINEAR_BOUND }).toEqual({ label, linear: true });
496	553	}
497		});
	554	}, 30_000);
498	555
499	556	test("random bytes never throw, in any language", () => {
500	557	// Deterministic: a seeded generator, so a failure is reproducible.