5dc9add · 5e28153 · 3bf0d17 · b29fc28
@@ -16,7 +16,7 @@
export interface WordDiff {
}
// Beyond this token count the O(n·m) LCS is not worth it; mark the whole line.
const MAX_TOKENS = 800;
export const MAX_TOKENS = 800;
function tokenize(line: string): string[] {
  return line.match(/\w+|\s+|[^\w\s]/g) ?? [];
@@ -0,0 +1,85 @@
/**
 * The intra-line word diff (src/web/worddiff.ts) draws the underlines in every
 * pull's diff: unchanged text stays unmarked, each side's segments rejoin to
 * its line, and a minified line degrades to one changed segment per side.
 */
import { describe, expect, test } from "bun:test";
import { MAX_TOKENS, wordDiff, type Segment } from "../src/web/worddiff";
const join = (s: Segment[]) => s.map((x) => x.text).join("");
const changedText = (s: Segment[]) =>
  s.filter((x) => x.changed).map((x) => x.text);
describe("wordDiff", () => {
  test("identical lines have no changed segment", () => {
    const d = wordDiff("const a = 1;", "const a = 1;");
    expect(d.old.some((s) => s.changed)).toBe(false);
    expect(d.new.some((s) => s.changed)).toBe(false);
  });
      
        junipercrew
        18d ago
        
      
      This case is resolved by the prefix/suffix trim alone. Add one where the changed word sits between repeated tokens, so the LCS table decides which tokens are marked.
    
  test("a one-word change marks only that word on each side", () => {
    const d = wordDiff("the quick brown fox", "the quick red fox");
    expect(changedText(d.old)).toEqual(["brown"]);
    expect(changedText(d.new)).toEqual(["red"]);
  });
  const cases: [string, string][] = [
    ["", ""],
    ["", "added line"],
    ["removed line", ""],
    ["a", "b"],
    ["foo(bar)", "foo(baz, qux)"],
    ["  indented", "    indented"],
    ["trailing ", "trailing"],
    ["héllo wörld", "héllo wörld!"],
    ["日本語 のテキスト", "日本語 の文章"],
    ["emoji 😀 here", "emoji 😎 here"],
    ["a b c d e", "e d c b a"],
    ["x\ty", "x y"],
  ];
  test("segments rejoin to each line over a dozen cases", () => {
    expect(cases.length).toBeGreaterThanOrEqual(12);
    for (const [o, n] of cases) {
      const d = wordDiff(o, n);
      expect(join(d.old)).toBe(o);
      expect(join(d.new)).toBe(n);
      expect(d.old.length).toBeGreaterThan(0);
      expect(d.new.length).toBeGreaterThan(0);
    }
  });
  // `w0 w1 …` tokenizes to one token per word plus one per space; a trailing
  // space adds one more, so any token count is reachable.
  const lineOfTokens = (count: number, tag: string) => {
      
        junipercrew
        18d ago
        outdated
      
      300 words gives about 599 tokens, so this hard-codes an assumption about MAX_TOKENS=800. Export the constant and build the line from it so the test fails if the limit moves.
    
    const words = Math.ceil(count / 2);
      
        junipercrew
        18d ago
        outdated
      
      Wall-clock bound can flake on a loaded gate machine. The shape assertions below already prove the degrade path; consider removing this.
    
    const text = Array.from({ length: words }, (_, i) => `${tag}${i}`).join(" ");
    return count % 2 === 0 ? text + " " : text;
  };
  test("a pair of exactly MAX_TOKENS tokens is still diffed word by word", () => {
    const o = lineOfTokens(MAX_TOKENS, "w");
    const n = o.replace("w0 ", "changed ");
    const d = wordDiff(o, n);
    expect(changedText(d.old)).toEqual(["w0"]);
    expect(changedText(d.new)).toEqual(["changed"]);
    expect(join(d.old)).toBe(o);
    expect(join(d.new)).toBe(n);
  });
  test("one token past MAX_TOKENS is one changed segment per side", () => {
    const o = lineOfTokens(MAX_TOKENS + 1, "w");
    const n = o.replace("w0 ", "changed ");
    const d = wordDiff(o, n);
    expect(d.old).toEqual([{ text: o, changed: true }]);
    expect(d.new).toEqual([{ text: n, changed: true }]);
  });
  test("a far longer pair is one changed segment per side", () => {
    const o = lineOfTokens(MAX_TOKENS * 3, "a");
    const n = lineOfTokens(MAX_TOKENS * 3, "b");
    const d = wordDiff(o, n);
    expect(d.old).toEqual([{ text: o, changed: true }]);
    expect(d.new).toEqual([{ text: n, changed: true }]);
  });
});
		@@ -16,7 +16,7 @@ export interface WordDiff {
16	16	}
17	17
18	18	// Beyond this token count the O(n·m) LCS is not worth it; mark the whole line.
19		const MAX_TOKENS = 800;
	19	export const MAX_TOKENS = 800;
20	20
21	21	function tokenize(line: string): string[] {
22	22	return line.match(/\w+\|\s+\|[^\w\s]/g) ?? [];
		@@ -0,0 +1,85 @@
	1	/**
	2	* The intra-line word diff (src/web/worddiff.ts) draws the underlines in every
	3	* pull's diff: unchanged text stays unmarked, each side's segments rejoin to
	4	* its line, and a minified line degrades to one changed segment per side.
	5	*/
	6	import { describe, expect, test } from "bun:test";
	7	import { MAX_TOKENS, wordDiff, type Segment } from "../src/web/worddiff";
	8
	9	const join = (s: Segment[]) => s.map((x) => x.text).join("");
	10	const changedText = (s: Segment[]) =>
	11	s.filter((x) => x.changed).map((x) => x.text);
	12
	13	describe("wordDiff", () => {
	14	test("identical lines have no changed segment", () => {
	15	const d = wordDiff("const a = 1;", "const a = 1;");
	16	expect(d.old.some((s) => s.changed)).toBe(false);
	17	expect(d.new.some((s) => s.changed)).toBe(false);
	18	});
		junipercrew 18d ago This case is resolved by the prefix/suffix trim alone. Add one where the changed word sits between repeated tokens, so the LCS table decides which tokens are marked.
	19
	20	test("a one-word change marks only that word on each side", () => {
	21	const d = wordDiff("the quick brown fox", "the quick red fox");
	22	expect(changedText(d.old)).toEqual(["brown"]);
	23	expect(changedText(d.new)).toEqual(["red"]);
	24	});
	25
	26	const cases: [string, string][] = [
	27	["", ""],
	28	["", "added line"],
	29	["removed line", ""],
	30	["a", "b"],
	31	["foo(bar)", "foo(baz, qux)"],
	32	[" indented", " indented"],
	33	["trailing ", "trailing"],
	34	["héllo wörld", "héllo wörld!"],
	35	["日本語のテキスト", "日本語の文章"],
	36	["emoji 😀 here", "emoji 😎 here"],
	37	["a b c d e", "e d c b a"],
	38	["x\ty", "x y"],
	39	];
	40
	41	test("segments rejoin to each line over a dozen cases", () => {
	42	expect(cases.length).toBeGreaterThanOrEqual(12);
	43	for (const [o, n] of cases) {
	44	const d = wordDiff(o, n);
	45	expect(join(d.old)).toBe(o);
	46	expect(join(d.new)).toBe(n);
	47	expect(d.old.length).toBeGreaterThan(0);
	48	expect(d.new.length).toBeGreaterThan(0);
	49	}
	50	});
	51
	52	// `w0 w1 …` tokenizes to one token per word plus one per space; a trailing
	53	// space adds one more, so any token count is reachable.
	54	const lineOfTokens = (count: number, tag: string) => {
		junipercrew 18d ago outdated 300 words gives about 599 tokens, so this hard-codes an assumption about MAX_TOKENS=800. Export the constant and build the line from it so the test fails if the limit moves.
	55	const words = Math.ceil(count / 2);
		junipercrew 18d ago outdated Wall-clock bound can flake on a loaded gate machine. The shape assertions below already prove the degrade path; consider removing this.
	56	const text = Array.from({ length: words }, (_, i) => `${tag}${i}`).join(" ");
	57	return count % 2 === 0 ? text + " " : text;
	58	};
	59
	60	test("a pair of exactly MAX_TOKENS tokens is still diffed word by word", () => {
	61	const o = lineOfTokens(MAX_TOKENS, "w");
	62	const n = o.replace("w0 ", "changed ");
	63	const d = wordDiff(o, n);
	64	expect(changedText(d.old)).toEqual(["w0"]);
	65	expect(changedText(d.new)).toEqual(["changed"]);
	66	expect(join(d.old)).toBe(o);
	67	expect(join(d.new)).toBe(n);
	68	});
	69
	70	test("one token past MAX_TOKENS is one changed segment per side", () => {
	71	const o = lineOfTokens(MAX_TOKENS + 1, "w");
	72	const n = o.replace("w0 ", "changed ");
	73	const d = wordDiff(o, n);
	74	expect(d.old).toEqual([{ text: o, changed: true }]);
	75	expect(d.new).toEqual([{ text: n, changed: true }]);
	76	});
	77
	78	test("a far longer pair is one changed segment per side", () => {
	79	const o = lineOfTokens(MAX_TOKENS * 3, "a");
	80	const n = lineOfTokens(MAX_TOKENS * 3, "b");
	81	const d = wordDiff(o, n);
	82	expect(d.old).toEqual([{ text: o, changed: true }]);
	83	expect(d.new).toEqual([{ text: n, changed: true }]);
	84	});
	85	});