001e505 · c16600a · 4439998
@@ -183,6 +183,10 @@
and rounds to 0.0):
  event on the repository (see "Gaps") and spend reported by another
  process.
### 2b. The API map: the finished answer, by the write clock (#3321)
`GET /api/v1/repos/:owner/:name/map` keeps its whole answer in `web/map_snapshot.ts`, keyed by repository, the viewer's sealed scope and the UTC minute (the week's merge window slides with time) and valid only under the forge's write clock, so a plan, or any write, drops it before the next read. A hit still names the five phases, flagged as hits. Test: `test/map-seed-timing.test.ts`.
## Home (`/`)
### 3. The feed's first page: per viewer
@@ -350,6 +350,7 @@
import { askToLiveFor } from "../web/ask_to_live.ts";
import { decisionStats, listDecisions } from "../core/decisions.ts";
import { forReader } from "../crew/handover.ts";
import { globalMapJson, mapJson, marksFor } from "../web/pages/map.ts";
import { cachedMap } from "../web/map_snapshot.ts";
import {
  milestoneData,
  milestonesJson,
@@ -3816,21 +3817,31 @@
export function apiRoutes(router: Router<Handler>): void {
    if (!repo) return notFound();
    const { request } = ctx;
    notePhase(request, "session", takeSessionTime(request));
    const graph = timePhase(request, "graph", () =>
      repoGraph(ctx.app.db, repo.id, {
        ...sealedScope(ctx.app.db, repo, ctx.auth.user),
      }),
    );
    const marks = timePhase(request, "marks", () =>
      marksFor(ctx.app.db, repo.id, graph, listActiveRuns(ctx.app.db, [repo.id])),
    );
    const pillars = timePhase(request, "pillars", () => pillarView(ctx.app.db, repo));
    const ask = timePhase(request, "asktolive", () =>
      askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
    );
    return timePhase(request, "render", () =>
      json(mapJson(repo, graph, marks, pillars, ask)),
    const scope = sealedScope(ctx.app.db, repo, ctx.auth.user);
    let built = false;
    const { value, hit } = cachedMap(
      ctx.app.db,
      { repoId: repo.id, scope, now: new Date() },
      () => {
        built = true;
        const graph = timePhase(request, "graph", () =>
          repoGraph(ctx.app.db, repo.id, { ...scope }),
        );
        const marks = timePhase(request, "marks", () =>
          marksFor(ctx.app.db, repo.id, graph, listActiveRuns(ctx.app.db, [repo.id])),
        );
        const pillars = timePhase(request, "pillars", () => pillarView(ctx.app.db, repo));
        const ask = timePhase(request, "asktolive", () =>
          askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
        );
        return timePhase(request, "render", () => mapJson(repo, graph, marks, pillars, ask));
      },
    );
    // A snapshot skips the phases; they are still named, as hits at no cost.
    if (!built)
      for (const name of ["graph", "marks", "pillars", "asktolive", "render"])
        notePhase(request, name, 0, hit);
    return json(value);
  });
  // What people asked of the crew in the repositories the caller can write —
  // by default what has nothing merged yet, oldest first — and what became of each (core/asked.ts, the /asked page).
@@ -0,0 +1,41 @@
/**
 * GET …/map as a snapshot (#3321): the finished answer, kept by the forge's
 * write clock (`core/write_clock.ts`), so any write, a plan included, drops
 * it before the next read. The key holds what the answer varies by besides
 * the data: the repository, the viewer's sealed scope and the minute (the
 * week's merge window slides with time, not with writes). Nothing here is
 * truth; an entry that is gone costs one rebuild.
 */
import type { Database } from "bun:sqlite";
import { writeStamp } from "../core/write_clock.ts";
import { Stamped } from "../kernel/stamped.ts";
import { scopeKey } from "./map_cache.ts";
const MAX_AGE_MS = 60_000;
const caches = new WeakMap<Database, Stamped<Record<string, unknown>>>();
/** The snapshot cache of `db`: for tests, which count its hits. */
export function mapSnapshots(db: Database): Stamped<Record<string, unknown>> {
  let cache = caches.get(db);
  if (cache === undefined) {
    cache = new Stamped({ maxEntries: 64, maxAgeMs: MAX_AGE_MS });
    caches.set(db, cache);
  }
  return cache;
}
/** `build()` once per repository, scope and minute while nothing has been written. */
export function cachedMap(
  db: Database,
  key: { repoId: string; scope: { sealed?: true; sealedBy?: string }; now: Date },
  build: () => Record<string, unknown>,
): { value: Record<string, unknown>; hit: boolean } {
  const stamp = writeStamp(db);
      
        sentinelcrew
        19d ago
        
      
      Correct, but measure it in production. This is the forge-wide write clock, and on a busy forge it moves with every run's cost and every event, in any repository. So the production hit rate may be low, and the 960 ms build is still paid on most reads, especially the plan-then-read path agents use. After this lands, check production Server-Timing for graph;desc=hit. If hits are rare, the follow-up is what the issue named first: make the slowest phase fast with one bounded query, or use a per-repository counter such as repoHeads plus the runs that marksFor reads.
    
  if (stamp === undefined) return { value: build(), hit: false };
  const cache = mapSnapshots(db);
  const before = cache.hits;
  const k = `map:${key.repoId}:${scopeKey(key.scope)}:${key.now.toISOString().slice(0, 16)}`;
  const value = cache.get(k, stamp, build);
  return { value, hit: cache.hits > before };
}
@@ -0,0 +1,111 @@
/**
 * GET …/map on 1,000 closed and 200 open issues, 400 edges, 9 pillars and a
 * week of merges answers fast when warm, with the same bytes as a cold
 * build, and a plan shows at once (#3321).
 */
import { baselineMs, expectBudget } from "./helpers/budget.ts";
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { createRepo } from "../src/actions/repos.ts";
import { addDependency } from "../src/core/dependencies.ts";
import { createIssue } from "../src/core/issues.ts";
import { getRepoById, updateRepoSettings } from "../src/core/repos.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { mapSnapshots } from "../src/web/map_snapshot.ts";
import { seedAdmin } from "./helpers/admin.ts";
let dataDir = "";
let app: App, servers: Servers, bearer = "", repoId = "";
const get = async (): Promise<{ body: string; ms: number; timing: string }> => {
  const t0 = performance.now();
  const res = await fetch(`${servers.url}/api/v1/repos/alice/notes/map`, { headers: { authorization: `Bearer ${bearer}` } });
  const body = await res.text();
  return { body, ms: performance.now() - t0, timing: res.headers.get("server-timing") ?? "" };
};
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-mapseed-"));
  app = createApp({ dataDir, port: 0 });
  servers = await serve(app);
  const admin = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
  if (!admin.ok) throw new Error(admin.error);
  const userId = admin.value.id;
  const made = await createRepo(app, admin.value, { name: "notes", visibility: "public", description: "" });
  if (!made.ok) throw new Error(String(made.error));
  repoId = made.value.id;
  updateRepoSettings(app.db, getRepoById(app.db, repoId)!, { plansNeedNod: false });
  const mk = (title: string, kind: "issue" | "pull" = "issue", body = "") => {
    const r = createIssue(app.db, { repoId, authorId: userId, title, body, kind });
    if (!r.ok) throw new Error(r.error);
    return r.value;
  };
  const umbrellas = Array.from({ length: 9 }, (_, i) => mk(`P${i + 1} Pillar ${i + 1}: umbrella`));
  const work = Array.from({ length: 1191 }, (_, i) => mk(`Issue ${i}`));
  app.db.transaction(() => {
    // Each pillar waits for work (200 edges); another 200 chain the work.
    for (let i = 0; i < 200; i++) addDependency(app.db, umbrellas[i % 9]!, work[i]!, userId);
    for (let i = 200; i < 400; i++) addDependency(app.db, work[i + 1]!, work[i]!, userId);
    // 1,000 closed, the rest open (191 + 9 umbrellas = 200).
    for (const w of work.slice(0, 1000))
      app.db.run("UPDATE issue SET state = 'closed' WHERE id = ?1", [w.id]);
    // 7 days of merges: 3 a day... 10 a day, each closing a closed issue.
    for (let d = 0; d < 7; d++)
      for (let n = 0; n < 10; n++) {
        const closes = work[d * 10 + n]!;
        const pull = mk(`Pull ${d}-${n}`, "pull", `Closes #${closes.number}`);
        const at = new Date(Date.now() - (d * 86_400_000 + n * 3_600_000 + 60_000)).toISOString();
        app.db.run(
          `INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at, merged_oid)
           VALUES (?1, 'main', ?2, 'a', 'b', ?3, 'c')`,
          [pull.id, `b${d}-${n}`, at],
        );
      }
  })();
  const minted = await fetch(`${servers.url}/api/v1/tokens`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ adminConfirm: true, username: "alice", password: "correct-horse-battery", name: "cli", scope: "admin" }),
  });
  bearer = ((await minted.json()) as { token: string }).token;
});
afterAll(async () => {
  servers.stop();
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
test("warm, the seed's map answers within its budget with the bytes of a cold build", async () => {
  mapSnapshots(app.db).clear();
  const cold = await get();
  const data = JSON.parse(cold.body) as { issues: unknown[]; edges: unknown[]; pillars: { merged?: number }[] };
  expect(data.issues.length).toBeGreaterThan(500);
  expect(data.edges.length).toBe(400);
  expect(data.pillars.length).toBeGreaterThanOrEqual(9);
  const hits = mapSnapshots(app.db).hits;
  const warm = await get();
  expect(mapSnapshots(app.db).hits).toBe(hits + 1);
  expect(warm.body).toBe(cold.body);
  let slowest = 0;
  for (let i = 0; i < 5; i++) slowest = Math.max(slowest, (await get()).ms);
  expectBudget(slowest, { ms: 150, against: await baselineMs(servers.url) });
  expect(warm.timing).toContain("asktolive");
});
test("a plan is on the next read", async () => {
  await get();
  const before = JSON.parse((await get()).body) as { issues: { title: string }[] };
  const res = await fetch(`${servers.url}/api/v1/repos/alice/notes/plan`, {
    method: "POST",
    headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" },
    body: JSON.stringify({ issues: [{ key: "a", title: "Freshly planned", pillar: "P1" }] }),
  });
  expect(res.status).toBe(201);
  const after = JSON.parse((await get()).body) as { issues: { title: string }[] };
  expect(before.issues.some((i) => i.title === "Freshly planned")).toBe(false);
  expect(after.issues.some((i) => i.title === "Freshly planned")).toBe(true);
});
		@@ -183,6 +183,10 @@ and rounds to 0.0):
183	183	event on the repository (see "Gaps") and spend reported by another
184	184	process.
185	185
	186	### 2b. The API map: the finished answer, by the write clock (#3321)
	187
	188	`GET /api/v1/repos/:owner/:name/map` keeps its whole answer in `web/map_snapshot.ts`, keyed by repository, the viewer's sealed scope and the UTC minute (the week's merge window slides with time) and valid only under the forge's write clock, so a plan, or any write, drops it before the next read. A hit still names the five phases, flagged as hits. Test: `test/map-seed-timing.test.ts`.
	189
186	190	## Home (`/`)
187	191
188	192	### 3. The feed's first page: per viewer
		@@ -350,6 +350,7 @@ import { askToLiveFor } from "../web/ask_to_live.ts";
350	350	import { decisionStats, listDecisions } from "../core/decisions.ts";
351	351	import { forReader } from "../crew/handover.ts";
352	352	import { globalMapJson, mapJson, marksFor } from "../web/pages/map.ts";
	353	import { cachedMap } from "../web/map_snapshot.ts";
353	354	import {
354	355	milestoneData,
355	356	milestonesJson,
		@@ -3816,21 +3817,31 @@ export function apiRoutes(router: Router<Handler>): void {
3816	3817	if (!repo) return notFound();
3817	3818	const { request } = ctx;
3818	3819	notePhase(request, "session", takeSessionTime(request));
3819		const graph = timePhase(request, "graph", () =>
3820		repoGraph(ctx.app.db, repo.id, {
3821		...sealedScope(ctx.app.db, repo, ctx.auth.user),
3822		}),
3823		);
3824		const marks = timePhase(request, "marks", () =>
3825		marksFor(ctx.app.db, repo.id, graph, listActiveRuns(ctx.app.db, [repo.id])),
3826		);
3827		const pillars = timePhase(request, "pillars", () => pillarView(ctx.app.db, repo));
3828		const ask = timePhase(request, "asktolive", () =>
3829		askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
3830		);
3831		return timePhase(request, "render", () =>
3832		json(mapJson(repo, graph, marks, pillars, ask)),
	3820	const scope = sealedScope(ctx.app.db, repo, ctx.auth.user);
	3821	let built = false;
	3822	const { value, hit } = cachedMap(
	3823	ctx.app.db,
	3824	{ repoId: repo.id, scope, now: new Date() },
	3825	() => {
	3826	built = true;
	3827	const graph = timePhase(request, "graph", () =>
	3828	repoGraph(ctx.app.db, repo.id, { ...scope }),
	3829	);
	3830	const marks = timePhase(request, "marks", () =>
	3831	marksFor(ctx.app.db, repo.id, graph, listActiveRuns(ctx.app.db, [repo.id])),
	3832	);
	3833	const pillars = timePhase(request, "pillars", () => pillarView(ctx.app.db, repo));
	3834	const ask = timePhase(request, "asktolive", () =>
	3835	askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
	3836	);
	3837	return timePhase(request, "render", () => mapJson(repo, graph, marks, pillars, ask));
	3838	},
3833	3839	);
	3840	// A snapshot skips the phases; they are still named, as hits at no cost.
	3841	if (!built)
	3842	for (const name of ["graph", "marks", "pillars", "asktolive", "render"])
	3843	notePhase(request, name, 0, hit);
	3844	return json(value);
3834	3845	});
3835	3846	// What people asked of the crew in the repositories the caller can write —
3836	3847	// by default what has nothing merged yet, oldest first — and what became of each (core/asked.ts, the /asked page).
		@@ -0,0 +1,41 @@
	1	/**
	2	* GET …/map as a snapshot (#3321): the finished answer, kept by the forge's
	3	* write clock (`core/write_clock.ts`), so any write, a plan included, drops
	4	* it before the next read. The key holds what the answer varies by besides
	5	* the data: the repository, the viewer's sealed scope and the minute (the
	6	* week's merge window slides with time, not with writes). Nothing here is
	7	* truth; an entry that is gone costs one rebuild.
	8	*/
	9	import type { Database } from "bun:sqlite";
	10	import { writeStamp } from "../core/write_clock.ts";
	11	import { Stamped } from "../kernel/stamped.ts";
	12	import { scopeKey } from "./map_cache.ts";
	13
	14	const MAX_AGE_MS = 60_000;
	15
	16	const caches = new WeakMap<Database, Stamped<Record<string, unknown>>>();
	17
	18	/** The snapshot cache of `db`: for tests, which count its hits. */
	19	export function mapSnapshots(db: Database): Stamped<Record<string, unknown>> {
	20	let cache = caches.get(db);
	21	if (cache === undefined) {
	22	cache = new Stamped({ maxEntries: 64, maxAgeMs: MAX_AGE_MS });
	23	caches.set(db, cache);
	24	}
	25	return cache;
	26	}
	27
	28	/** `build()` once per repository, scope and minute while nothing has been written. */
	29	export function cachedMap(
	30	db: Database,
	31	key: { repoId: string; scope: { sealed?: true; sealedBy?: string }; now: Date },
	32	build: () => Record<string, unknown>,
	33	): { value: Record<string, unknown>; hit: boolean } {
	34	const stamp = writeStamp(db);
		sentinelcrew 19d ago Correct, but measure it in production. This is the forge-wide write clock, and on a busy forge it moves with every run's cost and every event, in any repository. So the production hit rate may be low, and the 960 ms build is still paid on most reads, especially the plan-then-read path agents use. After this lands, check production `Server-Timing` for `graph;desc=hit`. If hits are rare, the follow-up is what the issue named first: make the slowest phase fast with one bounded query, or use a per-repository counter such as `repoHeads` plus the runs that `marksFor` reads.
	35	if (stamp === undefined) return { value: build(), hit: false };
	36	const cache = mapSnapshots(db);
	37	const before = cache.hits;
	38	const k = `map:${key.repoId}:${scopeKey(key.scope)}:${key.now.toISOString().slice(0, 16)}`;
	39	const value = cache.get(k, stamp, build);
	40	return { value, hit: cache.hits > before };
	41	}
		@@ -0,0 +1,111 @@
	1	/**
	2	* GET …/map on 1,000 closed and 200 open issues, 400 edges, 9 pillars and a
	3	* week of merges answers fast when warm, with the same bytes as a cold
	4	* build, and a plan shows at once (#3321).
	5	*/
	6	import { baselineMs, expectBudget } from "./helpers/budget.ts";
	7	import { afterAll, beforeAll, expect, test } from "bun:test";
	8	import { mkdtemp, rm } from "node:fs/promises";
	9	import { tmpdir } from "node:os";
	10	import { join } from "node:path";
	11	import { createApp, type App } from "../src/app.ts";
	12	import { createRepo } from "../src/actions/repos.ts";
	13	import { addDependency } from "../src/core/dependencies.ts";
	14	import { createIssue } from "../src/core/issues.ts";
	15	import { getRepoById, updateRepoSettings } from "../src/core/repos.ts";
	16	import { serve, type Servers } from "../src/http/server.ts";
	17	import { mapSnapshots } from "../src/web/map_snapshot.ts";
	18	import { seedAdmin } from "./helpers/admin.ts";
	19
	20	let dataDir = "";
	21	let app: App, servers: Servers, bearer = "", repoId = "";
	22
	23	const get = async (): Promise<{ body: string; ms: number; timing: string }> => {
	24	const t0 = performance.now();
	25	const res = await fetch(`${servers.url}/api/v1/repos/alice/notes/map`, { headers: { authorization: `Bearer ${bearer}` } });
	26	const body = await res.text();
	27	return { body, ms: performance.now() - t0, timing: res.headers.get("server-timing") ?? "" };
	28	};
	29
	30	beforeAll(async () => {
	31	dataDir = await mkdtemp(join(tmpdir(), "forge-mapseed-"));
	32	app = createApp({ dataDir, port: 0 });
	33	servers = await serve(app);
	34	const admin = await seedAdmin(app, { username: "alice", email: "[email protected]", password: "correct-horse-battery" });
	35	if (!admin.ok) throw new Error(admin.error);
	36	const userId = admin.value.id;
	37	const made = await createRepo(app, admin.value, { name: "notes", visibility: "public", description: "" });
	38	if (!made.ok) throw new Error(String(made.error));
	39	repoId = made.value.id;
	40	updateRepoSettings(app.db, getRepoById(app.db, repoId)!, { plansNeedNod: false });
	41	const mk = (title: string, kind: "issue" \| "pull" = "issue", body = "") => {
	42	const r = createIssue(app.db, { repoId, authorId: userId, title, body, kind });
	43	if (!r.ok) throw new Error(r.error);
	44	return r.value;
	45	};
	46	const umbrellas = Array.from({ length: 9 }, (_, i) => mk(`P${i + 1} Pillar ${i + 1}: umbrella`));
	47	const work = Array.from({ length: 1191 }, (_, i) => mk(`Issue ${i}`));
	48	app.db.transaction(() => {
	49	// Each pillar waits for work (200 edges); another 200 chain the work.
	50	for (let i = 0; i < 200; i++) addDependency(app.db, umbrellas[i % 9]!, work[i]!, userId);
	51	for (let i = 200; i < 400; i++) addDependency(app.db, work[i + 1]!, work[i]!, userId);
	52	// 1,000 closed, the rest open (191 + 9 umbrellas = 200).
	53	for (const w of work.slice(0, 1000))
	54	app.db.run("UPDATE issue SET state = 'closed' WHERE id = ?1", [w.id]);
	55	// 7 days of merges: 3 a day... 10 a day, each closing a closed issue.
	56	for (let d = 0; d < 7; d++)
	57	for (let n = 0; n < 10; n++) {
	58	const closes = work[d * 10 + n]!;
	59	const pull = mk(`Pull ${d}-${n}`, "pull", `Closes #${closes.number}`);
	60	const at = new Date(Date.now() - (d * 86_400_000 + n * 3_600_000 + 60_000)).toISOString();
	61	app.db.run(
	62	`INSERT INTO pull (issue_id, base_branch, head_branch, base_oid, head_oid, merged_at, merged_oid)
	63	VALUES (?1, 'main', ?2, 'a', 'b', ?3, 'c')`,
	64	[pull.id, `b${d}-${n}`, at],
	65	);
	66	}
	67	})();
	68	const minted = await fetch(`${servers.url}/api/v1/tokens`, {
	69	method: "POST",
	70	headers: { "content-type": "application/json" },
	71	body: JSON.stringify({ adminConfirm: true, username: "alice", password: "correct-horse-battery", name: "cli", scope: "admin" }),
	72	});
	73	bearer = ((await minted.json()) as { token: string }).token;
	74	});
	75
	76	afterAll(async () => {
	77	servers.stop();
	78	app.close();
	79	await rm(dataDir, { recursive: true, force: true });
	80	});
	81
	82	test("warm, the seed's map answers within its budget with the bytes of a cold build", async () => {
	83	mapSnapshots(app.db).clear();
	84	const cold = await get();
	85	const data = JSON.parse(cold.body) as { issues: unknown[]; edges: unknown[]; pillars: { merged?: number }[] };
	86	expect(data.issues.length).toBeGreaterThan(500);
	87	expect(data.edges.length).toBe(400);
	88	expect(data.pillars.length).toBeGreaterThanOrEqual(9);
	89	const hits = mapSnapshots(app.db).hits;
	90	const warm = await get();
	91	expect(mapSnapshots(app.db).hits).toBe(hits + 1);
	92	expect(warm.body).toBe(cold.body);
	93	let slowest = 0;
	94	for (let i = 0; i < 5; i++) slowest = Math.max(slowest, (await get()).ms);
	95	expectBudget(slowest, { ms: 150, against: await baselineMs(servers.url) });
	96	expect(warm.timing).toContain("asktolive");
	97	});
	98
	99	test("a plan is on the next read", async () => {
	100	await get();
	101	const before = JSON.parse((await get()).body) as { issues: { title: string }[] };
	102	const res = await fetch(`${servers.url}/api/v1/repos/alice/notes/plan`, {
	103	method: "POST",
	104	headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" },
	105	body: JSON.stringify({ issues: [{ key: "a", title: "Freshly planned", pillar: "P1" }] }),
	106	});
	107	expect(res.status).toBe(201);
	108	const after = JSON.parse((await get()).body) as { issues: { title: string }[] };
	109	expect(before.issues.some((i) => i.title === "Freshly planned")).toBe(false);
	110	expect(after.issues.some((i) => i.title === "Freshly planned")).toBe(true);
	111	});