58eb4cc · 3924a95
@@ -506,6 +506,10 @@
Where the whole forge waits is a forge admin's: `/flow` shows each stage from re
what "ready now" means to a person and to the crew. The map also shows each
pillar's ask-to-live: the median hours from an issue opening to its change live
in production, over the last 7 days (also in the map and milestones API and MCP).
It reads the `deploy_pull` record a production deploy writes when it finishes,
so a page view runs no git; an issue closed by several pulls counts once, at
its first time live. On boot the forge records what the last 7 days of earlier
production deploys carried, so work that went live before the record is counted.
`docs/STUDIO.md` is the front door from the top of this page in full: the five
words and where each is decided, the box, the three lists, a tell, a run
@@ -244,7 +244,6 @@
export const GATE: readonly string[] = [
  "test/pillars.test.ts",
  "test/carried.test.ts",
  "test/carried-deploy.test.ts",
  "test/reaches.test.ts",
  "test/milestones.test.ts",
  "test/mcp.test.ts",
  "test/name-marks.test.ts",
@@ -11,6 +11,9 @@
import type { Database } from "bun:sqlite";
import { runLocal } from "../git/local.ts";
import { repoDir } from "../git/repos.ts";
import { PILLAR_DAYS } from "./pillars.ts";
import { getRepoById } from "./repos.ts";
const FULL_OID = /^[0-9a-f]{40,64}$/;
/** Most commits one walk reads; past this a deploy records what it saw. */
@@ -68,6 +71,44 @@
export async function recordCarried(
  return carried.map((p) => p.number);
}
/**
 * Record what earlier production deploys carried, for the deploys of the
 * ask-to-live window that have no record yet (they finished before deploys
 * wrote one). A boot step, never a page view; a deploy that carries nothing
 * is walked again on each boot, bounded by the window. Returns how many
 * deploys it recorded something for.
 */
export async function backfillCarried(
  db: Database,
  reposRoot: string,
  now = new Date(),
): Promise<number> {
  const since = new Date(now.getTime() - PILLAR_DAYS * 86_400_000).toISOString();
  const runs = db
    .query<{ id: string; repoId: string; oid: string }, [string]>(
      `SELECT id, repo_id AS repoId, commit_oid AS oid FROM run
        WHERE kind = 'deploy' AND env = 'production' AND state = 'ok' AND commit_oid != ''
          AND COALESCE(serving_at, finished_at) >= ?1
          AND NOT EXISTS (SELECT 1 FROM deploy_pull WHERE run_id = run.id)
        ORDER BY COALESCE(serving_at, finished_at)`,
    )
    .all(since);
  let recorded = 0;
  for (const run of runs) {
    const repo = getRepoById(db, run.repoId);
    if (repo === undefined) continue;
    const dir = repoDir(reposRoot, repo.ownerName, repo.name);
    const carried = await recordCarried(db, {
      runId: run.id,
      repoId: run.repoId,
      commitOid: run.oid,
      dir,
    });
    if (carried.length > 0) recorded++;
  }
  return recorded;
}
/** The pull numbers a deploy run carries, ascending. */
export function carriedBy(db: Database, runId: string): number[] {
  return db
@@ -259,6 +259,7 @@
export function mergedByPillar(
}
interface MergedPull {
  readonly id: string;
  readonly closes: readonly number[];
  readonly mergedAt: string;
  readonly mergedOid: string;
@@ -291,8 +292,8 @@
function mergedPulls(
  scope: SealedScope = {},
): { pulls: MergedPull[]; issues: Map<number, ClosedIssue> } {
  const rows = db
    .query<{ body: string; at: string; oid: string | null }, string[]>(
      `SELECT i.body, p.merged_at AS at, p.merged_oid AS oid
    .query<{ id: string; body: string; at: string; oid: string | null }, string[]>(
      `SELECT i.id, i.body, p.merged_at AS at, p.merged_oid AS oid
         FROM pull p JOIN issue i ON i.id = p.issue_id
        WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL AND p.merged_at >= ?2
          AND ${sealedClause("i", scope, 3)}
@@ -300,6 +301,7 @@
function mergedPulls(
    )
    .all(repoId, since, ...sealedParams(scope));
  const pulls = rows.map((row) => ({
    id: row.id,
    closes: closesFromBody(row.body),
    mergedAt: row.at,
    mergedOid: row.oid ?? "",
@@ -342,9 +344,6 @@
function pillarOf(
// ---- ask-to-live: an issue opening to its change in production (#1185) ----
/** Whether `ancestor` is reachable from `descendant`; false when unsure. The git lives in src/git, handed in. */
export type Reaches = (ancestor: string, descendant: string) => Promise<boolean>;
/** How long the asking took, for one pillar's merged work. */
export interface AskToLive {
  /** Median hours from an issue opening to its change live in production. */
@@ -359,54 +358,55 @@
export interface AskToLiveCount {
  readonly askToLive: AskToLive | undefined;
}
/** Most ancestry questions one request may put to git; a pull left unanswered counts for nothing. */
const ASK_GIT_CALLS = 120;
/**
 * Per pillar, over the pulls merged since an instant: the median and the
 * slowest ask-to-live, from the creation of the issue that pull closed to
 * the first production deploy that went live on a commit containing the
 * pull's merge. A pull not yet live counts for nothing. Every name has a
 * row, in order; one read of deploys however many pulls merged.
 * the first production deploy that went live carrying the pull (the
 * `deploy_pull` record, so no git runs here). A pull not yet live counts
 * for nothing; an issue closed by several pulls counts once, at its first
 * time live. Every name has a row, in order; two reads however many pulls.
 */
export async function askToLiveByPillar(
export function askToLiveByPillar(
  db: Database,
  repoId: string,
  set: Pillars,
  served: ReadonlyMap<string, string>,
  since: string,
  reaches: Reaches,
  scope: SealedScope = {},
): Promise<AskToLiveCount[]> {
): AskToLiveCount[] {
  const { pulls, issues } = mergedPulls(db, repoId, since, scope);
  const deploys = db
    .query<{ oid: string; at: string }, [string, string]>(
      `SELECT commit_oid AS oid, COALESCE(serving_at, finished_at) AS at FROM run
        WHERE repo_id = ?1 AND kind = 'deploy' AND env = 'production' AND state = 'ok'
          AND commit_oid != '' AND COALESCE(serving_at, finished_at) >= ?2
        ORDER BY COALESCE(serving_at, finished_at), id`,
  const live = new Map<string, string[]>();
  for (const row of db
    .query<{ id: string; at: string }, [string, string]>(
      `SELECT dp.issue_id AS id, COALESCE(r.serving_at, r.finished_at) AS at
         FROM deploy_pull dp JOIN run r ON r.id = dp.run_id
        WHERE r.repo_id = ?1 AND r.kind = 'deploy' AND r.env = 'production' AND r.state = 'ok'
          AND COALESCE(r.serving_at, r.finished_at) >= ?2
        ORDER BY COALESCE(r.serving_at, r.finished_at), r.id`,
    )
    .all(repoId, since);
  const asked = new Map<string, { number: number; hours: number }[]>();
  let budget = ASK_GIT_CALLS;
    .all(repoId, since)) {
    const list = live.get(row.id) ?? [];
    list.push(row.at);
    live.set(row.id, list);
  }
  // One entry per asked issue: its first time live over every pull that closed it.
  const first = new Map<number, { pillar: string; createdAt: string; at: string }>();
  for (const pull of pulls) {
    const ask = pillarOf(pull.closes, issues, served);
    if (ask === undefined || pull.mergedOid === "") continue;
    let live: { oid: string; at: string } | undefined;
    for (const d of deploys) {
      if (d.at < pull.mergedAt) continue;
      if (budget-- <= 0) break;
      if (await reaches(pull.mergedOid, d.oid)) {
        live = d;
        break;
      }
    }
    if (live === undefined) continue;
    const hours = (Date.parse(live.at) - Date.parse(ask.createdAt)) / 3_600_000;
    if (ask === undefined) continue;
    const at = live.get(pull.id)?.find((t) => t >= pull.mergedAt);
    if (at === undefined) continue;
    const seen = first.get(ask.number);
    if (seen === undefined || at < seen.at)
      first.set(ask.number, { pillar: ask.pillar, createdAt: ask.createdAt, at });
  }
  const asked = new Map<string, { number: number; hours: number }[]>();
  for (const [number, f] of first) {
    const hours = (Date.parse(f.at) - Date.parse(f.createdAt)) / 3_600_000;
    if (!Number.isFinite(hours)) continue;
    const list = asked.get(ask.pillar) ?? [];
    list.push({ number: ask.number, hours: Math.max(0, hours) });
    asked.set(ask.pillar, list);
    const list = asked.get(f.pillar) ?? [];
    list.push({ number, hours: Math.max(0, hours) });
    asked.set(f.pillar, list);
  }
  return pillarNames(set).map((pillar) => {
    const list = (asked.get(pillar) ?? []).sort((a, b) => a.hours - b.hours);
@@ -488,22 +488,20 @@
export function pillarView(
  };
}
/** A view's ask-to-live over the last `PILLAR_DAYS` days; opt-in, for it reads git. */
/** A view's ask-to-live over the last `PILLAR_DAYS` days; opt-in, for it reads the deploy record. */
export function askToLiveOf(
  db: Database,
  repo: Repo,
  view: PillarView,
  reaches: Reaches,
  scope: SealedScope = {},
  now = new Date(),
): Promise<AskToLiveCount[]> {
): AskToLiveCount[] {
  return askToLiveByPillar(
    db,
    repo.id,
    view.set,
    view.of,
    pillarWindowStart(now),
    reaches,
    scope,
  );
}
@@ -861,56 +861,6 @@
export async function isAncestor(
  return out.ok;
}
const FULL_OID = /^[0-9a-f]{40,64}$/;
const KNOWN_LIMIT = 1000;
const KNOWN_DIRS = 200;
/**
 * Whether one commit contains another, for a repository's directory.
 *
 * Only a real answer is remembered — an ancestry between two commits never
 * changes — and the oldest answer goes first when the memory is full. A
 * missing object or directory is "no" now and asked again later, because it
 * may be mended. The promise never rejects.
 */
export function reachesIn(
  dir: string,
): (ancestor: string, descendant: string) => Promise<boolean> {
  return async (ancestor, descendant) => {
    if (!FULL_OID.test(ancestor) || !FULL_OID.test(descendant)) return false;
    if (ancestor === descendant) return true;
    const key = `${ancestor}\0${descendant}`;
    const known = reachKnown.get(dir)?.get(key);
    if (known !== undefined) return known;
    try {
      const out = await runLocal(
        ["merge-base", "--is-ancestor", ancestor, descendant],
        { cwd: dir },
      );
      if (out.ok) return remember(dir, key, true);
      return out.error.code === 1 ? remember(dir, key, false) : false;
    } catch {
      return false;
    }
  };
}
/** Answers per repository directory, so one busy repository cannot evict another's. */
const reachKnown = new Map<string, Map<string, boolean>>();
function remember(dir: string, key: string, value: boolean): boolean {
  let known = reachKnown.get(dir);
  if (known === undefined) {
    if (reachKnown.size >= KNOWN_DIRS)
      reachKnown.delete(reachKnown.keys().next().value as string);
    known = new Map();
    reachKnown.set(dir, known);
  }
  if (known.size >= KNOWN_LIMIT) known.delete(known.keys().next().value as string);
  known.set(key, value);
  return value;
}
/** `rev-list --left-right --count a...b` — what each side has alone. */
export async function aheadBehind(
  dir: string,
@@ -2712,7 +2712,7 @@
export function apiRoutes(router: Router<Handler>): void {
          listActiveRuns(ctx.app.db, [repo.id]),
        ),
        pillars,
        await askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
        askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
      ),
    );
  });
@@ -1415,7 +1415,7 @@
const TOOLS: readonly Tool[] = [
            listActiveRuns(ctx.app.db, [repo.id]),
          ),
          pillars,
          await askToLiveFor(ctx.app, repo, pillars, user),
          askToLiveFor(ctx.app, repo, pillars, user),
        ),
      );
    },
@@ -9,6 +9,7 @@
import { handover } from "./release/swap.ts";
import { parentUrlProblem } from "./core/parent_releases.ts";
import { releaseEnv } from "./release/env.ts";
import { createApp } from "./app.ts";
import { backfillCarried } from "./core/carried.ts";
import { reconcile } from "./core/reconcile.ts";
import { syncAllPillars } from "./actions/pillars.ts";
import { firstBoot } from "./spawn/first-boot.ts";
@@ -265,6 +266,10 @@
const apps = appDomain?.ok
    })
  : undefined;
await apps?.start();
// What deploys before the record carried, so ask-to-live is not blank for them.
void backfillCarried(app.db, app.reposRoot).catch((error) =>
  app.log.error("could not backfill what deploys carried", { error: String(error) }),
);
// A repository's own check, on every push a pull or the default branch takes.
const checks =
  process.env.FORGE_CHECKS === "0"
@@ -1,7 +1,8 @@
/**
 * A repository's pillars' ask-to-live, for the surfaces that show it (the map,
 * the milestones, their data). Opt-in and async: it asks git which deploys
 * carry which merges, so only the callers that print it pay for it.
 * the milestones, their data). Opt-in, so only the callers that print it pay
      
        junipercrew
        29d ago
        outdated
      
      Tidy the wrapped comment so it reads as one sentence; the old 'async' remark is gone but the line break is awkward.
    
 * for it; it reads which deploys carry which merges from `deploy_pull` and
 * runs no git.
 */
import type { App } from "../app.ts";
@@ -13,21 +14,18 @@
import {
import type { Repo } from "../core/repos.ts";
import { sealedScope } from "../core/sealed.ts";
import type { User } from "../core/users.ts";
import { reachesIn } from "../git/local.ts";
import { repoDir } from "../git/repos.ts";
export function askToLiveFor(
  app: App,
  repo: Repo,
  view: PillarView | undefined,
  user: User | undefined,
): Promise<AskToLiveCount[]> {
  if (view === undefined) return Promise.resolve([]);
): AskToLiveCount[] {
  if (view === undefined) return [];
  return askToLiveOf(
    app.db,
    repo,
    view,
    reachesIn(repoDir(app.reposRoot, repo.ownerName, repo.name)),
    sealedScope(app.db, repo, user),
  );
}
@@ -1419,7 +1419,7 @@
export async function mapPage(ctx: Ctx): Promise<Response> {
    graph.issues.map((i) => wordFor(i, readySet, marks)),
  );
  const pillars = pillarView(db, repo);
  const ask = await askToLiveFor(ctx.app, repo, pillars, ctx.auth.user);
  const ask = askToLiveFor(ctx.app, repo, pillars, ctx.auth.user);
  const body = html`${repoHeader(ctx, repo, "map")}
    <div class="page-head mt">
      <h1>Map</h1>
@@ -519,7 +519,7 @@
export async function milestonesJson(
  const plan = readPlan(ctx.app.db, repo, {
    spend: canSeeOwnerFacts(ctx.app.db, repo, ctx.auth.user),
  });
  const ask = await askByUmbrella(ctx, repo);
  const ask = askByUmbrella(ctx, repo);
  return {
    repo: { owner: repo.ownerName, name: repo.name },
    limbs: plan.limbs.map((limb) => ({
@@ -530,18 +530,18 @@
export async function milestonesJson(
}
/** Each pillar's ask-to-live, by its umbrella's number: what its limb shows too. */
async function askByUmbrella(
function askByUmbrella(
  ctx: Ctx,
  repo: Repo,
  only?: number,
): Promise<Map<number, AskToLive | null>> {
): Map<number, AskToLive | null> {
  const out = new Map<number, AskToLive | null>();
  const view = pillarView(ctx.app.db, repo);
  if (view === undefined) return out;
  // A milestone that is no pillar's umbrella has none: ask git for nothing.
  // A milestone that is no pillar's umbrella has none: read nothing.
  if (only !== undefined && !view.set.pillars.some((p) => p.umbrella.number === only))
    return out;
  const ask = await askToLiveFor(ctx.app, repo, view, ctx.auth.user);
  const ask = askToLiveFor(ctx.app, repo, view, ctx.auth.user);
  for (const pillar of view.set.pillars)
    out.set(
      pillar.umbrella.number,
@@ -569,7 +569,7 @@
export async function milestoneData(
    : {
        repo: { owner: repo.ownerName, name: repo.name },
        ...milestoneJson(milestone, owner),
        ...askOf(await askByUmbrella(ctx, repo, number), number),
        ...askOf(askByUmbrella(ctx, repo, number), number),
        // Whether the caller watches it; a caller who cannot has no answer.
        ...(viewer?.kind === "user"
          ? {
@@ -10,7 +10,7 @@
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { mergePull, openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { carriedBy, recordCarried } from "../src/core/carried.ts";
import { backfillCarried, carriedBy, recordCarried } from "../src/core/carried.ts";
import { getPull } from "../src/core/pulls.ts";
import { repoDir } from "../src/git/repos.ts";
import type { Repo } from "../src/core/repos.ts";
@@ -110,4 +110,17 @@
describe("carried", () => {
      }),
    ).toEqual([]);
  });
  test("backfill records what a recent production deploy carried before the record existed, once", async () => {
    app.db
      .query(
        `INSERT INTO run (id, repo_id, kind, env, commit_oid, state, serving_at, finished_at)
         VALUES ('old-prod', ?1, 'deploy', 'production', ?2, 'ok', ?3, ?3)`,
      )
      .run(repo.id, merged[0]!.oid, new Date().toISOString());
    expect(await backfillCarried(app.db, app.reposRoot)).toBe(1);
    expect(carriedBy(app.db, "old-prod")).toEqual([merged[0]!.number]);
    // Already recorded: not walked again.
    expect(await backfillCarried(app.db, app.reposRoot)).toBe(0);
  });
});
@@ -477,25 +477,45 @@
describe("pillars", () => {
    setCreated(parts[0]!, 10);
    setMerged(queuePull.id, 8);
    setMerged(composerPull.id, 6);
    const deploy = (env: string, oid: string, state: string, hoursAgo: number) =>
    // A deploy and the merged pulls it carries (the `deploy_pull` record).
    const deploy = (
      env: string,
      oid: string,
      state: string,
      hoursAgo: number,
      carries: { id: string; oid: string }[],
    ) => {
      const id = `ask-${nextId++}`;
      app.db
        .query(
          `INSERT INTO run (id, repo_id, kind, env, commit_oid, state, serving_at, finished_at)
           VALUES (?1, ?2, 'deploy', ?3, ?4, ?5, ?6, ?6)`,
        )
        .run(`ask-${nextId++}`, repo.id, env, oid, state, at(hoursAgo));
        .run(id, repo.id, env, oid, state, at(hoursAgo));
      for (const p of carries)
        app.db
          .query(
            `INSERT INTO deploy_pull (run_id, issue_id, pull_number, merged_oid)
             VALUES (?1, ?2, 0, ?3)`,
          )
          .run(id, p.id, p.oid);
    };
    // A preview and a failed deploy are not production; they count for nothing.
    deploy("pull-9", queuePull.oid, "ok", 19);
    deploy("production", composerPull.oid, "failed", 5.5);
    deploy("pull-9", queuePull.oid, "ok", 19, [sandboxPull, queuePull]);
    deploy("production", composerPull.oid, "failed", 5.5, [composerPull]);
    // Before the merge: it cannot hold it.
    deploy("production", sandboxPull.oid, "ok", 25);
    deploy("production", sandboxPull.oid, "ok", 25, [sandboxPull]);
    // The first live deploy holding sandbox's merge: 30h asked, live 12h later.
      
        junipercrew
        29d ago
        
      
      This comment is stale: nothing asks git ancestry any more; the deploy_pull rows say what a deploy holds. Reword it.
    
    // It runs a later commit on main, so only git ancestry says it holds it.
    deploy("production", queuePull.oid, "ok", 18);
    // The record, not the commit it ran, says which merges it holds.
    deploy("production", queuePull.oid, "ok", 18, [sandboxPull]);
    // Holds queue's merge, and sandbox's before it: queue was asked 10h ago, live 5h.
    deploy("production", queuePull.oid, "ok", 5);
    deploy("production", queuePull.oid, "ok", 5, [sandboxPull, queuePull]);
    const view = pillarView(app.db, fresh(), new Date(T))!;
    const asked = await askToLiveFor(app, fresh(), view, alice);
    // The record answers: with no repository on disk, no git can have run.
    const realRoot = app.reposRoot;
    (app as { reposRoot: string }).reposRoot = "/nonexistent";
    const asked = askToLiveFor(app, fresh(), view, alice);
    (app as { reposRoot: string }).reposRoot = realRoot;
    const ask = (name: string) =>
      asked.find((a) => a.pillar === name)!.askToLive;
    // sandbox: 30h -> deploy at 18h ago = 12h; queue: 10h -> 5h ago = 5h.
@@ -587,6 +607,33 @@
describe("pillars", () => {
    }
  });
  test("ask-to-live counts an issue once when two merged pulls close it, at its first time live", async () => {
    // Continues the tests above: the queue pull now also closes sandbox.
    const queueIssue = app.db
      .query<{ id: string; body: string }, []>(
        "SELECT id, body FROM issue WHERE title = 'Land queue'",
      )
      .get()!;
    app.db
      .query("UPDATE issue SET body = ?1 WHERE id = ?2")
      .run(`Closes #${sandbox.number}`, queueIssue.id);
    try {
      const view = pillarView(app.db, fresh(), new Date())!;
      const trustAsk = askToLiveFor(app, fresh(), view, alice).find(
        (a) => a.pillar === "P0 Trust",
      )!.askToLive;
      // sandbox once, live 12h after it was asked (the first of its two pulls to go live).
      expect(trustAsk).toEqual({
        medianHours: 12,
        slowest: { number: sandbox.number, hours: 12 },
      });
    } finally {
      app.db
        .query("UPDATE issue SET body = ?1 WHERE id = ?2")
        .run(queueIssue.body, queueIssue.id);
    }
  });
  test("a repository with no pillars shows none of it", async () => {
    const made = await createRepo(app, alice, {
      name: "plain",
@@ -1,31 +0,0 @@
import { afterAll, expect, test } from "bun:test";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { reachesIn } from "../src/git/local.ts";
const dir = mkdtempSync(join(tmpdir(), "forge-reaches-"));
afterAll(() => rmSync(dir, { recursive: true, force: true }));
const git = (...args: string[]) => {
  const out = Bun.spawnSync(
    ["git", "-c", "user.name=t", "-c", "user.email=t@t", ...args],
    { cwd: dir },
  );
  return out.stdout.toString().trim();
};
test("reachesIn: a later commit holds an earlier one, not the other way; a broken repository is no, not a throw", async () => {
  git("init", "-q");
  git("commit", "-q", "--allow-empty", "-m", "one");
  const first = git("rev-parse", "HEAD");
  git("commit", "-q", "--allow-empty", "-m", "two");
  const second = git("rev-parse", "HEAD");
  const reaches = reachesIn(dir);
  expect(await reaches(first, second)).toBe(true);
  expect(await reaches(second, first)).toBe(false);
  expect(await reaches(first, first)).toBe(true);
  expect(await reaches("f".repeat(40), second)).toBe(false);
  expect(await reaches("--all", second)).toBe(false);
  expect(await reachesIn(join(dir, "missing"))(first, second)).toBe(false);
});
		@@ -506,6 +506,10 @@ Where the whole forge waits is a forge admin's: `/flow` shows each stage from re
506	506	what "ready now" means to a person and to the crew. The map also shows each
507	507	pillar's ask-to-live: the median hours from an issue opening to its change live
508	508	in production, over the last 7 days (also in the map and milestones API and MCP).
	509	It reads the `deploy_pull` record a production deploy writes when it finishes,
	510	so a page view runs no git; an issue closed by several pulls counts once, at
	511	its first time live. On boot the forge records what the last 7 days of earlier
	512	production deploys carried, so work that went live before the record is counted.
509	513
510	514	`docs/STUDIO.md` is the front door from the top of this page in full: the five
511	515	words and where each is decided, the box, the three lists, a tell, a run
		@@ -244,7 +244,6 @@ export const GATE: readonly string[] = [
244	244	"test/pillars.test.ts",
245	245	"test/carried.test.ts",
246	246	"test/carried-deploy.test.ts",
247		"test/reaches.test.ts",
248	247	"test/milestones.test.ts",
249	248	"test/mcp.test.ts",
250	249	"test/name-marks.test.ts",
		@@ -11,6 +11,9 @@
11	11
12	12	import type { Database } from "bun:sqlite";
13	13	import { runLocal } from "../git/local.ts";
	14	import { repoDir } from "../git/repos.ts";
	15	import { PILLAR_DAYS } from "./pillars.ts";
	16	import { getRepoById } from "./repos.ts";
14	17
15	18	const FULL_OID = /^[0-9a-f]{40,64}$/;
16	19	/** Most commits one walk reads; past this a deploy records what it saw. */
		@@ -68,6 +71,44 @@ export async function recordCarried(
68	71	return carried.map((p) => p.number);
69	72	}
70	73
	74	/**
	75	* Record what earlier production deploys carried, for the deploys of the
	76	* ask-to-live window that have no record yet (they finished before deploys
	77	* wrote one). A boot step, never a page view; a deploy that carries nothing
	78	* is walked again on each boot, bounded by the window. Returns how many
	79	* deploys it recorded something for.
	80	*/
	81	export async function backfillCarried(
	82	db: Database,
	83	reposRoot: string,
	84	now = new Date(),
	85	): Promise<number> {
	86	const since = new Date(now.getTime() - PILLAR_DAYS * 86_400_000).toISOString();
	87	const runs = db
	88	.query<{ id: string; repoId: string; oid: string }, [string]>(
	89	`SELECT id, repo_id AS repoId, commit_oid AS oid FROM run
	90	WHERE kind = 'deploy' AND env = 'production' AND state = 'ok' AND commit_oid != ''
	91	AND COALESCE(serving_at, finished_at) >= ?1
	92	AND NOT EXISTS (SELECT 1 FROM deploy_pull WHERE run_id = run.id)
	93	ORDER BY COALESCE(serving_at, finished_at)`,
	94	)
	95	.all(since);
	96	let recorded = 0;
	97	for (const run of runs) {
	98	const repo = getRepoById(db, run.repoId);
	99	if (repo === undefined) continue;
	100	const dir = repoDir(reposRoot, repo.ownerName, repo.name);
	101	const carried = await recordCarried(db, {
	102	runId: run.id,
	103	repoId: run.repoId,
	104	commitOid: run.oid,
	105	dir,
	106	});
	107	if (carried.length > 0) recorded++;
	108	}
	109	return recorded;
	110	}
	111
71	112	/** The pull numbers a deploy run carries, ascending. */
72	113	export function carriedBy(db: Database, runId: string): number[] {
73	114	return db
		@@ -259,6 +259,7 @@ export function mergedByPillar(
259	259	}
260	260
261	261	interface MergedPull {
	262	readonly id: string;
262	263	readonly closes: readonly number[];
263	264	readonly mergedAt: string;
264	265	readonly mergedOid: string;
		@@ -291,8 +292,8 @@ function mergedPulls(
291	292	scope: SealedScope = {},
292	293	): { pulls: MergedPull[]; issues: Map<number, ClosedIssue> } {
293	294	const rows = db
294		.query<{ body: string; at: string; oid: string \| null }, string[]>(
295		`SELECT i.body, p.merged_at AS at, p.merged_oid AS oid
	295	.query<{ id: string; body: string; at: string; oid: string \| null }, string[]>(
	296	`SELECT i.id, i.body, p.merged_at AS at, p.merged_oid AS oid
296	297	FROM pull p JOIN issue i ON i.id = p.issue_id
297	298	WHERE i.repo_id = ?1 AND p.merged_at IS NOT NULL AND p.merged_at >= ?2
298	299	AND ${sealedClause("i", scope, 3)}
		@@ -300,6 +301,7 @@ function mergedPulls(
300	301	)
301	302	.all(repoId, since, ...sealedParams(scope));
302	303	const pulls = rows.map((row) => ({
	304	id: row.id,
303	305	closes: closesFromBody(row.body),
304	306	mergedAt: row.at,
305	307	mergedOid: row.oid ?? "",
		@@ -342,9 +344,6 @@ function pillarOf(
342	344
343	345	// ---- ask-to-live: an issue opening to its change in production (#1185) ----
344	346
345		/** Whether `ancestor` is reachable from `descendant`; false when unsure. The git lives in src/git, handed in. */
346		export type Reaches = (ancestor: string, descendant: string) => Promise<boolean>;
347
348	347	/** How long the asking took, for one pillar's merged work. */
349	348	export interface AskToLive {
350	349	/** Median hours from an issue opening to its change live in production. */
		@@ -359,54 +358,55 @@ export interface AskToLiveCount {
359	358	readonly askToLive: AskToLive \| undefined;
360	359	}
361	360
362		/** Most ancestry questions one request may put to git; a pull left unanswered counts for nothing. */
363		const ASK_GIT_CALLS = 120;
364
365	361	/**
366	362	* Per pillar, over the pulls merged since an instant: the median and the
367	363	* slowest ask-to-live, from the creation of the issue that pull closed to
368		* the first production deploy that went live on a commit containing the
369		* pull's merge. A pull not yet live counts for nothing. Every name has a
370		* row, in order; one read of deploys however many pulls merged.
	364	* the first production deploy that went live carrying the pull (the
	365	* `deploy_pull` record, so no git runs here). A pull not yet live counts
	366	* for nothing; an issue closed by several pulls counts once, at its first
	367	* time live. Every name has a row, in order; two reads however many pulls.
371	368	*/
372		export async function askToLiveByPillar(
	369	export function askToLiveByPillar(
373	370	db: Database,
374	371	repoId: string,
375	372	set: Pillars,
376	373	served: ReadonlyMap<string, string>,
377	374	since: string,
378		reaches: Reaches,
379	375	scope: SealedScope = {},
380		): Promise<AskToLiveCount[]> {
	376	): AskToLiveCount[] {
381	377	const { pulls, issues } = mergedPulls(db, repoId, since, scope);
382		const deploys = db
383		.query<{ oid: string; at: string }, [string, string]>(
384		`SELECT commit_oid AS oid, COALESCE(serving_at, finished_at) AS at FROM run
385		WHERE repo_id = ?1 AND kind = 'deploy' AND env = 'production' AND state = 'ok'
386		AND commit_oid != '' AND COALESCE(serving_at, finished_at) >= ?2
387		ORDER BY COALESCE(serving_at, finished_at), id`,
	378	const live = new Map<string, string[]>();
	379	for (const row of db
	380	.query<{ id: string; at: string }, [string, string]>(
	381	`SELECT dp.issue_id AS id, COALESCE(r.serving_at, r.finished_at) AS at
	382	FROM deploy_pull dp JOIN run r ON r.id = dp.run_id
	383	WHERE r.repo_id = ?1 AND r.kind = 'deploy' AND r.env = 'production' AND r.state = 'ok'
	384	AND COALESCE(r.serving_at, r.finished_at) >= ?2
	385	ORDER BY COALESCE(r.serving_at, r.finished_at), r.id`,
388	386	)
389		.all(repoId, since);
390		const asked = new Map<string, { number: number; hours: number }[]>();
391		let budget = ASK_GIT_CALLS;
	387	.all(repoId, since)) {
	388	const list = live.get(row.id) ?? [];
	389	list.push(row.at);
	390	live.set(row.id, list);
	391	}
	392	// One entry per asked issue: its first time live over every pull that closed it.
	393	const first = new Map<number, { pillar: string; createdAt: string; at: string }>();
392	394	for (const pull of pulls) {
393	395	const ask = pillarOf(pull.closes, issues, served);
394		if (ask === undefined \|\| pull.mergedOid === "") continue;
395		let live: { oid: string; at: string } \| undefined;
396		for (const d of deploys) {
397		if (d.at < pull.mergedAt) continue;
398		if (budget-- <= 0) break;
399		if (await reaches(pull.mergedOid, d.oid)) {
400		live = d;
401		break;
402		}
403		}
404		if (live === undefined) continue;
405		const hours = (Date.parse(live.at) - Date.parse(ask.createdAt)) / 3_600_000;
	396	if (ask === undefined) continue;
	397	const at = live.get(pull.id)?.find((t) => t >= pull.mergedAt);
	398	if (at === undefined) continue;
	399	const seen = first.get(ask.number);
	400	if (seen === undefined \|\| at < seen.at)
	401	first.set(ask.number, { pillar: ask.pillar, createdAt: ask.createdAt, at });
	402	}
	403	const asked = new Map<string, { number: number; hours: number }[]>();
	404	for (const [number, f] of first) {
	405	const hours = (Date.parse(f.at) - Date.parse(f.createdAt)) / 3_600_000;
406	406	if (!Number.isFinite(hours)) continue;
407		const list = asked.get(ask.pillar) ?? [];
408		list.push({ number: ask.number, hours: Math.max(0, hours) });
409		asked.set(ask.pillar, list);
	407	const list = asked.get(f.pillar) ?? [];
	408	list.push({ number, hours: Math.max(0, hours) });
	409	asked.set(f.pillar, list);
410	410	}
411	411	return pillarNames(set).map((pillar) => {
412	412	const list = (asked.get(pillar) ?? []).sort((a, b) => a.hours - b.hours);
		@@ -488,22 +488,20 @@ export function pillarView(
488	488	};
489	489	}
490	490
491		/** A view's ask-to-live over the last `PILLAR_DAYS` days; opt-in, for it reads git. */
	491	/** A view's ask-to-live over the last `PILLAR_DAYS` days; opt-in, for it reads the deploy record. */
492	492	export function askToLiveOf(
493	493	db: Database,
494	494	repo: Repo,
495	495	view: PillarView,
496		reaches: Reaches,
497	496	scope: SealedScope = {},
498	497	now = new Date(),
499		): Promise<AskToLiveCount[]> {
	498	): AskToLiveCount[] {
500	499	return askToLiveByPillar(
501	500	db,
502	501	repo.id,
503	502	view.set,
504	503	view.of,
505	504	pillarWindowStart(now),
506		reaches,
507	505	scope,
508	506	);
509	507	}
		@@ -861,56 +861,6 @@ export async function isAncestor(
861	861	return out.ok;
862	862	}
863	863
864		const FULL_OID = /^[0-9a-f]{40,64}$/;
865		const KNOWN_LIMIT = 1000;
866		const KNOWN_DIRS = 200;
867
868		/**
869		* Whether one commit contains another, for a repository's directory.
870		*
871		* Only a real answer is remembered — an ancestry between two commits never
872		* changes — and the oldest answer goes first when the memory is full. A
873		* missing object or directory is "no" now and asked again later, because it
874		* may be mended. The promise never rejects.
875		*/
876		export function reachesIn(
877		dir: string,
878		): (ancestor: string, descendant: string) => Promise<boolean> {
879		return async (ancestor, descendant) => {
880		if (!FULL_OID.test(ancestor) \|\| !FULL_OID.test(descendant)) return false;
881		if (ancestor === descendant) return true;
882		const key = `${ancestor}\0${descendant}`;
883		const known = reachKnown.get(dir)?.get(key);
884		if (known !== undefined) return known;
885		try {
886		const out = await runLocal(
887		["merge-base", "--is-ancestor", ancestor, descendant],
888		{ cwd: dir },
889		);
890		if (out.ok) return remember(dir, key, true);
891		return out.error.code === 1 ? remember(dir, key, false) : false;
892		} catch {
893		return false;
894		}
895		};
896		}
897
898		/** Answers per repository directory, so one busy repository cannot evict another's. */
899		const reachKnown = new Map<string, Map<string, boolean>>();
900
901		function remember(dir: string, key: string, value: boolean): boolean {
902		let known = reachKnown.get(dir);
903		if (known === undefined) {
904		if (reachKnown.size >= KNOWN_DIRS)
905		reachKnown.delete(reachKnown.keys().next().value as string);
906		known = new Map();
907		reachKnown.set(dir, known);
908		}
909		if (known.size >= KNOWN_LIMIT) known.delete(known.keys().next().value as string);
910		known.set(key, value);
911		return value;
912		}
913
914	864	/** `rev-list --left-right --count a...b` — what each side has alone. */
915	865	export async function aheadBehind(
916	866	dir: string,
		@@ -2712,7 +2712,7 @@ export function apiRoutes(router: Router<Handler>): void {
2712	2712	listActiveRuns(ctx.app.db, [repo.id]),
2713	2713	),
2714	2714	pillars,
2715		await askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
	2715	askToLiveFor(ctx.app, repo, pillars, ctx.auth.user),
2716	2716	),
2717	2717	);
2718	2718	});
		@@ -1415,7 +1415,7 @@ const TOOLS: readonly Tool[] = [
1415	1415	listActiveRuns(ctx.app.db, [repo.id]),
1416	1416	),
1417	1417	pillars,
1418		await askToLiveFor(ctx.app, repo, pillars, user),
	1418	askToLiveFor(ctx.app, repo, pillars, user),
1419	1419	),
1420	1420	);
1421	1421	},
		@@ -9,6 +9,7 @@ import { handover } from "./release/swap.ts";
9	9	import { parentUrlProblem } from "./core/parent_releases.ts";
10	10	import { releaseEnv } from "./release/env.ts";
11	11	import { createApp } from "./app.ts";
	12	import { backfillCarried } from "./core/carried.ts";
12	13	import { reconcile } from "./core/reconcile.ts";
13	14	import { syncAllPillars } from "./actions/pillars.ts";
14	15	import { firstBoot } from "./spawn/first-boot.ts";
		@@ -265,6 +266,10 @@ const apps = appDomain?.ok
265	266	})
266	267	: undefined;
267	268	await apps?.start();
	269	// What deploys before the record carried, so ask-to-live is not blank for them.
	270	void backfillCarried(app.db, app.reposRoot).catch((error) =>
	271	app.log.error("could not backfill what deploys carried", { error: String(error) }),
	272	);
268	273	// A repository's own check, on every push a pull or the default branch takes.
269	274	const checks =
270	275	process.env.FORGE_CHECKS === "0"
		@@ -1,7 +1,8 @@
1	1	/**
2	2	* A repository's pillars' ask-to-live, for the surfaces that show it (the map,
3		* the milestones, their data). Opt-in and async: it asks git which deploys
4		* carry which merges, so only the callers that print it pay for it.
	3	* the milestones, their data). Opt-in, so only the callers that print it pay
		junipercrew 29d ago outdated Tidy the wrapped comment so it reads as one sentence; the old 'async' remark is gone but the line break is awkward.
	4	* for it; it reads which deploys carry which merges from `deploy_pull` and
	5	* runs no git.
5	6	*/
6	7
7	8	import type { App } from "../app.ts";
		@@ -13,21 +14,18 @@ import {
13	14	import type { Repo } from "../core/repos.ts";
14	15	import { sealedScope } from "../core/sealed.ts";
15	16	import type { User } from "../core/users.ts";
16		import { reachesIn } from "../git/local.ts";
17		import { repoDir } from "../git/repos.ts";
18	17
19	18	export function askToLiveFor(
20	19	app: App,
21	20	repo: Repo,
22	21	view: PillarView \| undefined,
23	22	user: User \| undefined,
24		): Promise<AskToLiveCount[]> {
25		if (view === undefined) return Promise.resolve([]);
	23	): AskToLiveCount[] {
	24	if (view === undefined) return [];
26	25	return askToLiveOf(
27	26	app.db,
28	27	repo,
29	28	view,
30		reachesIn(repoDir(app.reposRoot, repo.ownerName, repo.name)),
31	29	sealedScope(app.db, repo, user),
32	30	);
33	31	}
		@@ -1419,7 +1419,7 @@ export async function mapPage(ctx: Ctx): Promise<Response> {
1419	1419	graph.issues.map((i) => wordFor(i, readySet, marks)),
1420	1420	);
1421	1421	const pillars = pillarView(db, repo);
1422		const ask = await askToLiveFor(ctx.app, repo, pillars, ctx.auth.user);
	1422	const ask = askToLiveFor(ctx.app, repo, pillars, ctx.auth.user);
1423	1423	const body = html`${repoHeader(ctx, repo, "map")}
1424	1424	<div class="page-head mt">
1425	1425	<h1>Map</h1>