b1a3d5c · 51fe491 · c3f1e7b · 361cde0 · 465bc93 · 8f61dff
@@ -37,6 +37,7 @@
  allowed).
- **System memory** — `/system`, for a forge admin who is a person (anyone else gets 403): the forge's RSS beside its JS heap (`heapStats()` from bun:jsc), hourly mean and max for the last 24 h, drawn on the server. Sampled once a minute into `memory_hour` in `app.db` (a week kept). RSS rising while the heap stays flat is native memory growth.  In the terminal the same admin has a **System memory** command in the palette: the last 24 hours as text rows (hour, RSS mean/max, heap mean/max in MiB), read-only. The account menu links `/system` for that admin.
- **Flow alarms** — a firing alarm (red main, a stuck approved pull, an idle crew, no release for 90 min) shows as one linked line above home's feed and on `/boards` (admin), under the vitals line of the terminal's Flow dialog and home, as one phone notification when it starts (not repeated while it stands), and in the `flow` tool's `alarms` (MCP, `/api/v1/telemetry/flow`; an agent's cut to public repositories). Nothing firing draws nothing. The dialog also lists the idle crew pulls (number, repository, hours since the last push), oldest first, at most 20 read and five drawn with `+N more`; none draws nothing, and an agent's list is cut to public repositories.
- **Clog watch** — the minute tick (`src/flow/clog_tick.ts`) reads the flow; when a clog starts the admin gets one phone notification, and when it clears one `clog.cleared` note (cause, time to clear, peak stranded $) lands once on the admin's feed and as a line on the terminal's home. The watch state is in the database, so a restart repeats neither.
- **Boards** — `/boards` also shows the pipeline (a row per stage, the bottleneck named) and the held counts (why each autonomous repository's ready issues wait; private repositories included), for the same admin who is a person; they sit as two cards under the board list, not as panels of a board, because they are not saved queries. The terminal's Boards screen does not draw them: its door to the same rows is the **Flow** dialog. `/boards/data`, for a forge admin who is a person (anyone else
  gets a 404 on the page): the query is the URL (`?source=runs&aggregate=count&by=day&fcolumn=state&fvalue=failed`),
  drawn server-side as the chart chosen beside it (`chart=line|bar|area|number|table|heat`)
@@ -170,6 +170,8 @@
export function classifiedAudience(kind: string): EventAudience | undefined {
    kind === "repo.upstream_follow" ||
    kind === "repo.upstream_checked" ||
    kind === "repo.crew_outsiders" ||
    // A clog starting names the forge's own pile-up: the owner's to hear (#6524).
      
        sentinelcrew
        18d ago
        
      
      Nit: this prefix also covers clog.cleared, so the comment should say "a clog starting or clearing".
    
    kind.startsWith("clog.") ||
    // A file held against new work is its admins' book-keeping (#6522).
    kind === "repo.file_held" ||
    kind === "repo.file_released"
@@ -0,0 +1,11 @@
// The clog watch survives a restart (#6524): the one standing clog (since
// when, the reasons that started it, quiet readings so far). The cleared
// note rides the event outbox as `clog.cleared`.
export default `
  CREATE TABLE IF NOT EXISTS clog_state (
    id            INTEGER PRIMARY KEY CHECK (id = 1),
    since         INTEGER NOT NULL,
    first_reasons TEXT NOT NULL,
    quiet         INTEGER NOT NULL DEFAULT 0
  );
  `;
@@ -0,0 +1,5 @@
// The most the stranded spend reached while a clog stood (#6524), so the
// cleared note can say "at peak" truthfully. Additive; zero before this column.
export default `
  ALTER TABLE clog_state ADD COLUMN peak_usd REAL NOT NULL DEFAULT 0;
  `;
@@ -41,6 +41,7 @@
import { health } from "../http/health.ts";
import { forgeVitals } from "../actions/health.ts";
import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch } from "./incidents.ts";
import { firedAlarms, type FiredAlarm } from "../flow/alarms.ts";
import { loadClog } from "../flow/clog_tick.ts";
import { forgeCeiling } from "../flow/ceiling.ts";
import { askLinesFor } from "../web/studio/asks.ts";
@@ -56,6 +57,7 @@
const BACKOFF_MAX_MS = HOUR_MS;
/** Events after which the set of asks may have changed. */
const WAKES = new Set([
  "clog.started",
  "run.finished",
  "pull.opened",
  "pull.pushed",
@@ -262,6 +264,22 @@
export function alarmLines(
  });
}
/**
 * The owner hears once when a clog (#6524) starts: the id is the clog's own
 * start, so the one standing clog is the same ask on every tick and a later
 * one is new. The state is the database's, so a restart never says it twice.
 */
export function clogLines(
  app: App,
  user: User,
): { id: string; line: string; items: readonly string[]; link: string }[] {
  if (!user.isAdmin || user.kind !== "user") return [];
  const clog = loadClog(app.db);
  if (clog === null) return [];
  const id = `clog:${clog.since}`;
  return [{ id, line: `Clogged: ${clog.firstReasons.join("; ")}.`, items: [id], link: "/" }];
}
export interface Notifier {
  /** Resolves when the work queued so far is done. For tests and shutdown. */
  settled(): Promise<void>;
@@ -448,6 +466,7 @@
export function startNotifier(app: App, options: NotifyOptions): Notifier {
      ...mirrorLines(app, user, now()),
      ...redLines(app, user, now()),
      ...alarmLines(app, user, now(), options.alarms?.()),
      ...clogLines(app, user),
    ];
    const before = seen.get(userId);
    // What is gone may ask again later, and then it is new again. Seen by
@@ -30,6 +30,7 @@
import {
import { getRunErrorLine } from "../core/runs.ts";
import { getUserById, type User } from "../core/users.ts";
import { watchedBy } from "../core/watches.ts";
import { CLOG_NOTE_WINDOW_MS, flattenNote } from "../flow/clog_note.ts";
import { chooseCandidates } from "./choose.ts";
import { writeStamp } from "../core/write_clock.ts";
import { Stamped } from "../kernel/stamped.ts";
@@ -379,6 +380,7 @@
export function candidatesFor(
    if (!repo) continue;
    const fold = folded.get(e.id);
    if (fold === "skip") continue;
    if (e.kind === "clog.cleared" && now - Date.parse(e.createdAt) > CLOG_NOTE_WINDOW_MS) continue;
    const c = fromEvent(db, e, repo, owns(repo), readsLogs(repo), subjects);
    if (c)
      out.push({
@@ -686,6 +688,12 @@
function fromEvent(
        what: said === "" ? `replied on “${on}”` : `on “${on}”: ${clip(said)}`,
      };
    }
    case "clog.cleared": {
      const note = typeof p["note"] === "string" ? p["note"] : "";
      if (note === "") return undefined;
      const what = clip(flattenNote(note), 300);
      return { ...common, kind: "map", link: base(repo), what, said: what };
    }
    case "issue.blocked":
    case "issue.unblocked":
      return {
@@ -1,6 +1,6 @@
/**
 * The note posted when a clog clears (#5930): the cause on one line, how long
 * the clog took to be noticed and to clear, and the dollars stranded at the
 * the clog took to be noticed (only when that was measured) and to clear, and the dollars stranded at the
 * peak. Pure: plain data in, markdown out. Posting it is another module's job.
 */
@@ -25,14 +25,23 @@
export function duration(ms: number): string {
  return `${(Math.round(min / 6) / 10).toString()} h`;
}
/** How long a cleared clog's note stays on the feed and the terminal's home line. */
export const CLOG_NOTE_WINDOW_MS = 24 * 3_600_000;
/** The note on one line: its blank line and bullets become " · ". */
export function flattenNote(note: string): string {
  return note.replace(/\s*\n+\s*-?\s*/g, " · ").trim();
      
        junipercrew
        18d ago
        
      
      Skip this line when detectedAt === startedAt rather than print a figure that is always 0.
    
}
export function clogNote(input: ClogNoteInput): string {
  const why = input.reasons.map((r) => r.trim()).filter(Boolean);
  const lines = [
    `Clog cleared. Cause: ${why.length ? why.join("; ") : "not recorded"}.`,
    "",
    `- Time to detect: ${duration(input.detectedAt - input.startedAt)}`,
    `- Time to clear: ${duration(input.clearedAt - input.detectedAt)}`,
  ];
  // Detected the moment it started means detection was not measured: print no figure rather than a false 0.
  if (input.detectedAt !== input.startedAt) lines.push(`- Time to detect: ${duration(input.detectedAt - input.startedAt)}`);
  lines.push(`- Time to clear: ${duration(input.clearedAt - input.detectedAt)}`);
  const usd = input.strandedUsd;
  if (typeof usd === "number" && Number.isFinite(usd) && usd > 0) {
    lines.push(`- Stranded at peak: $${usd.toLocaleString("en-US", { maximumFractionDigits: 2 })}`);
@@ -0,0 +1,115 @@
/**
 * The minute's clog tick (#6524): reads the flow, asks `clogOf` whether it is
 * clogged, steps the watch (`clogStep`) and acts on the one event: "started"
 * is told on the bus once, "cleared" puts `clogNote` on the bus once (the feed
 * and the terminal read it from there). The watch state lives in `clog_state`, so a restart reads where it
 * left off and never repeats either.
 */
import type { Database } from "bun:sqlite";
import { listEvents, type EventBus } from "../core/events.ts";
import { getRepo } from "../core/repos.ts";
import { createLog } from "../kernel/log.ts";
import { ownRepoPath } from "../kernel/own_repo.ts";
import { clogOf } from "./clog.ts";
import { CLOG_NOTE_WINDOW_MS, clogNote, flattenNote } from "./clog_note.ts";
import { clogStep, type ClogEvent, type ClogState } from "./clog_watch.ts";
import { readOldestApproved } from "./oldest_approved.ts";
import { readStranded } from "./stranded.ts";
import { readVitals } from "./vitals.ts";
const HOUR_MS = 3_600_000;
const log = createLog("clog");
interface StateRow {
  since: number;
  first_reasons: string;
  quiet: number;
  peak_usd: number;
}
export function loadClog(db: Database): ClogState | null {
  const row = db.query<StateRow, []>("SELECT since, first_reasons, quiet, peak_usd FROM clog_state WHERE id = 1").get();
  if (!row) return null;
  const parsed: unknown = JSON.parse(row.first_reasons);
  const firstReasons = Array.isArray(parsed) ? parsed.filter((r): r is string => typeof r === "string") : [];
  return { since: row.since, firstReasons, ...(row.quiet > 0 ? { quiet: row.quiet } : {}), peakUsd: row.peak_usd };
}
function saveClog(db: Database, state: ClogState | null): void {
  if (!state) {
    db.query("DELETE FROM clog_state WHERE id = 1").run();
    return;
  }
  db.query(
    `INSERT INTO clog_state (id, since, first_reasons, quiet, peak_usd) VALUES (1, ?1, ?2, ?3, ?4)
     ON CONFLICT (id) DO UPDATE SET since = ?1, first_reasons = ?2, quiet = ?3, peak_usd = ?4`,
  ).run(state.since, JSON.stringify(state.firstReasons), state.quiet ?? 0, state.peakUsd ?? 0);
}
/** Pulls opened and landed over the last 2 h, forge-wide, as the clog thresholds read them. */
function twoHours(db: Database, now: Date): { opened: number; landed: number } {
  const from = new Date(now.getTime() - 2 * HOUR_MS).toISOString();
  const to = now.toISOString();
  const row = db
    .query<{ opened: number; landed: number }, [string, string]>(
      `SELECT
         (SELECT COUNT(*) FROM pull p JOIN issue i ON i.id = p.issue_id
           WHERE i.created_at >= ?1 AND i.created_at <= ?2) AS opened,
         (SELECT COUNT(*) FROM pull p JOIN repo r ON r.id = (SELECT repo_id FROM issue WHERE id = p.issue_id)
           WHERE p.base_branch = r.default_branch AND p.merged_at >= ?1 AND p.merged_at <= ?2) AS landed`,
    )
    .get(from, to);
  return { opened: row?.opened ?? 0, landed: row?.landed ?? 0 };
}
      
        junipercrew
        18d ago
        outdated
      
      The replace(/\s*\n+\s*-?\s*/g, " · ") flattening is duplicated in src/feed/items.ts. Export one helper (next to clogNote) and use it in both. The 24 h cutoff here is also not applied on the web feed, so the two doors disagree about how long the note lives.
    
export interface ClogTick {
  readonly event: ClogEvent | null;
      
        junipercrew
        18d ago
        outdated
      
      The comment says first line only, but flattenNote returns the whole note on one line. Fix the comment.
    
}
/** The newest cleared clog's note within a day, on one line: the terminal's door to what the feed shows. */
export function lastClearedNote(db: Database, now = new Date()): string | undefined {
  const [e] = listEvents(db, { kind: "clog.cleared", limit: 1 });
  const note = e?.payload["note"];
  if (typeof note !== "string" || now.getTime() - Date.parse(e!.createdAt) > CLOG_NOTE_WINDOW_MS) return undefined;
  return flattenNote(note);
}
/** One tick. With no own repository there is nobody to tell, and the watch still steps. */
export function tickClog(db: Database, bus: Pick<EventBus, "append">, now = new Date()): ClogTick {
      
        sentinelcrew
        18d ago
        outdated
      
      loadClog runs outside the transaction that saves and posts. If two processes tick while a rollout overlaps, both can read the same prev and both insert a clog_post, so the note is posted twice. Do the read, the step and the write in one db.transaction so the clear stays once-only.
    
  const vitals = readVitals(db, now);
  const { opened, landed } = twoHours(db, now);
  const oldest = readOldestApproved(db, now);
  const reading = clogOf({
      
        sentinelcrew
        18d ago
        outdated
      
      While a clog stands, saveClog upserts an unchanged row every minute. Each upsert moves total_changes(), and with it the forge-wide write clock, so every cached page is invalidated once a minute for as long as the clog lasts. Write only when step.state differs from prev.
    
    conflicted: vitals.conflicted,
    opened,
    landed,
    oldestApprovedHours: oldest ? Math.floor(oldest.hours) : 0,
  }).reasons;
  const [owner, name] = ownRepoPath().split("/");
  const repo = owner && name ? getRepo(db, owner, name) : undefined;
  // Read, step and write in one transaction: two overlapping processes cannot both clear.
  return db.transaction((): ClogTick => {
    const prev = loadClog(db);
    const step = clogStep(prev, reading, now.getTime());
    const usd = readStranded(db).usd;
    // The peak is the most seen while the clog stood: by the clear the pile is gone.
    const state = step.state && { ...step.state, peakUsd: Math.max(prev?.peakUsd ?? 0, usd) };
    if (JSON.stringify(state) !== JSON.stringify(prev)) saveClog(db, state);
      
        junipercrew
        18d ago
        outdated
      
      detectedAt equals startedAt here, so the note always reports "Time to detect: 0 min". Omit that line until detection is measured.
    
    if (step.event === "cleared" && !repo) log.warn("clog cleared but the own repository is missing; no note posted");
    if (step.event === "cleared" && repo && prev) {
      const note = clogNote({
        reasons: prev.firstReasons,
        startedAt: prev.since,
        detectedAt: prev.since,
        clearedAt: now.getTime(),
        strandedUsd: Math.max(prev.peakUsd ?? 0, usd),
      });
      bus.append({ kind: "clog.cleared", repoId: repo.id, payload: { note, since: prev.since } });
    }
    if (step.event === "started" && repo) {
      bus.append({ kind: "clog.started", repoId: repo.id, payload: { since: now.getTime() } });
    }
    return { event: step.event };
  }).immediate();
}
@@ -14,6 +14,8 @@
export interface ClogState {
  readonly firstReasons: readonly string[];
  /** Quiet readings in a row so far; absent when zero. */
  readonly quiet?: number;
  /** The most stranded dollars seen while it stood; kept by the tick, not by the step. */
  readonly peakUsd?: number;
}
export type ClogEvent = "started" | "cleared";
@@ -30,7 +32,8 @@
export function clogStep(prev: ClogState | null, reading: readonly string[], now
    if (reading.length === 0) return { state: null, event: null };
    return { state: { since: now, firstReasons: [...reading] }, event: "started" };
  }
  if (reading.length > 0) return { state: { since: prev.since, firstReasons: prev.firstReasons }, event: null };
  if (reading.length > 0)
    return { state: { since: prev.since, firstReasons: prev.firstReasons, ...(prev.peakUsd === undefined ? {} : { peakUsd: prev.peakUsd }) }, event: null };
  const quiet = (prev.quiet ?? 0) + 1;
  if (quiet >= QUIET_TO_CLEAR) return { state: null, event: "cleared", since: prev.since };
  return { state: { ...prev, quiet }, event: null };
@@ -54,6 +54,7 @@
import { typeSafeDecider } from "./crew/decide.ts";
import { graphInstagram } from "./crew/instagram.ts";
import { peopleOnCrews } from "./core/agents.ts";
import { startNotifier } from "./crew/notify.ts";
import { tickClog } from "./flow/clog_tick.ts";
import { sweepStaleAsks } from "./crew/stale_asks.ts";
import { recordAskLedger } from "./web/studio/asks.ts";
import { startPromptRoutine } from "./crew/prompts.ts";
@@ -550,6 +551,15 @@
const processTick = (): void => {
    app.log.warn("process census failed", { error: String(error) });
  }
};
// The clog watch (#6524) reads the flow every minute; its state is the database's.
const clogTick = (): void => {
  try {
    tickClog(app.db, app.bus);
  } catch (error) {
    app.log.warn("clog tick failed", { error: String(error) });
  }
};
const clogTimer = setInterval(clogTick, 60_000);
const processTimer = setInterval(processTick, 60_000);
processTick();
// A new version proves itself against the public URL once (release/prove.ts).
@@ -668,6 +678,7 @@
function drain(signal: NodeJS.Signals): Promise<unknown> {
  clearInterval(staleTimer);
  clearInterval(followTimer);
  clearInterval(processTimer);
  clearInterval(clogTimer);
  clearInterval(leaseTimer);
  // The platform's SIGTERM: the listener stays open while routers learn
  // this process is going. A person's Ctrl-C does not wait for routers.
@@ -107,6 +107,7 @@
import { listPinned } from "../../actions/board-pins.ts";
import { migrateScreen } from "./migrate.ts";
import { newRepoScreen } from "./newrepo.ts";
import { firedAlarms } from "../../flow/alarms.ts";
import { lastClearedNote } from "../../flow/clog_tick.ts";
import { forgeCeiling } from "../../flow/ceiling.ts";
import { alarmIdleCrew } from "../../flow/alarm_idle_crew.ts";
import { noReleaseAlarm } from "../../flow/alarm_no_release.ts";
@@ -302,6 +303,7 @@
export function dashboardScreen(f: Session, ctx: Ctx): Screen {
    lane = forgeLane(f.app, f.user);
    decider = deciderLine(f.app, f.user);
    const commit = running().commit;
    const cleared = lastClearedNote(f.db, now);
      
        sentinelcrew
        18d ago
        
      
      Nit: lastClearedNote runs for every viewer, but only an admin's figures use it. It can move inside the admin branch to skip one query per render for everyone else. This is not a leak, because the result is dropped for non-admins.
    
    figures =
      f.user?.isAdmin !== true
        ? []
@@ -309,6 +311,7 @@
export function dashboardScreen(f: Session, ctx: Ctx): Screen {
            ...kpiLines(readKpis(f.db, now, kpiForge(f.app)), commit === null ? null : { sha: commit, since: ago(BOOTED) }),
            vitalsLine(readVitals(f.db, now)),
            ...firedAlarms(f.db, now, { ceilingUsd: forgeCeiling(f.app.config) }).map((a) => a.says),
      
        junipercrew
        18d ago
        outdated
      
      This line is the terminal door to the clear note and nothing renders it in a test. Add a dashboard test: admin sees the note, non-admin sees nothing. [x].flatMap(...) is also an indirect way to write a conditional spread; bind the note to a const and spread note === undefined ? [] : [note].
    
            ...(cleared === undefined ? [] : [cleared]),
          ];
    // "Yours" is ownership plus the orgs you are on a team of; typing in the
    // filter reaches every repository you can see.
@@ -31,3 +31,9 @@
test("a negative span reads as zero", () => {
test("no reasons still reads", () => {
  expect(clogNote({ ...base, reasons: [] })).toContain("Cause: not recorded.");
});
test("a detection that was not measured prints no time to detect", () => {
  const note = clogNote({ ...base, detectedAt: base.startedAt });
  expect(note).not.toContain("Time to detect");
  expect(note).toContain("- Time to clear: 70 min");
});
@@ -0,0 +1,163 @@
/** The minute's clog tick (#6524): start notifies once, clear posts once, a restart repeats neither. */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { addToCrew, createAgent } from "../src/actions/crew.ts";
import { openPull } from "../src/actions/pulls.ts";
import { commitToBranch, createRepo } from "../src/actions/repos.ts";
import { createUser, getUserByName, type User } from "../src/core/users.ts";
import { setNotifyUrl, startNotifier, type Notifier } from "../src/crew/notify.ts";
import { candidatesFor } from "../src/feed/items.ts";
import { lastClearedNote, loadClog, tickClog } from "../src/flow/clog_tick.ts";
import { quietLogs } from "./helpers/globals.ts";
quietLogs();
const MIN = 60_000;
let dataDir = "";
let app: App;
let alice: User;
let was: string | undefined;
const posts: string[] = [];
const notifiers: Notifier[] = [];
const boot = async (): Promise<Notifier> => {
  const n = startNotifier(app, {
    forgeUrl: "https://forge.example.test",
    drainingSince: () => null,
    send: async (_u, body) => {
      posts.push(body);
      return true;
    },
  });
  notifiers.push(n);
  await n.settled();
  return n;
};
beforeAll(async () => {
  was = process.env.FORGE_SENTINEL_REPO;
  process.env.FORGE_SENTINEL_REPO = "alice/forge";
  dataDir = await mkdtemp(join(tmpdir(), "forge-clog-tick-"));
  app = createApp({ dataDir, port: 0 });
  const a = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "alice",
    email: "[email protected]",
    password: "a-long-clog-tick-passphrase",
    adminIfFirst: true,
  });
  if (!a.ok) throw new Error(a.error);
  alice = a.value;
  setNotifyUrl(app.db, alice.id, "https://ntfy.example.test/root-topic");
  await createAgent(app, alice, { username: "bot" });
  const made = await createRepo(app, alice, { name: "forge", visibility: "public" });
  if (!made.ok) throw new Error(String(made.error));
  const repo = made.value;
  await commitToBranch(app, alice, repo, { message: "seed", files: [{ path: "README.md", content: "# f\n" }] });
  addToCrew(app, alice, repo, "bot");
  // Ten pulls opened and none landed over 2 h: a clog by the landing ratio.
  for (let i = 0; i < 10; i++) {
    await commitToBranch(app, alice, repo, {
      branch: `bot/${i}`,
      base: "main",
      message: `p${i}`,
      files: [{ path: `F${i}.md`, content: `${i}\n` }],
    });
    const opened = await openPull(app, getUserByName(app.db, "bot")!, repo, {
      title: `p${i}`,
      body: "",
      baseBranch: "main",
      headBranch: `bot/${i}`,
    });
    if (!opened.ok) throw new Error(String(opened.error));
  }
}, 60_000);
afterAll(async () => {
  for (const n of notifiers) await n.close();
  if (was === undefined) delete process.env.FORGE_SENTINEL_REPO;
  else process.env.FORGE_SENTINEL_REPO = was;
  await rm(dataDir, { recursive: true, force: true });
});
test("across ticks, start notifies once and clear posts once, through a restart", async () => {
  const t0 = new Date();
  const first = await boot();
  expect(tickClog(app.db, app.bus, t0).event).toBe("started");
  await first.settled();
  expect(posts).toHaveLength(1);
  expect(posts[0]).toContain("Clogged:");
  // Still clogged: no second event, no second notification.
  expect(tickClog(app.db, app.bus, new Date(t0.getTime() + MIN)).event).toBeNull();
  await first.settled();
  expect(posts).toHaveLength(1);
  // A restart: the watch state is the database's, and the new notifier says nothing.
  await first.close();
  expect(loadClog(app.db)?.since).toBe(t0.getTime());
  const second = await boot();
  expect(tickClog(app.db, app.bus, new Date(t0.getTime() + 2 * MIN)).event).toBeNull();
  await second.settled();
  expect(posts).toHaveLength(1);
  // The pile leaves the 2 h window: two quiet ticks clear it, once.
  const later = t0.getTime() + 3 * 60 * MIN;
  expect(tickClog(app.db, app.bus, new Date(later)).event).toBeNull();
  expect(tickClog(app.db, app.bus, new Date(later + MIN)).event).toBe("cleared");
  expect(tickClog(app.db, app.bus, new Date(later + 2 * MIN)).event).toBeNull();
  expect(app.db.query("SELECT COUNT(*) AS n FROM event WHERE kind = 'clog.cleared'").get()).toEqual({ n: 1 });
  expect(loadClog(app.db)).toBeNull();
  await second.settled();
  expect(posts).toHaveLength(1);
  // The note is on the admin's feed, once.
  const mine = candidatesFor(app.db, alice, later).filter((c) => c.what.startsWith("Clog cleared"));
  expect(mine).toHaveLength(1);
  expect(mine[0]!.what).toContain("Clog cleared. Cause:");
  expect(mine[0]!.what).not.toContain("Time to detect");
  // The terminal's home reads the same note from the same event.
  expect(lastClearedNote(app.db, new Date(later))).toContain("Clog cleared. Cause:");
  expect(lastClearedNote(app.db, new Date(later + 25 * 60 * MIN))).toBeUndefined();
  // Both doors drop the note after the same window.
  const dayOn = Date.now() + 25 * 60 * MIN;
  expect(candidatesFor(app.db, alice, dayOn).filter((c) => c.what.startsWith("Clog cleared"))).toHaveLength(0);
  const bob = await createUser(app.db, {
    origin: "seed",
    trusted: true,
    username: "bob",
    email: "[email protected]",
    password: "a-long-clog-tick-passphrase",
  });
  if (!bob.ok) throw new Error(bob.error);
  expect(candidatesFor(app.db, bob.value, later).filter((c) => c.what.startsWith("Clog cleared"))).toHaveLength(0);
}, 30_000);
test("the note reports the peak stranded dollars, not the figure at the clear", () => {
  const far = Date.now() + 20 * 60 * MIN;
  // A clog that stood with 50 dollars stranded; by now nothing is stranded.
  app.db.query("INSERT INTO clog_state (id, since, first_reasons, quiet, peak_usd) VALUES (1, ?1, ?2, 0, 50)").run(far - 30 * MIN, "[\"x\"]");
  expect(tickClog(app.db, app.bus, new Date(far)).event).toBeNull();
  expect(loadClog(app.db)?.peakUsd).toBe(50);
  expect(tickClog(app.db, app.bus, new Date(far + MIN)).event).toBe("cleared");
  expect(lastClearedNote(app.db, new Date(far + MIN))).toContain("Stranded at peak: $50");
});
test("a clear with no own repository still clears the watch and posts no note", () => {
  const was = process.env.FORGE_SENTINEL_REPO;
  process.env.FORGE_SENTINEL_REPO = "alice/nowhere";
  try {
    const far = Date.now() + 10 * 60 * 60 * MIN;
    app.db.query("INSERT INTO clog_state (id, since, first_reasons, quiet) VALUES (1, ?1, ?2, 1)").run(far - MIN, "[\"x\"]");
    expect(tickClog(app.db, app.bus, new Date(far)).event).toBe("cleared");
    expect(loadClog(app.db)).toBeNull();
    expect(app.db.query("SELECT COUNT(*) AS n FROM event WHERE kind = 'clog.cleared'").get()).toEqual({ n: 2 });
  } finally {
    process.env.FORGE_SENTINEL_REPO = was;
  }
});
@@ -77,6 +77,15 @@
test("a stuck approved pull draws the stuck line for an admin; a non-admin reads
  expect(await firstScreen(bob)).toContain("alice/open#1 has sat approved");
});
test("a cleared clog's note is on the admin's home line and nobody else's", async () => {
  forge.db.run(
    "INSERT INTO event (id, kind, repo_id, actor_id, payload, created_at) VALUES ('clog1', 'clog.cleared', NULL, NULL, ?1, ?2)",
    [JSON.stringify({ note: "Clog cleared. Cause: the pile.\n\n- Time to clear: 3 min\n" }), new Date().toISOString()],
  );
  expect(await firstScreen(alice)).toContain("Clog cleared. Cause: the pile.");
  expect(await firstScreen(bob)).not.toContain("Clog cleared");
});
test("the dashboard's since-you-were-here shows a release and a failed gate", async () => {
  const db = forge.db;
  const now = new Date().toISOString();
		@@ -37,6 +37,7 @@
37	37	allowed).
38	38	- System memory — `/system`, for a forge admin who is a person (anyone else gets 403): the forge's RSS beside its JS heap (`heapStats()` from bun:jsc), hourly mean and max for the last 24 h, drawn on the server. Sampled once a minute into `memory_hour` in `app.db` (a week kept). RSS rising while the heap stays flat is native memory growth. In the terminal the same admin has a System memory command in the palette: the last 24 hours as text rows (hour, RSS mean/max, heap mean/max in MiB), read-only. The account menu links `/system` for that admin.
39	39	- Flow alarms — a firing alarm (red main, a stuck approved pull, an idle crew, no release for 90 min) shows as one linked line above home's feed and on `/boards` (admin), under the vitals line of the terminal's Flow dialog and home, as one phone notification when it starts (not repeated while it stands), and in the `flow` tool's `alarms` (MCP, `/api/v1/telemetry/flow`; an agent's cut to public repositories). Nothing firing draws nothing. The dialog also lists the idle crew pulls (number, repository, hours since the last push), oldest first, at most 20 read and five drawn with `+N more`; none draws nothing, and an agent's list is cut to public repositories.
	40	- Clog watch — the minute tick (`src/flow/clog_tick.ts`) reads the flow; when a clog starts the admin gets one phone notification, and when it clears one `clog.cleared` note (cause, time to clear, peak stranded $) lands once on the admin's feed and as a line on the terminal's home. The watch state is in the database, so a restart repeats neither.
40	41	- Boards — `/boards` also shows the pipeline (a row per stage, the bottleneck named) and the held counts (why each autonomous repository's ready issues wait; private repositories included), for the same admin who is a person; they sit as two cards under the board list, not as panels of a board, because they are not saved queries. The terminal's Boards screen does not draw them: its door to the same rows is the Flow dialog. `/boards/data`, for a forge admin who is a person (anyone else
41	42	gets a 404 on the page): the query is the URL (`?source=runs&aggregate=count&by=day&fcolumn=state&fvalue=failed`),
42	43	drawn server-side as the chart chosen beside it (`chart=line\|bar\|area\|number\|table\|heat`)
		@@ -170,6 +170,8 @@ export function classifiedAudience(kind: string): EventAudience \| undefined {
170	170	kind === "repo.upstream_follow" \|\|
171	171	kind === "repo.upstream_checked" \|\|
172	172	kind === "repo.crew_outsiders" \|\|
	173	// A clog starting names the forge's own pile-up: the owner's to hear (#6524).
		sentinelcrew 18d ago Nit: this prefix also covers `clog.cleared`, so the comment should say "a clog starting or clearing".
	174	kind.startsWith("clog.") \|\|
173	175	// A file held against new work is its admins' book-keeping (#6522).
174	176	kind === "repo.file_held" \|\|
175	177	kind === "repo.file_released"
		@@ -0,0 +1,11 @@
	1	// The clog watch survives a restart (#6524): the one standing clog (since
	2	// when, the reasons that started it, quiet readings so far). The cleared
	3	// note rides the event outbox as `clog.cleared`.
	4	export default `
	5	CREATE TABLE IF NOT EXISTS clog_state (
	6	id INTEGER PRIMARY KEY CHECK (id = 1),
	7	since INTEGER NOT NULL,
	8	first_reasons TEXT NOT NULL,
	9	quiet INTEGER NOT NULL DEFAULT 0
	10	);
	11	`;
		@@ -0,0 +1,5 @@
	1	// The most the stranded spend reached while a clog stood (#6524), so the
	2	// cleared note can say "at peak" truthfully. Additive; zero before this column.
	3	export default `
	4	ALTER TABLE clog_state ADD COLUMN peak_usd REAL NOT NULL DEFAULT 0;
	5	`;
		@@ -41,6 +41,7 @@ import { health } from "../http/health.ts";
41	41	import { forgeVitals } from "../actions/health.ts";
42	42	import { RECOVERED, drainIncident, nextPage, observe, recoveredLine, type Watch } from "./incidents.ts";
43	43	import { firedAlarms, type FiredAlarm } from "../flow/alarms.ts";
	44	import { loadClog } from "../flow/clog_tick.ts";
44	45	import { forgeCeiling } from "../flow/ceiling.ts";
45	46	import { askLinesFor } from "../web/studio/asks.ts";
46	47
		@@ -56,6 +57,7 @@ const BACKOFF_MAX_MS = HOUR_MS;
56	57
57	58	/** Events after which the set of asks may have changed. */
58	59	const WAKES = new Set([
	60	"clog.started",
59	61	"run.finished",
60	62	"pull.opened",
61	63	"pull.pushed",
		@@ -262,6 +264,22 @@ export function alarmLines(
262	264	});
263	265	}
264	266
	267	/**
	268	* The owner hears once when a clog (#6524) starts: the id is the clog's own
	269	* start, so the one standing clog is the same ask on every tick and a later
	270	* one is new. The state is the database's, so a restart never says it twice.
	271	*/
	272	export function clogLines(
	273	app: App,
	274	user: User,
	275	): { id: string; line: string; items: readonly string[]; link: string }[] {
	276	if (!user.isAdmin \|\| user.kind !== "user") return [];
	277	const clog = loadClog(app.db);
	278	if (clog === null) return [];
	279	const id = `clog:${clog.since}`;
	280	return [{ id, line: `Clogged: ${clog.firstReasons.join("; ")}.`, items: [id], link: "/" }];
	281	}
	282
265	283	export interface Notifier {
266	284	/** Resolves when the work queued so far is done. For tests and shutdown. */
267	285	settled(): Promise<void>;
		@@ -448,6 +466,7 @@ export function startNotifier(app: App, options: NotifyOptions): Notifier {
448	466	...mirrorLines(app, user, now()),
449	467	...redLines(app, user, now()),
450	468	...alarmLines(app, user, now(), options.alarms?.()),
	469	...clogLines(app, user),
451	470	];
452	471	const before = seen.get(userId);
453	472	// What is gone may ask again later, and then it is new again. Seen by
		@@ -30,6 +30,7 @@ import {
30	30	import { getRunErrorLine } from "../core/runs.ts";
31	31	import { getUserById, type User } from "../core/users.ts";
32	32	import { watchedBy } from "../core/watches.ts";
	33	import { CLOG_NOTE_WINDOW_MS, flattenNote } from "../flow/clog_note.ts";
33	34	import { chooseCandidates } from "./choose.ts";
34	35	import { writeStamp } from "../core/write_clock.ts";
35	36	import { Stamped } from "../kernel/stamped.ts";
		@@ -379,6 +380,7 @@ export function candidatesFor(
379	380	if (!repo) continue;
380	381	const fold = folded.get(e.id);
381	382	if (fold === "skip") continue;
	383	if (e.kind === "clog.cleared" && now - Date.parse(e.createdAt) > CLOG_NOTE_WINDOW_MS) continue;
382	384	const c = fromEvent(db, e, repo, owns(repo), readsLogs(repo), subjects);
383	385	if (c)
384	386	out.push({
		@@ -686,6 +688,12 @@ function fromEvent(
686	688	what: said === "" ? `replied on “${on}”` : `on “${on}”: ${clip(said)}`,
687	689	};
688	690	}
	691	case "clog.cleared": {
	692	const note = typeof p["note"] === "string" ? p["note"] : "";
	693	if (note === "") return undefined;
	694	const what = clip(flattenNote(note), 300);
	695	return { ...common, kind: "map", link: base(repo), what, said: what };
	696	}
689	697	case "issue.blocked":
690	698	case "issue.unblocked":
691	699	return {
		@@ -1,6 +1,6 @@
1	1	/**
2	2	* The note posted when a clog clears (#5930): the cause on one line, how long
3		* the clog took to be noticed and to clear, and the dollars stranded at the
	3	* the clog took to be noticed (only when that was measured) and to clear, and the dollars stranded at the
4	4	* peak. Pure: plain data in, markdown out. Posting it is another module's job.
5	5	*/
6	6
		@@ -25,14 +25,23 @@ export function duration(ms: number): string {
25	25	return `${(Math.round(min / 6) / 10).toString()} h`;
26	26	}
27	27
	28	/** How long a cleared clog's note stays on the feed and the terminal's home line. */
	29	export const CLOG_NOTE_WINDOW_MS = 24 * 3_600_000;
	30
	31	/** The note on one line: its blank line and bullets become " · ". */
	32	export function flattenNote(note: string): string {
	33	return note.replace(/\s\n+\s-?\s*/g, " · ").trim();
		junipercrew 18d ago Skip this line when detectedAt === startedAt rather than print a figure that is always 0.
	34	}
	35
28	36	export function clogNote(input: ClogNoteInput): string {
29	37	const why = input.reasons.map((r) => r.trim()).filter(Boolean);
30	38	const lines = [
31	39	`Clog cleared. Cause: ${why.length ? why.join("; ") : "not recorded"}.`,
32	40	"",
33		`- Time to detect: ${duration(input.detectedAt - input.startedAt)}`,
34		`- Time to clear: ${duration(input.clearedAt - input.detectedAt)}`,
35	41	];
	42	// Detected the moment it started means detection was not measured: print no figure rather than a false 0.
	43	if (input.detectedAt !== input.startedAt) lines.push(`- Time to detect: ${duration(input.detectedAt - input.startedAt)}`);
	44	lines.push(`- Time to clear: ${duration(input.clearedAt - input.detectedAt)}`);
36	45	const usd = input.strandedUsd;
37	46	if (typeof usd === "number" && Number.isFinite(usd) && usd > 0) {
38	47	lines.push(`- Stranded at peak: $${usd.toLocaleString("en-US", { maximumFractionDigits: 2 })}`);
		@@ -0,0 +1,115 @@
	1	/**
	2	* The minute's clog tick (#6524): reads the flow, asks `clogOf` whether it is
	3	* clogged, steps the watch (`clogStep`) and acts on the one event: "started"
	4	* is told on the bus once, "cleared" puts `clogNote` on the bus once (the feed
	5	* and the terminal read it from there). The watch state lives in `clog_state`, so a restart reads where it
	6	* left off and never repeats either.
	7	*/
	8
	9	import type { Database } from "bun:sqlite";
	10	import { listEvents, type EventBus } from "../core/events.ts";
	11	import { getRepo } from "../core/repos.ts";
	12	import { createLog } from "../kernel/log.ts";
	13	import { ownRepoPath } from "../kernel/own_repo.ts";
	14	import { clogOf } from "./clog.ts";
	15	import { CLOG_NOTE_WINDOW_MS, clogNote, flattenNote } from "./clog_note.ts";
	16	import { clogStep, type ClogEvent, type ClogState } from "./clog_watch.ts";
	17	import { readOldestApproved } from "./oldest_approved.ts";
	18	import { readStranded } from "./stranded.ts";
	19	import { readVitals } from "./vitals.ts";
	20
	21	const HOUR_MS = 3_600_000;
	22	const log = createLog("clog");
	23
	24	interface StateRow {
	25	since: number;
	26	first_reasons: string;
	27	quiet: number;
	28	peak_usd: number;
	29	}
	30
	31	export function loadClog(db: Database): ClogState \| null {
	32	const row = db.query<StateRow, []>("SELECT since, first_reasons, quiet, peak_usd FROM clog_state WHERE id = 1").get();
	33	if (!row) return null;
	34	const parsed: unknown = JSON.parse(row.first_reasons);
	35	const firstReasons = Array.isArray(parsed) ? parsed.filter((r): r is string => typeof r === "string") : [];
	36	return { since: row.since, firstReasons, ...(row.quiet > 0 ? { quiet: row.quiet } : {}), peakUsd: row.peak_usd };
	37	}
	38
	39	function saveClog(db: Database, state: ClogState \| null): void {
	40	if (!state) {
	41	db.query("DELETE FROM clog_state WHERE id = 1").run();
	42	return;
	43	}
	44	db.query(
	45	`INSERT INTO clog_state (id, since, first_reasons, quiet, peak_usd) VALUES (1, ?1, ?2, ?3, ?4)
	46	ON CONFLICT (id) DO UPDATE SET since = ?1, first_reasons = ?2, quiet = ?3, peak_usd = ?4`,
	47	).run(state.since, JSON.stringify(state.firstReasons), state.quiet ?? 0, state.peakUsd ?? 0);
	48	}
	49
	50	/** Pulls opened and landed over the last 2 h, forge-wide, as the clog thresholds read them. */
	51	function twoHours(db: Database, now: Date): { opened: number; landed: number } {
	52	const from = new Date(now.getTime() - 2 * HOUR_MS).toISOString();
	53	const to = now.toISOString();
	54	const row = db
	55	.query<{ opened: number; landed: number }, [string, string]>(
	56	`SELECT
	57	(SELECT COUNT(*) FROM pull p JOIN issue i ON i.id = p.issue_id
	58	WHERE i.created_at >= ?1 AND i.created_at <= ?2) AS opened,
	59	(SELECT COUNT(*) FROM pull p JOIN repo r ON r.id = (SELECT repo_id FROM issue WHERE id = p.issue_id)
	60	WHERE p.base_branch = r.default_branch AND p.merged_at >= ?1 AND p.merged_at <= ?2) AS landed`,
	61	)
	62	.get(from, to);
	63	return { opened: row?.opened ?? 0, landed: row?.landed ?? 0 };
	64	}
	65
		junipercrew 18d ago outdated The `replace(/\s\n+\s-?\s*/g, " · ")` flattening is duplicated in `src/feed/items.ts`. Export one helper (next to `clogNote`) and use it in both. The 24 h cutoff here is also not applied on the web feed, so the two doors disagree about how long the note lives.
	66	export interface ClogTick {
	67	readonly event: ClogEvent \| null;
		junipercrew 18d ago outdated The comment says first line only, but flattenNote returns the whole note on one line. Fix the comment.
	68	}
	69
	70	/** The newest cleared clog's note within a day, on one line: the terminal's door to what the feed shows. */
	71	export function lastClearedNote(db: Database, now = new Date()): string \| undefined {
	72	const [e] = listEvents(db, { kind: "clog.cleared", limit: 1 });
	73	const note = e?.payload["note"];
	74	if (typeof note !== "string" \|\| now.getTime() - Date.parse(e!.createdAt) > CLOG_NOTE_WINDOW_MS) return undefined;
	75	return flattenNote(note);
	76	}
	77
	78	/** One tick. With no own repository there is nobody to tell, and the watch still steps. */
	79	export function tickClog(db: Database, bus: Pick<EventBus, "append">, now = new Date()): ClogTick {
		sentinelcrew 18d ago outdated `loadClog` runs outside the transaction that saves and posts. If two processes tick while a rollout overlaps, both can read the same `prev` and both insert a `clog_post`, so the note is posted twice. Do the read, the step and the write in one `db.transaction` so the clear stays once-only.
	80	const vitals = readVitals(db, now);
	81	const { opened, landed } = twoHours(db, now);
	82	const oldest = readOldestApproved(db, now);
	83	const reading = clogOf({
		sentinelcrew 18d ago outdated While a clog stands, `saveClog` upserts an unchanged row every minute. Each upsert moves `total_changes()`, and with it the forge-wide write clock, so every cached page is invalidated once a minute for as long as the clog lasts. Write only when `step.state` differs from `prev`.
	84	conflicted: vitals.conflicted,
	85	opened,
	86	landed,
	87	oldestApprovedHours: oldest ? Math.floor(oldest.hours) : 0,
	88	}).reasons;
	89	const [owner, name] = ownRepoPath().split("/");
	90	const repo = owner && name ? getRepo(db, owner, name) : undefined;
	91	// Read, step and write in one transaction: two overlapping processes cannot both clear.
	92	return db.transaction((): ClogTick => {
	93	const prev = loadClog(db);
	94	const step = clogStep(prev, reading, now.getTime());
	95	const usd = readStranded(db).usd;
	96	// The peak is the most seen while the clog stood: by the clear the pile is gone.
	97	const state = step.state && { ...step.state, peakUsd: Math.max(prev?.peakUsd ?? 0, usd) };
	98	if (JSON.stringify(state) !== JSON.stringify(prev)) saveClog(db, state);
		junipercrew 18d ago outdated detectedAt equals startedAt here, so the note always reports "Time to detect: 0 min". Omit that line until detection is measured.
	99	if (step.event === "cleared" && !repo) log.warn("clog cleared but the own repository is missing; no note posted");
	100	if (step.event === "cleared" && repo && prev) {
	101	const note = clogNote({
	102	reasons: prev.firstReasons,
	103	startedAt: prev.since,
	104	detectedAt: prev.since,
	105	clearedAt: now.getTime(),
	106	strandedUsd: Math.max(prev.peakUsd ?? 0, usd),
	107	});
	108	bus.append({ kind: "clog.cleared", repoId: repo.id, payload: { note, since: prev.since } });
	109	}
	110	if (step.event === "started" && repo) {
	111	bus.append({ kind: "clog.started", repoId: repo.id, payload: { since: now.getTime() } });
	112	}
	113	return { event: step.event };
	114	}).immediate();
	115	}
		@@ -14,6 +14,8 @@ export interface ClogState {
14	14	readonly firstReasons: readonly string[];
15	15	/** Quiet readings in a row so far; absent when zero. */
16	16	readonly quiet?: number;
	17	/** The most stranded dollars seen while it stood; kept by the tick, not by the step. */
	18	readonly peakUsd?: number;
17	19	}
18	20
19	21	export type ClogEvent = "started" \| "cleared";
		@@ -30,7 +32,8 @@ export function clogStep(prev: ClogState \| null, reading: readonly string[], now
30	32	if (reading.length === 0) return { state: null, event: null };
31	33	return { state: { since: now, firstReasons: [...reading] }, event: "started" };
32	34	}
33		if (reading.length > 0) return { state: { since: prev.since, firstReasons: prev.firstReasons }, event: null };
	35	if (reading.length > 0)
	36	return { state: { since: prev.since, firstReasons: prev.firstReasons, ...(prev.peakUsd === undefined ? {} : { peakUsd: prev.peakUsd }) }, event: null };
34	37	const quiet = (prev.quiet ?? 0) + 1;
35	38	if (quiet >= QUIET_TO_CLEAR) return { state: null, event: "cleared", since: prev.since };
36	39	return { state: { ...prev, quiet }, event: null };
		@@ -54,6 +54,7 @@ import { typeSafeDecider } from "./crew/decide.ts";
54	54	import { graphInstagram } from "./crew/instagram.ts";
55	55	import { peopleOnCrews } from "./core/agents.ts";
56	56	import { startNotifier } from "./crew/notify.ts";
	57	import { tickClog } from "./flow/clog_tick.ts";
57	58	import { sweepStaleAsks } from "./crew/stale_asks.ts";
58	59	import { recordAskLedger } from "./web/studio/asks.ts";
59	60	import { startPromptRoutine } from "./crew/prompts.ts";
		@@ -550,6 +551,15 @@ const processTick = (): void => {
550	551	app.log.warn("process census failed", { error: String(error) });
551	552	}
552	553	};
	554	// The clog watch (#6524) reads the flow every minute; its state is the database's.
	555	const clogTick = (): void => {
	556	try {
	557	tickClog(app.db, app.bus);
	558	} catch (error) {
	559	app.log.warn("clog tick failed", { error: String(error) });
	560	}
	561	};
	562	const clogTimer = setInterval(clogTick, 60_000);
553	563	const processTimer = setInterval(processTick, 60_000);
554	564	processTick();
555	565	// A new version proves itself against the public URL once (release/prove.ts).
		@@ -668,6 +678,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> {
668	678	clearInterval(staleTimer);
669	679	clearInterval(followTimer);
670	680	clearInterval(processTimer);
	681	clearInterval(clogTimer);
671	682	clearInterval(leaseTimer);
672	683	// The platform's SIGTERM: the listener stays open while routers learn
673	684	// this process is going. A person's Ctrl-C does not wait for routers.