9edbc52 · 2257349 · 7114870
@@ -137,7 +137,7 @@
bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
| `land_past_gate` | — | Land one pull past a broken gate, with a recorded reason; an owner's only | 321 |
| `lift` | — | Lift an issue out from behind the Later gate | 214 |
| `limb` | `defer`, `resume`, `close`, `raise`, `lower`, `replan`, `takeback`, `undo` | Act on a limb by `action` | 532 |
| `limits` | — | Every limit the owner can hit | 1025 |
| `limits` | — | Every limit the owner can hit | 1077 |
| `maintenance` | `status`, `prepare`, `resume`, `pause-duty`, `resume-duty` | Prepare the forge for a power-off, check the drain, or end maintenance (admin) | 308 |
| `make_it_yours` | — | Fork an app you can see: code and history, following the original | 337 |
| `map` | — | An app's plan: ready or waiting, critical path, crew, parkedTwice; `done` adds closed; `first: N` ranks N first | 375 |
@@ -6,13 +6,13 @@
`src/system/retention_deploys.ts` keeps the rows of:
- deploys from the last `DEPLOY_PULL_KEEP_DAYS` (14) days, and
- deploys from the last `retention.deploy_pull_days` days (14 by default, `DEPLOY_PULL_KEEP_DAYS`; an admin moves it on `/admin/limits`, never below 7, the week ask-to-live reads), and
- the deploy each app is running now (its newest successful production deploy), however old.
Every other deploy's rows are deleted by the crew tick in batches of at most `DEPLOY_PULL_BATCH` (5,000) rows. The tick reads the candidate rows first, then deletes them by rowid in one short transaction. It runs one batch per tick while each batch comes back full, then one batch an hour after that. When it deletes something, the log says `trimmed deploy_pull`.
## Asking for an older deploy
You don't have to do anything. Open `/<owner>/<repo>/runs/<id>` for a production deploy older than 14 days, or call the run API, or open the app's deploys screen in the terminal. The forge walks that deploy's commit in git (`carriedFor` / `carriedForRuns` in `src/core/carried.ts`), finds the merged pulls it contains and saves them again. You get the same list as before the trim, and later views read the saved rows.
You don't have to do anything. Open `/<owner>/<repo>/runs/<id>` for a production deploy older than the window, or call the run API, or open the app's deploys screen in the terminal. The forge walks that deploy's commit in git (`carriedFor` / `carriedForRuns` in `src/core/carried.ts`), finds the merged pulls it contains and saves them again. You get the same list as before the trim, and later views read the saved rows.
Each running forge walks a deploy at most once at a time; views of the same deploy that arrive together share one walk. If a deploy carries no pulls, it is walked once and the empty answer is remembered for as long as the forge runs, for up to 10,000 deploys. If git can't walk the commit, the forge waits a minute before trying again. Preview deploys and deploys that failed or are still running are never walked.
@@ -0,0 +1,15 @@
# How long the event log keeps what it holds
`event` is the forge's log of what happened. `src/system/retention_events.ts` says which kinds are state read back at any age and are never trimmed: `issue.*`, `pull.*`, `repo.*`, `user.*`, `push`, `app.egress.*`, `keeper_*` and a few marks. Every other kind is trimmed once it is older than `retention.event_days` (#12008, #12052).
## The window
The default is 30 days (`KEEP_DAYS`). That is the longest fixed window any page or tool asks of a kind that is not kept: a board's `30d` range. The home feed, the digest and the affinity weights read a week or less. The events page, the events API, the MCP `events` tool, the Atom feed and SSE replay page back through the log, so for unkept kinds they answer the window and nothing older.
It was 180 days. On a forge younger than that, which production is (it started 2026-08-03), the rule trimmed nothing while `event` held 578 MB of a 4.6 GiB file.
An admin moves the window on `/admin/limits` (**Days the event log keeps what no page reads at any age**), from 7 to 3,650 days. The sampler's next 15-second tick reads the new value.
## How it trims
The sampler's tick deletes at most 1,000 rows, a chunk of 1,000 per transaction, oldest first. When a pass reaches rows inside the window, it waits a day before starting again. A webhook delivery that is still pending keeps its event.
@@ -30,6 +30,7 @@
import { FINE_MS, record, type Point } from "./ring.ts";
import { readHwmon, readTopology, temperatures } from "./thermal.ts";
import { during } from "../perf/jobs.ts";
import { withoutWaiting } from "../kernel/db.ts";
import { limitValue } from "../settings/limits.ts";
import { pruneEvents, TICK_ROWS } from "../system/retention_events.ts";
import { drainSamples } from "../system/retention_samples.ts";
@@ -176,7 +177,7 @@
export function startSampler(
    // The same tick prunes one batch of old events (`system/retention_events.ts`); a busy
    // database skips it, and the next tick tries again.
    try {
      withoutWaiting(app.db, () => pruneEvents(app.db, now, TICK_ROWS));
      withoutWaiting(app.db, () => pruneEvents(app.db, now, TICK_ROWS, limitValue(app.db, "retention.event_days") * 86_400_000));
    } catch (error) {
      app.log.warn("event prune failed", { error: String(error) });
    }
@@ -41,6 +41,7 @@
import {
import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
import { claimedClones, sweepOrphanClones } from "./orphans.ts";
import { pruneRunFiles } from "../runs/prune.ts";
import { limitValue } from "../settings/limits.ts";
import { deployRetention } from "../system/retention_deploys.ts";
import { crewRoot, isClaimed } from "./workdir.ts";
import { resolveOpen } from "../core/decisions.ts";
@@ -274,7 +275,7 @@
export async function startCrew(
  // same clock as the pump, so a pick becomes a run on the next tick.
  let lastAutonomy = 0;
  // deploy_pull keeps recent deploys only (#12009): a capped batch a tick while rows remain.
  const trimDeploys = deployRetention(app.db);
  const trimDeploys = deployRetention(app.db, undefined, () => limitValue(app.db, "retention.deploy_pull_days"));
  // The sweeps a tick started and has not seen settle: `stop()` waits for
  // them, so none reads `app.db` after its caller closed it (#9977).
  const sweeps = new Set<Promise<unknown>>();
@@ -21,6 +21,8 @@
import {
  type CapacityKey,
} from "../core/capacity.ts";
import { err, ok, type Result } from "../kernel/result.ts";
import { DEPLOY_PULL_KEEP_DAYS } from "../system/retention_deploys.ts";
import { KEEP_DAYS as EVENT_KEEP_DAYS } from "../system/retention_events.ts";
export type LimitKey =
  | "crew.concurrency"
@@ -55,7 +57,9 @@
export type LimitKey =
  | "model_review.slack_pct"
  | "shell_idle_minutes"
  | "billing.grace_days"
  | "billing.retention_days";
  | "billing.retention_days"
  | "retention.event_days"
  | "retention.deploy_pull_days";
export interface LimitDef {
  readonly label: string;
@@ -339,6 +343,22 @@
export const LIMITS: Record<LimitKey, LimitDef> = {
    max: 365,
    reason: "A forge that lapsed may come back, so its data waits this long before anything is allowed to remove it; paying again makes it active at once.",
  },
  "retention.event_days": {
    label: "Days the event log keeps what no page reads at any age",
    unit: "days",
    default: EVENT_KEEP_DAYS,
    min: 7,
    max: 3_650,
    reason: "Issue, pull, push and account events are kept at any age; the rest feeds windows of 30 days at most, and kept longer they filled the database past the 2 GiB Bun maps.",
  },
  "retention.deploy_pull_days": {
    label: "Days a deploy keeps the list of pulls it carried",
    unit: "days",
    default: DEPLOY_PULL_KEEP_DAYS,
    min: 7,
    max: 3_650,
    reason: "The list is an index over git, rebuilt when an older deploy is opened, and it was the largest table in the database; a week is what ask-to-live reads.",
  },
};
export const LIMIT_KEYS = Object.keys(LIMITS) as LimitKey[];
@@ -17,7 +17,11 @@
import type { Database } from "bun:sqlite";
import { transact } from "../kernel/db.ts";
/** Deploys this recent keep their rows. Above `PILLAR_DAYS` (7), the window ask-to-live reads. */
/**
 * Deploys this recent keep their rows, by default; an admin moves it with the
 * `retention.deploy_pull_days` limit (`settings/limits.ts`), never below
 * `PILLAR_DAYS` (7), the window ask-to-live reads.
 */
export const DEPLOY_PULL_KEEP_DAYS = 14;
/** The least time between two slices of one pass. */
const DEPLOY_PULL_PACE_MS = 10_000;
@@ -47,6 +51,7 @@
export function pruneDeploys(
  now: Date = new Date(),
  rowCap: number = DEPLOY_PULL_SCAN,
  after = 0,
  keepDays: number = DEPLOY_PULL_KEEP_DAYS,
): Pruned & { readonly next: number } {
  const cap = Math.max(0, Math.floor(rowCap));
  if (cap === 0) return { deleted: 0, more: false, next: after };
@@ -56,7 +61,7 @@
export function pruneDeploys(
    )
    .all(after, cap);
  if (slice.length === 0) return { deleted: 0, more: false, next: 0 };
  const cutoff = new Date(now.getTime() - DEPLOY_PULL_KEEP_DAYS * 86_400_000).toISOString();
  const cutoff = new Date(now.getTime() - keepDays * 86_400_000).toISOString();
  const runIds = [...new Set(slice.map((row) => row.run_id))];
  const old = new Set<string>();
  const isOld = db.query<{ id: string; repo_id: string }, [string, string]>(
@@ -94,11 +99,13 @@
export function pruneDeploys(
/**
 * The crew tick's view of the rule: a batch on each tick while the last one
 * was full, and otherwise one an hour. Returns the batch's result, or
 * undefined when this tick runs none.
 * undefined when this tick runs none. `keepDays` is read as each slice runs,
 * so a changed setting governs the next slice.
 */
export function deployRetention(
  db: Database,
  every: number = 60 * 60_000,
  keepDays: () => number = () => DEPLOY_PULL_KEEP_DAYS,
): (now?: Date) => Pruned | undefined {
  let more = true;
  let last = 0;
@@ -107,7 +114,7 @@
export function deployRetention(
    // Mid-pass, one slice at most every `DEPLOY_PULL_PACE_MS`; a finished pass waits `every`.
    if (now.getTime() - last < (more ? DEPLOY_PULL_PACE_MS : every)) return undefined;
    last = now.getTime();
    const result = pruneDeploys(db, now, DEPLOY_PULL_SCAN, after);
    const result = pruneDeploys(db, now, DEPLOY_PULL_SCAN, after, keepDays());
    more = result.more;
    after = result.next;
    return result;
@@ -3,19 +3,20 @@
 *
 * Every reader of `event`, and the oldest row it can reach:
 *
 * Windowed by age — the longest decides KEEP_MS:
 * - `flow/estimate_base.ts`: reopened issues, 90 days (the base rate's widest level).
 * - `core/team-metrics.ts`: reopens for the team ledger, 7 days.
 * - `web/pages/security.ts`: deleted repositories, 30 days.
 * Windowed by age — the longest over a kind not KEPT decides KEEP_DAYS:
 * - `boards/sources.ts`: spawn timings by day over a board's range; the longest fixed one is 30d
 *   (`boards/store.ts` RANGES), and "all" reaches as far as the rule keeps.
 * - `flow/estimate_base.ts`: reopened issues, 90 days, and `core/team-metrics.ts`: reopens, 7 days —
 *   `issue.*`, KEPT at any age, so no window binds them.
 * - `web/pages/security.ts`: deleted repositories, 30 days (`repo.*`, KEPT).
 * - `web/studio/since.ts`: "since you were here", the digest window (days).
 * - `feed/items.ts`, `feed/quiet.ts`: the home feed, its newest 300 and a 24 h quiet window.
 * - `feed/affinity.ts`: a viewer's newest 200, weighed by a 7-day half-life.
 * - `actions/mail.ts`: mail ceiling notices, 24 h. `crew/stale_pulls.ts`: closes, 72 h.
 * - `crew/landed.ts`: comments since a pull merged (read when it merges).
 * - `boards/sources.ts`: spawn timings by day, a board's range.
 *
 * Feeds, audit views and replay cursors — they page back through every row, so the rule sets
 * their reach: they answer the last KEEP_MS, and nothing they list newer than that changes:
 * their reach: they answer the window the rule keeps, and nothing they list newer than that changes:
 * - `core/events.ts` `listEvents`/`listEventsVisibleTo`, under the events page and
 *   `/api/v1/repos/…/events` (`core/events_page.ts`), the MCP `events` tool
 *   (`http/mcp/tools/events.ts`), the TUI events screen, the Atom feed (`web/feeds.ts`, newest
@@ -41,13 +42,20 @@
 * - `pinned.check`, `clog.cleared`, `webhook.failing`: the newest of each, or a streak's mark.
 * Newest-row readers (`web/map_cache.ts`, `spawn/clients.ts`) read MAX and never reach the old end.
 *
 * A new reader that reaches past KEEP_MS names its kind in KEPT, or it is windowed.
 * A new reader that reaches past the window names its kind in KEPT, or it is windowed.
 */
import type { Database } from "bun:sqlite";
const DAY_MS = 86_400_000;
/** Twice the longest windowed reader (estimate_base's 90 days). */
export const KEEP_MS = 180 * DAY_MS;
/**
 * The default window, in days; an admin moves it with the `retention.event_days` limit
 * (`settings/limits.ts`). 30: the longest fixed window a reader asks of an unkept kind (a
 * board's 30d range). It was 180 (#12008), which on a forge younger than 180 days trims
 * nothing, while `event` was 578 MB of a 4.6 GiB file (#12052).
 */
export const KEEP_DAYS = 30;
/** The default window, in milliseconds. */
export const KEEP_MS = KEEP_DAYS * DAY_MS;
/** One batch: small enough to hold the write lock for milliseconds. */
export const BATCH_ROWS = 5_000;
/** Rows deleted per transaction: about 7 µs a row across the table's indexes. */
@@ -70,17 +78,22 @@
const passes = new WeakMap<Database, Pass>();
/**
 * One batch: walks at most `rowCap` rows from where the last batch stopped, in `seq` order (the
 * order they were written, so oldest first), and deletes those older than KEEP_MS that no reader
 * order they were written, so oldest first), and deletes those older than `keepMs` (KEEP_MS unless the setting says otherwise) that no reader
 * reaches at any age. `more` is whether rows past the rule are still to walk; when a pass meets
 * a row inside the rule it ends and rests REST_MS before starting from the oldest again. `lockMs`
 * is the longest the delete held the write lock (one CHUNK_ROWS transaction).
 */
export function pruneEvents(db: Database, now: number, rowCap: number = BATCH_ROWS): { deleted: number; more: boolean; lockMs: number } {
export function pruneEvents(
  db: Database,
  now: number,
  rowCap: number = BATCH_ROWS,
  keepMs: number = KEEP_MS,
): { deleted: number; more: boolean; lockMs: number } {
  const cap = Math.max(1, Math.min(BATCH_ROWS, Math.floor(rowCap)));
  const pass = passes.get(db) ?? { after: 0, restUntil: 0 };
  passes.set(db, pass);
  if (now < pass.restUntil) return { deleted: 0, more: false, lockMs: 0 };
  const cutoff = new Date(now - KEEP_MS).toISOString();
  const cutoff = new Date(now - keepMs).toISOString();
  const rows = db
    .query<{ seq: number; rowid: number; kind: string; created_at: string }, [number, number]>(
      "SELECT seq, rowid, kind, created_at FROM event WHERE seq > ?1 ORDER BY seq LIMIT ?2",
@@ -30,6 +30,7 @@
import {
  deployRetention,
  pruneDeploys,
} from "../src/system/retention_deploys.ts";
import { LIMITS, limitValue, setLimit } from "../src/settings/limits.ts";
quietLogs();
@@ -262,4 +263,28 @@
describe("deploy_pull retention", () => {
    expect(tick(at(60))).toBeUndefined();
    expect(tick(new Date(now.getTime() + 62 * 60_000))).toMatchObject({ deleted: 0, more: false });
  });
  test("the keep window is the setting's: deploys past it go, deploys inside it stay, each slice bounded", async () => {
    app.db.query("DELETE FROM deploy_pull").run();
    app.db.query("DELETE FROM run").run();
    app.db.query("DELETE FROM app").run();
    // Inside the default window, past a 7-day setting.
    deploy("past", merged[0]!.oid, ago(9));
    deploy("inside", merged[1]!.oid, ago(3));
    for (const id of ["past", "inside"]) await record(id, merged[1]!.oid);
    expect(pruneDeploys(app.db, now, 100)).toMatchObject({ deleted: 0 });
    const alice = getUserByName(app.db, "alice")!;
    expect(setLimit(app.db, "retention.deploy_pull_days", 7, alice.id).ok).toBe(true);
    const tick = deployRetention(app.db, undefined, () => limitValue(app.db, "retention.deploy_pull_days"));
    const r = tick(now)!;
    expect(r).toMatchObject({ deleted: 2, more: false });
    expect(rows("past")).toBe(0);
    expect(rows("inside")).toBe(2);
    // One row a slice: a slice never reads more than its cap.
    deploy("past2", merged[0]!.oid, ago(10));
    await record("past2", merged[1]!.oid);
    expect(pruneDeploys(app.db, now, 1, 0, 7)).toMatchObject({ more: true });
    expect(LIMITS["retention.deploy_pull_days"].default).toBe(DEPLOY_PULL_KEEP_DAYS);
    setLimit(app.db, "retention.deploy_pull_days", null, alice.id);
  });
});
@@ -4,6 +4,7 @@
import { migrations } from "../src/core/schema.ts";
import { listEvents } from "../src/core/events.ts";
import { countNotes } from "../src/maintain/keeper/record.ts";
import { openDb } from "../src/kernel/db.ts";
import { LIMITS } from "../src/settings/limits.ts";
import { BATCH_ROWS, KEEP_MS, keptKind, pruneEvents, REST_MS } from "../src/system/retention_events.ts";
const DAY = 86_400_000;
@@ -43,7 +44,9 @@
describe("pruneEvents", () => {
    put(db, "keeper_said", KEEP_MS + 10 * DAY, null, { key: "k1" });
    put(db, "push", KEEP_MS + 5 * DAY);
    const recent: string[] = [];
    for (let d = 170; d >= 0; d -= 10) recent.push(put(db, d % 20 === 0 ? "secret.read" : "issue.reopened", d * DAY, "r1", { number: d }));
    // Eighteen rows spread over the window, oldest first, alternating a windowed and a kept kind.
    for (let i = 17; i >= 0; i--)
      recent.push(put(db, i % 2 === 0 ? "secret.read" : "issue.reopened", Math.floor((i * (KEEP_MS - DAY)) / 17), "r1", { number: i }));
    const cutoff = iso(NOW - KEEP_MS);
    const readers = () => ({
@@ -95,7 +98,22 @@
describe("pruneEvents", () => {
    for (const k of ["issue.closed", "pull.pushed", "repo.detached", "user.created", "push", "keeper_lull", "forge_down", "app.egress.granted"])
      expect(keptKind(k)).toBe(true);
    for (const k of ["secret.read", "forge.spawn.live", "mail.ceiling", "measurement"]) expect(keptKind(k)).toBe(false);
    expect(KEEP_MS).toBeGreaterThanOrEqual(90 * DAY);
    // The longest fixed window over an unkept kind is a board's 30d range (`boards/store.ts`).
    expect(KEEP_MS).toBeGreaterThanOrEqual(30 * DAY);
    expect(LIMITS["retention.event_days"].default * DAY).toBe(KEEP_MS);
  });
  test("the keep window is the caller's: rows past it go, rows inside it stay, a batch is bounded", () => {
    const db = seeded();
    const keep = 60 * DAY;
    const past = [put(db, "secret.read", keep + 2 * DAY), put(db, "secret.read", keep + DAY)];
    // Past the default window, inside this one.
    const inside = [put(db, "secret.read", KEEP_MS + DAY), put(db, "secret.read", DAY)];
    expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 1, more: true });
    expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 1, more: true });
    expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 0, more: false });
    for (const id of past) expect(db.query("SELECT 1 FROM event WHERE id = ?1").get(id)).toBeNull();
    for (const id of inside) expect(db.query("SELECT 1 FROM event WHERE id = ?1").get(id)).not.toBeNull();
  });
  test("one 5,000-row batch holds the write lock under 50 ms", () => {
		@@ -137,7 +137,7 @@ bytes in `tools/list`. After changing a tool run `bun scripts/mcp-table.ts`;
137	137	\| `land_past_gate` \| — \| Land one pull past a broken gate, with a recorded reason; an owner's only \| 321 \|
138	138	\| `lift` \| — \| Lift an issue out from behind the Later gate \| 214 \|
139	139	\| `limb` \| `defer`, `resume`, `close`, `raise`, `lower`, `replan`, `takeback`, `undo` \| Act on a limb by `action` \| 532 \|
140		\| `limits` \| — \| Every limit the owner can hit \| 1025 \|
	140	\| `limits` \| — \| Every limit the owner can hit \| 1077 \|
141	141	\| `maintenance` \| `status`, `prepare`, `resume`, `pause-duty`, `resume-duty` \| Prepare the forge for a power-off, check the drain, or end maintenance (admin) \| 308 \|
142	142	\| `make_it_yours` \| — \| Fork an app you can see: code and history, following the original \| 337 \|
143	143	\| `map` \| — \| An app's plan: ready or waiting, critical path, crew, parkedTwice; `done` adds closed; `first: N` ranks N first \| 375 \|
		@@ -6,13 +6,13 @@
6	6
7	7	`src/system/retention_deploys.ts` keeps the rows of:
8	8
9		- deploys from the last `DEPLOY_PULL_KEEP_DAYS` (14) days, and
	9	- deploys from the last `retention.deploy_pull_days` days (14 by default, `DEPLOY_PULL_KEEP_DAYS`; an admin moves it on `/admin/limits`, never below 7, the week ask-to-live reads), and
10	10	- the deploy each app is running now (its newest successful production deploy), however old.
11	11
12	12	Every other deploy's rows are deleted by the crew tick in batches of at most `DEPLOY_PULL_BATCH` (5,000) rows. The tick reads the candidate rows first, then deletes them by rowid in one short transaction. It runs one batch per tick while each batch comes back full, then one batch an hour after that. When it deletes something, the log says `trimmed deploy_pull`.
13	13
14	14	## Asking for an older deploy
15	15
16		You don't have to do anything. Open `/<owner>/<repo>/runs/<id>` for a production deploy older than 14 days, or call the run API, or open the app's deploys screen in the terminal. The forge walks that deploy's commit in git (`carriedFor` / `carriedForRuns` in `src/core/carried.ts`), finds the merged pulls it contains and saves them again. You get the same list as before the trim, and later views read the saved rows.
	16	You don't have to do anything. Open `/<owner>/<repo>/runs/<id>` for a production deploy older than the window, or call the run API, or open the app's deploys screen in the terminal. The forge walks that deploy's commit in git (`carriedFor` / `carriedForRuns` in `src/core/carried.ts`), finds the merged pulls it contains and saves them again. You get the same list as before the trim, and later views read the saved rows.
17	17
18	18	Each running forge walks a deploy at most once at a time; views of the same deploy that arrive together share one walk. If a deploy carries no pulls, it is walked once and the empty answer is remembered for as long as the forge runs, for up to 10,000 deploys. If git can't walk the commit, the forge waits a minute before trying again. Preview deploys and deploys that failed or are still running are never walked.
		@@ -0,0 +1,15 @@
	1	# How long the event log keeps what it holds
	2
	3	`event` is the forge's log of what happened. `src/system/retention_events.ts` says which kinds are state read back at any age and are never trimmed: `issue.`, `pull.`, `repo.`, `user.`, `push`, `app.egress.`, `keeper_` and a few marks. Every other kind is trimmed once it is older than `retention.event_days` (#12008, #12052).
	4
	5	## The window
	6
	7	The default is 30 days (`KEEP_DAYS`). That is the longest fixed window any page or tool asks of a kind that is not kept: a board's `30d` range. The home feed, the digest and the affinity weights read a week or less. The events page, the events API, the MCP `events` tool, the Atom feed and SSE replay page back through the log, so for unkept kinds they answer the window and nothing older.
	8
	9	It was 180 days. On a forge younger than that, which production is (it started 2026-08-03), the rule trimmed nothing while `event` held 578 MB of a 4.6 GiB file.
	10
	11	An admin moves the window on `/admin/limits` (Days the event log keeps what no page reads at any age), from 7 to 3,650 days. The sampler's next 15-second tick reads the new value.
	12
	13	## How it trims
	14
	15	The sampler's tick deletes at most 1,000 rows, a chunk of 1,000 per transaction, oldest first. When a pass reaches rows inside the window, it waits a day before starting again. A webhook delivery that is still pending keeps its event.
		@@ -30,6 +30,7 @@ import { FINE_MS, record, type Point } from "./ring.ts";
30	30	import { readHwmon, readTopology, temperatures } from "./thermal.ts";
31	31	import { during } from "../perf/jobs.ts";
32	32	import { withoutWaiting } from "../kernel/db.ts";
	33	import { limitValue } from "../settings/limits.ts";
33	34	import { pruneEvents, TICK_ROWS } from "../system/retention_events.ts";
34	35	import { drainSamples } from "../system/retention_samples.ts";
35	36
		@@ -176,7 +177,7 @@ export function startSampler(
176	177	// The same tick prunes one batch of old events (`system/retention_events.ts`); a busy
177	178	// database skips it, and the next tick tries again.
178	179	try {
179		withoutWaiting(app.db, () => pruneEvents(app.db, now, TICK_ROWS));
	180	withoutWaiting(app.db, () => pruneEvents(app.db, now, TICK_ROWS, limitValue(app.db, "retention.event_days") * 86_400_000));
180	181	} catch (error) {
181	182	app.log.warn("event prune failed", { error: String(error) });
182	183	}
		@@ -41,6 +41,7 @@ import {
41	41	import { closesFromBody, getPull, pullDir, type Pull } from "../core/pulls.ts";
42	42	import { claimedClones, sweepOrphanClones } from "./orphans.ts";
43	43	import { pruneRunFiles } from "../runs/prune.ts";
	44	import { limitValue } from "../settings/limits.ts";
44	45	import { deployRetention } from "../system/retention_deploys.ts";
45	46	import { crewRoot, isClaimed } from "./workdir.ts";
46	47	import { resolveOpen } from "../core/decisions.ts";
		@@ -274,7 +275,7 @@ export async function startCrew(
274	275	// same clock as the pump, so a pick becomes a run on the next tick.
275	276	let lastAutonomy = 0;
276	277	// deploy_pull keeps recent deploys only (#12009): a capped batch a tick while rows remain.
277		const trimDeploys = deployRetention(app.db);
	278	const trimDeploys = deployRetention(app.db, undefined, () => limitValue(app.db, "retention.deploy_pull_days"));
278	279	// The sweeps a tick started and has not seen settle: `stop()` waits for
279	280	// them, so none reads `app.db` after its caller closed it (#9977).
280	281	const sweeps = new Set<Promise<unknown>>();
		@@ -21,6 +21,8 @@ import {
21	21	type CapacityKey,
22	22	} from "../core/capacity.ts";
23	23	import { err, ok, type Result } from "../kernel/result.ts";
	24	import { DEPLOY_PULL_KEEP_DAYS } from "../system/retention_deploys.ts";
	25	import { KEEP_DAYS as EVENT_KEEP_DAYS } from "../system/retention_events.ts";
24	26
25	27	export type LimitKey =
26	28	\| "crew.concurrency"
		@@ -55,7 +57,9 @@ export type LimitKey =
55	57	\| "model_review.slack_pct"
56	58	\| "shell_idle_minutes"
57	59	\| "billing.grace_days"
58		\| "billing.retention_days";
	60	\| "billing.retention_days"
	61	\| "retention.event_days"
	62	\| "retention.deploy_pull_days";
59	63
60	64	export interface LimitDef {
61	65	readonly label: string;
		@@ -339,6 +343,22 @@ export const LIMITS: Record<LimitKey, LimitDef> = {
339	343	max: 365,
340	344	reason: "A forge that lapsed may come back, so its data waits this long before anything is allowed to remove it; paying again makes it active at once.",
341	345	},
	346	"retention.event_days": {
	347	label: "Days the event log keeps what no page reads at any age",
	348	unit: "days",
	349	default: EVENT_KEEP_DAYS,
	350	min: 7,
	351	max: 3_650,
	352	reason: "Issue, pull, push and account events are kept at any age; the rest feeds windows of 30 days at most, and kept longer they filled the database past the 2 GiB Bun maps.",
	353	},
	354	"retention.deploy_pull_days": {
	355	label: "Days a deploy keeps the list of pulls it carried",
	356	unit: "days",
	357	default: DEPLOY_PULL_KEEP_DAYS,
	358	min: 7,
	359	max: 3_650,
	360	reason: "The list is an index over git, rebuilt when an older deploy is opened, and it was the largest table in the database; a week is what ask-to-live reads.",
	361	},
342	362	};
343	363
344	364	export const LIMIT_KEYS = Object.keys(LIMITS) as LimitKey[];
		@@ -17,7 +17,11 @@
17	17	import type { Database } from "bun:sqlite";
18	18	import { transact } from "../kernel/db.ts";
19	19
20		/** Deploys this recent keep their rows. Above `PILLAR_DAYS` (7), the window ask-to-live reads. */
	20	/**
	21	* Deploys this recent keep their rows, by default; an admin moves it with the
	22	* `retention.deploy_pull_days` limit (`settings/limits.ts`), never below
	23	* `PILLAR_DAYS` (7), the window ask-to-live reads.
	24	*/
21	25	export const DEPLOY_PULL_KEEP_DAYS = 14;
22	26	/** The least time between two slices of one pass. */
23	27	const DEPLOY_PULL_PACE_MS = 10_000;
		@@ -47,6 +51,7 @@ export function pruneDeploys(
47	51	now: Date = new Date(),
48	52	rowCap: number = DEPLOY_PULL_SCAN,
49	53	after = 0,
	54	keepDays: number = DEPLOY_PULL_KEEP_DAYS,
50	55	): Pruned & { readonly next: number } {
51	56	const cap = Math.max(0, Math.floor(rowCap));
52	57	if (cap === 0) return { deleted: 0, more: false, next: after };
		@@ -56,7 +61,7 @@ export function pruneDeploys(
56	61	)
57	62	.all(after, cap);
58	63	if (slice.length === 0) return { deleted: 0, more: false, next: 0 };
59		const cutoff = new Date(now.getTime() - DEPLOY_PULL_KEEP_DAYS * 86_400_000).toISOString();
	64	const cutoff = new Date(now.getTime() - keepDays * 86_400_000).toISOString();
60	65	const runIds = [...new Set(slice.map((row) => row.run_id))];
61	66	const old = new Set<string>();
62	67	const isOld = db.query<{ id: string; repo_id: string }, [string, string]>(
		@@ -94,11 +99,13 @@ export function pruneDeploys(
94	99	/**
95	100	* The crew tick's view of the rule: a batch on each tick while the last one
96	101	* was full, and otherwise one an hour. Returns the batch's result, or
97		* undefined when this tick runs none.
	102	* undefined when this tick runs none. `keepDays` is read as each slice runs,
	103	* so a changed setting governs the next slice.
98	104	*/
99	105	export function deployRetention(
100	106	db: Database,
101	107	every: number = 60 * 60_000,
	108	keepDays: () => number = () => DEPLOY_PULL_KEEP_DAYS,
102	109	): (now?: Date) => Pruned \| undefined {
103	110	let more = true;
104	111	let last = 0;
		@@ -107,7 +114,7 @@ export function deployRetention(
107	114	// Mid-pass, one slice at most every `DEPLOY_PULL_PACE_MS`; a finished pass waits `every`.
108	115	if (now.getTime() - last < (more ? DEPLOY_PULL_PACE_MS : every)) return undefined;
109	116	last = now.getTime();
110		const result = pruneDeploys(db, now, DEPLOY_PULL_SCAN, after);
	117	const result = pruneDeploys(db, now, DEPLOY_PULL_SCAN, after, keepDays());
111	118	more = result.more;
112	119	after = result.next;
113	120	return result;
		@@ -3,19 +3,20 @@
3	3	*
4	4	* Every reader of `event`, and the oldest row it can reach:
5	5	*
6		* Windowed by age — the longest decides KEEP_MS:
7		* - `flow/estimate_base.ts`: reopened issues, 90 days (the base rate's widest level).
8		* - `core/team-metrics.ts`: reopens for the team ledger, 7 days.
9		* - `web/pages/security.ts`: deleted repositories, 30 days.
	6	* Windowed by age — the longest over a kind not KEPT decides KEEP_DAYS:
	7	* - `boards/sources.ts`: spawn timings by day over a board's range; the longest fixed one is 30d
	8	* (`boards/store.ts` RANGES), and "all" reaches as far as the rule keeps.
	9	* - `flow/estimate_base.ts`: reopened issues, 90 days, and `core/team-metrics.ts`: reopens, 7 days —
	10	* `issue.*`, KEPT at any age, so no window binds them.
	11	* - `web/pages/security.ts`: deleted repositories, 30 days (`repo.*`, KEPT).
10	12	* - `web/studio/since.ts`: "since you were here", the digest window (days).
11	13	* - `feed/items.ts`, `feed/quiet.ts`: the home feed, its newest 300 and a 24 h quiet window.
12	14	* - `feed/affinity.ts`: a viewer's newest 200, weighed by a 7-day half-life.
13	15	* - `actions/mail.ts`: mail ceiling notices, 24 h. `crew/stale_pulls.ts`: closes, 72 h.
14	16	* - `crew/landed.ts`: comments since a pull merged (read when it merges).
15		* - `boards/sources.ts`: spawn timings by day, a board's range.
16	17	*
17	18	* Feeds, audit views and replay cursors — they page back through every row, so the rule sets
18		* their reach: they answer the last KEEP_MS, and nothing they list newer than that changes:
	19	* their reach: they answer the window the rule keeps, and nothing they list newer than that changes:
19	20	* - `core/events.ts` `listEvents`/`listEventsVisibleTo`, under the events page and
20	21	* `/api/v1/repos/…/events` (`core/events_page.ts`), the MCP `events` tool
21	22	* (`http/mcp/tools/events.ts`), the TUI events screen, the Atom feed (`web/feeds.ts`, newest
		@@ -41,13 +42,20 @@
41	42	* - `pinned.check`, `clog.cleared`, `webhook.failing`: the newest of each, or a streak's mark.
42	43	* Newest-row readers (`web/map_cache.ts`, `spawn/clients.ts`) read MAX and never reach the old end.
43	44	*
44		* A new reader that reaches past KEEP_MS names its kind in KEPT, or it is windowed.
	45	* A new reader that reaches past the window names its kind in KEPT, or it is windowed.
45	46	*/
46	47	import type { Database } from "bun:sqlite";
47	48
48	49	const DAY_MS = 86_400_000;
49		/** Twice the longest windowed reader (estimate_base's 90 days). */
50		export const KEEP_MS = 180 * DAY_MS;
	50	/**
	51	* The default window, in days; an admin moves it with the `retention.event_days` limit
	52	* (`settings/limits.ts`). 30: the longest fixed window a reader asks of an unkept kind (a
	53	* board's 30d range). It was 180 (#12008), which on a forge younger than 180 days trims
	54	* nothing, while `event` was 578 MB of a 4.6 GiB file (#12052).
	55	*/
	56	export const KEEP_DAYS = 30;
	57	/** The default window, in milliseconds. */
	58	export const KEEP_MS = KEEP_DAYS * DAY_MS;
51	59	/** One batch: small enough to hold the write lock for milliseconds. */
52	60	export const BATCH_ROWS = 5_000;
53	61	/** Rows deleted per transaction: about 7 µs a row across the table's indexes. */
		@@ -70,17 +78,22 @@ const passes = new WeakMap<Database, Pass>();
70	78
71	79	/**
72	80	* One batch: walks at most `rowCap` rows from where the last batch stopped, in `seq` order (the
73		* order they were written, so oldest first), and deletes those older than KEEP_MS that no reader
	81	* order they were written, so oldest first), and deletes those older than `keepMs` (KEEP_MS unless the setting says otherwise) that no reader
74	82	* reaches at any age. `more` is whether rows past the rule are still to walk; when a pass meets
75	83	* a row inside the rule it ends and rests REST_MS before starting from the oldest again. `lockMs`
76	84	* is the longest the delete held the write lock (one CHUNK_ROWS transaction).
77	85	*/
78		export function pruneEvents(db: Database, now: number, rowCap: number = BATCH_ROWS): { deleted: number; more: boolean; lockMs: number } {
	86	export function pruneEvents(
	87	db: Database,
	88	now: number,
	89	rowCap: number = BATCH_ROWS,
	90	keepMs: number = KEEP_MS,
	91	): { deleted: number; more: boolean; lockMs: number } {
79	92	const cap = Math.max(1, Math.min(BATCH_ROWS, Math.floor(rowCap)));
80	93	const pass = passes.get(db) ?? { after: 0, restUntil: 0 };
81	94	passes.set(db, pass);
82	95	if (now < pass.restUntil) return { deleted: 0, more: false, lockMs: 0 };
83		const cutoff = new Date(now - KEEP_MS).toISOString();
	96	const cutoff = new Date(now - keepMs).toISOString();
84	97	const rows = db
85	98	.query<{ seq: number; rowid: number; kind: string; created_at: string }, [number, number]>(
86	99	"SELECT seq, rowid, kind, created_at FROM event WHERE seq > ?1 ORDER BY seq LIMIT ?2",
		@@ -30,6 +30,7 @@ import {
30	30	deployRetention,
31	31	pruneDeploys,
32	32	} from "../src/system/retention_deploys.ts";
	33	import { LIMITS, limitValue, setLimit } from "../src/settings/limits.ts";
33	34
34	35	quietLogs();
35	36
		@@ -262,4 +263,28 @@ describe("deploy_pull retention", () => {
262	263	expect(tick(at(60))).toBeUndefined();
263	264	expect(tick(new Date(now.getTime() + 62 * 60_000))).toMatchObject({ deleted: 0, more: false });
264	265	});
	266
	267	test("the keep window is the setting's: deploys past it go, deploys inside it stay, each slice bounded", async () => {
	268	app.db.query("DELETE FROM deploy_pull").run();
	269	app.db.query("DELETE FROM run").run();
	270	app.db.query("DELETE FROM app").run();
	271	// Inside the default window, past a 7-day setting.
	272	deploy("past", merged[0]!.oid, ago(9));
	273	deploy("inside", merged[1]!.oid, ago(3));
	274	for (const id of ["past", "inside"]) await record(id, merged[1]!.oid);
	275	expect(pruneDeploys(app.db, now, 100)).toMatchObject({ deleted: 0 });
	276	const alice = getUserByName(app.db, "alice")!;
	277	expect(setLimit(app.db, "retention.deploy_pull_days", 7, alice.id).ok).toBe(true);
	278	const tick = deployRetention(app.db, undefined, () => limitValue(app.db, "retention.deploy_pull_days"));
	279	const r = tick(now)!;
	280	expect(r).toMatchObject({ deleted: 2, more: false });
	281	expect(rows("past")).toBe(0);
	282	expect(rows("inside")).toBe(2);
	283	// One row a slice: a slice never reads more than its cap.
	284	deploy("past2", merged[0]!.oid, ago(10));
	285	await record("past2", merged[1]!.oid);
	286	expect(pruneDeploys(app.db, now, 1, 0, 7)).toMatchObject({ more: true });
	287	expect(LIMITS["retention.deploy_pull_days"].default).toBe(DEPLOY_PULL_KEEP_DAYS);
	288	setLimit(app.db, "retention.deploy_pull_days", null, alice.id);
	289	});
265	290	});
		@@ -4,6 +4,7 @@ import { migrations } from "../src/core/schema.ts";
4	4	import { listEvents } from "../src/core/events.ts";
5	5	import { countNotes } from "../src/maintain/keeper/record.ts";
6	6	import { openDb } from "../src/kernel/db.ts";
	7	import { LIMITS } from "../src/settings/limits.ts";
7	8	import { BATCH_ROWS, KEEP_MS, keptKind, pruneEvents, REST_MS } from "../src/system/retention_events.ts";
8	9
9	10	const DAY = 86_400_000;
		@@ -43,7 +44,9 @@ describe("pruneEvents", () => {
43	44	put(db, "keeper_said", KEEP_MS + 10 * DAY, null, { key: "k1" });
44	45	put(db, "push", KEEP_MS + 5 * DAY);
45	46	const recent: string[] = [];
46		for (let d = 170; d >= 0; d -= 10) recent.push(put(db, d % 20 === 0 ? "secret.read" : "issue.reopened", d * DAY, "r1", { number: d }));
	47	// Eighteen rows spread over the window, oldest first, alternating a windowed and a kept kind.
	48	for (let i = 17; i >= 0; i--)
	49	recent.push(put(db, i % 2 === 0 ? "secret.read" : "issue.reopened", Math.floor((i * (KEEP_MS - DAY)) / 17), "r1", { number: i }));
47	50
48	51	const cutoff = iso(NOW - KEEP_MS);
49	52	const readers = () => ({
		@@ -95,7 +98,22 @@ describe("pruneEvents", () => {
95	98	for (const k of ["issue.closed", "pull.pushed", "repo.detached", "user.created", "push", "keeper_lull", "forge_down", "app.egress.granted"])
96	99	expect(keptKind(k)).toBe(true);
97	100	for (const k of ["secret.read", "forge.spawn.live", "mail.ceiling", "measurement"]) expect(keptKind(k)).toBe(false);
98		expect(KEEP_MS).toBeGreaterThanOrEqual(90 * DAY);
	101	// The longest fixed window over an unkept kind is a board's 30d range (`boards/store.ts`).
	102	expect(KEEP_MS).toBeGreaterThanOrEqual(30 * DAY);
	103	expect(LIMITS["retention.event_days"].default * DAY).toBe(KEEP_MS);
	104	});
	105
	106	test("the keep window is the caller's: rows past it go, rows inside it stay, a batch is bounded", () => {
	107	const db = seeded();
	108	const keep = 60 * DAY;
	109	const past = [put(db, "secret.read", keep + 2 * DAY), put(db, "secret.read", keep + DAY)];
	110	// Past the default window, inside this one.
	111	const inside = [put(db, "secret.read", KEEP_MS + DAY), put(db, "secret.read", DAY)];
	112	expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 1, more: true });
	113	expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 1, more: true });
	114	expect(pruneEvents(db, NOW, 1, keep)).toMatchObject({ deleted: 0, more: false });
	115	for (const id of past) expect(db.query("SELECT 1 FROM event WHERE id = ?1").get(id)).toBeNull();
	116	for (const id of inside) expect(db.query("SELECT 1 FROM event WHERE id = ?1").get(id)).not.toBeNull();
99	117	});
100	118
101	119	test("one 5,000-row batch holds the write lock under 50 ms", () => {