Public: anyone can read this; the crew can change it.

Per-account limits on live previews, concurrent runs and concurrent checks, refused with the count before anything starts #2184

Merged opened this

Check · Review · Land · Live done — Landed and live in production.

Merged

scout/forge:scout/918-per-account-limits-on-live-previews-conc → main 6 commits fork 4 threads · 4 unresolved

  • Merged. scout/918-per-account-limits-on-live-previews-conc was merged into main as 0d3e2d9.

Review

Sign in to review this change.
9 files changed +659 −15
README.md +1 −1

Diff not shown — a line 4,193 characters wide (limit 2,000). +1 −1

View file
src/apps/proxy.ts +1 −1
@@ -1016,7 +1016,7 @@ function unavailable(
10161016 reason ??
10171017 (deploying
10181018 ? "A new version is starting."
1019 : state === "failed"
1019 : state === "failed" || (state === "asleep" && status?.error !== undefined)
10201020 ? (status?.error ?? "The last deploy failed.")
10211021 : gone
10221022 ? "This preview is gone: its pull was closed, and its data with it."
src/apps/runtime.ts +64
@@ -31,6 +31,7 @@ import { requeueAbandonedChecks } from "../checks/service.ts";
3131 import { getPull, listPulls, type Pull } from "../core/pulls.ts";
3232 import { pullMayRun } from "../core/pull_trust.ts";
3333 import { canEnterApp, getRepoById, type Repo } from "../core/repos.ts";
34 import { runningLimitSentence } from "../settings/quotas.ts";
3435 import type { User } from "../core/users.ts";
3536 import {
3637 cancelRun,
@@ -321,6 +322,8 @@ interface Environment {
321322 dormant?:
322323 | { since: string; idleSince: string; commitOid: string }
323324 | undefined;
325 /** Why the last wake did not happen (the owner's preview limit), for the page a visitor sees. */
326 wakeRefused?: string | undefined;
324327 /** One wake at a time; every request that arrives meanwhile waits on it. */
325328 waking?: Promise<void> | undefined;
326329 /** When something last came through the proxy for this environment. */
@@ -583,6 +586,7 @@ export class AppsRuntime {
583586 url,
584587 runId: state.asleep.runId,
585588 since: state.asleep.since,
589 error: state.wakeRefused,
586590 };
587591 }
588592 return {
@@ -988,6 +992,30 @@ export class AppsRuntime {
988992 return state;
989993 }
990994
995 /**
996 * Previews of the account's apps that are up, `except` aside, and those
997 * coming up whose run is older than `before`: of two deploying together
998 * the older counts against the younger, never both against each other.
999 * With no `before`, every one coming up counts (a wake has no run).
1000 */
1001 private livePreviews(
1002 ownerId: string,
1003 except: Environment,
1004 before?: string,
1005 ): number {
1006 let n = 0;
1007 for (const other of this.envs.values()) {
1008 if (other === except || other.env === "production") continue;
1009 const up =
1010 other.live !== undefined ||
1011 (other.deploying !== undefined &&
1012 (before === undefined || other.deploying.run.id < before));
1013 if (!up) continue;
1014 if (getRepoById(this.app.db, other.app.repoId)?.ownerId === ownerId) n++;
1015 }
1016 return n;
1017 }
1018
9911019 private async execute(
9921020 state: Environment,
9931021 repo: Repo,
@@ -1002,6 +1030,28 @@ export class AppsRuntime {
10021030 const write = (line: string): void =>
10031031 appendRunLog(this.app.config.dataDir, run.id, line);
10041032 announceRun(this.app, claimed);
1033 // An account's live previews are bounded (`settings/quotas.ts`): a
1034 // preview past the limit does not start, and its run — the check the
1035 // pull shows — fails saying why, so the pull never waits for it.
1036 const refusal =
1037 state.env === "production" || state.live !== undefined
1038 ? undefined
1039 : runningLimitSentence(
1040 this.app.db,
1041 "previews",
1042 "live previews",
1043 repo.ownerId,
1044 () => this.livePreviews(repo.ownerId, state, run.id),
1045 );
1046 if (refusal !== undefined) {
1047 state.deploying = undefined;
1048 lease();
1049 const message = `not started: ${refusal}. Close a pull or stop a preview to make room.`;
1050 write(message);
1051 const done = finishRun(this.app.db, run.id, { state: "failed", summary: message });
1052 if (done) announceRun(this.app, done);
1053 return;
1054 }
10051055 write(
10061056 `deploy ${state.env} of ${repo.ownerName}/${repo.name} at ${run.commitOid.slice(0, 7)} (${run.reason})`,
10071057 );
@@ -1519,6 +1569,20 @@ export class AppsRuntime {
15191569 await this.teardown(record, env);
15201570 return undefined;
15211571 }
1572 // A wake starts a process too: past the account's preview limit it stays asleep.
1573 const full = runningLimitSentence(
1574 this.app.db,
1575 "previews",
1576 "live previews",
1577 repo.ownerId,
1578 () => this.livePreviews(repo.ownerId, state),
1579 );
1580 if (full !== undefined) {
1581 log.info("preview stays asleep", { app: record.slug, env, limit: full });
1582 state.wakeRefused = `This preview stays asleep: its owner is at ${full}. Close a pull or stop a preview to make room.`;
1583 return undefined;

This wake refusal and the wakeRefused text shown by the proxy have no test. Add one that wakes an asleep preview at the limit and asserts that it stays asleep and that the page says why.

1584 }
1585 state.wakeRefused = undefined;
15221586 state.waking = (async () => {
15231587 const write = (line: string): void => {
15241588 if (asleep.runId !== "")
src/checks/service.ts +32 −1
@@ -57,6 +57,7 @@ import { transact } from "../kernel/db.ts";
5757 import { createLog } from "../kernel/log.ts";
5858 import { holdLease, STALE_AFTER_MS } from "../runs/lease.ts";
5959 import { appendRunLog } from "../runs/log.ts";
60 import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
6061 import { exportTree, readFileAt } from "../apps/checkout.ts";
6162 import { parseCheck } from "../apps/manifest.ts";
6263 import { accountActive } from "../core/accounts.ts";
@@ -250,6 +251,8 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
250251 string,
251252 { promise: Promise<void>; abort: AbortController }
252253 >();
254 /** Checks already told why they wait, so the log says it once. */
255 const waiting = new Set<string>();
253256 let stopped = false;
254257
255258 sweepCheckouts(app);
@@ -331,6 +334,7 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
331334 /** Run a queued check here, whichever forge queued it. */
332335 function start(run: Run, repo: Repo): void {
333336 if (inFlight.has(run.id)) return;
337 waiting.delete(run.id);
334338 const abort = new AbortController();
335339 const promise = execute(run, repo, abort.signal)
336340 .catch((error) =>
@@ -348,6 +352,24 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
348352 inFlight.set(run.id, { promise, abort });
349353 }
350354
355 /**
356 * An account at its concurrent-checks limit (`settings/quotas.ts`) keeps
357 * the check queued, the count in its log once, until one of its own ends.
358 */
359 function atLimit(run: Run): boolean {
360 const owner = getRepoById(app.db, run.repoId)?.ownerId;
361 if (owner === undefined) return false;
outdated

This block is inserted between schedule()s doc comment and schedule(); move atLimit above that comment.

362 const full = runningLimitSentence(app.db, "checks", "concurrent checks", owner, () =>
363 runningRuns(app.db, "check", owner, inFlight.keys()),
364 );
365 if (full === undefined) return false;
366 if (!waiting.has(run.id)) {
367 waiting.add(run.id);
368 appendRunLog(app.config.dataDir, run.id, `queued, not started: ${full}`);
369 }
370 return true;
371 }
372
351373 /**
352374 * Fill every free slot with the queued check `pickNext` says deserves it —
353375 * one the forge before this one never reached, one a forge stopping
@@ -368,7 +390,16 @@ export function startChecks(app: App, options: ChecksOptions): ChecksHandle {
368390 (repoId) => redMain(app.db, repoId),
369391 options.redHoldMs ?? 0,
370392 );
371 const next = pickNext(queued);
393 // The limit is asked at the point of starting, one pick at a time:
394 // a started check is in flight before the next is asked about, so a
395 // burst of one account's checks cannot pass together.
396 let rest = queued;
397 let next = pickNext(rest);
398 while (next !== undefined && atLimit(next)) {
399 const held = next;
400 rest = rest.filter((run) => run !== held);
401 next = pickNext(rest);
402 }
372403 if (!next) return;
373404 const repo = getRepoById(app.db, next.repoId);
374405 if (!repo) return;
src/crew/dispatch.ts +26
@@ -30,6 +30,8 @@ import {
3030 import { getPull, type Pull } from "../core/pulls.ts";
3131 import { resolveOpen } from "../core/decisions.ts";
3232 import { accountActive } from "../core/accounts.ts";
33 import { runningLimitSentence, runningRuns } from "../settings/quotas.ts";
34 import { appendRunLog } from "../runs/log.ts";
3335 import { decideAndRecord } from "./ledger.ts";
3436 import { getRepoById, type Repo } from "../core/repos.ts";
3537 import { listReviews } from "../core/reviews.ts";
@@ -1085,6 +1087,9 @@ function sweep(app: App): void {
10851087 }
10861088 }
10871089
1090 /** Per app, the runs already told why they wait, so the log says it once. */
1091 const notedBy = new WeakMap<App, Set<string>>();
outdated

Module-global set that never shrinks. Remove ids when the run starts or finishes, and keep it per app.

1092
10881093 /** Take queued runs, oldest first, up to the limit — one per issue at a time. */
10891094 function pump(
10901095 app: App,
@@ -1100,11 +1105,32 @@ function pump(
11001105 let slots =
11011106 capacityCount(app.db, "crew.concurrency", options.concurrency) -
11021107 runningIssues.size;
1108 const started: string[] = [];
1109 const noted = notedBy.get(app) ?? new Set<string>();
1110 notedBy.set(app, noted);
11031111 for (const queued of listQueued(app.db, "agent", 50)) {
11041112 if (slots <= 0) break;
11051113 if (queued.issueId !== null && runningIssues.has(queued.issueId)) continue;
11061114 const fresh = getRun(app.db, queued.id);
11071115 if (!fresh || fresh.state !== "queued") continue;
1116 // An account at its concurrent-runs limit keeps this run queued, with
1117 // the count in its log once, until one of its runs ends.
1118 const owner = getRepoById(app.db, fresh.repoId)?.ownerId;
1119 const full =
1120 owner === undefined
1121 ? undefined
1122 : runningLimitSentence(app.db, "runs", "concurrent runs", owner, () =>
1123 runningRuns(app.db, "agent", owner, [...inFlight.keys(), ...started]),
1124 );
1125 if (full !== undefined) {
1126 if (!noted.has(fresh.id)) {
1127 noted.add(fresh.id);
1128 appendRunLog(app.config.dataDir, fresh.id, `queued, not started: ${full}`);
1129 }
1130 continue;
1131 }
1132 started.push(fresh.id);
1133 noted.delete(fresh.id);
11081134 slots--;
11091135 if (queued.issueId !== null) runningIssues.add(queued.issueId);
11101136 const running = executeAgentRun(app, fresh, options)
src/settings/quotas.ts +67 −2
@@ -6,14 +6,18 @@
66 * (`actions/app_limit.ts`); creating, forking and copying a repository ask the
77 * repositories limit first (`actions/repos.ts`). `storage` is megabytes
88 * of repositories plus app data (`core/storage.ts`), asked on a push, create
9 * and fork. Room for more keys is the `QUOTA_KEYS` list.
9 * and fork. The running limits — live `previews`, concurrent agent `runs` and
10 * concurrent `checks` — bound what an account has going at once; each is asked
11 * before the thing starts (`apps/runtime.ts`, `crew/dispatch.ts`,
12 * `checks/service.ts`) and never of a forge admin. Room for more keys is the
13 * `QUOTA_KEYS` list.
1014 */
1115
1216 import type { Database } from "bun:sqlite";
1317 import { err, ok, type Result } from "../kernel/result.ts";
1418 import { writeSetting } from "./limits.ts";
1519
16 export const QUOTA_KEYS = ["repositories", "apps", "storage"] as const;
20 export const QUOTA_KEYS = ["repositories", "apps", "storage", "previews", "runs", "checks"] as const;
1721 export type QuotaKey = (typeof QUOTA_KEYS)[number];
1822 export type Quotas = Record<QuotaKey, number | null>;
1923
@@ -52,6 +56,9 @@ export function getQuotas(db: Database): Quotas {
5256 repositories: getQuota(db, "repositories"),
5357 apps: getQuota(db, "apps"),
5458 storage: getQuota(db, "storage"),
59 previews: getQuota(db, "previews"),
60 runs: getQuota(db, "runs"),
61 checks: getQuota(db, "checks"),
5562 };
5663 }
5764
@@ -74,6 +81,64 @@ export function repositoryLimitSentence(db: Database, ownerId: string): string |
7481 return owned >= limit ? `${owned} of ${limit} repositories` : undefined;
7582 }
7683
84 /** Whether `ownerId` is a forge admin: the forge's own account is never limited. */
85 function exempt(db: Database, ownerId: string): boolean {
86 return (
87 db
88 .query<{ is_admin: number }, [string]>("SELECT is_admin FROM user WHERE id = ?1")
89 .get(ownerId)?.is_admin === 1
90 );
91 }
92
93 /**
94 * The sentence a refused start says, or nothing when there is room: `held`
95 * is how many of `noun` ("live previews", "runs", "checks") the account
96 * has going now, asked against the `key` limit.
97 */
98 export function runningLimitSentence(
99 db: Database,
100 key: "previews" | "runs" | "checks",
101 noun: string,
102 ownerId: string,
103 held: () => number,
104 ): string | undefined {
105 const limit = getQuota(db, key);
106 if (limit === null || exempt(db, ownerId)) return undefined;
107 const n = held();
108 return n >= limit ? `${n} of ${limit} ${noun}` : undefined;
109 }
110
111 /**
112 * Runs of `kind` running now for repositories `ownerId` owns, plus the
113 * `also` ids (started a moment ago, not yet claimed in the database).
114 */
115 export function runningRuns(
116 db: Database,
117 kind: "agent" | "check",
118 ownerId: string,
119 also: Iterable<string> = [],
120 ): number {
121 const ids = new Set(
122 db
123 .query<{ id: string }, [string, string]>(
124 `SELECT run.id FROM run JOIN repo ON repo.id = run.repo_id
125 WHERE run.kind = ?1 AND run.state = 'running' AND repo.owner_id = ?2`,
126 )
127 .all(kind, ownerId)
128 .map((r) => r.id),
129 );
130 for (const id of also) {
131 const row = db
132 .query<{ owner_id: string }, [string, string]>(
133 `SELECT repo.owner_id FROM run JOIN repo ON repo.id = run.repo_id
134 WHERE run.id = ?1 AND run.kind = ?2`,
135 )
136 .get(id, kind);
137 if (row?.owner_id === ownerId) ids.add(id);
138 }
139 return ids.size;
140 }
141
77142 /** Set `key` (`null` clears it) and keep the change. */
78143 export function setQuota(
79144 db: Database,
src/web/pages/admin-limits.ts +4 −2
@@ -86,8 +86,10 @@ export function limitsPage(ctx: Ctx): Response {
8686 const quotaCard = card(
8787 html`<div class="card-body">
8888 <p class="muted small">
89 What one account may own. Empty is no limit. Storage is in
90 megabytes: repositories plus app data. Admins are not held to these.
89 What one account may own (repositories, apps), have running at
90 once (previews, runs, checks) and store (storage, in megabytes:
91 repositories plus app data). Empty is no limit; a forge admin is
92 never held to any of them.
9193 </p>
9294 <form class="row-actions" method="post" action="/admin/quotas">
9395 ${QUOTA_KEYS.map(
test/quotas-running.test.ts added +454
@@ -0,0 +1,454 @@
1 /**
2 * The running limits (#918): a fresh account may have only so many live
3 * previews, concurrent agent runs and concurrent checks. One past the limit
4 * does not start — a preview's run fails, a run or check stays queued — and
5 * its words name the limit and the count ("1 of 1 live previews"). The
6 * owner's own (admin) account starts the same things freely. Every wait is
7 * for a named state or a flag file, never a guess at timing.
8 */
9
10 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
11 import { chmodSync, existsSync, writeFileSync } from "node:fs";
12 import { mkdtemp, rm } from "node:fs/promises";
13 import { tmpdir } from "node:os";
14 import { join } from "node:path";
15 import { createApp, type App } from "../src/app.ts";
16 import { newApp } from "../src/actions/apps.ts";
17 import { addToCrew, createAgent } from "../src/actions/crew.ts";
18 import { assignIssue, openIssue } from "../src/actions/issues.ts";
19 import { openPull } from "../src/actions/pulls.ts";
20 import { commitToBranch, createRepo } from "../src/actions/repos.ts";
21 import type { EnvName } from "../src/apps/data.ts";
22 import { AppsRuntime } from "../src/apps/runtime.ts";
23 import { startChecks } from "../src/checks/service.ts";
24 import { getRepo, type Repo } from "../src/core/repos.ts";
25 import { createRun, listRuns, type Run } from "../src/core/runs.ts";
26 import { createUser, type User } from "../src/core/users.ts";
27 import { startCrew } from "../src/crew/dispatch.ts";
28 import { serve, type Servers } from "../src/http/server.ts";
29 import { readRunLog } from "../src/runs/log.ts";
30 import { QUOTA_KEYS, getQuota, runningLimitSentence, setQuota } from "../src/settings/quotas.ts";
31
32 async function until<T>(read: () => T | undefined, ms = 60_000): Promise<T> {
33 const deadline = Date.now() + ms;
34 while (Date.now() < deadline) {
35 const value = read();
36 if (value !== undefined) return value;
37 await Bun.sleep(50);
38 }
39 throw new Error("did not happen in time");
40 }
41
42 async function twoUsers(app: App): Promise<{ alice: User; bob: User }> {
43 const users: User[] = [];
44 for (const username of ["alice", "bob"]) {
45 const made = await createUser(app.db, {
46 trusted: true,
47 username,
48 email: `${username}@forge.local`,
49 password: "a-good-passphrase",
50 adminIfFirst: username === "alice",
51 });
52 if (!made.ok) throw new Error(made.error);
53 users.push(made.value);
54 }
55 return { alice: users[0]!, bob: users[1]! };
56 }
57
58 describe("the limits themselves", () => {
59 test("each running limit is a quota key, unset by default, and never asked of a forge admin", async () => {
60 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-unit-"));
61 const app = createApp({ dataDir, port: 0 });
62 try {
63 const { alice, bob } = await twoUsers(app);
64 for (const key of ["previews", "runs", "checks"] as const) {
65 expect(QUOTA_KEYS).toContain(key);
66 expect(getQuota(app.db, key)).toBeNull();
67 expect(runningLimitSentence(app.db, key, "things", bob.id, () => 9)).toBeUndefined();
68 setQuota(app.db, key, 3, alice.id);
69 expect(runningLimitSentence(app.db, key, "things", bob.id, () => 2)).toBeUndefined();
70 expect(runningLimitSentence(app.db, key, "things", bob.id, () => 3)).toBe("3 of 3 things");
71 expect(runningLimitSentence(app.db, key, "things", alice.id, () => 99)).toBeUndefined();
72 }
73 } finally {
74 app.close();
75 await rm(dataDir, { recursive: true, force: true });
76 }
77 });
78 });
79
80 describe("live previews", () => {
81 test("a fresh account's second preview is refused with the count on its run; the owner's two both run", async () => {
82 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-previews-"));
83 const app = createApp({ dataDir, port: 0 });
84 const servers = await serve(app);
85 const runtime = new AppsRuntime(app, {
86 domain: "apps.test",
87 scheme: "http",
88 publicPort: servers.port,
89 forgeUrl: servers.url,
90 });
91 await runtime.start();
92 try {
93 const { alice, bob } = await twoUsers(app);
94 setQuota(app.db, "previews", 1, alice.id);
95 const pullOf = async (who: User, repo: Repo, name: string): Promise<number> => {
96 const branch = await commitToBranch(app, who, repo, {
97 branch: `feature/${name}`,
98 base: "main",
99 message: name,
100 files: [{ path: `${name}.md`, content: `${name}\n` }],
101 });
102 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
103 const opened = await openPull(app, who, repo, {
104 title: name,
105 baseBranch: "main",
106 headBranch: `feature/${name}`,
107 });
108 if (!opened.ok) throw new Error(String(opened.error));
109 return opened.value.number;
110 };
111 const state = (record: Parameters<typeof runtime.status>[0], n: number) =>
112 runtime.status(record).find((e) => e.env === `pull-${n}`)?.state;
113 const deployOf = (repo: Repo, n: number): Run | undefined =>
114 listRuns(app.db, repo.id, { kind: "deploy", limit: 20 }).find((r) => r.env === `pull-${n}`);
115
116 const mine = await newApp(app, bob, { name: "bobs", description: "x", visibility: "public" });
117 if (!mine.ok) throw new Error(String(mine.error));
118 const first = await pullOf(bob, mine.value.repo, "one");
119 await until(() => (state(mine.value.app, first) === "running" ? true : undefined));
120 const second = await pullOf(bob, mine.value.repo, "two");
121 const refused = await until(() => {
122 const run = deployOf(mine.value.repo, second);
123 return run?.state === "failed" ? run : undefined;
124 });
125 expect(refused.summary).toContain("1 of 1 live previews");
126 expect(readRunLog(dataDir, refused.id).lines.join("\n")).toContain("1 of 1 live previews");
127 expect(state(mine.value.app, second)).not.toBe("running");
128 expect(state(mine.value.app, first)).toBe("running");
129
130 const theirs = await newApp(app, alice, { name: "alices", description: "x", visibility: "public" });
131 if (!theirs.ok) throw new Error(String(theirs.error));
132 const a = await pullOf(alice, theirs.value.repo, "a");
133 const b = await pullOf(alice, theirs.value.repo, "b");
134 for (const n of [a, b])
135 await until(() => (state(theirs.value.app, n) === "running" ? true : undefined));
136 } finally {
137 await runtime.stop();
138 servers.stop();
139 app.close();
140 await rm(dataDir, { recursive: true, force: true });
141 }
142 }, 180_000);
143 });
144
145 describe("two previews deploying together", () => {
146 // A regression guard: the deploys are decided one after the other, so this passes without the ordering too.
147 test("at a limit of one, the older deploy starts and only the younger is refused", async () => {
148 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-burst-previews-"));
149 const app = createApp({ dataDir, port: 0 });
150 const servers = await serve(app);
151 const runtime = new AppsRuntime(app, {
152 domain: "apps.test",
153 scheme: "http",
154 publicPort: servers.port,
155 forgeUrl: servers.url,
156 });
157 await runtime.start();
158 try {
159 const { alice, bob } = await twoUsers(app);
160 setQuota(app.db, "previews", 1, alice.id);
161 const made = await newApp(app, bob, { name: "bobs", description: "x", visibility: "public" });
162 if (!made.ok) throw new Error(String(made.error));
163 const { repo, app: record } = made.value;
164 for (const name of ["one", "two"]) {
165 const branch = await commitToBranch(app, bob, repo, {
166 branch: `feature/${name}`,
167 base: "main",
168 message: name,
169 files: [{ path: `${name}.md`, content: `${name}\n` }],
170 });
171 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
172 }
173 // Both pulls open in the same breath, so both deploys are in flight together.
174 const opened = await Promise.all(
175 ["one", "two"].map((name) =>
176 openPull(app, bob, repo, { title: name, baseBranch: "main", headBranch: `feature/${name}` }),
177 ),
178 );
179 const numbers = opened.map((o) => {
180 if (!o.ok) throw new Error(String(o.error));
181 return o.value.number;
182 });
183 const deploys = () =>
184 numbers.map((n) => listRuns(app.db, repo.id, { kind: "deploy", limit: 20 }).find((r) => r.env === `pull-${n}`));
185 await until(() => {
186 const runs = deploys();
187 return runs.every((r) => r !== undefined && ["failed", "ok", "running"].includes(r.state)) ? true : undefined;
188 });
189 const states = () => runtime.status(record).filter((e) => e.env.startsWith("pull-")).map((e) => e.state);
190 await until(() => (states().includes("running") ? true : undefined));
191 const failed = deploys().filter((r) => r?.state === "failed");
192 expect(failed).toHaveLength(1);
193 expect(failed[0]!.summary).toContain("1 of 1 live previews");
194 expect(states().filter((s) => s === "running")).toHaveLength(1);
195 } finally {
196 await runtime.stop();
197 servers.stop();
198 app.close();
199 await rm(dataDir, { recursive: true, force: true });
200 }
201 }, 180_000);
202 });
203
204 describe("a sleeping preview woken at the limit", () => {
205 test("stays asleep, and the page a visitor sees says why", async () => {
206 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-wake-"));
207 const app = createApp({ dataDir, port: 0 });
208 const servers = await serve(app);
209 const runtime = new AppsRuntime(app, {
210 domain: "apps.test",
211 scheme: "http",
212 publicPort: servers.port,
213 forgeUrl: servers.url,
214 // Previews rest after a few seconds unasked; production is left alone.
215 sleepAfterMs: { production: 3_600_000, preview: 2_500 },
216 sweepEveryMs: 100,
217 });
218 await runtime.start();
219 try {
220 const { alice, bob } = await twoUsers(app);
221 const made = await newApp(app, bob, { name: "bobs", description: "x", visibility: "public" });
222 if (!made.ok) throw new Error(String(made.error));
223 const { repo, app: record } = made.value;
224 const envs: EnvName[] = [];
225 for (const name of ["one", "two"]) {
226 const branch = await commitToBranch(app, bob, repo, {
227 branch: `feature/${name}`,
228 base: "main",
229 message: name,
230 files: [{ path: `${name}.md`, content: `${name}\n` }],
231 });
232 if (!branch.ok) throw new Error(JSON.stringify(branch.error));
233 const opened = await openPull(app, bob, repo, {
234 title: name,
235 baseBranch: "main",
236 headBranch: `feature/${name}`,
237 });
238 if (!opened.ok) throw new Error(String(opened.error));
239 envs.push(`pull-${opened.value.number}`);
240 }
241 const status = (env: EnvName) => runtime.status(record).find((e) => e.env === env);
242 // No limit yet: both come up, then both rest, unasked.
243 for (const env of envs) await until(() => (status(env)?.state === "asleep" ? true : undefined));
244 // The limit is set only now, so the two asleep are the whole story: one wake fits, the next does not.
245 setQuota(app.db, "previews", 1, alice.id);
246 expect(await runtime.wake(record, envs[0]!)).toBeDefined();
247 expect(status(envs[0]!)?.state).toBe("running");
248
249 const url = new URL(status(envs[1]!)!.url);
250 const page = await fetch(`${servers.url}/`, { headers: { host: url.host }, redirect: "manual" });
251 const html = await page.text();
252 expect(status(envs[1]!)?.state).toBe("asleep");
253 expect(status(envs[1]!)?.port).toBeUndefined();
254 expect(status(envs[1]!)?.error).toContain("1 of 1 live previews");
255 expect(page.status).toBeGreaterThanOrEqual(500);
256 expect(html).toContain("stays asleep");
257 expect(html).toContain("1 of 1 live previews");
258 } finally {
259 await runtime.stop();
260 servers.stop();
261 app.close();
262 await rm(dataDir, { recursive: true, force: true });
263 }
264 }, 180_000);
265 });
266
267 describe("concurrent checks", () => {
268 // A regression guard: `schedule()` re-reads what is in flight after every start, so this passes without per-pick asking too.
269 test("two checks of one account queued together do not start together", async () => {
270 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-burst-checks-"));
271 const app = createApp({ dataDir, port: 0 });
272 const servers = await serve(app);
273 const flag = (name: string) => join(dataDir, `go-${name}`);
274 let checks: ReturnType<typeof startChecks> | undefined;
275 try {
276 const { alice, bob } = await twoUsers(app);
277 setQuota(app.db, "checks", 1, alice.id);
278 const runs: { repo: Repo; run: Run }[] = [];
279 // Queued before the checks service exists, so its first look sees both.
280 for (const name of ["x1", "x2"]) {
281 const made = await createRepo(app, bob, { name, visibility: "public" });
282 if (!made.ok) throw new Error(String(made.error));
283 const pushed = await commitToBranch(app, bob, made.value, {
284 message: "Check me",
285 files: [
286 {
287 path: "forge.json",
288 content: JSON.stringify({ check: { run: `while [ ! -f ${flag(name)} ]; do sleep 0.05; done` } }),
289 },
290 ],
291 });
292 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
293 const run = createRun(app.db, {
294 repoId: made.value.id,
295 kind: "check",
296 env: "production",
297 commitOid: pushed.value.commitOid,
298 reason: "push",
299 });
300 runs.push({ repo: made.value, run });
301 }
302 checks = startChecks(app, { forgeUrl: servers.url, concurrency: 4 });
303 const stateOf = (i: number) => listRuns(app.db, runs[i]!.repo.id, { kind: "check", limit: 5 })[0]?.state;
304 const said = (i: number) => readRunLog(dataDir, runs[i]!.run.id).lines.join("\n").includes("1 of 1 concurrent checks");
305 // One has started and the other has been told why it waits: both were weighed.
306 await until(() => (said(0) || said(1) ? true : undefined));
307 const states = [stateOf(0), stateOf(1)].sort();
308 expect(states).toEqual(["queued", "running"]);
309 for (const name of ["x1", "x2"]) writeFileSync(flag(name), "");
310 await until(() => (stateOf(0) === "ok" && stateOf(1) === "ok" ? true : undefined));
311 } finally {
312 for (const name of ["x1", "x2"]) writeFileSync(flag(name), "");
313 await checks?.stop();
314 servers.stop();
315 app.close();
316 await rm(dataDir, { recursive: true, force: true });
317 }
318 }, 180_000);
319
320 test("a fresh account's second check waits queued, naming the count; the owner's two run at once; a freed slot starts the waiter", async () => {
321 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-checks-"));
322 const app = createApp({ dataDir, port: 0 });
323 const servers = await serve(app);
324 const checks = startChecks(app, { forgeUrl: servers.url, concurrency: 4 });
325 const flag = (name: string) => join(dataDir, `go-${name}`);
326 try {
327 const { alice, bob } = await twoUsers(app);
328 setQuota(app.db, "checks", 1, alice.id);
329 const push = async (who: User, name: string): Promise<Repo> => {
330 const made = await createRepo(app, who, { name, visibility: "public" });
331 if (!made.ok) throw new Error(String(made.error));
332 const pushed = await commitToBranch(app, who, made.value, {
333 message: "Check me",
334 files: [
335 {
336 path: "forge.json",
337 content: JSON.stringify({
338 check: { run: `while [ ! -f ${flag(name)} ]; do sleep 0.05; done` },
339 }),
340 },
341 ],
342 });
343 if (!pushed.ok) throw new Error(JSON.stringify(pushed.error));
344 return made.value;
345 };
346 const checkOf = (repo: Repo): Run | undefined =>
347 listRuns(app.db, repo.id, { kind: "check", limit: 10 })[0];
348 const stateOf = (repo: Repo) => checkOf(repo)?.state;
349
350 const b1 = await push(bob, "b1");
351 await until(() => (stateOf(b1) === "running" ? true : undefined));
352 const b2 = await push(bob, "b2");
353 const waiting = await until(() => {
354 const run = checkOf(b2);
355 return run !== undefined && readRunLog(dataDir, run.id).lines.join("\n").includes("1 of 1 concurrent checks")
356 ? run
357 : undefined;
358 });
359 expect(stateOf(b2)).toBe("queued");
360 expect(waiting.state).toBe("queued");
361
362 const a1 = await push(alice, "a1");
363 const a2 = await push(alice, "a2");
364 await until(() => (stateOf(a1) === "running" && stateOf(a2) === "running" ? true : undefined));
365
366 // The slot frees: bob's waiter is started, never dropped.
367 writeFileSync(flag("b1"), "");
368 await until(() => (stateOf(b1) === "ok" ? true : undefined));
369 await until(() => (stateOf(b2) === "running" ? true : undefined));
370 for (const name of ["b2", "a1", "a2"]) writeFileSync(flag(name), "");
371 await until(() => ([b2, a1, a2].every((r) => stateOf(r) === "ok") ? true : undefined));
372 expect(existsSync(flag("b2"))).toBe(true);
373 } finally {
374 await checks.stop();
375 servers.stop();
376 app.close();
377 await rm(dataDir, { recursive: true, force: true });
378 }
379 }, 180_000);
380 });
381
382 describe("concurrent runs", () => {
383 test("a fresh account's second run waits queued, naming the count, until its first ends", async () => {
384 const dataDir = await mkdtemp(join(tmpdir(), "forge-qrun-runs-"));
385 const app = createApp({ dataDir, port: 0 });
386 const servers = await serve(app);
387 const flag = join(dataDir, "go");
388 const fake = join(dataDir, "fake-claude");
389 writeFileSync(
390 fake,
391 `#!/bin/sh
392 if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
393 IFS= read -r first
394 echo '{"type":"system","subtype":"init","session_id":"s","model":"fake"}'
395 while [ ! -f ${flag} ]; do sleep 0.05; done
396 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}'
397 `,
398 );
399 chmodSync(fake, 0o755);
400 const crew = await startCrew(app, {
401 forgeUrl: servers.url,
402 localUrl: servers.url,
403 binary: fake,
404 concurrency: 4,
405 budgetUsd: 1,
406 runTimeoutMs: 60_000,
407 });
408 try {
409 const { alice, bob } = await twoUsers(app);
410 setQuota(app.db, "runs", 1, alice.id);
411 const assignTwo = async (who: User, repoName: string): Promise<Repo> => {
412 const made = await createRepo(app, who, { name: repoName, visibility: "private" });
413 if (!made.ok) throw new Error(String(made.error));
414 const seeded = await commitToBranch(app, who, made.value, {
415 message: "Start",
416 files: [{ path: "README.md", content: "# x\n" }],
417 });
418 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
419 for (const name of [`${repoName}-bot`, `${repoName}-bee`]) {
420 const agent = await createAgent(app, alice, { username: name, brief: "Be brief." });
421 if (!agent.ok) throw new Error(String(agent.error));
422 expect(addToCrew(app, who, made.value, name).ok).toBe(true);
423 const issue = openIssue(app, who, made.value, { title: `Plant ${name}`, body: "Along the fence." });
424 if (!issue.ok) throw new Error(String(issue.error));
425 expect(assignIssue(app, who, made.value, issue.value, agent.value.user).ok).toBe(true);
426 }
427 return getRepo(app.db, who.username, repoName)!;
428 };
429 const runsOf = (repo: Repo) => listRuns(app.db, repo.id, { kind: "agent", limit: 10 });
430 const count = (repo: Repo, state: string) => runsOf(repo).filter((r) => r.state === state).length;
431
432 const mine = await assignTwo(bob, "garden");
433 await until(() => (count(mine, "running") === 1 && count(mine, "queued") === 1 ? true : undefined));
434 const waiting = runsOf(mine).find((r) => r.state === "queued")!;
435 await until(() =>
436 readRunLog(dataDir, waiting.id).lines.join("\n").includes("1 of 1 concurrent runs") ? true : undefined,
437 );
438 expect(count(mine, "running")).toBe(1);
439
440 const theirs = await assignTwo(alice, "orchard");
441 await until(() => (count(theirs, "running") === 2 ? true : undefined));
442
443 writeFileSync(flag, "");
444 await until(() => (runsOf(mine).every((r) => r.state !== "queued" && r.state !== "running") ? true : undefined));
445 expect(runsOf(mine)).toHaveLength(2);
446 } finally {
447 writeFileSync(flag, "");
448 await crew.stop();
449 servers.stop();
450 app.close();
451 await rm(dataDir, { recursive: true, force: true });
452 }
453 }, 180_000);
454 });
test/quotas.test.ts +10 −8
@@ -26,6 +26,8 @@ import { forkScreen, newRepoScreen } from "../src/tui/screens/newrepo.ts";
2626 import { headlessTerminal } from "../src/tui/term/terminal.ts";
2727 import { settleApp } from "./helpers/tui.ts";
2828
29 const NONE = { repositories: null, apps: null, storage: null, previews: null, runs: null, checks: null };
30
2931 describe("quotas", () => {
3032 let dataDir: string;
3133 let app: App;
@@ -64,19 +66,19 @@ describe("quotas", () => {
6466
6567 test("limits start unset, an admin sets one and clears it, each change is kept", async () => {
6668 const t = createToken(app.db, alice.id, "t", "admin").token;
67 expect(getQuotas(app.db)).toEqual({ repositories: null, apps: null, storage: null });
69 expect(getQuotas(app.db)).toEqual(NONE);
6870 expect(await (await api(t, "GET")).json()).toMatchObject({
69 quotas: { repositories: null, apps: null },
71 quotas: NONE,
7072 changes: [],
7173 });
7274
7375 const set = await api(t, "PATCH", { quotas: { repositories: 5, apps: "3" } });
7476 expect(set.status).toBe(200);
75 expect(getQuotas(app.db)).toEqual({ repositories: 5, apps: 3, storage: null });
77 expect(getQuotas(app.db)).toEqual({ ...NONE, repositories: 5, apps: 3 });
7678
7779 const cleared = await api(t, "PATCH", { quotas: { repositories: null, apps: "" } });
7880 expect(cleared.status).toBe(200);
79 expect(getQuotas(app.db)).toEqual({ repositories: null, apps: null, storage: null });
81 expect(getQuotas(app.db)).toEqual(NONE);
8082
8183 const changes = app.db
8284 .query<{ n: number }, []>(
@@ -122,7 +124,7 @@ describe("quotas", () => {
122124 const bobAdmin = createToken(app.db, bob.id, "t", "admin").token;
123125 expect((await rpc(bobAdmin, "quotas")).error).toContain("no such tool");
124126
125 expect((await rpc(ro, "quotas")).value.quotas).toEqual({ repositories: null, apps: null, storage: null });
127 expect((await rpc(ro, "quotas")).value.quotas).toEqual(NONE);
126128 const dry = await rpc(admin, "quotas", { key: "apps", value: "9" });
127129 expect(dry.value.says).toContain("Would set");
128130 expect(getQuotas(app.db).apps).toBeNull();
@@ -157,7 +159,7 @@ describe("quotas", () => {
157159 }
158160 expect((await api(t, "PATCH", { quotas: { nope: 1 } })).status).toBe(422);
159161 expect((await api(t, "PATCH", {})).status).toBe(422);
160 expect(getQuotas(app.db)).toEqual({ repositories: null, apps: null, storage: null });
162 expect(getQuotas(app.db)).toEqual(NONE);
161163 });
162164
163165 test("the admin page form sets a quota and refuses a bad one", async () => {
@@ -180,9 +182,9 @@ describe("quotas", () => {
180182 body: new URLSearchParams(form),
181183 });
182184 await post({ repositories: "7", apps: "" });
183 expect(getQuotas(app.db)).toEqual({ repositories: 7, apps: null, storage: null });
185 expect(getQuotas(app.db)).toEqual({ ...NONE, repositories: 7 });
184186 await post({ repositories: "0", apps: "2" });
185 expect(getQuotas(app.db)).toEqual({ repositories: 7, apps: null, storage: null });
187 expect(getQuotas(app.db)).toEqual({ ...NONE, repositories: 7 });
186188 const page = await fetch(`${servers.url}/admin/limits`, { headers: { cookie } });
187189 const text = await page.text();
188190 expect(text).toContain('name="repositories"');

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs