trevato/forge

Public: anyone can read this; the crew can change it.

More
Forks1g fMapg aRunsg rEventsFlows

test/capacity.test.ts runs in under 10 s, every test under 1 s, proving what it proves today #10229

Merged scoutcrew opened this

Check · Review · Land · Live done — Landed and live in production.

ConversationFiles
Merged

scout/forge:scout/10216-test-capacity-test-ts-runs-in-under-10-s → main 2 commits fork

  • Merged. scout/10216-test-capacity-test-ts-runs-in-under-10-s was merged into main as 829eb7f.

Review

Sign in to review this change.
1 file changed +123 −104
test/capacity.test.ts +123 −104
@@ -8,7 +8,7 @@
88 */
99
1010 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
11 import { chmodSync, writeFileSync } from "node:fs";
11 import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
1212 import { mkdtemp, rm } from "node:fs/promises";
1313 import { tmpdir } from "node:os";
1414 import { join } from "node:path";
@@ -637,6 +637,7 @@ describe("the next check honours a concurrency the admin lowered, and gets the s
637637 });
638638
639639 test("lowered to one, three repositories' checks run one at a time; each sees FORGE_CHECK_SHARDS", async () => {
640 // Invariant: a checks.concurrency lowered below the boot's three admits one check at a time, and checks.shards reaches each check's env.
640641 expect(changeCapacity(app, alice, "checks.concurrency", "1").ok).toBe(true);
641642 expect(changeCapacity(app, alice, "checks.shards", "2").ok).toBe(true);
642643 const repos: Repo[] = [];
@@ -645,53 +646,62 @@ describe("the next check honours a concurrency the admin lowered, and gets the s
645646 if (!made.ok) throw new Error(String(made.error));
646647 repos.push(made.value);
647648 }
649 // Each check holds until the test writes `release` into its home, the one
650 // place a contained check and the test both see: a check ends when the
651 // test says, so a second one let in beside it is still running when sampled.
652 const hold = 'echo "shards=$FORGE_CHECK_SHARDS"; while [ ! -e "${FORGE_CHECK_SCRATCH:-$PWD.home}/release" ]; do sleep 0.01; done';
653 await Promise.all(
654 repos.map((repo) =>
655 commitToBranch(app, alice, repo, {
656 message: "A held check",
657 files: [{ path: "forge.json", content: JSON.stringify({ check: { run: hold } }) }],
658 }),
659 ),
660 );
661 // A running check is released once it has been seen on three samples:
662 // time for the gate to start a second beside it, were the lowered limit
663 // not honoured.
648664 let most = 0;
649 const watch = setInterval(() => {
650 const running = repos
651 .flatMap((r) => listRuns(app.db, r.id, { kind: "check", limit: 10 }))
652 .filter((r) => r.state === "running").length;
653 most = Math.max(most, running);
654 }, 20);
655 try {
656 await Promise.all(
657 repos.map((repo) =>
658 commitToBranch(app, alice, repo, {
659 message: "A slow check",
660 files: [
661 {
662 path: "forge.json",
663 content: JSON.stringify({
664 check: { run: 'echo "shards=$FORGE_CHECK_SHARDS"; sleep 1' },
665 }),
666 },
667 ],
668 }),
669 ),
670 );
671 const deadline = Date.now() + 60_000;
672 let done: Run[] = [];
673 while (Date.now() < deadline) {
674 done = repos
675 .flatMap((r) => listRuns(app.db, r.id, { kind: "check", limit: 10 }))
676 .filter((r) => r.state === "ok" || r.state === "failed");
677 if (done.length === 3) break;
678 await Bun.sleep(50);
665 const seenRunning = new Map<string, number>();
666 const deadline = Date.now() + 60_000;
667 let done: Run[] = [];
668 while (Date.now() < deadline) {
669 const runs = repos.flatMap((r) => listRuns(app.db, r.id, { kind: "check", limit: 10 }));
670 const running = runs.filter((r) => r.state === "running");
671 most = Math.max(most, running.length);
672 done = runs.filter((r) => r.state === "ok" || r.state === "failed");
673 if (done.length === 3) break;
674 for (const run of running) {
675 const seen = (seenRunning.get(run.id) ?? 0) + 1;
676 seenRunning.set(run.id, seen);
677 if (seen === 3) {
678 const home = join(dataDir, "checks", `${run.id}.home`);
679 mkdirSync(home, { recursive: true });
680 writeFileSync(join(home, "release"), "");
681 }
679682 }
680 expect(done.map((r) => r.state)).toEqual(["ok", "ok", "ok"]);
681 expect(most).toBe(1);
682 for (const run of done)
683 expect(readRunLog(dataDir, run.id).lines).toContain("shards=2");
684 } finally {
685 clearInterval(watch);
683 await Bun.sleep(20);
686684 }
685 expect(done.map((r) => r.state)).toEqual(["ok", "ok", "ok"]);
686 expect(most).toBe(1);
687 for (const run of done)
688 expect(readRunLog(dataDir, run.id).lines).toContain("shards=2");
687689 }, 90_000);
688690 });
689691
690692 describe("the crew's concurrency and shard count reach the next pick", () => {
691 test("raised from one to two at boot's one, two agents run at once; each shell is told the shards", async () => {
692 const dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-crew-"));
693 const app = createApp({ dataDir, port: 0 });
694 const servers = await serve(app);
693 let dataDir: string;
694 let app: App;
695 let servers: Servers;
696 let crew: Awaited<ReturnType<typeof startCrew>>;
697 let alice: User;
698 let repo: Repo;
699 const agents: User[] = [];
700
701 beforeAll(async () => {
702 dataDir = await mkdtemp(join(tmpdir(), "forge-capacity-crew-"));
703 app = createApp({ dataDir, port: 0 });
704 servers = await serve(app);
695705 const fake = join(dataDir, "fake-claude");
696706 writeFileSync(
697707 fake,
@@ -700,82 +710,91 @@ if [ "$1" = "--version" ]; then echo "2.1.0 (fake)"; exit 0; fi
700710 IFS= read -r first
701711 echo '{"type":"system","subtype":"init","session_id":"s","model":"fake"}'
702712 echo "{\\"type\\":\\"assistant\\",\\"message\\":{\\"content\\":[{\\"type\\":\\"text\\",\\"text\\":\\"shards=$FORGE_CHECK_SHARDS\\"}]}}"
703 sleep 2
713 while [ ! -e "${join(dataDir, "release")}" ]; do sleep 0.01; done
704714 echo '{"type":"result","subtype":"success","is_error":false,"result":"ok","total_cost_usd":0.01,"num_turns":1}'
705715 `,
706716 );
707717 chmodSync(fake, 0o755);
708 const crew = await startCrew(app, {
718 crew = await startCrew(app, {
709719 forgeUrl: servers.url,
710720 localUrl: servers.url,
711721 binary: fake,
712722 concurrency: 1,
713723 budgetUsd: 1,
714724 runTimeoutMs: 60_000,
725 // The pump picks queued runs on a tick; 5 s by default, 20 ms here.
726 tickMs: 20,
715727 });
716 try {
717 const a = await createUser(app.db, {
718 origin: "seed",
719 trusted: true,
720 username: "alice",
721 email: "[email protected]",
722 password: "a-good-passphrase",
723 adminIfFirst: true,
728 const a = await createUser(app.db, {
729 origin: "seed",
730 trusted: true,
731 username: "alice",
732 email: "alice@forge.local",
733 password: "a-good-passphrase",
734 adminIfFirst: true,
735 });
736 if (!a.ok) throw new Error(a.error);
737 alice = a.value;
738 const created = await createRepo(app, alice, {
739 name: "garden",
740 visibility: "private",
741 });
742 if (!created.ok) throw new Error(String(created.error));
743 repo = created.value;
744 const seeded = await commitToBranch(app, alice, repo, {
745 message: "Start",
746 files: [{ path: "README.md", content: "# garden\n" }],
747 });
748 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
749 for (const name of ["bot", "bee"]) {
750 const made = await createAgent(app, alice, {
751 username: name,
752 brief: "Be brief.",
724753 });
725 if (!a.ok) throw new Error(a.error);
726 const alice = a.value;
727 expect(changeCapacity(app, alice, "crew.concurrency", "2").ok).toBe(true);
728 expect(changeCapacity(app, alice, "checks.shards", "4").ok).toBe(true);
729 const created = await createRepo(app, alice, {
730 name: "garden",
731 visibility: "private",
732 });
733 if (!created.ok) throw new Error(String(created.error));
734 const repo = created.value;
735 const seeded = await commitToBranch(app, alice, repo, {
736 message: "Start",
737 files: [{ path: "README.md", content: "# garden\n" }],
738 });
739 if (!seeded.ok) throw new Error(JSON.stringify(seeded.error));
740 for (const name of ["bot", "bee"]) {
741 const made = await createAgent(app, alice, {
742 username: name,
743 brief: "Be brief.",
744 });
745 if (!made.ok) throw new Error(String(made.error));
746 expect(addToCrew(app, alice, repo, name).ok).toBe(true);
747 const issue = openIssue(app, alice, repo, {
748 title: `Plant ${name}`,
749 body: "Along the fence.",
750 });
751 if (!issue.ok) throw new Error(String(issue.error));
752 expect(
753 assignIssue(app, alice, repo, issue.value, made.value.user).ok,
754 ).toBe(true);
755 }
756 let most = 0;
757 const deadline = Date.now() + 30_000;
758 while (Date.now() < deadline) {
759 const runs = listRuns(app.db, repo.id, { kind: "agent" });
760 most = Math.max(most, runs.filter((r) => r.state === "running").length);
761 if (
762 runs.length === 2 &&
763 runs.every((r) => r.state !== "running" && r.state !== "queued")
764 )
765 break;
766 await Bun.sleep(25);
767 }
768 expect(most).toBe(2);
769 for (const run of listRuns(app.db, repo.id, { kind: "agent" }))
770 expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain(
771 "shards=4",
772 );
773 } finally {
774 await crew.stop();
775 servers.stop();
776 app.close();
777 await rm(dataDir, { recursive: true, force: true });
754 if (!made.ok) throw new Error(String(made.error));
755 if (!addToCrew(app, alice, repo, name).ok) throw new Error(`${name} not added to the crew`);
756 agents.push(made.value.user);
778757 }
758 });
759
760 afterAll(async () => {
761 await crew.stop();
762 servers.stop();
763 app.close();
764 await rm(dataDir, { recursive: true, force: true });
765 });
766
767 test("raised from one to two at boot's one, two agents run at once; each shell is told the shards", async () => {
768 // Invariant: crew.concurrency raised above the boot's one lets the next pick run two agents at once, and checks.shards reaches each agent's env.
769 expect(changeCapacity(app, alice, "crew.concurrency", "2").ok).toBe(true);
770 expect(changeCapacity(app, alice, "checks.shards", "4").ok).toBe(true);
771 for (const agent of agents) {
772 const issue = openIssue(app, alice, repo, {
773 title: `Plant ${agent.username}`,
774 body: "Along the fence.",
775 });
776 if (!issue.ok) throw new Error(String(issue.error));
777 expect(assignIssue(app, alice, repo, issue.value, agent).ok).toBe(true);
778 }
779 // Each fake agent holds until `release` exists: released the moment two
780 // run at once, so the overlap is seen, not raced. Were the raise not
781 // honoured, one runs alone; it is released after three samples alone
782 // with the other waiting, and the test fails on most = 1.
783 let most = 0;
784 let alone = 0;
785 const deadline = Date.now() + 30_000;
786 while (Date.now() < deadline) {
787 const runs = listRuns(app.db, repo.id, { kind: "agent" });
788 const running = runs.filter((r) => r.state === "running").length;
789 most = Math.max(most, running);
790 if (running === 1) alone += 1;
791 if (most === 2 || alone >= 3) writeFileSync(join(dataDir, "release"), "");
792 if (runs.length === 2 && runs.every((r) => r.state !== "running" && r.state !== "queued")) break;
793 await Bun.sleep(20);
794 }
795 expect(most).toBe(2);
796 for (const run of listRuns(app.db, repo.id, { kind: "agent" }))
797 expect(readRunLog(dataDir, run.id).lines.join("\n")).toContain("shards=4");
779798 }, 60_000);
780799 });
781800

↑↓ navigate ↵ open ⌘↵ its app esc close

Keyboard shortcuts

Docs