Public: anyone can read this; the crew can change it.
Pulls waiting for the disk stay with the forge that parked them: one forge's recheck never drops another's #6293
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/6289-pulls-waiting-for-the-disk-stay-with-the → main 1 commit fork
2 files
changed +39
−18
src/crew/dispatch.ts +17 −18
| @@ -752,9 +752,9 @@ function waitOnMain( | ||
| 752 | 752 | // The check machine's disk was full when it failed (#5453): the machine's red. |
| 753 | 753 | if (diskRed(run)) { |
| 754 | 754 | if (typeof commitOid === "string" && typeof env === "string") { |
| 755 | const waiting = waitingOnDisk.get(repo.id) ?? new Map(); | |
| 755 | const waiting = waitingOnDisk.get(app.db) ?? new Map(); | |
| 756 | 756 | waiting.set(issue.id, { env, commitOid, repo }); |
| 757 | waitingOnDisk.set(repo.id, waiting); | |
| 757 | waitingOnDisk.set(app.db, waiting); | |
| 758 | 758 | } |
| 759 | 759 | log.info("check failed while the check machine's disk was full; waiting for the disk", { |
| 760 | 760 | issue: issue.number, |
| @@ -777,27 +777,26 @@ function waitOnMain( | ||
| 777 | 777 | return true; |
| 778 | 778 | } |
| 779 | 779 | |
| 780 | /** Pulls whose check failed on a full disk, per repository; memory only, like {@link waitingOnMain}. */ | |
| 781 | const waitingOnDisk = new Map<string, Map<string, { env: string; commitOid: string; repo: Repo }>>(); | |
| 780 | /** Pulls whose check failed on a full disk, per forge (database), so one forge's recheck never drops another's; memory only. */ | |
| 781 | const waitingOnDisk = new WeakMap<object, Map<string, { env: string; commitOid: string; repo: Repo }>>(); | |
| 782 | 782 | |
| 783 | 783 | /** Once the disk is back above its floor, the pulls that waited for it are checked again. */ |
| 784 | 784 | export function recheckForDisk(app: App): void { |
| 785 | if (waitingOnDisk.size === 0 || !diskRecovered(app.config.dataDir)) return; | |
| 785 | const waiting = waitingOnDisk.get(app.db); | |
| 786 | if (waiting === undefined || waiting.size === 0 || !diskRecovered(app.config.dataDir)) return; | |
| 786 | 787 | const checks = checksFor(app); |
| 787 | 788 | if (checks === undefined) return; |
| 788 | const all = [...waitingOnDisk.values()]; | |
| 789 | waitingOnDisk.clear(); | |
| 790 | for (const waiting of all) | |
| 791 | for (const [issueId, head] of waiting) { | |
| 792 | const issue = getIssueById(app.db, issueId); | |
| 793 | const pull = issue ? getPull(app.db, head.repo.id, issue.number) : undefined; | |
| 794 | if (!pull || pull.state !== "open" || pull.headOid !== head.commitOid) continue; | |
| 795 | void checks | |
| 796 | .request(head.repo, head.env, head.commitOid, "retry", issueId) | |
| 797 | .catch((error) => | |
| 798 | log.warn("a pull waiting for the disk could not be checked again", { issue: issue?.number, error: String(error) }), | |
| 799 | ); | |
| 800 | } | |
| 789 | waitingOnDisk.delete(app.db); | |
| 790 | for (const [issueId, head] of waiting) { | |
| 791 | const issue = getIssueById(app.db, issueId); | |
| 792 | const pull = issue ? getPull(app.db, head.repo.id, issue.number) : undefined; | |
| 793 | if (!pull || pull.state !== "open" || pull.headOid !== head.commitOid) continue; | |
| 794 | void checks | |
| 795 | .request(head.repo, head.env, head.commitOid, "retry", issueId) | |
| 796 | .catch((error) => | |
| 797 | log.warn("a pull waiting for the disk could not be checked again", { issue: issue?.number, error: String(error) }), | |
| 798 | ); | |
| 799 | } | |
| 801 | 800 | } |
| 802 | 801 | |
| 803 | 802 | /** Main's check finished: when it passed, the pulls that waited are checked again. */ |
test/checks-disk-red.test.ts +22
| @@ -118,6 +118,28 @@ describe("a red on a full check machine", () => { | ||
| 118 | 118 | expect(requested).toHaveLength(3); |
| 119 | 119 | }); |
| 120 | 120 | |
| 121 | test("another forge's recheck leaves this forge's waiting pulls alone", async () => { | |
| 122 | const other = await startStandInCrew(); | |
| 123 | try { | |
| 124 | const asked: unknown[][] = []; | |
| 125 | other.checks.request = async (...args: unknown[]) => { | |
| 126 | asked.push(args); | |
| 127 | return undefined; | |
| 128 | }; | |
| 129 | const pull = await spentPull(); | |
| 130 | requested = []; | |
| 131 | announceRun(app, failed(pull, 40)); | |
| 132 | diskReading.read = reading(300); | |
| 133 | recheckForDisk(other.app); | |
| 134 | expect(requested).toHaveLength(0); | |
| 135 | expect(asked).toHaveLength(0); | |
| 136 | recheckForDisk(app); | |
| 137 | expect(requested.map((r) => r[4])).toEqual([pull.id]); | |
| 138 | } finally { | |
| 139 | await stopStandInCrew(other); | |
| 140 | } | |
| 141 | }, 30_000); | |
| 142 | ||
| 121 | 143 | test("the card ends when the disk is back above the floor, not an hour later", () => { |
| 122 | 144 | expect(cards(30)).toHaveLength(1); |
| 123 | 145 | expect(cards(500)).toHaveLength(0); |
new above
offline