Public: anyone can read this; the crew can change it.
The forge calls an absorbed app's scheduled tool on time, once, and remembers how it went #9962
mason/forge:mason/9926-the-forge-calls-an-absorbed-app-s-schedu → main 3 commits fork
5 files
changed +203
−0
src/apps/feature_schedule.ts added +107
| @@ -0,0 +1,107 @@ | ||
| 1 | /** | |
| 2 | * An absorbed app's declared schedule, kept (#9926). Every minute the forge | |
| 3 | * looks at each absorbed app whose production is running and calls each tool | |
| 4 | * whose interval has passed, through the same loopback call the `features` | |
| 5 | * tool makes (`callTool`: POST /forge/tools/<name>, 64 KB, 10 s). The last | |
| 6 | * fire of each (app, tool) is kept: when, ok or the error's sentence, how long. | |
| 7 | * An asleep or stopped app is skipped, never woken; an un-absorbed app is never called. | |
| 8 | * No model, no crew: a clock and a loopback call. | |
| 9 | */ | |
| 10 | ||
| 11 | import type { Database } from "bun:sqlite"; | |
| 12 | import type { App } from "../app.ts"; | |
| 13 | import { declaredFeature } from "../actions/features.ts"; | |
| 14 | import { getAppById } from "../core/apps.ts"; | |
| 15 | import { listAbsorbed } from "../core/features.ts"; | |
| 16 | import { getRepoById } from "../core/repos.ts"; | |
| 17 | import type { Result } from "../kernel/result.ts"; | |
| 18 | import { callTool } from "./feature_tools.ts"; | |
| 19 | import { appsRuntime } from "./runtime.ts"; | |
| 20 | import type { ScheduleEntry } from "./feature.ts"; | |
| 21 | ||
| 22 | export interface Fired { | |
| 23 | readonly at: number; | |
| 24 | readonly ok: boolean; | |
| 25 | readonly error: string | null; | |
| 26 | readonly durationMs: number; | |
| 27 | } | |
| 28 | ||
| 29 | export const TICK_MS = 60_000; | |
| 30 | ||
| 31 | /** Whether an entry is due: never fired, or its interval has passed since the last fire. */ | |
| 32 | export function due(entry: ScheduleEntry, lastFired: number | undefined, now: number): boolean { | |
| 33 | return lastFired === undefined || now - lastFired >= entry.everyMs; | |
| 34 | } | |
| 35 | ||
| 36 | /** What `fireDue` reaches outside the database; tests replace them. */ | |
| 37 | export interface FireDeps { | |
| 38 | schedule(app: App, appId: string): Promise<readonly ScheduleEntry[]>; | |
| 39 | port(app: App, appId: string): number | undefined; | |
| 40 | call(port: number, tool: string): Promise<Result<unknown, string>>; | |
| 41 | } | |
| 42 | ||
| 43 | const real: FireDeps = { | |
| 44 | async schedule(app, appId) { | |
| 45 | const record = getAppById(app.db, appId); | |
| 46 | const repo = record && getRepoById(app.db, record.repoId); | |
| 47 | if (repo === undefined) return []; | |
| 48 | const declared = await declaredFeature(app, repo); | |
| 49 | return declared.ok ? (declared.value?.schedule ?? []) : []; | |
| 50 | }, | |
| 51 | port(app, appId) { | |
| 52 | const record = getAppById(app.db, appId); | |
| 53 | const status = record && appsRuntime(app)?.envStatus(record, "production"); | |
| 54 | return status && status.state === "running" ? status.port : undefined; | |
| 55 | }, | |
| 56 | call: (port, tool) => callTool(port, tool, {}), | |
| 57 | }; | |
| 58 | ||
| 59 | export function lastFire(db: Database, appId: string, tool: string): Fired | undefined { | |
| 60 | const row = db | |
| 61 | .query<{ fired_at: number; ok: number; error: string | null; duration_ms: number }, [string, string]>( | |
| 62 | "SELECT fired_at, ok, error, duration_ms FROM app_feature_fire WHERE app_id = ?1 AND tool = ?2", | |
| 63 | ) | |
| 64 | .get(appId, tool); | |
| 65 | return row === null ? undefined : { at: row.fired_at, ok: row.ok === 1, error: row.error, durationMs: row.duration_ms }; | |
| 66 | } | |
| 67 | ||
| 68 | const flying = new Set<string>(); | |
| 69 | ||
| 70 | /** Call every due scheduled tool of every running absorbed app; how many were called. */ | |
| 71 | export async function fireDue(app: App, now: number, deps: FireDeps = real): Promise<number> { | |
| 72 | let fired = 0; | |
| 73 | for (const absorbed of listAbsorbed(app.db)) { | |
| 74 | const port = deps.port(app, absorbed.appId); | |
| 75 | if (port === undefined) continue; | |
| 76 | for (const entry of await deps.schedule(app, absorbed.appId)) { | |
| 77 | const slot = `${absorbed.appId}/${entry.tool}`; | |
| 78 | if (flying.has(slot) || !due(entry, lastFire(app.db, absorbed.appId, entry.tool)?.at, now)) continue; | |
| 79 | flying.add(slot); | |
| 80 | try { | |
| 81 | const started = performance.now(); | |
| 82 | const answer = await deps.call(port, entry.tool); | |
| 83 | app.db | |
| 84 | .query("INSERT OR REPLACE INTO app_feature_fire (app_id, tool, fired_at, ok, error, duration_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6)") | |
| 85 | .run(absorbed.appId, entry.tool, now, answer.ok ? 1 : 0, answer.ok ? null : answer.error, Math.round(performance.now() - started)); | |
| 86 | fired++; | |
| 87 | } finally { | |
| 88 | flying.delete(slot); | |
| 89 | } | |
| 90 | } | |
| 91 | } | |
| 92 | return fired; | |
| 93 | } | |
| 94 | ||
| 95 | /** The tick, for the server: unref'd, and `stop` clears it. Never started under `bun test` unless a test calls it. */ | |
| 96 | export function startFeatureSchedule(app: App, everyMs: number = TICK_MS): { stop(): void } { | |
| 97 | let running = false; | |
| 98 | const tick = setInterval(() => { | |
| 99 | if (running) return; | |
| 100 | running = true; | |
| 101 | fireDue(app, Date.now()) | |
| 102 | .catch((error) => app.log.warn("scheduled tools did not finish", { error: String(error) })) | |
| 103 | .finally(() => (running = false)); | |
| 104 | }, everyMs); | |
| 105 | tick.unref(); | |
| 106 | return { stop: () => clearInterval(tick) }; | |
| 107 | } |
src/core/migration-pins/20261109050000-feature-fire.pin added +1
| @@ -0,0 +1,1 @@ | ||
| 1 | bb1ba65968b7cfe2c2ec3243d70ddf33ce650480a5415531353b8ea72b66fc3c |
src/core/migrations/20261109050000-feature-fire.ts added +14
| @@ -0,0 +1,14 @@ | ||
| 1 | // The last time the forge called an absorbed app's scheduled tool (#9926, | |
| 2 | // `apps/feature_schedule.ts`): when, whether it answered, and how long it took. | |
| 3 | // One row per (app, tool); a new fire replaces it. Additive. | |
| 4 | export default ` | |
| 5 | CREATE TABLE app_feature_fire ( | |
| 6 | app_id TEXT NOT NULL, | |
| 7 | tool TEXT NOT NULL, | |
| 8 | fired_at INTEGER NOT NULL, | |
| 9 | ok INTEGER NOT NULL, | |
| 10 | error TEXT, | |
| 11 | duration_ms INTEGER NOT NULL, | |
| 12 | PRIMARY KEY (app_id, tool) | |
| 13 | ); | |
| 14 | `; |
src/http/server.ts +4
| @@ -473,6 +473,7 @@ import { createTrafficWriter } from "../perf/traffic_store.ts"; | ||
| 473 | 473 | import { startGithubPull } from "../perf/github_traffic.ts"; |
| 474 | 474 | import { appSourcesPage } from "../web/pages/sources.ts"; |
| 475 | 475 | import { createViewWriter } from "../perf/views_store.ts"; |
| 476 | import { startFeatureSchedule } from "../apps/feature_schedule.ts"; | |
| 476 | 477 | import { createReadWriter, readOf } from "../perf/reads.ts"; |
| 477 | 478 | import { parseServerTiming } from "../perf/record.ts"; |
| 478 | 479 | import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts"; |
| @@ -2310,6 +2311,8 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2310 | 2311 | // Page views are buffered by the beacon doors and written here; GitHub's traffic is pulled daily when a token is set. |
| 2311 | 2312 | const viewWriter = createViewWriter(app.db, app.log); |
| 2312 | 2313 | const githubPull = startGithubPull(app.db, app.log); |
| 2314 | // Absorbed apps' declared schedules; a test that wants the tick calls `startFeatureSchedule` itself. | |
| 2315 | const featureTick = process.env["NODE_ENV"] === "test" ? undefined : startFeatureSchedule(app); | |
| 2313 | 2316 | const readWriter = createReadWriter(app.db, readLedger, app.log); |
| 2314 | 2317 | const counted = ( |
| 2315 | 2318 | owner: App, |
| @@ -2632,6 +2635,7 @@ export async function serve(app: App): Promise<Servers> { | ||
| 2632 | 2635 | trafficWriter.stop(); |
| 2633 | 2636 | viewWriter.stop(); |
| 2634 | 2637 | githubPull?.stop(); |
| 2638 | featureTick?.stop(); | |
| 2635 | 2639 | readWriter.stop(); |
| 2636 | 2640 | // A child's hooks stop answering before its database closes. |
| 2637 | 2641 | for (const key of [...new Set([...childHttps.keys(), ...childInternals.keys()])]) letGo(key); |
test/feature-schedule-fire.test.ts added +77
| @@ -0,0 +1,77 @@ | ||
| 1 | /** | |
| 2 | * An absorbed app's declared schedule runs (#9926): due or not, once per window, | |
| 3 | * an asleep app skipped, a failing tool kept with its sentence, an un-absorbed | |
| 4 | * app never called. The clock is injected; nothing waits on wall time. | |
| 5 | */ | |
| 6 | ||
| 7 | import { afterAll, beforeAll, expect, test } from "bun:test"; | |
| 8 | import { mkdtemp, rm } from "node:fs/promises"; | |
| 9 | import { tmpdir } from "node:os"; | |
| 10 | import { join } from "node:path"; | |
| 11 | import { createApp, type App } from "../src/app.ts"; | |
| 12 | import { due, fireDue, lastFire, type FireDeps } from "../src/apps/feature_schedule.ts"; | |
| 13 | import { err, ok } from "../src/kernel/result.ts"; | |
| 14 | import { absorb, unabsorb } from "../src/core/features.ts"; | |
| 15 | ||
| 16 | const HOUR = 3_600_000; | |
| 17 | let dir: string; | |
| 18 | let app: App; | |
| 19 | beforeAll(async () => { | |
| 20 | dir = await mkdtemp(join(tmpdir(), "forge-fire-")); | |
| 21 | app = createApp({ dataDir: dir, port: 0, host: "forge.test" }); | |
| 22 | }); | |
| 23 | afterAll(async () => { | |
| 24 | app.db.close(); | |
| 25 | await rm(dir, { recursive: true, force: true }); | |
| 26 | }); | |
| 27 | ||
| 28 | const calls: string[] = []; | |
| 29 | let running = true; | |
| 30 | let answer: ReturnType<typeof ok> | ReturnType<typeof err<string>> = ok({}); | |
| 31 | const deps: FireDeps = { | |
| 32 | schedule: async () => [{ tool: "scan", everyMs: HOUR }], | |
| 33 | port: () => (running ? 4000 : undefined), | |
| 34 | call: async (_port, tool) => { | |
| 35 | calls.push(tool); | |
| 36 | return answer; | |
| 37 | }, | |
| 38 | }; | |
| 39 | ||
| 40 | test("due: never fired, or the interval has passed", () => { | |
| 41 | const entry = { tool: "scan", everyMs: HOUR }; | |
| 42 | expect(due(entry, undefined, 5)).toBe(true); | |
| 43 | expect(due(entry, 1000, 1000 + HOUR - 1)).toBe(false); | |
| 44 | expect(due(entry, 1000, 1000 + HOUR)).toBe(true); | |
| 45 | }); | |
| 46 | ||
| 47 | test("fires once per window, again in the next, and keeps how it went", async () => { | |
| 48 | absorb(app.db, "a1", "u", "Radar", "/"); | |
| 49 | expect(await fireDue(app, 10 * HOUR, deps)).toBe(1); | |
| 50 | expect(await fireDue(app, 10 * HOUR + 1000, deps)).toBe(0); | |
| 51 | expect(calls).toEqual(["scan"]); | |
| 52 | expect(lastFire(app.db, "a1", "scan")).toMatchObject({ at: 10 * HOUR, ok: true, error: null }); | |
| 53 | expect(await fireDue(app, 11 * HOUR, deps)).toBe(1); | |
| 54 | expect(calls).toEqual(["scan", "scan"]); | |
| 55 | }); | |
| 56 | ||
| 57 | test("a failing tool is recorded with its sentence", async () => { | |
| 58 | answer = err("the app did not answer"); | |
| 59 | await fireDue(app, 20 * HOUR, deps); | |
| 60 | expect(lastFire(app.db, "a1", "scan")).toMatchObject({ at: 20 * HOUR, ok: false, error: "the app did not answer" }); | |
| 61 | answer = ok({}); | |
| 62 | }); | |
| 63 | ||
| 64 | test("an asleep or stopped app is skipped", async () => { | |
| 65 | running = false; | |
| 66 | const before = calls.length; | |
| 67 | expect(await fireDue(app, 40 * HOUR, deps)).toBe(0); | |
| 68 | expect(calls.length).toBe(before); | |
| 69 | running = true; | |
| 70 | }); | |
| 71 | ||
| 72 | test("an un-absorbed app is never called", async () => { | |
| 73 | unabsorb(app.db, "a1"); | |
| 74 | const before = calls.length; | |
| 75 | expect(await fireDue(app, 60 * HOUR, deps)).toBe(0); | |
| 76 | expect(calls.length).toBe(before); | |
| 77 | }); |
new above
offline