Public: anyone can read this; the crew can change it.
Checking an upstream fetches only the vetted address, sends a token only to its exact https origin, and runs one check at a time #4046
Check · Review · Land · Live done — Landed and live in production.
scout/forge:scout/3977-checking-an-upstream-fetches-only-the-ve → main 2 commits fork 2 threads · 2 unresolved
4 files
changed +346
−3
src/actions/upstream_follow.ts added +174
| @@ -0,0 +1,174 @@ | ||
| 1 | /** | |
| 2 | * Checking a followed upstream (#3442, layer 2): one `ls-remote`, a fetch | |
| 3 | * only when the tip moved, into `refs/upstream-follow/<branch>` — not a | |
| 4 | * branch, so nothing of ours moves. What it found is noted on the follow row | |
| 5 | * (upstream tip, commits behind). It opens nothing. | |
| 6 | * | |
| 7 | * The fetch is the mirror's: the host is vetted and the vetted address | |
| 8 | * pinned (`vettedEnv`), https and http only. A connected origin's token goes | |
| 9 | * only to the exact https origin it was connected at. One check runs per | |
| 10 | * repository at a time, whoever asks. | |
| 11 | */ | |
| 12 | ||
| 13 | import type { App } from "../app.ts"; | |
| 14 | import { getOriginFor } from "../core/origins.ts"; | |
| 15 | import { canWrite, getRepoById, type Repo } from "../core/repos.ts"; | |
| 16 | import { storageStale } from "../core/storage.ts"; | |
| 17 | import { getFollow, listFollowing, noteCheck } from "../core/upstream_follow.ts"; | |
| 18 | import { getUserById } from "../core/users.ts"; | |
| 19 | import { runGitCapture } from "../git/env.ts"; | |
| 20 | import { isSafeOid } from "../git/guard.ts"; | |
| 21 | import { aheadCount, resolveOid } from "../git/merge.ts"; | |
| 22 | import { REMOTE_PROTOCOLS } from "../git/remote-env.ts"; | |
| 23 | import { repoDir } from "../git/repos.ts"; | |
| 24 | import { err, ok, type Result } from "../kernel/result.ts"; | |
| 25 | import { providerForOrigin } from "../origins/credential.ts"; | |
| 26 | import { redact, secretsIn } from "../origins/credentials.ts"; | |
| 27 | import { vettedEnv } from "../origins/git-mirror.ts"; | |
| 28 | ||
| 29 | /** How often a following repository is checked on its own. */ | |
| 30 | export const FOLLOW_EVERY_MS = 30 * 60_000; | |
| 31 | /** One check's whole budget: ls-remote and fetch together. */ | |
| 32 | const CHECK_TIMEOUT_MS = 120_000; | |
| 33 | ||
| 34 | /** Where the upstream tip is staged: not a branch, so a check moves nothing of ours. */ | |
| 35 | export const stagedRef = (branch: string): string => `refs/upstream-follow/${branch}`; | |
| 36 | ||
| 37 | export type CheckError = "off" | "nobody" | "unreadable" | "no_branch" | "git" | "busy"; | |
| 38 | ||
| 39 | /** What each refusal says, at every door. */ | |
| 40 | export const CHECK_REFUSALS: Record<CheckError, string> = { | |
| 41 | off: "Following an upstream is off for this repository.", | |
| 42 | nobody: "The person who turned following on can no longer write here; turn it on again.", | |
| 43 | unreadable: "The upstream could not be read.", | |
| 44 | no_branch: "The upstream has no such branch.", | |
| 45 | git: "Git could not read the staged upstream.", | |
| 46 | busy: "Already checking this upstream", | |
| 47 | }; | |
| 48 | ||
| 49 | /** What one check read. */ | |
| 50 | export interface Checked { | |
| 51 | readonly tip: string; | |
| 52 | readonly behind: number; | |
| 53 | } | |
| 54 | ||
| 55 | /** | |
| 56 | * Whether an origin's token may travel to this upstream: only to the exact | |
| 57 | * https origin (scheme, host and port) the person connected. An `http://` | |
| 58 | * upstream on the same host is fetched anonymously. | |
| 59 | */ | |
| 60 | export function mayCarryCredential(upstreamUrl: string, webUrl: string): boolean { | |
| 61 | try { | |
| 62 | const up = new URL(upstreamUrl); | |
| 63 | return up.protocol === "https:" && up.origin === new URL(webUrl).origin; | |
| 64 | } catch { | |
| 65 | return false; | |
| 66 | } | |
| 67 | } | |
| 68 | ||
| 69 | /** Repositories with a check in flight: one at a time each, whoever asked. */ | |
| 70 | const checking = new Set<string>(); | |
| 71 | ||
| 72 | /** | |
| 73 | * One check, of one repository. The claim is taken before the first await, | |
| 74 | * so a double click and the clock never both run: the second is `busy`. | |
| 75 | */ | |
| 76 | export async function checkUpstream(app: App, repo: Repo): Promise<Result<Checked, CheckError>> { | |
| 77 | if (checking.has(repo.id)) return err("busy"); | |
| 78 | checking.add(repo.id); | |
| 79 | try { | |
| 80 | return await checkOnce(app, repo); | |
| 81 | } finally { | |
| 82 | checking.delete(repo.id); | |
| 83 | } | |
| 84 | } | |
| 85 | ||
| 86 | /** | |
| 87 | * The clock's pass: every repository whose following is on and was last | |
| 88 | * checked more than `FOLLOW_EVERY_MS` ago, one after another. Never throws | |
| 89 | * into the timer; a repository busy with a person's check is skipped. | |
| 90 | */ | |
| 91 | export async function checkDue(app: App, now = Date.now()): Promise<number> { | |
| 92 | let checked = 0; | |
| 93 | for (const follow of listFollowing(app.db)) { | |
| 94 | if (follow.checkedAt !== null && now - Date.parse(follow.checkedAt) < FOLLOW_EVERY_MS) continue; | |
| 95 | const repo = getRepoById(app.db, follow.repoId); | |
| 96 | if (repo === undefined) continue; | |
| 97 | try { | |
| 98 | if ((await checkUpstream(app, repo)).ok) checked++; | |
| 99 | } catch (error) { | |
| 100 | app.log.error("upstream check failed", { repo: repo.id, error: String(error) }); | |
| 101 | } | |
| 102 | } | |
| 103 | return checked; | |
| 104 | } | |
| 105 | ||
| 106 | async function checkOnce(app: App, repo: Repo): Promise<Result<Checked, CheckError>> { | |
| 107 | const follow = getFollow(app.db, repo.id); | |
| 108 | if (follow === undefined || !follow.enabled || repo.archivedAt !== null || repo.origin !== null) | |
| 109 | return err("off"); | |
| 110 | const at = new Date().toISOString(); | |
| 111 | const failed = (error: CheckError, why = ""): Result<Checked, CheckError> => { | |
| 112 | noteCheck(app.db, repo.id, { at, error: why || CHECK_REFUSALS[error] }); | |
| 113 | return err(error); | |
| 114 | }; | |
| 115 | const by = getUserById(app.db, follow.byId); | |
| 116 | if (by === undefined || !canWrite(app.db, repo, by)) return failed("nobody"); | |
| 117 | ||
| 118 | // The credential of a private upstream: the connected origin's, scoped to | |
| 119 | // this one git invocation, and only when the upstream is that origin's https origin. | |
| 120 | let authEnv: Record<string, string> = {}; | |
| 121 | if (follow.originId !== "") { | |
| 122 | const origin = getOriginFor(app.db, by.id, follow.originId); | |
| 123 | const provider = origin === undefined ? undefined : providerForOrigin(app, origin); | |
| 124 | if (origin !== undefined && provider?.ok && mayCarryCredential(follow.url, origin.webUrl)) | |
| 125 | authEnv = await provider.value.gitAuthEnv(follow.url); | |
| 126 | } | |
| 127 | // Vetted as the mirror fetch is, and pinned to the address that passed. | |
| 128 | const pinned = await vettedEnv(follow.url, authEnv, REMOTE_PROTOCOLS); | |
| 129 | if (!pinned.ok) return failed("unreadable", pinned.error.message); | |
| 130 | const env = pinned.value; | |
| 131 | const said = (stderr: string): string => | |
| 132 | redact(stderr.trim(), ...secretsIn(authEnv)).split("\n").find((l) => l.trim() !== "") ?? ""; | |
| 133 | ||
| 134 | const dir = repoDir(app.reposRoot, repo.ownerName, repo.name); | |
| 135 | const deadline = Date.now() + CHECK_TIMEOUT_MS; | |
| 136 | const remaining = (): number => Math.max(1_000, deadline - Date.now()); | |
| 137 | ||
| 138 | const listed = await runGitCapture( | |
| 139 | ["ls-remote", "--", follow.url, `refs/heads/${follow.branch}`], | |
| 140 | { cwd: dir, env, timeoutMs: remaining() }, | |
| 141 | ); | |
| 142 | if (listed.code !== 0) | |
| 143 | return failed("unreadable", `The upstream could not be read: ${said(listed.stderr) || "no reason given"}`); | |
| 144 | const tip = listed.stdout.split("\t")[0]?.trim() ?? ""; | |
| 145 | if (!isSafeOid(tip)) return failed("no_branch"); | |
| 146 | ||
| 147 | // Fetch only when the tip moved: the staged ref is what the last fetch brought. | |
| 148 | const ref = stagedRef(follow.branch); | |
| 149 | if ((await resolveOid(dir, ref)) !== tip) { | |
| 150 | const fetched = await runGitCapture( | |
| 151 | [ | |
Parse by exact ref name: split the lines and pick the one whose second field is |
||
| 152 | "fetch", | |
| 153 | "--no-tags", | |
| 154 | "--no-write-fetch-head", | |
| 155 | "--no-recurse-submodules", | |
| 156 | "--quiet", | |
| 157 | "--", | |
| 158 | follow.url, | |
| 159 | `+refs/heads/${follow.branch}:${ref}`, | |
| 160 | ], | |
| 161 | { cwd: dir, env, timeoutMs: remaining() }, | |
| 162 | ); | |
| 163 | if (fetched.code !== 0) | |
| 164 | return failed("unreadable", `The upstream could not be fetched: ${said(fetched.stderr) || "no reason given"}`); | |
| 165 | storageStale(app, repo.ownerId); | |
| 166 | } | |
| 167 | const staged = await resolveOid(dir, ref); | |
| 168 | if (staged === undefined) return failed("git"); | |
| 169 | const main = await resolveOid(dir, `refs/heads/${repo.defaultBranch}`); | |
| 170 | if (main === undefined) return failed("git", "The default branch has no commits yet."); | |
| 171 | const behind = await aheadCount(dir, main, staged); | |
| 172 | noteCheck(app.db, repo.id, { at, tip: staged, behind, error: "" }); | |
| 173 | return ok({ tip: staged, behind }); | |
| 174 | } |
src/main.ts +5
| @@ -27,6 +27,7 @@ import { runDelivery } from "./core/webhooks.ts"; | ||
| 27 | 27 | import { isHookName, runHookClient } from "./git/hooks.ts"; |
| 28 | 28 | import { drainStreams } from "./http/sse.ts"; |
| 29 | 29 | import { serve } from "./http/server.ts"; |
| 30 | import { checkDue } from "./actions/upstream_follow.ts"; | |
| 30 | 31 | import { runSync } from "./origins/sync.ts"; |
| 31 | 32 | import { startForgeSsh } from "./tui/ssh.ts"; |
| 32 | 33 | import { AppsRuntime } from "./apps/runtime.ts"; |
| @@ -491,6 +492,9 @@ const tickAsks = async (): Promise<void> => { | ||
| 491 | 492 | app.log.warn("degraded history: vitals failed", { error: String(error) }); |
| 492 | 493 | } |
| 493 | 494 | }; |
| 495 | // Followed upstreams are read on a slow tick (`actions/upstream_follow.ts`): | |
| 496 | // the tip and how far behind land on the follow row; nothing is opened. | |
| 497 | const followTimer = setInterval(() => void checkDue(app), 60_000); | |
| 494 | 498 | const staleTimer = setInterval(() => void tickAsks(), 120_000); |
| 495 | 499 | void tickAsks(); |
| 496 | 500 | |
| @@ -637,6 +641,7 @@ function drain(signal: NodeJS.Signals): Promise<unknown> { | ||
| 637 | 641 | clearInterval(upstreamTimer); |
| 638 | 642 | clearInterval(syncTimer); |
| 639 | 643 | clearInterval(staleTimer); |
| 644 | clearInterval(followTimer); | |
| 640 | 645 | clearInterval(processTimer); |
| 641 | 646 | clearInterval(leaseTimer); |
| 642 | 647 | // The platform's SIGTERM: the listener stays open while routers learn |
src/origins/git-mirror.ts +3 −2
| @@ -57,9 +57,10 @@ export const MIRROR_UNRESOLVED = "the remote's host could not be resolved"; | ||
| 57 | 57 | * Vet the clone URL's host and return git's environment pinned to the |
| 58 | 58 | * address that passed — or the one sentence that ends the sync. |
| 59 | 59 | */ |
| 60 | async function vettedEnv( | |
| 60 | export async function vettedEnv( | |
| 61 | 61 | url: string, |
| 62 | 62 | authEnv: Record<string, string>, |
| 63 | protocols?: string, | |
| 63 | 64 | ): Promise<Result<Record<string, string>, GitMirrorError>> { |
| 64 | 65 | const host = new URL(url).hostname; |
| 65 | 66 | const vetted = await vetHost(host, { ...outbound, trusted: trustedHosts() }); |
| @@ -67,7 +68,7 @@ async function vettedEnv( | ||
| 67 | 68 | return err({ |
| 68 | 69 | message: vetted.error === "refused" ? MIRROR_REFUSED : MIRROR_UNRESOLVED, |
| 69 | 70 | }); |
| 70 | return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!))); | |
| 71 | return ok(remoteGitEnv(authEnv, pinFor(url, vetted.value[0]!), protocols)); | |
| 71 | 72 | } |
| 72 | 73 | |
| 73 | 74 | export interface RefSnapshot { |
test/upstream_follow.test.ts +164 −1
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | /** |
| 2 | 2 | * A repository's upstream setting (#3442, layer 1): stored as a row, https |
| 3 | * or http only, and an admin's to read. | |
| 3 | * or http only, and an admin's to read. Layer 2: its check — one at a time, | |
| 4 | * the vetted address, a token only to its exact https origin. | |
| 4 | 5 | */ |
| 5 | 6 | |
| 6 | 7 | import { afterAll, beforeAll, describe, expect, test } from "bun:test"; |
| @@ -13,6 +14,12 @@ import { audienceFilter } from "../src/core/event_audience.ts"; | ||
| 13 | 14 | import { listEvents } from "../src/core/events.ts"; |
| 14 | 15 | import type { Repo } from "../src/core/repos.ts"; |
| 15 | 16 | import { createUser, type User } from "../src/core/users.ts"; |
| 17 | import { checkDue, checkUpstream, mayCarryCredential } from "../src/actions/upstream_follow.ts"; | |
| 18 | import { createOrigin } from "../src/core/origins.ts"; | |
| 19 | import { runGit } from "../src/git/env.ts"; | |
| 20 | import { REMOTE_PROTOCOLS } from "../src/git/remote-env.ts"; | |
| 21 | import { repoDir } from "../src/git/repos.ts"; | |
| 22 | import { MIRROR_REFUSED, provideMirrorOutboundForTest, vettedEnv } from "../src/origins/git-mirror.ts"; | |
| 16 | 23 | import { |
| 17 | 24 | getFollow, |
| 18 | 25 | listFollowing, |
| @@ -26,6 +33,8 @@ let app: App; | ||
| 26 | 33 | let alice: User; |
| 27 | 34 | let guest: User; |
| 28 | 35 | let repo: Repo; |
| 36 | let target: Repo; | |
| 37 | let dir = ""; | |
| 29 | 38 | |
| 30 | 39 | const ok = <T>(r: { ok: true; value: T } | { ok: false; error: unknown }): T => { |
| 31 | 40 | if (!r.ok) throw new Error(JSON.stringify(r.error)); |
| @@ -128,3 +137,157 @@ describe("who hears of it", () => { | ||
| 128 | 137 | expect(seen(guest)).not.toContain(secret); |
| 129 | 138 | }); |
| 130 | 139 | }); |
| 140 | ||
| 141 | describe("checking an upstream (layer 2)", () => { | |
| 142 | beforeAll(async () => { | |
| 143 | target = ok(await createRepo(app, alice, { name: "checked", visibility: "public" })); | |
| 144 | dir = repoDir(app.reposRoot, target.ownerName, target.name); | |
| 145 | provideMirrorOutboundForTest({ allow: ["127.0.0.1"] }); | |
| 146 | }); | |
| 147 | afterAll(() => provideMirrorOutboundForTest({})); | |
| 148 | /** A server that records every request and answers with `refs` as a dumb-http `info/refs`. */ | |
| 149 | function upstream(refs: () => string, delayMs = 0) { | |
| 150 | const seen: Request[] = []; | |
| 151 | const server = Bun.serve({ | |
| 152 | port: 0, | |
| 153 | hostname: "127.0.0.1", | |
| 154 | async fetch(req) { | |
| 155 | seen.push(req); | |
| 156 | if (delayMs > 0) await Bun.sleep(delayMs); | |
| 157 | return new URL(req.url).pathname.endsWith("/info/refs") | |
| 158 | ? new Response(refs(), { headers: { "content-type": "text/plain" } }) | |
| 159 | : new Response("no", { status: 404 }); | |
| 160 | }, | |
| 161 | }); | |
| 162 | return { seen, server, url: `http://127.0.0.1:${server.port}/o/r.git` }; | |
| 163 | } | |
| 164 | ||
| 165 | const follow = (url: string, originId = "") => | |
| 166 | putFollow(app.db, { repoId: target.id, url, branch: "main", enabled: true, byId: alice.id, originId }); | |
| 167 | ||
| 168 | describe("one check at a time", () => { | |
| 169 | test("a second caller while one runs is refused", async () => { | |
| 170 | const up = upstream(() => "", 150); | |
| 171 | try { | |
| 172 | follow(up.url); | |
| 173 | const [a, b] = await Promise.all([checkUpstream(app, target), checkUpstream(app, target)]); | |
| 174 | expect(!b.ok && b.error).toBe("busy"); | |
| 175 | expect(a.ok).toBe(false); | |
| 176 | expect(!a.ok && a.error).not.toBe("busy"); | |
| 177 | // The claim is let go: the next one runs. | |
| 178 | const c = await checkUpstream(app, target); | |
| 179 | expect(!c.ok && c.error).not.toBe("busy"); | |
| 180 | } finally { | |
| 181 | up.server.stop(true); | |
| 182 | } | |
| 183 | }); | |
| 184 | }); | |
| 185 | ||
| 186 | describe("the vetted address", () => { | |
| 187 | test("is pinned in git's environment, with only https and http", async () => { | |
| 188 | const env = ok(await vettedEnv("http://127.0.0.1:8123/o/r.git", {}, REMOTE_PROTOCOLS)); | |
| 189 | expect(env["GIT_ALLOW_PROTOCOL"]).toBe("https:http"); | |
| 190 | const entries = Object.entries(env).filter(([, v]) => v === "127.0.0.1:8123:127.0.0.1"); | |
| 191 | expect(entries.length).toBe(1); | |
| 192 | expect(env[entries[0]![0].replace("VALUE", "KEY")]).toBe("http.curloptResolve"); | |
| 193 | }); | |
| 194 | ||
| 195 | test("a private host is refused unless trusted, and nothing is fetched", async () => { | |
| 196 | provideMirrorOutboundForTest({}); | |
| 197 | const up = upstream(() => ""); | |
| 198 | try { | |
| 199 | follow(up.url); | |
| 200 | const r = await checkUpstream(app, target); | |
| 201 | expect(!r.ok && r.error).toBe("unreadable"); | |
| 202 | expect(getFollow(app.db, target.id)?.error).toBe(MIRROR_REFUSED); | |
| 203 | expect(up.seen.length).toBe(0); | |
| 204 | } finally { | |
| 205 | provideMirrorOutboundForTest({ allow: ["127.0.0.1"] }); | |
| 206 | up.server.stop(true); | |
| 207 | } | |
| 208 | }); | |
| 209 | }); | |
| 210 | ||
| 211 | describe("the credential", () => { | |
| 212 | test("goes only to the same scheme, host and port, over https", () => { | |
| 213 | const web = "https://git.example.com"; | |
| 214 | expect(mayCarryCredential("https://git.example.com/a/b.git", web)).toBe(true); | |
| 215 | expect(mayCarryCredential("http://git.example.com/a/b.git", web)).toBe(false); | |
| 216 | expect(mayCarryCredential("https://git.example.com:8443/a/b.git", web)).toBe(false); | |
| 217 | expect(mayCarryCredential("https://evil.example.com/a/b.git", web)).toBe(false); | |
| 218 | expect(mayCarryCredential("https://git.example.com.evil.com/a", web)).toBe(false); | |
| 219 | expect(mayCarryCredential("https://git.example.com/a", "http://git.example.com")).toBe(false); | |
| 220 | expect(mayCarryCredential("not a url", web)).toBe(false); | |
| 221 | }); | |
| 222 | ||
| 223 | test("an http upstream on a connected host is fetched with no Authorization", async () => { | |
| 224 | const up = upstream(() => ""); | |
| 225 | try { | |
| 226 | const origin = ok( | |
| 227 | createOrigin(app.db, dataDir, { | |
| 228 | userId: alice.id, | |
| 229 | kind: "forgejo", | |
| 230 | apiUrl: `http://127.0.0.1:${up.server.port}/api/v1`, | |
| 231 | webUrl: `http://127.0.0.1:${up.server.port}`, | |
| 232 | login: "alice", | |
| 233 | token: "s3cret-token", | |
| 234 | }), | |
| 235 | ); | |
| 236 | follow(up.url, origin.id); | |
| 237 | await checkUpstream(app, target); | |
| 238 | expect(up.seen.length).toBeGreaterThan(0); | |
| 239 | for (const req of up.seen) expect(req.headers.get("authorization")).toBeNull(); | |
| 240 | } finally { | |
| 241 | up.server.stop(true); | |
| 242 | } | |
| 243 | }); | |
| 244 | }); | |
| 245 | ||
| 246 | describe("what a check records", () => { | |
| 247 | test("the upstream tip and how far behind, without opening anything", async () => { | |
| 248 | const git = async (args: string[]) => ok(await runGit(args, { cwd: dir, env: ident })).trim(); | |
| 249 | const ident = { | |
| 250 | GIT_AUTHOR_NAME: "t", | |
| 251 | GIT_AUTHOR_EMAIL: "t@x", | |
| 252 | GIT_COMMITTER_NAME: "t", | |
| 253 | GIT_COMMITTER_EMAIL: "t@x", | |
| 254 | }; | |
| 255 | const empty = await git(["hash-object", "-t", "tree", "-w", "--stdin"]).catch(() => ""); | |
| 256 | const tree = empty || "4b825dc642cb9eb9a060e54bf8d69288fbee4904"; | |
| 257 | const c1 = await git(["commit-tree", tree, "-m", "one"]); | |
| 258 | const c2 = await git(["commit-tree", tree, "-p", c1, "-m", "two"]); | |
| 259 | await git(["update-ref", `refs/heads/${target.defaultBranch}`, c1]); | |
| 260 | // Already staged, so the check reads the tip and has nothing to fetch. | |
| 261 | await git(["update-ref", "refs/upstream-follow/main", c2]); | |
| 262 | const up = upstream(() => `${c2}\trefs/heads/main\n`); | |
| 263 | try { | |
| 264 | follow(up.url); | |
| 265 | const r = ok(await checkUpstream(app, target)); | |
| 266 | expect(r).toEqual({ tip: c2, behind: 1 }); | |
| 267 | const row = getFollow(app.db, target.id)!; | |
| 268 | expect(row.tip).toBe(c2); | |
| 269 | expect(row.behind).toBe(1); | |
| 270 | expect(row.error).toBe(""); | |
| 271 | expect(row.pendingId).toBeNull(); | |
| 272 | expect(row.opened).toBe(0); | |
| 273 | } finally { | |
| 274 | up.server.stop(true); | |
| 275 | } | |
| 276 | }); | |
| 277 | }); | |
| 278 | ||
| 279 | test("the clock checks a due following once and leaves a fresh one alone", async () => { | |
| 280 | const up = upstream(() => ""); | |
| 281 | try { | |
| 282 | follow(up.url); | |
| 283 | await checkDue(app, Date.now() + 31 * 60_000); | |
| 284 | const seen = up.seen.length; | |
| 285 | expect(seen).toBeGreaterThan(0); | |
| 286 | // Just checked: not due again. | |
| 287 | await checkDue(app, Date.now()); | |
| 288 | expect(up.seen.length).toBe(seen); | |
| 289 | } finally { | |
| 290 | up.server.stop(true); | |
| 291 | } | |
| 292 | }); | |
| 293 | }); |
new above
offline
Add the closing full stop to this message, as the other refusals have.