ee2574d · 18255e5 · b2f6f69 · f788bb7 · cad8a8a
@@ -137,6 +137,7 @@
Machine routes are not pages and have no entry: `/audit.csv`,
- `/settings/origins` — `pages/origins.ts` (1,497 lines) · other forges this one reads and mirrors from · `/migrate` · none seen.
- `/settings/security` — `pages/security.ts` · the Shield and the evidence an admin needs to act with no AI · `/settings/traffic`, `/settings/backups` · none seen.
- `/settings/traffic` — `pages/traffic.ts` · who used the answering time in the last 15 minutes · `/settings/usage`, `/settings/telemetry` · none seen.
- `/settings/traffic/scrapes` — `pages/scrapes.ts` · what was read by repository and path, which networks and addresses did it, a bounded capture, the abuse export · `/settings/traffic`, `/settings/security` · none seen.
## Operator views — why each is a page (#2696)
@@ -0,0 +1,12 @@
# Scrape evidence (#9406)
An admin sees what was read and works out who is doing it, at `/settings/traffic/scrapes` (linked from Traffic) and in the terminal: `:` → "Scrapes" (a forge admin who is a person; read-only there).
- **Reads.** For 1, 3 or 7 days: per repository, the paths read, the commits and branches, how much of the repository was covered, and files whose names look like keys or credentials. Counted in memory as requests finish (`perf/reads.ts`), written once a minute to `read_hour`, kept 7 days. No address is kept in it. A stored hour holds at most 500 distinct (repository, ref, path) rows; later ones fold into `(others)` for their repository, so an attacker walking paths cannot grow the table, and the report groups in SQL.
- **Networks.** Address prefixes show name, operator, country and abuse contact from the forge's own `network` table, filled by pasting `range, name, operator, country, abuse contact` lines. No visitor address goes to a third party. Where the table has no entry, the page says the evidence stops there.
- **Capture.** One click starts a capture: 10 minutes, 2000 requests, in memory only, no cookies, tokens or query strings, signed-in people skipped. Requests that look alike are grouped across addresses with a stated confidence and reasons. Full addresses exist only inside a capture and are gone when it expires or is stopped.
- **Bot score and country** (`cf-bot-score`, `x-bot-score`, `cf-ipcountry`) are believed only when the request came through a trusted proxy (`peer.ts`, `wasViaEdge`); from anyone else they are ignored.
- **Export.** A group exports as a plain-text abuse report with its limits stated.
- Admin only; agent accounts are refused; every mutation is same-origin.
Not done: lookups against a public registry (by design, nothing leaves the forge), and blocking from this page.
@@ -165,6 +165,8 @@
const READER_KINDS: ReadonlySet<string> = new Set([
  "issue.confirmed",
  "capacity.changed",
  "shield.changed",
  "traffic.capture_started",
  "traffic.networks_imported",
  "crew.governed",
]);
@@ -0,0 +1,1 @@
d90e8888280398516a6b00d80d951bad10ef7c23e8f7af59628a6fee590d77c3
@@ -0,0 +1,26 @@
// Evidence for telling a scraper from a reader (`perf/reads.ts`, `perf/network.ts`).
// `read_hour`: what was read from repositories, by the hour for 7 days: repository,
// ref (a branch or a commit), path, how many reads and how many were anonymous.
// No address is kept here. `network`: the forge's own database of who runs an
// address range (name, operator, country, abuse contact), filled by an admin, so no
// visitor address ever goes to a third party. New tables only. Additive.
export default `
  CREATE TABLE read_hour (
    hour      INTEGER NOT NULL,
    repo      TEXT    NOT NULL,
    ref       TEXT    NOT NULL,
    path      TEXT    NOT NULL,
    requests  INTEGER NOT NULL,
    anonymous INTEGER NOT NULL,
    PRIMARY KEY (hour, repo, ref, path)
  ) WITHOUT ROWID;
  CREATE TABLE network (
    cidr     TEXT PRIMARY KEY,
    name     TEXT NOT NULL,
    operator TEXT NOT NULL,
    country  TEXT NOT NULL,
    abuse    TEXT NOT NULL,
    source   TEXT NOT NULL,
    added_at INTEGER NOT NULL
  ) WITHOUT ROWID;
  `;
@@ -61,6 +61,7 @@
export function resolveClient(
const clients = new WeakMap<Request, string>();
const internals = new WeakSet<Request>();
const edged = new WeakSet<Request>();
/** Headers a proxy names a client in. The tunnel always sets its own, so no stranger can leave them all out. */
const CLIENT_HEADERS = ["cf-connecting-ip", "x-forwarded-for", "x-real-ip", "forwarded"];
@@ -85,8 +86,13 @@
export const notePeer = (
): void => {
  clients.set(request, resolveClient(address, request.headers, trust));
  if (isInternalPeer(address, request.headers, trust)) internals.add(request);
  else if (address !== undefined && (trust.proxies.length > 0 ? inCidrs(address, trust.proxies) : LOOPBACK.has(address)))
    edged.add(request);
};
/** The request came through a proxy the operator trusts, so the headers an edge adds (bot score, country) are its, not the client's. */
export const wasViaEdge = (request: Request): boolean => edged.has(request);
export const wasInternal = (request: Request): boolean => internals.has(request);
export function clientAddress(request: Request): string {
@@ -458,14 +458,15 @@
import {
  takeDbTime,
  type RouteTiming,
} from "../core/route_timing.ts";
import { loopWatch, perfRecord } from "../perf/live.ts";
import { countryOf, repoOf, signatureOf } from "../perf/traffic.ts";
import { captureStore, loopWatch, perfRecord, readLedger } from "../perf/live.ts";
import { classOf, countryOf, repoOf, signatureOf } from "../perf/traffic.ts";
import { createTrafficWriter } from "../perf/traffic_store.ts";
import { createReadWriter, readOf } from "../perf/reads.ts";
      
        sentinelcrew
        6d ago
        outdated
      
      Duplicate imports from perf/live.ts and perf/traffic.ts: merge into the existing import lines.
    
import { parseServerTiming } from "../perf/record.ts";
import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts";
import { bearerChallenge, oauthRoutes } from "./oauth.ts";
import { flushAccess, recordAccess } from "./access_log.ts";
import { carryPeer, clientAddress, notePeer, wasInternal, type ProxyTrust } from "./peer.ts";
import { carryPeer, clientAddress, notePeer, wasInternal, wasViaEdge, type ProxyTrust } from "./peer.ts";
import { isProbe } from "../kernel/probe.ts";
import { takeStrangerRead } from "./stranger_reads.ts";
import { parseCidrs } from "../kernel/cidr.ts";
@@ -2229,6 +2230,7 @@
export async function serve(app: App): Promise<Servers> {
  const timings = new Map<Database, RouteTiming>();
  // The minute ledger's history: one write a minute, never on a request.
  const trafficWriter = createTrafficWriter(app.db, perfRecord.ledger, app.log);
  const readWriter = createReadWriter(app.db, readLedger, app.log);
  const counted = (
    owner: App,
    request: Request,
@@ -2255,6 +2257,16 @@
export async function serve(app: App): Promise<Servers> {
        },
      );
      loopWatch.finished(tag.pattern, started, end);
      // Evidence of what was read, and the running capture, if any (never a signed-in person's).
      const signedIn = tag.role !== "anon" && tag.role !== "crawler";
      const cls = classOf({ signedIn, crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request) });
      if (cls !== "internal" && response.status < 400) {
        const read = readOf(tag.pattern, new URL(request.url).pathname);
        if (read !== null) readLedger.note(Date.now(), read, !signedIn);
      }
      const at = Date.now();
      if (captureStore.active(at))
        captureStore.note(at, { request, address: clientAddress(request), viaEdge: wasViaEdge(request), status: response.status, ms, cls, pattern: tag.pattern });
    } catch {
      // A response the forge is only carrying keeps the headers it came with.
    }
@@ -2492,6 +2504,7 @@
export async function serve(app: App): Promise<Servers> {
      }
      timings.clear();
      trafficWriter.stop();
      readWriter.stop();
      // A child's hooks stop answering before its database closes.
      for (const key of [...new Set([...childHttps.keys(), ...childInternals.keys()])]) letGo(key);
      closeHostedChildren(app);
@@ -20,6 +20,7 @@
import { usagePage } from "../web/pages/usage.ts";
import { spendPage } from "../web/pages/spend-tab.ts";
import { securityPage, securityShieldSet } from "../web/pages/security.ts";
import { trafficPage } from "../web/pages/traffic.ts";
import { captureStart, captureStop, networksImport, scrapesExport, scrapesPage } from "../web/pages/scrapes.ts";
import {
  accountAct,
  retireAct,
@@ -189,6 +190,11 @@
export function settingsRoutes(router: Router<Handler>, knownPatterns: () => rea
  router.add("GET", "/settings/usage", (ctx) => usagePage(ctx, knownPatterns));
  router.add("GET", "/settings/spend", spendPage);
  router.add("GET", "/settings/traffic", trafficPage);
  router.add("GET", "/settings/traffic/scrapes", scrapesPage);
  router.add("GET", "/settings/traffic/scrapes/export", scrapesExport);
  router.add("POST", "/settings/traffic/scrapes/capture", captureStart);
  router.add("POST", "/settings/traffic/scrapes/stop", captureStop);
  router.add("POST", "/settings/traffic/scrapes/networks", networksImport);
  router.add("GET", "/settings/security", securityPage);
  router.add("POST", "/settings/security/shield", securityShieldSet);
  router.add("GET", "/settings/backups", backupsPage);
@@ -0,0 +1,100 @@
/**
 * One actor across addresses: captured requests grouped by what they share
 * (user agent, header names and order, language, accept, encoding,
 * sec-fetch-site), each group with its size, rate, addresses, the commits it
 * walked, its cadence and Cloudflare's bot score if sent, and a confidence
 * stated with the reasons for it.
 *
 * A group is a claim that requests look alike, not that one party sent them.
 * Traits every browser of one version shares prove little, so confidence rests
 * on the strong signals (a steady cadence, a walk through commits, a bot score)
 * and the page says what the evidence cannot show. Nothing here names anyone.
 */
import type { CapturedRequest } from "./capture.ts";
import { isCommit } from "./reads.ts";
export type Confidence = "low" | "medium" | "high";
export interface Group {
  readonly id: string;
  readonly ua: string;
  readonly language: string;
  readonly size: number;
  readonly addresses: readonly string[];
  readonly prefixes: readonly string[];
  readonly perMinute: number;
  readonly spanMs: number;
  readonly commits: readonly string[];
  readonly cadence: "steady" | "uneven" | "too few to say";
  readonly botScore: { readonly min: number; readonly max: number } | null;
  readonly confidence: Confidence;
  /** What the confidence rests on, and what it lacks. */
  readonly reasons: readonly string[];
  readonly requests: readonly CapturedRequest[];
}
const key = (r: CapturedRequest): string =>
  [r.ua, r.shape, r.language, r.accept, r.encoding, r.fetchSite].join("\u0000");
const idOf = (k: string): string => (Number(Bun.hash.xxHash32(k)) >>> 0).toString(16).padStart(8, "0");
/** Is the gap between requests steady: at least 10 gaps and a coefficient of variation under 0.5. */
function cadenceOf(times: readonly number[]): Group["cadence"] {
  if (times.length < 11) return "too few to say";
  const sorted = [...times].sort((a, b) => a - b);
  const gaps = sorted.slice(1).map((t, i) => t - sorted[i]!);
  const mean = gaps.reduce((a, b) => a + b, 0) / gaps.length;
  if (mean <= 0) return "steady";
  const sd = Math.sqrt(gaps.reduce((a, g) => a + (g - mean) ** 2, 0) / gaps.length);
  return sd / mean < 0.5 ? "steady" : "uneven";
}
export function groupRequests(entries: readonly CapturedRequest[]): Group[] {
  const by = new Map<string, CapturedRequest[]>();
  for (const e of entries) by.set(key(e), [...(by.get(key(e)) ?? []), e]);
  const groups: Group[] = [];
  for (const [k, list] of by) {
    const addresses = [...new Set(list.map((r) => r.address))];
    const prefixes = [...new Set(list.map((r) => r.prefix))];
    const times = list.map((r) => r.at);
    const spanMs = Math.max(...times) - Math.min(...times);
    const commits = [...new Set(list.flatMap((r) => (r.read !== null && isCommit(r.read.ref) ? [r.read.ref] : [])))];
    const scores = list.flatMap((r) => (r.botScore === null ? [] : [r.botScore]));
    const cadence = cadenceOf(times);
    const botScore = scores.length === 0 ? null : { min: Math.min(...scores), max: Math.max(...scores) };
    const strong = [cadence === "steady", commits.length >= 3, botScore !== null && botScore.max <= 29];
    const strongN = strong.filter(Boolean).length;
    const size = list.length;
    const confidence: Confidence =
      strongN >= 2 && size >= 20 ? "high" : strongN >= 1 && size >= 10 && addresses.length >= 2 ? "medium" : "low";
    const reasons = [
      addresses.length >= 2
        ? `${addresses.length} addresses sent requests with the same user agent and headers.`
        : "Only one address: nothing here spans addresses.",
      cadence === "steady" ? "Requests arrive at a steady pace." : cadence === "uneven" ? "Requests arrive unevenly, as people's do." : "Too few requests to judge their pace.",
      commits.length >= 3 ? `They read ${commits.length} different commits.` : "They did not walk a run of commits.",
      botScore === null ? "No bot score was sent." : `Cloudflare's bot score was ${botScore.min === botScore.max ? botScore.min : `${botScore.min} to ${botScore.max}`} (low means likely automated).`,
      size < 20 ? "Few requests: a small sample." : `${size} requests.`,
      confidence === "low" && strongN === 0 ? "Traits that browsers of one version share are weak evidence on their own." : "",
    ].filter((r) => r !== "");
    groups.push({
      id: idOf(k),
      ua: list[0]!.ua,
      language: list[0]!.language,
      size,
      addresses,
      prefixes,
      perMinute: Math.round((size / Math.max(1, spanMs / 60_000)) * 10) / 10,
      spanMs,
      commits,
      cadence,
      botScore,
      confidence,
      reasons,
      requests: list,
    });
  }
  const rank: Record<Confidence, number> = { high: 3, medium: 2, low: 1 };
  return groups.sort((a, b) => b.size - a.size || rank[b.confidence] - rank[a.confidence]);
}
@@ -0,0 +1,167 @@
/**
 * A capture: for a short while, keep what it takes to tell one actor from
 * another across addresses. The only place a full visitor address is ever
 * held, and only in memory, so a restart ends it as an expiry does.
 *
 * Bounded in time (`CAPTURE_MS`) and size (`CAPTURE_MAX` requests), and
 * expires by itself: `current` returns null once it is over and its requests
 * are dropped. It keeps no cookie, authorization header or query string: of
 * the headers, only their names, and the handful of values that tell clients
 * apart. Only requests no one was signed in behind are kept; a signed-in
 * person's requests are not an investigation's business.
 */
import { addressBytes } from "../kernel/cidr.ts";
import { prefixOf, type TrafficClass } from "./traffic.ts";
import { readOf, type Read } from "./reads.ts";
export const CAPTURE_MS = 10 * 60_000;
export const CAPTURE_MAX = 2_000;
export interface CapturedRequest {
  readonly n: number;
  readonly at: number;
  readonly address: string;
  readonly prefix: string;
  readonly method: string;
  readonly path: string;
  readonly status: number;
  readonly ms: number;
  readonly ua: string;
  /** Header names (never values) except the ones a proxy adds, in the order received. */
  readonly shape: string;
  readonly language: string;
  readonly accept: string;
  readonly encoding: string;
  readonly fetchSite: string;
  readonly country: string | null;
  /** Cloudflare's bot score, 1 to 99, when the edge sent one. */
  readonly botScore: number | null;
  readonly read: Read | null;
}
export interface Capture {
  readonly id: string;
  readonly startedAt: number;
  readonly expiresAt: number;
  readonly by: string;
  readonly entries: readonly CapturedRequest[];
  /** The size bound was reached; later requests were not kept. */
  readonly full: boolean;
}
/** What the server tells a capture of one finished request. */
export interface CaptureInput {
  readonly request: Request;
  readonly address: string;
  /** Came through a trusted edge: only then are its bot score and country headers believed. */
  readonly viaEdge: boolean;
  readonly status: number;
  readonly ms: number;
  readonly cls: TrafficClass;
  readonly pattern: string;
}
const PROXY_HEADER = /^(x-forwarded-|cf-|x-real-ip$|cdn-loop$|traceparent$|tracestate$|x-request-id$|forwarded$|via$|x-envoy-|x-amzn-)/;
const NEVER = /^(cookie|authorization|proxy-authorization|x-api-key|x-auth-token)$/;
const short = (s: string | null, max: number): string =>
  // eslint-disable-next-line no-control-regex
  (s ?? "").replace(/[\u0000-\u001f\u007f]/g, "").slice(0, max);
/** The Cloudflare bot score a header carries (`cf-bot-score` or `x-bot-score`): a whole number from 1 to 99, else null. */
export function botScoreOf(headers: Headers): number | null {
  const raw = headers.get("cf-bot-score") ?? headers.get("x-bot-score");
  if (raw === null || !/^\d{1,2}$/.test(raw.trim())) return null;
  const n = Number(raw);
  return n >= 1 && n <= 99 ? n : null;
}
export function describe(input: CaptureInput, n: number, at: number): CapturedRequest {
  const { request: r } = input;
  const url = new URL(r.url);
  const bytes = addressBytes(input.address);
  const names = [...r.headers.keys()].filter((h) => !PROXY_HEADER.test(h) && !NEVER.test(h));
  const country = input.viaEdge ? (r.headers.get("cf-ipcountry")?.toUpperCase() ?? "") : "";
  return {
    n,
    at,
    address: bytes === undefined ? "unknown" : short(input.address, 45),
    prefix: bytes === undefined ? "unknown" : prefixOf(bytes),
    method: short(r.method, 8),
    // The query is left out: it can carry a token.
    path: short(url.pathname, 300),
    status: input.status,
    ms: Math.round(input.ms),
    ua: short(r.headers.get("user-agent"), 200),
    shape: names.slice(0, 30).join(","),
    language: short(r.headers.get("accept-language"), 60),
    accept: short(r.headers.get("accept"), 80),
    encoding: short(r.headers.get("accept-encoding"), 40),
    fetchSite: short(r.headers.get("sec-fetch-site"), 16),
    country: /^[A-Z][A-Z0-9]$/.test(country) ? country : null,
    botScore: input.viaEdge ? botScoreOf(r.headers) : null,
    read: readOf(input.pattern, url.pathname),
  };
}
export interface CaptureStore {
  /** Begin a capture; if one is running, that one. */
  start(now: number, by: string): Capture;
  stop(): void;
  /** The running capture, or null once it has expired (its requests are dropped then). */
  current(now: number): Capture | null;
  /** Cheap: is anything being kept at all. */
  active(now: number): boolean;
  note(now: number, input: CaptureInput): void;
}
export function createCaptureStore(): CaptureStore {
  let id = "";
  let startedAt = 0;
  let expiresAt = 0;
  let by = "";
  let entries: CapturedRequest[] = [];
  let full = false;
  let timer: ReturnType<typeof setTimeout> | undefined;
  const drop = (): void => {
    clearTimeout(timer);
    id = "";
    entries = [];
    full = false;
    expiresAt = 0;
  };
  const live = (now: number): boolean => {
    if (id !== "" && now >= expiresAt) drop();
    return id !== "";
  };
  return {
    start(now, who) {
      if (!live(now)) {
        id = crypto.randomUUID().slice(0, 8);
        startedAt = now;
        expiresAt = now + CAPTURE_MS;
        by = who;
        entries = [];
        full = false;
        timer = setTimeout(drop, CAPTURE_MS + 1_000);
        timer.unref();
      }
      return this.current(now)!;
    },
    stop: drop,
    current(now) {
      return live(now) ? { id, startedAt, expiresAt, by, entries, full } : null;
    },
    active: live,
    note(now, input) {
      if (!live(now) || input.cls === "people" || input.cls === "internal") return;
      if (new URL(input.request.url).pathname.startsWith("/settings/traffic")) return;
      if (entries.length >= CAPTURE_MAX) {
        full = true;
        return;
      }
      entries.push(describe(input, entries.length + 1, now));
    },
  };
}
@@ -3,6 +3,8 @@
import { createBootStall } from "./boot_stall.ts";
import { createGitRecord } from "./git.ts";
import { createLoopWatch } from "./loop.ts";
import { createCaptureStore } from "./capture.ts";
import { createReadLedger } from "./reads.ts";
import { createPerfRecord } from "./record.ts";
export const perfRecord = createPerfRecord();
@@ -10,3 +12,6 @@
export const loopWatch = createLoopWatch();
export const gitRecord = createGitRecord();
/** The worst stall in this process's first five minutes, read by the release keeper (#8848). */
export const bootStall = createBootStall();
/** What was read from repositories (`reads.ts`), and the one bounded capture an admin may run (`capture.ts`). */
export const readLedger = createReadLedger();
export const captureStore = createCaptureStore();
@@ -0,0 +1,106 @@
/**
 * Who runs an address range, from the forge's own `network` table: name,
 * operator, country and abuse contact, entered by an admin (a registry
 * extract, an RIR whois answer they looked up themselves). A lookup is a
 * local scan of a small table, so no visitor address is sent to anyone.
 *
 * What it says is what the table says. A range with no entry has no network
 * here, and the page says so rather than guess. An entry narrower than a
 * prefix covers only part of it (`scope: "part"`).
 */
import type { Database } from "bun:sqlite";
import { addressBytes, inCidr, parseCidr, type Cidr } from "../kernel/cidr.ts";
export const MAX_IMPORT_LINES = 2_000;
export interface Network {
  readonly cidr: string;
  readonly name: string;
  readonly operator: string;
  readonly country: string;
  readonly abuse: string;
  /** Does the entry cover the whole of what was asked about, or only part of it. */
  readonly scope: "whole" | "part";
}
interface Row {
  cidr: string;
  name: string;
  operator: string;
  country: string;
  abuse: string;
}
export interface NetworkTable {
  /** The most specific entry covering an address or a prefix (`203.0.0.0/16`); null when none does. */
  find(addressOrPrefix: string): Network | null;
  size: number;
}
export function networkTable(db: Database): NetworkTable {
  const rows = db
    .query<Row, []>("SELECT cidr, name, operator, country, abuse FROM network")
    .all()
    .flatMap((r) => {
      const cidr = parseCidr(r.cidr);
      return cidr === undefined ? [] : [{ r, cidr }];
    });
  return {
    size: rows.length,
    find(text) {
      const [base, len] = text.split("/");
      if (base === undefined || addressBytes(base) === undefined) return null;
      const asked = len === undefined ? (addressBytes(base)!.length * 8) : Number(len);
      let best: { r: Row; cidr: Cidr } | undefined;
      for (const e of rows)
        if (inCidr(base, e.cidr) && (best === undefined || e.cidr.bits > best.cidr.bits)) best = e;
      if (best === undefined) return null;
      return { ...best.r, scope: best.cidr.bits <= asked ? "whole" : "part" };
    },
  };
}
const ABUSE = /^(?:[^\s@<>,]{1,64}@[^\s@<>,]{1,120}|https?:\/\/[^\s<>"]{4,200})$/;
const clean = (s: string, max: number): string =>
  // eslint-disable-next-line no-control-regex
  s.replace(/[\u0000-\u001f\u007f]/g, " ").trim().slice(0, max);
export interface Imported {
  readonly rows: readonly Omit<Network, "scope">[];
  /** `line N: why`, for each line refused. */
  readonly refused: readonly string[];
}
/** Lines of `range, name, operator, country, abuse contact`; a malformed line is named and skipped, never guessed at. */
export function parseNetworks(text: string): Imported {
  const rows: Omit<Network, "scope">[] = [];
  const refused: string[] = [];
  const lines = text.split(/\r?\n/);
  lines.slice(0, MAX_IMPORT_LINES).forEach((line, i) => {
    if (line.trim() === "" || line.trim().startsWith("#")) return;
    const f = line.split(",").map((x) => x.trim());
    const n = `line ${i + 1}`;
    if (f.length !== 5) return void refused.push(`${n}: want range, name, operator, country, abuse contact`);
    const [cidr, name, operator, country, abuse] = f as [string, string, string, string, string];
    if (parseCidr(cidr) === undefined || !cidr.includes("/")) return void refused.push(`${n}: not a range like 203.0.0.0/16`);
    if (name === "") return void refused.push(`${n}: no network name`);
    if (!/^[A-Za-z]{2}$/.test(country)) return void refused.push(`${n}: country is two letters`);
    if (abuse !== "" && !ABUSE.test(abuse)) return void refused.push(`${n}: abuse contact is an email address or a web address`);
    rows.push({ cidr, name: clean(name, 100), operator: clean(operator, 100), country: country.toUpperCase(), abuse });
  });
  if (lines.length > MAX_IMPORT_LINES) refused.push(`only the first ${MAX_IMPORT_LINES} lines were read`);
  return { rows, refused };
}
export function saveNetworks(db: Database, rows: Imported["rows"], by: string, now: number): void {
  const put = db.query(
    `INSERT INTO network (cidr, name, operator, country, abuse, source, added_at)
     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
     ON CONFLICT (cidr) DO UPDATE SET name = excluded.name, operator = excluded.operator,
       country = excluded.country, abuse = excluded.abuse, source = excluded.source, added_at = excluded.added_at`,
  );
  db.transaction(() => {
    for (const r of rows) put.run(r.cidr, r.name, r.operator, r.country, r.abuse, `admin ${by}`, now);
  })();
}
@@ -0,0 +1,312 @@
/**
      
        sentinelcrew
        6d ago
        
      
      HIGH: unbounded read_hour growth (cap is per drain, not per hour) and readsReport loads the whole window; cap distinct keys per hour and aggregate in SQL.
    
 * What was read from repositories, not only how much: repository, ref (branch
 * or commit) and path of every file, tree and commit page, counted in memory
 * as requests finish and written once a minute into `read_hour` (kept 7 days).
 * No address is kept here. A path that looks like a secret or a credential is
 * flagged (`secretPath`) so an admin sees a scraper going for keys.
 *
 * Bounded twice: at most `MAX_KEYS` distinct reads wait in memory between
 * writes, and a stored hour holds at most `HOUR_KEYS_MAX` distinct rows however
 * many minutes wrote to it; later ones fall under `OTHER_PATH` for their
 * repository. So the table cannot pass `HOUR_KEYS_MAX` x 168 rows plus the
 * folds, and the report reads it grouped in SQL, never row by row.
 */
import type { Database } from "bun:sqlite";
import { runGit } from "../git/env.ts";
import { withoutWaiting } from "../kernel/db.ts";
import { quietly } from "../core/write_clock.ts";
const HOUR_MS = 3_600_000;
const MAX_KEYS = 2_000;
/** Distinct (repo, ref, path) rows one hour may hold in the table; later ones fold into `(others)` for their repository. */
export const HOUR_KEYS_MAX = 500;
const FLUSH_MS = 60_000;
export const READ_DAYS_MAX = 7;
export const OTHER_PATH = "(others)";
export interface Read {
  readonly repo: string;
  readonly ref: string;
  readonly path: string;
}
const clean = (text: string, max: number): string =>
  // eslint-disable-next-line no-control-regex
  text.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, max);
function decode(text: string): string {
  try {
    return decodeURIComponent(text);
  } catch {
    return text;
  }
}
const FILE_ROUTE = /^\/:owner\/:name\/(blob|raw|tree|commit)\//;
const FILE_PATH = /^\/([^/]+)\/([^/]+)\/(blob|raw|tree|commit)\/([^/]+)(?:\/(.*))?$/;
/** The repository, ref and path a file, tree or commit route read; null for any other route. */
export function readOf(pattern: string, pathname: string): Read | null {
  if (!FILE_ROUTE.test(pattern)) return null;
  const m = FILE_PATH.exec(pathname);
  if (m === null) return null;
  const path = m[3] === "commit" ? "" : decode(m[5] ?? "");
  return {
    repo: clean(`${decode(m[1]!)}/${decode(m[2]!)}`, 120),
    ref: clean(decode(m[4]!), 80),
    path: clean(path, 300),
  };
}
/** A ref that is a commit id (7 to 64 hex characters) rather than a branch name. */
export const isCommit = (ref: string): boolean => /^[0-9a-f]{7,64}$/.test(ref);
const SECRET = [
  /(^|\/)\.env(\.[^/]*)?$/i,
  /(^|\/)id_(rsa|dsa|ecdsa|ed25519)(\.pub)?$/i,
  /\.(pem|key|p12|pfx|jks|kdbx|keystore)$/i,
  /(^|\/)\.(npmrc|netrc|pgpass|htpasswd|git-credentials|dockercfg|pypirc)$/i,
  /(^|\/)(credentials|secrets?)(\.[^/]*)?$/i,
  /(^|\/)\.(aws|ssh|gnupg|kube)\//i,
  /(^|\/)service[-_]account[^/]*\.json$/i,
  /(^|\/)wp-config\.php$/i,
  /(^|\/)\.docker\/config\.json$/i,
  /(^|\/)(terraform\.tfstate|\.tfvars|[^/]*\.tfvars)$/i,
];
/** The file's name or place says it holds a secret or a credential: a guess from the path, never from the content. */
export const secretPath = (path: string): boolean => SECRET.some((r) => r.test(path));
interface Count {
  requests: number;
  anonymous: number;
}
export interface ReadLedger {
  note(at: number, read: Read, anonymous: boolean): void;
  /** What waits to be written, for the page to add to what is stored. */
  pending(): ReadRow[];
  drain(): Map<number, Map<string, Count>>;
}
export interface ReadRow extends Read {
  readonly hour: number;
  readonly requests: number;
  readonly anonymous: number;
}
const SEP = "\u0000";
export function createReadLedger(): ReadLedger {
  let held = new Map<number, Map<string, Count>>();
  let size = 0;
  return {
    note(at, read, anonymous) {
      const hour = Math.floor(at / HOUR_MS);
      let table = held.get(hour);
      if (table === undefined) held.set(hour, (table = new Map()));
      let key = [read.repo, read.ref, read.path].join(SEP);
      if (!table.has(key)) {
        if (size >= MAX_KEYS) key = [read.repo, "", OTHER_PATH].join(SEP);
        else size++;
      }
      const c = table.get(key) ?? { requests: 0, anonymous: 0 };
      c.requests++;
      if (anonymous) c.anonymous++;
      table.set(key, c);
    },
    pending() {
      const rows: ReadRow[] = [];
      for (const [hour, table] of held)
        for (const [key, c] of table) {
          const [repo, ref, path] = key.split(SEP) as [string, string, string];
          rows.push({ hour, repo, ref, path, ...c });
        }
      return rows;
    },
    drain() {
      const out = held;
      held = new Map();
      size = 0;
      return out;
    },
  };
}
export interface ReadWriter {
  flush(): void;
  stop(): void;
}
export function createReadWriter(
  db: Database,
  ledger: ReadLedger,
  log: { warn(message: string, fields?: Record<string, unknown>): void },
  now: () => number = Date.now,
  flushMs: number = FLUSH_MS,
): ReadWriter {
  const flush = (): void => {
    const drained = ledger.drain();
    try {
      withoutWaiting(
        db,
        db.transaction(() =>
          quietly(db, () => {
            const put = db.query(
              `INSERT INTO read_hour (hour, repo, ref, path, requests, anonymous)
               VALUES (?1, ?2, ?3, ?4, ?5, ?6)
               ON CONFLICT (hour, repo, ref, path) DO UPDATE SET
                 requests = requests + excluded.requests,
                 anonymous = anonymous + excluded.anonymous`,
            );
            const held = db.query<{ n: number }, [number]>("SELECT COUNT(*) AS n FROM read_hour WHERE hour = ?1");
            const has = db.query<{ one: number }, [number, string, string, string]>(
              "SELECT 1 AS one FROM read_hour WHERE hour = ?1 AND repo = ?2 AND ref = ?3 AND path = ?4",
            );
            for (const [hour, table] of drained) {
              let rows = held.get(hour)?.n ?? 0;
              for (const [key, c] of table) {
                let [repo, ref, path] = key.split(SEP) as [string, string, string];
                if (has.get(hour, repo, ref, path) === null) {
                  if (rows >= HOUR_KEYS_MAX && path !== OTHER_PATH) {
                    ref = "";
                    path = OTHER_PATH;
                    if (has.get(hour, repo, ref, path) === null) rows++;
                  } else rows++;
                }
                put.run(hour, repo, ref, path, c.requests, c.anonymous);
              }
            }
            db.query("DELETE FROM read_hour WHERE hour < ?1").run(
              Math.floor(now() / HOUR_MS) - READ_DAYS_MAX * 24,
            );
          }),
        ),
      );
    } catch (error) {
      // A failed write loses that minute's reads; evidence, not a ledger.
      log.warn("could not write read history", { error: String(error) });
    }
  };
  const tick = setInterval(flush, flushMs);
  tick.unref();
  return {
    flush,
    stop() {
      clearInterval(tick);
      flush();
    },
  };
}
export interface PathRow {
  readonly path: string;
  readonly requests: number;
  readonly anonymous: number;
  readonly secret: boolean;
}
export interface RepoReads {
  readonly repo: string;
  readonly requests: number;
  readonly anonymous: number;
  /** Distinct file or tree paths read. */
  readonly paths: number;
  /** Most-read commits (by id), then branches. */
  readonly commits: readonly { ref: string; requests: number }[];
  readonly topPaths: readonly PathRow[];
  readonly secrets: readonly PathRow[];
  /** Files at HEAD, and the share of them the window read; filled by `addCoverage`. */
  files?: number | null;
  coverage?: number | null;
}
/** The window's reads grouped by repository, busiest first; `days` is cut to 1..7. */
export function readsReport(
  db: Database,
  ledger: ReadLedger,
  now: number,
  days: number,
): RepoReads[] {
  const d = Math.min(READ_DAYS_MAX, Math.max(1, Math.floor(days) || 1));
  const since = Math.floor(now / HOUR_MS) - d * 24 + 1;
  // Grouped in SQL: one row per repository and path, one per repository and ref.
  const byPath = db
    .query<ReadRow, [number]>(
      `SELECT 0 AS hour, repo, '' AS ref, path, SUM(requests) AS requests, SUM(anonymous) AS anonymous
       FROM read_hour WHERE hour >= ?1 GROUP BY repo, path`,
    )
    .all(since);
  const byRef = db
    .query<ReadRow, [number]>(
      `SELECT 0 AS hour, repo, ref, '' AS path, SUM(requests) AS requests, SUM(anonymous) AS anonymous
       FROM read_hour WHERE hour >= ?1 AND ref != '' GROUP BY repo, ref`,
    )
    .all(since);
  const waiting = ledger.pending().filter((r) => r.hour >= since);
  const rows = [...byPath, ...waiting];
  const byRepo = new Map<string, ReadRow[]>();
  for (const r of rows) {
    let list = byRepo.get(r.repo);
    if (list === undefined) byRepo.set(r.repo, (list = []));
    list.push(r);
  }
  const refRows = [...byRef, ...waiting.filter((r) => r.ref !== "")];
  const refsOf = new Map<string, Map<string, number>>();
  for (const r of refRows) {
    let m = refsOf.get(r.repo);
    if (m === undefined) refsOf.set(r.repo, (m = new Map()));
    m.set(r.ref, (m.get(r.ref) ?? 0) + r.requests);
  }
  const out: RepoReads[] = [];
  for (const [repo, list] of byRepo) {
    const paths = new Map<string, Count>();
    const refs = refsOf.get(repo) ?? new Map<string, number>();
    let requests = 0;
    let anonymous = 0;
    for (const r of list) {
      requests += r.requests;
      anonymous += r.anonymous;
      if (r.path !== "" && r.path !== OTHER_PATH) {
        const c = paths.get(r.path) ?? { requests: 0, anonymous: 0 };
        c.requests += r.requests;
        c.anonymous += r.anonymous;
        paths.set(r.path, c);
      }
    }
    const all: PathRow[] = [...paths]
      .map(([path, c]) => ({ path, ...c, secret: secretPath(path) }))
      .sort((a, b) => b.requests - a.requests || a.path.localeCompare(b.path));
    out.push({
      repo,
      requests,
      anonymous,
      paths: paths.size,
      commits: [...refs]
        .map(([ref, n]) => ({ ref, requests: n }))
        .sort((a, b) => Number(isCommit(b.ref)) - Number(isCommit(a.ref)) || b.requests - a.requests)
        .slice(0, 5),
      topPaths: all.slice(0, 5),
      secrets: all.filter((p) => p.secret).slice(0, 10),
    });
  }
  return out.sort((a, b) => b.requests - a.requests || a.repo.localeCompare(b.repo)).slice(0, 20);
}
/** How much of the repository the window's reads reached: distinct paths read over files at HEAD. Null when git cannot say. */
export async function addCoverage(
  report: RepoReads,
  dirOf: (repo: string) => string | null,
): Promise<RepoReads> {
  const dir = dirOf(report.repo);
  if (dir === null) return { ...report, files: null, coverage: null };
  const listed = await runGit(["ls-tree", "-r", "--name-only", "HEAD"], { cwd: dir });
  if (!listed.ok) return { ...report, files: null, coverage: null };
  const files = listed.value === "" ? 0 : listed.value.split("\n").length;
  return {
    ...report,
    files,
    coverage: files === 0 ? null : Math.min(1, report.paths / files),
  };
}
@@ -44,6 +44,7 @@
import { entry as profile } from "./profile.ts";
import { entry as quotas } from "./quotas.ts";
import { entry as resilience } from "./resilience.ts";
import { entry as scorecard } from "./scorecard.ts";
import { entry as scrapes } from "./scrapes.ts";
import { entry as secrets } from "./secrets.ts";
import { entry as sentinel } from "./sentinel.ts";
import { entry as subscriptions } from "./subscriptions.ts";
@@ -90,6 +91,7 @@
export const ENTRIES: readonly PaletteEntry[] = [
  people,
  quotas,
  limits,
  scrapes,
  teams,
  orgTeams,
  orgGrants,
@@ -0,0 +1,59 @@
/**
 * What was scraped, in the terminal, a forge admin's who is a person: the last
 * day's reads by repository (paths, commits, key-looking files) and the groups
 * of requests a running capture sees, as /settings/traffic/scrapes shows them.
 * Read-only: a capture is started and exported on the page, where the abuse
 * report is a download.
 */
import type { Ctx, Screen } from "../app/app.ts";
import { binding } from "../app/keys.ts";
import { groupRequests } from "../../perf/actors.ts";
import { captureStore, readLedger } from "../../perf/live.ts";
import { readsReport } from "../../perf/reads.ts";
import type { Session } from "../session.ts";
import { style } from "../term/style.ts";
import { truncate } from "../term/width.ts";
import { fill, len, split } from "../ui/layout.ts";
import { ListState, drawList, type ListRow } from "./common.ts";
import type { PaletteEntry } from "./entries.ts";
const may = (f: Session): boolean => f.user?.isAdmin === true && f.user.kind !== "agent";
export function scrapesScreen(f: Session, ctx: Ctx): Screen {
  const state = new ListState();
  const lines = (): ListRow[] => {
    if (!may(f)) return [];
    const rows: ListRow[] = readsReport(f.app.db, readLedger, Date.now(), 1).map((r) => ({
      title: r.repo,
      meta: `${r.requests} reads (${r.anonymous} anonymous) · ${r.paths} paths${r.secrets.length > 0 ? ` · ${r.secrets.length} key-looking: ${r.secrets.map((s) => s.path).slice(0, 3).join(", ")}` : ""}`,
    }));
    const cap = captureStore.current(Date.now());
    if (cap !== null)
      for (const g of groupRequests(cap.entries).filter((x) => x.size >= 3))
        rows.push({ title: `Group ${g.id}: ${g.size} requests from ${g.addresses.length} addresses`, meta: `confidence ${g.confidence}` });
    return rows;
  };
  return {
    title: "Scrapes",
    keys: [binding(["j", "k"], "move cursor")],
    render(area, s) {
      const c = ctx.theme.c;
      const [head, list] = split(area, "v", [len(1), fill(1)]);
      s.text(head!.x, head!.y, truncate("reads in the last day, and groups in a running capture; networks and the abuse export are on the page", head!.w), style(c.mutedFg, c.bg), head!.x + head!.w);
      drawList(s, list!, c, state, lines());
    },
    key(e) {
      return state.key(e, lines().length);
    },
  };
}
export const entry: PaletteEntry = {
  id: "scrapes",
  title: "Scrapes",
  hint: "what was read by repository, and groups in a running capture",
  after: ["crawlers", "limits"],
  offered: may,
  run: (session, ctx) => ctx.push(scrapesScreen(session, ctx)),
};
@@ -0,0 +1,302 @@
/**
 * /settings/traffic/scrapes: what was read, who it was, and whether one actor
 * sat behind several addresses (`perf/reads.ts`, `perf/network.ts`,
 * `perf/capture.ts`, `perf/actors.ts`). Admin only. Full addresses appear
 * here and in the export only while a capture holds them. Every statement
 * of identity stops at a network or a group, and says so.
 */
import { notePhase, takeSessionTime } from "../../core/route_timing.ts";
import type { Ctx } from "../../http/context.ts";
import { repoDir } from "../../git/repos.ts";
import { perfRecord, captureStore, readLedger } from "../../perf/live.ts";
import { groupRequests, type Group } from "../../perf/actors.ts";
import { CAPTURE_MAX, CAPTURE_MS } from "../../perf/capture.ts";
import { networkTable, parseNetworks, saveNetworks, type Network, type NetworkTable } from "../../perf/network.ts";
import { addCoverage, isCommit, READ_DAYS_MAX, readsReport, type RepoReads } from "../../perf/reads.ts";
import { trafficHistory } from "../../perf/traffic_store.ts";
import { notFoundPage, settingsLayout, signInFirst } from "../chrome.ts";
import { field, readForm } from "../form.ts";
import { html, type Html } from "../html.ts";
import { flashTo, page } from "../layout.ts";
import { badge, btn, card, empty, seg } from "../ui.ts";
import { accountSections } from "./origins.ts";
const DAYS = [1, 3, 7] as const;
const SHOWN_REQUESTS = 100;
const BASE = "/settings/traffic/scrapes";
/** What the evidence cannot show, said wherever a finding is. */
export const LIMITS =
  "This shows a network and a pattern of requests. It does not show who is behind either: a network's name is what the forge's own table says, requests that look alike may come from different people, and nothing here is proof of who did what.";
const days = (raw: string | null): number => {
  const n = Number(raw);
  return (DAYS as readonly number[]).includes(n) ? n : 1;
};
function admin(ctx: Ctx): Response | null {
  if (!ctx.auth.user) return signInFirst(ctx);
  if (!ctx.auth.user.isAdmin) return notFoundPage(ctx);
  return null;
}
function whose(n: Network | null): Html {
  if (n === null)
    return html`<span class="muted">No network on file: the evidence stops at this address range.</span>`;
  return html`${n.name}${n.operator ? html`, run by ${n.operator}` : null} (${n.country})${n.scope === "part"
      ? html` <span class="muted">— covers only part of this range</span>`
      : null}`;
}
function readsCard(r: RepoReads, from: number): Html {
  const commits = r.commits.filter((c) => isCommit(c.ref));
  return card(
    html`<p class="small">
        <strong class="break">${r.repo}</strong>: ${r.requests} reads, ${r.anonymous} anonymous,
        ${r.paths} distinct paths.
        ${r.coverage === undefined || r.coverage === null
          ? html`<span class="muted">Coverage unknown.</span>`
          : html`About ${Math.round(r.coverage * 100)}% of its ${r.files} files were read in the last ${from}
              ${from === 1 ? "day" : "days"}.`}
      </p>
      ${r.secrets.length === 0
        ? null
        : html`<p class="small">
              ${badge("Looks like a secret", { tone: "danger" })} Files read whose names look like keys or
              credentials (a guess from the name; the contents were not looked at):
            </p>
            <ul class="small">
              ${r.secrets.map((p) => html`<li class="break">${p.path} — ${p.requests}</li>`)}
            </ul>`}
      <p class="small muted mb">Most read:</p>
      <ul class="small">
        ${r.topPaths.map((p) => html`<li class="break">${p.path} — ${p.requests} (${p.anonymous} anonymous)</li>`)}
      </ul>
      ${commits.length === 0
        ? null
        : html`<p class="small muted">Commits read:
            ${commits.map((c) => html`<code>${c.ref.slice(0, 12)}</code> ${c.requests} `)}</p>`}`,
  );
}
function captureCard(now: number): Html {
  const c = captureStore.current(now);
  if (c === null)
    return card(
      html`<p class="small">
          A capture keeps full addresses and the shape of requests so one actor can be told across addresses.
          It lasts ${CAPTURE_MS / 60_000} minutes or ${CAPTURE_MAX} requests, keeps no cookies, tokens or
          query strings, skips signed-in people, and is deleted when it ends.
        </p>
        <form method="post" action="${BASE}/capture" class="inline">${btn("Start a capture", { variant: "primary" })}</form>`,
    );
  return card(
    html`<p class="small">
        Capture <code>${c.id}</code> is running: ${c.entries.length} requests kept${c.full ? " (full: later requests are not kept)" : ""},
        ends in ${Math.max(0, Math.ceil((c.expiresAt - now) / 60_000))} min and is then deleted.
        <a href="${BASE}">Refresh</a>
      </p>
      <form method="post" action="${BASE}/stop" class="inline">${btn("Stop and delete it", { size: "sm" })}</form>`,
  );
}
function groupRow(g: Group, net: NetworkTable): Html {
  return html`<tr>
    <td class="break">${g.ua || "(none)"}<br /><span class="muted small">${g.language}</span></td>
    <td class="num">${g.size}</td>
    <td class="num">${g.perMinute}/min</td>
    <td class="num">${g.addresses.length}</td>
    <td>${badge(g.confidence, { tone: g.confidence === "high" ? "danger" : "default" })}</td>
    <td class="small">${g.prefixes.slice(0, 3).map((p) => html`<div class="break">${p}: ${whose(net.find(p))}</div>`)}</td>
    <td><a href="${BASE}?group=${g.id}#group">Requests</a></td>
  </tr>`;
}
function groupDetail(g: Group, net: NetworkTable): Html {
  return card(
    html`<h2 id="group">Group ${g.id}</h2>
      <p class="small">${g.size} requests from ${g.addresses.length} addresses, confidence ${g.confidence}.</p>
      <ul class="small">${g.reasons.map((r) => html`<li>${r}</li>`)}</ul>
      <p class="small muted">${LIMITS}</p>
      ${btn("Export for an abuse report", { href: `${BASE}/export?group=${g.id}`, size: "sm" })}
      <div class="table-scroll">
        <table class="table" data-dense>
          <caption class="sr-only">Requests behind group ${g.id}</caption>
          <thead><tr><th scope="col">Address</th><th scope="col">Request</th><th scope="col" class="num">Status</th><th scope="col">Network</th></tr></thead>
          <tbody>
            ${g.requests.slice(0, SHOWN_REQUESTS).map(
              (r) => html`<tr>
                <td class="break">${r.address}</td>
                <td class="break">${r.method} ${r.path}</td>
                <td class="num">${r.status}</td>
                <td class="small">${whose(net.find(r.address))}</td>
              </tr>`,
            )}
          </tbody>
        </table>
      </div>
      ${g.size > SHOWN_REQUESTS ? html`<p class="small muted">First ${SHOWN_REQUESTS} of ${g.size} shown.</p>` : null}`,
  );
}
export async function scrapesPage(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  notePhase(ctx.request, "session", takeSessionTime(ctx.request));
  const rendering = performance.now();
  const now = Date.now();
  const q = new URL(ctx.request.url).searchParams;
  const d = days(q.get("days"));
  const net = networkTable(ctx.app.db);
  const reports = await Promise.all(
    readsReport(ctx.app.db, readLedger, now, d)
      .slice(0, 8)
      .map((r) => {
        const [owner, name] = r.repo.split("/");
        return addCoverage(r, () =>
          owner && name && /^[\w.-]+$/.test(owner) && /^[\w.-]+$/.test(name)
            ? repoDir(ctx.app.reposRoot, owner, name)
            : null,
        );
      }),
  );
  const prefixes = trafficHistory(ctx.app.db, perfRecord.ledger, now, "24h").tops.prefixes;
  const cap = captureStore.current(now);
  const groups = cap === null ? [] : groupRequests(cap.entries).filter((g) => g.size >= 3);
  const picked = groups.find((g) => g.id === q.get("group"));
  const body = html`<div class="mb">
      ${seg(DAYS.map((n) => ({ label: n === 1 ? "1 day" : `${n} days`, href: `${BASE}?days=${n}`, current: n === d })))}
    </div>
    <h2>What was read</h2>
    ${reports.length === 0
      ? empty({ icon: "users", title: "No file reads recorded in this window", note: `Reads are kept up to ${READ_DAYS_MAX} days.` })
      : reports.map((r) => readsCard(r, d))}
    <h2>Who it is</h2>
    ${card(
      prefixes.length === 0
        ? html`<p class="small muted">No address prefixes in the last 24 hours.</p>`
        : html`<div class="table-scroll"><table class="table" data-dense>
            <caption class="sr-only">Address prefixes and their networks</caption>
            <thead><tr><th scope="col">Prefix</th><th scope="col" class="num">Requests</th><th scope="col">Network (from the forge's own table)</th><th scope="col">Abuse contact</th></tr></thead>
            <tbody>${prefixes.map((p) => {
              const n = net.find(p.key);
              return html`<tr><td class="break">${p.key}</td><td class="num">${p.requests}</td><td class="small">${whose(n)}</td><td class="break small">${n?.abuse || "—"}</td></tr>`;
            })}</tbody></table></div>`,
    )}
    <h2>One actor across addresses</h2>
    ${captureCard(now)}
    ${cap === null
      ? null
      : groups.length === 0
        ? html`<p class="small muted">No group of three or more similar requests yet.</p>`
        : card(html`<div class="table-scroll"><table class="table" data-dense>
            <caption class="sr-only">Groups of requests that look alike</caption>
            <thead><tr><th scope="col">User agent</th><th scope="col" class="num">Requests</th><th scope="col" class="num">Rate</th><th scope="col" class="num">Addresses</th><th scope="col">Confidence</th><th scope="col">Networks</th><th scope="col"></th></tr></thead>
            <tbody>${groups.slice(0, 20).map((g) => groupRow(g, net))}</tbody></table></div>
            <p class="small muted">Groups share a user agent and the names and order of their headers. ${LIMITS}</p>`)}
    ${picked === undefined ? null : groupDetail(picked, net)}
    <h2>Networks on file</h2>
    ${card(html`<p class="small">${net.size} ranges. Lookups are local: no visitor address is sent to anyone. Paste lines of
        <code>range, name, operator, country, abuse contact</code> from a registry you looked up yourself.</p>
      <form method="post" action="${BASE}/networks">
        <textarea name="lines" rows="4" maxlength="200000" aria-label="Network ranges" placeholder="203.0.0.0/16, Example Hosting, Example Ltd, NL, [email protected]"></textarea>
        ${btn("Save networks", { size: "sm" })}
      </form>`)}`;
  notePhase(ctx.request, "render", performance.now() - rendering);
  return page({
    ctx,
    title: "Scrapes",
    body: html`<div class="page-head"><h1>Scrapes</h1></div>
      ${settingsLayout(accountSections("traffic", { admin: true }), body, "Account settings")}`,
  });
}
export async function captureStart(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  const user = ctx.auth.user!;
  if (user.kind === "agent") return notFoundPage(ctx);
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  const before = captureStore.current(Date.now());
  const c = captureStore.start(Date.now(), user.username);
  if (before === null)
    ctx.app.bus.append({ kind: "traffic.capture_started", actorId: user.id, payload: { id: c.id, minutes: CAPTURE_MS / 60_000 } });
  return flashTo(ctx, BASE, { message: `Capture ${c.id} is running.` });
}
export async function captureStop(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  const form = await readForm(ctx, "admin");
  if (form instanceof Response) return form;
  captureStore.stop();
  return flashTo(ctx, BASE, { message: "Capture deleted." });
}
export async function networksImport(ctx: Ctx): Promise<Response> {
  const denied = admin(ctx);
  if (denied) return denied;
  const user = ctx.auth.user!;
  if (user.kind === "agent") return notFoundPage(ctx);
  const form = await readForm(ctx, "admin", 300_000);
  if (form instanceof Response) return form;
  const got = parseNetworks(field(form, "lines"));
  saveNetworks(ctx.app.db, got.rows, user.username, Date.now());
  if (got.rows.length > 0)
    ctx.app.bus.append({ kind: "traffic.networks_imported", actorId: user.id, payload: { count: got.rows.length } });
  const note = got.refused.length === 0 ? "" : ` Refused: ${got.refused.slice(0, 3).join("; ")}.`;
  return flashTo(ctx, BASE, {
    ...(got.refused.length > 0 && got.rows.length === 0 ? { tone: "danger" as const } : {}),
    message: `Saved ${got.rows.length} ranges.${note}`,
  });
}
/** A plain-text report for one group: what was seen, the networks and their abuse contacts, and what it cannot show. */
export function reportOf(g: Group, net: NetworkTable, capture: { id: string; startedAt: number }): string {
  const nets = new Map<string, Network | null>(g.prefixes.map((p) => [p, net.find(p)]));
  const first = Math.min(...g.requests.map((r) => r.at));
  const last = Math.max(...g.requests.map((r) => r.at));
  return [
    `Abuse report draft: group ${g.id} (capture ${capture.id})`,
    `Window: ${new Date(first).toISOString()} to ${new Date(last).toISOString()}`,
    `Requests: ${g.size} (${g.perMinute} per minute) from ${g.addresses.length} addresses. Confidence: ${g.confidence}.`,
    `User agent: ${g.ua || "(none)"}`,
    "",
    "Why these requests were grouped:",
    ...g.reasons.map((r) => `- ${r}`),
    "",
    "Networks (from the forge's own table; send to each abuse contact):",
    ...[...nets].map(([p, n]) =>
      n === null
        ? `- ${p}: no network on file; the evidence stops here.`
        : `- ${p}: ${n.name}${n.operator ? `, ${n.operator}` : ""} (${n.country}); abuse contact: ${n.abuse || "none on file"}`,
    ),
    "",
    "Addresses:",
    ...g.addresses.map((a) => `- ${a}`),
    "",
    "Sample requests:",
    ...g.requests.slice(0, 50).map((r) => `${new Date(r.at).toISOString()} ${r.address} ${r.method} ${r.path} ${r.status}`),
    "",
    `Limits: ${LIMITS}`,
    "",
  ].join("\n");
}
export function scrapesExport(ctx: Ctx): Response {
  const denied = admin(ctx);
  if (denied) return denied;
  const cap = captureStore.current(Date.now());
  const id = new URL(ctx.request.url).searchParams.get("group");
  const g = cap === null ? undefined : groupRequests(cap.entries).find((x) => x.id === id);
  if (cap === null || g === undefined) return notFoundPage(ctx);
  return new Response(reportOf(g, networkTable(ctx.app.db), cap), {
    headers: {
      "content-type": "text/plain; charset=utf-8",
      "content-disposition": `attachment; filename="abuse-report-${g.id}.txt"`,
      "cache-control": "no-store",
    },
  });
}
@@ -171,6 +171,9 @@
export function trafficPage(ctx: Ctx): Response {
          ${top("Address prefix", tops.prefixes, t)}
          ${top("Country", tops.countries, t)}
          ${top("Request signature", tops.signatures, t)}
          <p class="mt">
            <a href="/settings/traffic/scrapes">What was scraped, and who: reads, networks, captures</a>
          </p>
          <p class="small muted mt">
            ${chosen === "15m"
              ? KPI_COUNTS.traffic
@@ -15,7 +15,7 @@
const doc = readFileSync("docs/design/audit.md", "utf8");
/** Routes that answer a file, a fragment, a redirect hop or a callback. */
const NOT_PAGES =
  /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|audit\.csv$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|home\/archived\/|settings\/backups\/live$|api\/v1\/)/;
  /^\/(assets\/|\.well-known\/|icons\/|og\/|live\/|__forge\/|go\/|new\/issue$|events$|system\/stream$|talk\/stream$|settings\/teams\/stream$|sw\.js$|robots\.txt$|favicon\.ico$|apple-touch-icon(-precomposed)?\.png$|sitemap\.xml$|manifest\.webmanifest$|opensearch\.xml$|llms\.txt$|openapi\.json$|releases\.json$|changelog\.atom$|audit\.csv$|settings\/traffic\/scrapes\/export$|fleet\/graph\/live$|fleet\/graph\.csv$|settings\/origins\/callback$|home\/archived\/|settings\/backups\/live$|api\/v1\/)/;
const getRoutes = [
  ...server.matchAll(/router\.add\(\s*"GET",\s*"([^"]+)"/g),
@@ -94,6 +94,7 @@
test("a site-admin person is offered the whole roster", () => {
    ["capacity", "Capacity", "concurrency and guards, live; e changes a settable one"],
    ["resilience", "Resilience", "where snapshots go and the mirror; each protection green, red or untested"],
    ["crawlers", "Crawlers", "their share of answering time, and who was refused"],
    ["scrapes", "Scrapes", "what was read by repository, and groups in a running capture"],
    ["api", "API", "the forge's endpoints; run a GET"],
    ["changelog", "Changelog", "the last 30 releases and their pulls"],
    ["releases", "Releases", "the live release and the kept ones"],
@@ -0,0 +1,225 @@
/**
 * What was scraped and who did it: reads by repository and path with secrets
 * flagged, networks from the forge's own table, requests grouped across
 * addresses with a stated confidence, and a bounded capture that keeps no
 * secrets and expires (#9406).
 */
import { afterAll, beforeAll, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createApp, type App } from "../src/app.ts";
import { wasViaEdge, notePeer } from "../src/http/peer.ts";
import { createSession } from "../src/core/sessions.ts";
import { createUser, type User } from "../src/core/users.ts";
import { serve, type Servers } from "../src/http/server.ts";
import { groupRequests } from "../src/perf/actors.ts";
import { botScoreOf, CAPTURE_MAX, CAPTURE_MS, createCaptureStore } from "../src/perf/capture.ts";
import { captureStore } from "../src/perf/live.ts";
import { networkTable, parseNetworks, saveNetworks } from "../src/perf/network.ts";
import { createReadLedger, createReadWriter, HOUR_KEYS_MAX, OTHER_PATH, readOf, readsReport, secretPath } from "../src/perf/reads.ts";
let dataDir: string;
let app: App;
let servers: Servers;
let admin: User;
let bob: User;
beforeAll(async () => {
  dataDir = await mkdtemp(join(tmpdir(), "forge-scrapes-"));
  app = createApp({ dataDir, port: 0 });
  servers = await serve(app);
  const person = async (username: string, isAdmin = false): Promise<User> => {
    const made = await createUser(app.db, { origin: "seed", trusted: true, username, email: `${username}@forge.local`, password: "correct-horse-battery", isAdmin });
    if (!made.ok) throw new Error(made.error);
    return made.value;
  };
  admin = await person("keeper", true);
  bob = await person("bob");
});
afterAll(async () => {
  captureStore.stop();
  servers.stop();
  app.close();
  await rm(dataDir, { recursive: true, force: true });
});
const NOW = 1_800_000_000_000;
const HOUR = 3_600_000;
const input = (path: string, address: string, headers: Record<string, string> = {}, cls: "deep" | "people" = "deep", pattern = "/:owner/:name/blob/:ref/*", viaEdge = true) => ({
  request: new Request(`http://forge.test${path}`, { headers: { "user-agent": "ScrapeBot/1.0", ...headers } }),
  address,
  viaEdge,
  status: 200,
  ms: 5,
  cls,
  pattern,
});
test("a file read names repository, ref and path; other routes read nothing", () => {
  expect(readOf("/:owner/:name/blob/:ref/*", "/alice/open/blob/main/src/a%20b.ts")).toEqual({ repo: "alice/open", ref: "main", path: "src/a b.ts" });
  expect(readOf("/:owner/:name/commit/:sha", "/alice/open/commit/abc1234")?.ref).toBe("abc1234");
  expect(readOf("/:owner/:name/issues", "/alice/open/issues")).toBeNull();
  expect(secretPath(".env")).toBe(true);
  expect(secretPath("deploy/id_rsa")).toBe(true);
  expect(secretPath("src/main.ts")).toBe(false);
});
test("reads are reported over a window of at most 7 days, secrets flagged, and no address kept", () => {
  const ledger = createReadLedger();
  ledger.note(NOW, { repo: "alice/open", ref: "abc1234", path: ".env" }, true);
  ledger.note(NOW, { repo: "alice/open", ref: "main", path: "src/a.ts" }, false);
  ledger.note(NOW - 2 * 24 * HOUR, { repo: "alice/open", ref: "main", path: "old.ts" }, true);
  const day = readsReport(app.db, ledger, NOW, 1);
  expect(day[0]?.requests).toBe(2);
  expect(day[0]?.secrets.map((s) => s.path)).toEqual([".env"]);
  expect(day[0]?.commits[0]?.ref).toBe("abc1234");
  expect(readsReport(app.db, ledger, NOW, 3)[0]?.requests).toBe(3);
  expect(readsReport(app.db, ledger, NOW, 99)[0]?.requests).toBe(3); // cut to 7 days
});
test("a network is found in the forge's own table, most specific first, and absence is absence", () => {
  const parsed = parseNetworks(
    "# comment\n203.0.0.0/8, Wide, Wide Inc, us, [email protected]\n203.0.0.0/16, Narrow, Narrow Ltd, NL, https://narrow.example/abuse\nbad line\n10.0.0.0/8, X, X, USA, [email protected]\n1.2.0.0/16, Y, Y, DE, not a contact",
  );
  expect(parsed.rows).toHaveLength(2);
  expect(parsed.refused).toHaveLength(3);
  saveNetworks(app.db, parsed.rows, "keeper", NOW);
  const table = networkTable(app.db);
  expect(table.find("203.0.0.0/16")).toMatchObject({ name: "Narrow", country: "NL", scope: "whole" });
  expect(table.find("203.5.0.0/16")).toMatchObject({ name: "Wide", scope: "whole" });
  expect(table.find("198.51.0.0/16")).toBeNull();
  expect(table.find("203.0.113.9")?.abuse).toBe("https://narrow.example/abuse");
});
test("a capture is bounded, keeps no cookie, token or query, skips people, and expires by itself", () => {
  const store = createCaptureStore();
  expect(store.active(NOW)).toBe(false);
  store.start(NOW, "keeper");
  store.note(NOW, input("/alice/open/blob/main/a.ts?token=SECRET", "203.0.113.9", { cookie: "forge_session=SECRET", authorization: "Bearer SECRET", "cf-bot-score": "5" }));
  store.note(NOW, input("/alice/open", "198.51.100.1", {}, "people"));
  const c = store.current(NOW)!;
  expect(c.entries).toHaveLength(1);
  const json = JSON.stringify(c);
  expect(json).not.toContain("SECRET");
  expect(c.entries[0]).toMatchObject({ prefix: "203.0.0.0/16", botScore: 5, address: "203.0.113.9" });
  expect(c.entries[0]!.shape).not.toContain("cookie");
  for (let i = 0; i < CAPTURE_MAX + 5; i++) store.note(NOW, input("/x", "203.0.113.9"));
  expect(store.current(NOW)!.entries).toHaveLength(CAPTURE_MAX);
  expect(store.current(NOW)!.full).toBe(true);
  expect(store.current(NOW + CAPTURE_MS)).toBeNull(); // expired: addresses gone
  expect(store.active(NOW + CAPTURE_MS)).toBe(false);
  store.note(NOW + CAPTURE_MS, input("/x", "203.0.113.9"));
  expect(store.current(NOW + CAPTURE_MS + 1)).toBeNull();
  expect(botScoreOf(new Headers({ "cf-bot-score": "150" }))).toBeNull();
});
test("requests that look alike are one group across addresses, with reasons and a confidence; a lone address is not called an actor", () => {
  const store = createCaptureStore();
  store.start(NOW, "keeper");
  for (let i = 0; i < 24; i++)
    store.note(NOW + i * 1000, input(`/alice/open/commit/${(0xabc1230 + i).toString(16)}`, `203.0.113.${i % 4}`, { "cf-bot-score": "3", "accept-language": "xx" }, "deep", "/:owner/:name/commit/:sha"));
  store.note(NOW, { ...input("/", "198.51.100.7", { "user-agent": "Other" }) });
  const groups = groupRequests(store.current(NOW + 30_000)!.entries);
  expect(groups).toHaveLength(2);
  const big = groups[0]!;
  expect(big.size).toBe(24);
  expect(big.addresses).toHaveLength(4);
  expect(big.cadence).toBe("steady");
  expect(big.commits.length).toBeGreaterThanOrEqual(3);
  expect(big.confidence).toBe("high");
  expect(big.reasons.join(" ")).toContain("bot score");
  expect(groups[1]!.confidence).toBe("low");
  expect(groups[1]!.reasons.join(" ")).toContain("Only one address");
});
test("the page, capture, import and export are an admin's and need a same-origin post", async () => {
  const cookie = `forge_session=${createSession(app.db, admin.id).token}`;
  const other = `forge_session=${createSession(app.db, bob.id).token}`;
  const url = `${servers.url}/settings/traffic/scrapes`;
  expect((await fetch(url, { headers: { cookie: other } })).status).toBe(404);
  const page = await fetch(url, { headers: { cookie } });
  expect(page.status).toBe(200);
  const text = await page.text();
  expect(text).toContain("Start a capture");
  expect(text).toContain("What was read");
  const post = (path: string, body: string, origin = servers.url) =>
    fetch(`${url}/${path}`, { method: "POST", redirect: "manual", headers: { cookie, origin, "content-type": "application/x-www-form-urlencoded" }, body });
  expect((await post("capture", "", "https://evil.example")).status).toBe(403);
  expect(captureStore.active(Date.now())).toBe(false);
  expect((await post("capture", "")).status).toBe(303);
  expect(captureStore.active(Date.now())).toBe(true);
  expect((await post("networks", `lines=${encodeURIComponent("203.0.0.0/16, <b>Evil</b>, Op, DE, [email protected]")}`)).status).toBe(303);
  for (let i = 0; i < 12; i++) captureStore.note(Date.now(), input(`/alice/open/blob/main/f${i}.ts`, `203.0.113.${i}`));
  const shown = await (await fetch(url, { headers: { cookie } })).text();
  const id = /group=([0-9a-f]{8})/.exec(shown)?.[1];
  expect(id).toBeDefined();
  expect(shown).toContain("&lt;b&gt;Evil&lt;/b&gt;");
  expect(shown).not.toContain("<b>Evil</b>");
  const detail = await (await fetch(`${url}?group=${id}`, { headers: { cookie } })).text();
  expect(detail).toContain("203.0.113.3");
  expect(detail).toContain("does not show who is behind");
  const report = await fetch(`${url}/export?group=${id}`, { headers: { cookie } });
  expect(report.headers.get("content-type")).toContain("text/plain");
  const reportText = await report.text();
  expect(reportText).toContain("[email protected]");
  expect(reportText).toContain("Limits:");
  expect((await fetch(`${url}/export?group=${id}`, { headers: { cookie: other } })).status).toBe(404);
  expect((await post("stop", "")).status).toBe(303);
  expect(captureStore.current(Date.now())).toBeNull();
  expect((await fetch(`${url}/export?group=${id}`, { headers: { cookie } })).status).toBe(404);
});
test("a stored hour holds at most HOUR_KEYS_MAX distinct reads however many minutes wrote it; the rest fold into (others)", () => {
  const ledger = createReadLedger();
  const writer = createReadWriter(app.db, ledger, { warn() {} }, () => NOW, 3_600_000);
  const hour = Math.floor((NOW + 5 * HOUR) / HOUR);
  const at = hour * HOUR;
  for (let minute = 0; minute < 4; minute++) {
    for (let i = 0; i < HOUR_KEYS_MAX; i++) ledger.note(at, { repo: "carol/wide", ref: "main", path: `m${minute}/f${i}.ts` }, true);
    writer.flush();
  }
  writer.stop();
  const n = app.db.query<{ n: number }, [number]>("SELECT COUNT(*) AS n FROM read_hour WHERE hour = ?1").get(hour)!.n;
  expect(n).toBeLessThanOrEqual(HOUR_KEYS_MAX + 1);
  const folded = app.db.query<{ requests: number }, [number, string]>("SELECT requests FROM read_hour WHERE hour = ?1 AND path = ?2").get(hour, OTHER_PATH)!;
  expect(folded.requests).toBe(3 * HOUR_KEYS_MAX);
  // nothing is lost from the totals
  expect(readsReport(app.db, createReadLedger(), at, 1).find((r) => r.repo === "carol/wide")?.requests).toBe(4 * HOUR_KEYS_MAX);
});
test("a bot score and country are believed only from a trusted edge", () => {
  const store = createCaptureStore();
  store.start(NOW, "keeper");
  store.note(NOW, input("/x", "203.0.113.1", { "cf-bot-score": "5", "cf-ipcountry": "NL" }, "deep", "/:owner/:name/blob/:ref/*", false));
  store.note(NOW, input("/x", "203.0.113.2", { "cf-bot-score": "5", "cf-ipcountry": "NL" }));
  const [forged, edged] = store.current(NOW)!.entries;
  expect(forged).toMatchObject({ botScore: null, country: null });
  expect(edged).toMatchObject({ botScore: 5, country: "NL" });
  const viaProxy = new Request("http://forge.test/");
  notePeer(viaProxy, "127.0.0.1", { proxies: [] });
  const direct = new Request("http://forge.test/", { headers: { "cf-bot-score": "5" } });
  notePeer(direct, "198.51.100.9", { proxies: [] });
  expect(wasViaEdge(viaProxy) || wasViaEdge(direct)).toBe(false); // loopback with no client named is the forge itself; a stranger is no edge
  const named = new Request("http://forge.test/", { headers: { "cf-connecting-ip": "203.0.113.5" } });
  notePeer(named, "127.0.0.1", { proxies: [] });
  expect(wasViaEdge(named)).toBe(true);
});
test("an agent account, even an admin's, cannot start a capture or import networks", async () => {
  const made = await createUser(app.db, { origin: "seed", trusted: true, username: "botty", email: "[email protected]", password: "correct-horse-battery", isAdmin: true });
  if (!made.ok) throw new Error(made.error);
  app.db.query("UPDATE user SET is_agent = 1 WHERE id = ?1").run(made.value.id);
  const cookie = `forge_session=${createSession(app.db, made.value.id).token}`;
  const url = `${servers.url}/settings/traffic/scrapes`;
  const post = (path: string, body = "") =>
    fetch(`${url}/${path}`, { method: "POST", redirect: "manual", headers: { cookie, origin: servers.url, "content-type": "application/x-www-form-urlencoded" }, body });
  expect((await post("capture")).status).toBe(404);
  expect(captureStore.active(Date.now())).toBe(false);
  expect((await post("networks", "lines=" + encodeURIComponent("198.51.100.0/24, A, B, DE, [email protected]"))).status).toBe(404);
  expect(networkTable(app.db).find("198.51.100.5")).toBeNull();
});
@@ -29,6 +29,7 @@
const NOT_PAGES = new Set([
  "/system/stream",
  "/settings/teams/stream",
  "/fleet/graph/live",
  "/settings/traffic/scrapes/export",
  "/settings/backups/live",
  "/:owner/:name/stream",
]);
		@@ -137,6 +137,7 @@ Machine routes are not pages and have no entry: `/audit.csv`,
137	137	- `/settings/origins` — `pages/origins.ts` (1,497 lines) · other forges this one reads and mirrors from · `/migrate` · none seen.
138	138	- `/settings/security` — `pages/security.ts` · the Shield and the evidence an admin needs to act with no AI · `/settings/traffic`, `/settings/backups` · none seen.
139	139	- `/settings/traffic` — `pages/traffic.ts` · who used the answering time in the last 15 minutes · `/settings/usage`, `/settings/telemetry` · none seen.
	140	- `/settings/traffic/scrapes` — `pages/scrapes.ts` · what was read by repository and path, which networks and addresses did it, a bounded capture, the abuse export · `/settings/traffic`, `/settings/security` · none seen.
140	141
141	142	## Operator views — why each is a page (#2696)
142	143
		@@ -0,0 +1,12 @@
	1	# Scrape evidence (#9406)
	2
	3	An admin sees what was read and works out who is doing it, at `/settings/traffic/scrapes` (linked from Traffic) and in the terminal: `:` → "Scrapes" (a forge admin who is a person; read-only there).
	4
	5	- Reads. For 1, 3 or 7 days: per repository, the paths read, the commits and branches, how much of the repository was covered, and files whose names look like keys or credentials. Counted in memory as requests finish (`perf/reads.ts`), written once a minute to `read_hour`, kept 7 days. No address is kept in it. A stored hour holds at most 500 distinct (repository, ref, path) rows; later ones fold into `(others)` for their repository, so an attacker walking paths cannot grow the table, and the report groups in SQL.
	6	- Networks. Address prefixes show name, operator, country and abuse contact from the forge's own `network` table, filled by pasting `range, name, operator, country, abuse contact` lines. No visitor address goes to a third party. Where the table has no entry, the page says the evidence stops there.
	7	- Capture. One click starts a capture: 10 minutes, 2000 requests, in memory only, no cookies, tokens or query strings, signed-in people skipped. Requests that look alike are grouped across addresses with a stated confidence and reasons. Full addresses exist only inside a capture and are gone when it expires or is stopped.
	8	- Bot score and country (`cf-bot-score`, `x-bot-score`, `cf-ipcountry`) are believed only when the request came through a trusted proxy (`peer.ts`, `wasViaEdge`); from anyone else they are ignored.
	9	- Export. A group exports as a plain-text abuse report with its limits stated.
	10	- Admin only; agent accounts are refused; every mutation is same-origin.
	11
	12	Not done: lookups against a public registry (by design, nothing leaves the forge), and blocking from this page.
		@@ -165,6 +165,8 @@ const READER_KINDS: ReadonlySet<string> = new Set([
165	165	"issue.confirmed",
166	166	"capacity.changed",
167	167	"shield.changed",
	168	"traffic.capture_started",
	169	"traffic.networks_imported",
168	170	"crew.governed",
169	171	]);
170	172
		@@ -0,0 +1,1 @@
	1	d90e8888280398516a6b00d80d951bad10ef7c23e8f7af59628a6fee590d77c3
		@@ -0,0 +1,26 @@
	1	// Evidence for telling a scraper from a reader (`perf/reads.ts`, `perf/network.ts`).
	2	// `read_hour`: what was read from repositories, by the hour for 7 days: repository,
	3	// ref (a branch or a commit), path, how many reads and how many were anonymous.
	4	// No address is kept here. `network`: the forge's own database of who runs an
	5	// address range (name, operator, country, abuse contact), filled by an admin, so no
	6	// visitor address ever goes to a third party. New tables only. Additive.
	7	export default `
	8	CREATE TABLE read_hour (
	9	hour INTEGER NOT NULL,
	10	repo TEXT NOT NULL,
	11	ref TEXT NOT NULL,
	12	path TEXT NOT NULL,
	13	requests INTEGER NOT NULL,
	14	anonymous INTEGER NOT NULL,
	15	PRIMARY KEY (hour, repo, ref, path)
	16	) WITHOUT ROWID;
	17	CREATE TABLE network (
	18	cidr TEXT PRIMARY KEY,
	19	name TEXT NOT NULL,
	20	operator TEXT NOT NULL,
	21	country TEXT NOT NULL,
	22	abuse TEXT NOT NULL,
	23	source TEXT NOT NULL,
	24	added_at INTEGER NOT NULL
	25	) WITHOUT ROWID;
	26	`;
		@@ -61,6 +61,7 @@ export function resolveClient(
61	61
62	62	const clients = new WeakMap<Request, string>();
63	63	const internals = new WeakSet<Request>();
	64	const edged = new WeakSet<Request>();
64	65
65	66	/** Headers a proxy names a client in. The tunnel always sets its own, so no stranger can leave them all out. */
66	67	const CLIENT_HEADERS = ["cf-connecting-ip", "x-forwarded-for", "x-real-ip", "forwarded"];
		@@ -85,8 +86,13 @@ export const notePeer = (
85	86	): void => {
86	87	clients.set(request, resolveClient(address, request.headers, trust));
87	88	if (isInternalPeer(address, request.headers, trust)) internals.add(request);
	89	else if (address !== undefined && (trust.proxies.length > 0 ? inCidrs(address, trust.proxies) : LOOPBACK.has(address)))
	90	edged.add(request);
88	91	};
89	92
	93	/** The request came through a proxy the operator trusts, so the headers an edge adds (bot score, country) are its, not the client's. */
	94	export const wasViaEdge = (request: Request): boolean => edged.has(request);
	95
90	96	export const wasInternal = (request: Request): boolean => internals.has(request);
91	97
92	98	export function clientAddress(request: Request): string {
		@@ -458,14 +458,15 @@ import {
458	458	takeDbTime,
459	459	type RouteTiming,
460	460	} from "../core/route_timing.ts";
461		import { loopWatch, perfRecord } from "../perf/live.ts";
462		import { countryOf, repoOf, signatureOf } from "../perf/traffic.ts";
	461	import { captureStore, loopWatch, perfRecord, readLedger } from "../perf/live.ts";
	462	import { classOf, countryOf, repoOf, signatureOf } from "../perf/traffic.ts";
463	463	import { createTrafficWriter } from "../perf/traffic_store.ts";
	464	import { createReadWriter, readOf } from "../perf/reads.ts";
		sentinelcrew 6d ago outdated Duplicate imports from perf/live.ts and perf/traffic.ts: merge into the existing import lines.
464	465	import { parseServerTiming } from "../perf/record.ts";
465	466	import { methodOf, noteUsage, roleOf, surfaceOf, takeUsage } from "./usage_tag.ts";
466	467	import { bearerChallenge, oauthRoutes } from "./oauth.ts";
467	468	import { flushAccess, recordAccess } from "./access_log.ts";
468		import { carryPeer, clientAddress, notePeer, wasInternal, type ProxyTrust } from "./peer.ts";
	469	import { carryPeer, clientAddress, notePeer, wasInternal, wasViaEdge, type ProxyTrust } from "./peer.ts";
469	470	import { isProbe } from "../kernel/probe.ts";
470	471	import { takeStrangerRead } from "./stranger_reads.ts";
471	472	import { parseCidrs } from "../kernel/cidr.ts";
		@@ -2229,6 +2230,7 @@ export async function serve(app: App): Promise<Servers> {
2229	2230	const timings = new Map<Database, RouteTiming>();
2230	2231	// The minute ledger's history: one write a minute, never on a request.
2231	2232	const trafficWriter = createTrafficWriter(app.db, perfRecord.ledger, app.log);
	2233	const readWriter = createReadWriter(app.db, readLedger, app.log);
2232	2234	const counted = (
2233	2235	owner: App,
2234	2236	request: Request,
		@@ -2255,6 +2257,16 @@ export async function serve(app: App): Promise<Servers> {
2255	2257	},
2256	2258	);
2257	2259	loopWatch.finished(tag.pattern, started, end);
	2260	// Evidence of what was read, and the running capture, if any (never a signed-in person's).
	2261	const signedIn = tag.role !== "anon" && tag.role !== "crawler";
	2262	const cls = classOf({ signedIn, crawler: wasCrawler(request), deepStranger: wasDeepStranger(request), internal: wasInternal(request) });
	2263	if (cls !== "internal" && response.status < 400) {
	2264	const read = readOf(tag.pattern, new URL(request.url).pathname);
	2265	if (read !== null) readLedger.note(Date.now(), read, !signedIn);
	2266	}
	2267	const at = Date.now();
	2268	if (captureStore.active(at))
	2269	captureStore.note(at, { request, address: clientAddress(request), viaEdge: wasViaEdge(request), status: response.status, ms, cls, pattern: tag.pattern });
2258	2270	} catch {
2259	2271	// A response the forge is only carrying keeps the headers it came with.
2260	2272	}
		@@ -2492,6 +2504,7 @@ export async function serve(app: App): Promise<Servers> {
2492	2504	}
2493	2505	timings.clear();
2494	2506	trafficWriter.stop();
	2507	readWriter.stop();
2495	2508	// A child's hooks stop answering before its database closes.
2496	2509	for (const key of [...new Set([...childHttps.keys(), ...childInternals.keys()])]) letGo(key);
2497	2510	closeHostedChildren(app);
		@@ -20,6 +20,7 @@ import { usagePage } from "../web/pages/usage.ts";
20	20	import { spendPage } from "../web/pages/spend-tab.ts";
21	21	import { securityPage, securityShieldSet } from "../web/pages/security.ts";
22	22	import { trafficPage } from "../web/pages/traffic.ts";
	23	import { captureStart, captureStop, networksImport, scrapesExport, scrapesPage } from "../web/pages/scrapes.ts";
23	24	import {
24	25	accountAct,
25	26	retireAct,
		@@ -189,6 +190,11 @@ export function settingsRoutes(router: Router<Handler>, knownPatterns: () => rea
189	190	router.add("GET", "/settings/usage", (ctx) => usagePage(ctx, knownPatterns));
190	191	router.add("GET", "/settings/spend", spendPage);
191	192	router.add("GET", "/settings/traffic", trafficPage);
	193	router.add("GET", "/settings/traffic/scrapes", scrapesPage);
	194	router.add("GET", "/settings/traffic/scrapes/export", scrapesExport);
	195	router.add("POST", "/settings/traffic/scrapes/capture", captureStart);
	196	router.add("POST", "/settings/traffic/scrapes/stop", captureStop);
	197	router.add("POST", "/settings/traffic/scrapes/networks", networksImport);
192	198	router.add("GET", "/settings/security", securityPage);
193	199	router.add("POST", "/settings/security/shield", securityShieldSet);
194	200	router.add("GET", "/settings/backups", backupsPage);
		@@ -0,0 +1,100 @@
	1	/**
	2	* One actor across addresses: captured requests grouped by what they share
	3	* (user agent, header names and order, language, accept, encoding,
	4	* sec-fetch-site), each group with its size, rate, addresses, the commits it
	5	* walked, its cadence and Cloudflare's bot score if sent, and a confidence
	6	* stated with the reasons for it.
	7	*
	8	* A group is a claim that requests look alike, not that one party sent them.
	9	* Traits every browser of one version shares prove little, so confidence rests
	10	* on the strong signals (a steady cadence, a walk through commits, a bot score)
	11	* and the page says what the evidence cannot show. Nothing here names anyone.
	12	*/
	13
	14	import type { CapturedRequest } from "./capture.ts";
	15	import { isCommit } from "./reads.ts";
	16
	17	export type Confidence = "low" \| "medium" \| "high";
	18
	19	export interface Group {
	20	readonly id: string;
	21	readonly ua: string;
	22	readonly language: string;
	23	readonly size: number;
	24	readonly addresses: readonly string[];
	25	readonly prefixes: readonly string[];
	26	readonly perMinute: number;
	27	readonly spanMs: number;
	28	readonly commits: readonly string[];
	29	readonly cadence: "steady" \| "uneven" \| "too few to say";
	30	readonly botScore: { readonly min: number; readonly max: number } \| null;
	31	readonly confidence: Confidence;
	32	/** What the confidence rests on, and what it lacks. */
	33	readonly reasons: readonly string[];
	34	readonly requests: readonly CapturedRequest[];
	35	}
	36
	37	const key = (r: CapturedRequest): string =>
	38	[r.ua, r.shape, r.language, r.accept, r.encoding, r.fetchSite].join("\u0000");
	39
	40	const idOf = (k: string): string => (Number(Bun.hash.xxHash32(k)) >>> 0).toString(16).padStart(8, "0");
	41
	42	/** Is the gap between requests steady: at least 10 gaps and a coefficient of variation under 0.5. */
	43	function cadenceOf(times: readonly number[]): Group["cadence"] {
	44	if (times.length < 11) return "too few to say";
	45	const sorted = [...times].sort((a, b) => a - b);
	46	const gaps = sorted.slice(1).map((t, i) => t - sorted[i]!);
	47	const mean = gaps.reduce((a, b) => a + b, 0) / gaps.length;
	48	if (mean <= 0) return "steady";
	49	const sd = Math.sqrt(gaps.reduce((a, g) => a + (g - mean) ** 2, 0) / gaps.length);
	50	return sd / mean < 0.5 ? "steady" : "uneven";
	51	}
	52
	53	export function groupRequests(entries: readonly CapturedRequest[]): Group[] {
	54	const by = new Map<string, CapturedRequest[]>();
	55	for (const e of entries) by.set(key(e), [...(by.get(key(e)) ?? []), e]);
	56	const groups: Group[] = [];
	57	for (const [k, list] of by) {
	58	const addresses = [...new Set(list.map((r) => r.address))];
	59	const prefixes = [...new Set(list.map((r) => r.prefix))];
	60	const times = list.map((r) => r.at);
	61	const spanMs = Math.max(...times) - Math.min(...times);
	62	const commits = [...new Set(list.flatMap((r) => (r.read !== null && isCommit(r.read.ref) ? [r.read.ref] : [])))];
	63	const scores = list.flatMap((r) => (r.botScore === null ? [] : [r.botScore]));
	64	const cadence = cadenceOf(times);
	65	const botScore = scores.length === 0 ? null : { min: Math.min(...scores), max: Math.max(...scores) };
	66	const strong = [cadence === "steady", commits.length >= 3, botScore !== null && botScore.max <= 29];
	67	const strongN = strong.filter(Boolean).length;
	68	const size = list.length;
	69	const confidence: Confidence =
	70	strongN >= 2 && size >= 20 ? "high" : strongN >= 1 && size >= 10 && addresses.length >= 2 ? "medium" : "low";
	71	const reasons = [
	72	addresses.length >= 2
	73	? `${addresses.length} addresses sent requests with the same user agent and headers.`
	74	: "Only one address: nothing here spans addresses.",
	75	cadence === "steady" ? "Requests arrive at a steady pace." : cadence === "uneven" ? "Requests arrive unevenly, as people's do." : "Too few requests to judge their pace.",
	76	commits.length >= 3 ? `They read ${commits.length} different commits.` : "They did not walk a run of commits.",
	77	botScore === null ? "No bot score was sent." : `Cloudflare's bot score was ${botScore.min === botScore.max ? botScore.min : `${botScore.min} to ${botScore.max}`} (low means likely automated).`,
	78	size < 20 ? "Few requests: a small sample." : `${size} requests.`,
	79	confidence === "low" && strongN === 0 ? "Traits that browsers of one version share are weak evidence on their own." : "",
	80	].filter((r) => r !== "");
	81	groups.push({
	82	id: idOf(k),
	83	ua: list[0]!.ua,
	84	language: list[0]!.language,
	85	size,
	86	addresses,
	87	prefixes,
	88	perMinute: Math.round((size / Math.max(1, spanMs / 60_000)) * 10) / 10,
	89	spanMs,
	90	commits,
	91	cadence,
	92	botScore,
	93	confidence,
	94	reasons,
	95	requests: list,
	96	});
	97	}
	98	const rank: Record<Confidence, number> = { high: 3, medium: 2, low: 1 };
	99	return groups.sort((a, b) => b.size - a.size \|\| rank[b.confidence] - rank[a.confidence]);
	100	}
		@@ -0,0 +1,167 @@
	1	/**
	2	* A capture: for a short while, keep what it takes to tell one actor from
	3	* another across addresses. The only place a full visitor address is ever
	4	* held, and only in memory, so a restart ends it as an expiry does.
	5	*
	6	* Bounded in time (`CAPTURE_MS`) and size (`CAPTURE_MAX` requests), and
	7	* expires by itself: `current` returns null once it is over and its requests
	8	* are dropped. It keeps no cookie, authorization header or query string: of
	9	* the headers, only their names, and the handful of values that tell clients
	10	* apart. Only requests no one was signed in behind are kept; a signed-in
	11	* person's requests are not an investigation's business.
	12	*/
	13
	14	import { addressBytes } from "../kernel/cidr.ts";
	15	import { prefixOf, type TrafficClass } from "./traffic.ts";
	16	import { readOf, type Read } from "./reads.ts";
	17
	18	export const CAPTURE_MS = 10 * 60_000;
	19	export const CAPTURE_MAX = 2_000;
	20
	21	export interface CapturedRequest {
	22	readonly n: number;
	23	readonly at: number;
	24	readonly address: string;
	25	readonly prefix: string;
	26	readonly method: string;
	27	readonly path: string;
	28	readonly status: number;
	29	readonly ms: number;
	30	readonly ua: string;
	31	/** Header names (never values) except the ones a proxy adds, in the order received. */
	32	readonly shape: string;
	33	readonly language: string;
	34	readonly accept: string;
	35	readonly encoding: string;
	36	readonly fetchSite: string;
	37	readonly country: string \| null;
	38	/** Cloudflare's bot score, 1 to 99, when the edge sent one. */
	39	readonly botScore: number \| null;
	40	readonly read: Read \| null;
	41	}
	42
	43	export interface Capture {
	44	readonly id: string;
	45	readonly startedAt: number;
	46	readonly expiresAt: number;
	47	readonly by: string;
	48	readonly entries: readonly CapturedRequest[];
	49	/** The size bound was reached; later requests were not kept. */
	50	readonly full: boolean;
	51	}
	52
	53	/** What the server tells a capture of one finished request. */
	54	export interface CaptureInput {
	55	readonly request: Request;
	56	readonly address: string;
	57	/** Came through a trusted edge: only then are its bot score and country headers believed. */
	58	readonly viaEdge: boolean;
	59	readonly status: number;
	60	readonly ms: number;
	61	readonly cls: TrafficClass;
	62	readonly pattern: string;
	63	}
	64
	65	const PROXY_HEADER = /^(x-forwarded-\|cf-\|x-real-ip$\|cdn-loop$\|traceparent$\|tracestate$\|x-request-id$\|forwarded$\|via$\|x-envoy-\|x-amzn-)/;
	66	const NEVER = /^(cookie\|authorization\|proxy-authorization\|x-api-key\|x-auth-token)$/;
	67
	68	const short = (s: string \| null, max: number): string =>
	69	// eslint-disable-next-line no-control-regex
	70	(s ?? "").replace(/[\u0000-\u001f\u007f]/g, "").slice(0, max);
	71
	72	/** The Cloudflare bot score a header carries (`cf-bot-score` or `x-bot-score`): a whole number from 1 to 99, else null. */
	73	export function botScoreOf(headers: Headers): number \| null {
	74	const raw = headers.get("cf-bot-score") ?? headers.get("x-bot-score");
	75	if (raw === null \|\| !/^\d{1,2}$/.test(raw.trim())) return null;
	76	const n = Number(raw);
	77	return n >= 1 && n <= 99 ? n : null;
	78	}
	79
	80	export function describe(input: CaptureInput, n: number, at: number): CapturedRequest {
	81	const { request: r } = input;
	82	const url = new URL(r.url);
	83	const bytes = addressBytes(input.address);
	84	const names = [...r.headers.keys()].filter((h) => !PROXY_HEADER.test(h) && !NEVER.test(h));
	85	const country = input.viaEdge ? (r.headers.get("cf-ipcountry")?.toUpperCase() ?? "") : "";
	86	return {
	87	n,
	88	at,
	89	address: bytes === undefined ? "unknown" : short(input.address, 45),
	90	prefix: bytes === undefined ? "unknown" : prefixOf(bytes),
	91	method: short(r.method, 8),
	92	// The query is left out: it can carry a token.
	93	path: short(url.pathname, 300),
	94	status: input.status,
	95	ms: Math.round(input.ms),
	96	ua: short(r.headers.get("user-agent"), 200),
	97	shape: names.slice(0, 30).join(","),
	98	language: short(r.headers.get("accept-language"), 60),
	99	accept: short(r.headers.get("accept"), 80),
	100	encoding: short(r.headers.get("accept-encoding"), 40),
	101	fetchSite: short(r.headers.get("sec-fetch-site"), 16),
	102	country: /^[A-Z][A-Z0-9]$/.test(country) ? country : null,
	103	botScore: input.viaEdge ? botScoreOf(r.headers) : null,
	104	read: readOf(input.pattern, url.pathname),
	105	};
	106	}
	107
	108	export interface CaptureStore {
	109	/** Begin a capture; if one is running, that one. */
	110	start(now: number, by: string): Capture;
	111	stop(): void;
	112	/** The running capture, or null once it has expired (its requests are dropped then). */
	113	current(now: number): Capture \| null;
	114	/** Cheap: is anything being kept at all. */
	115	active(now: number): boolean;
	116	note(now: number, input: CaptureInput): void;
	117	}
	118
	119	export function createCaptureStore(): CaptureStore {
	120	let id = "";
	121	let startedAt = 0;
	122	let expiresAt = 0;
	123	let by = "";
	124	let entries: CapturedRequest[] = [];
	125	let full = false;
	126	let timer: ReturnType<typeof setTimeout> \| undefined;
	127	const drop = (): void => {
	128	clearTimeout(timer);
	129	id = "";
	130	entries = [];
	131	full = false;
	132	expiresAt = 0;
	133	};
	134	const live = (now: number): boolean => {
	135	if (id !== "" && now >= expiresAt) drop();
	136	return id !== "";
	137	};
	138	return {
	139	start(now, who) {
	140	if (!live(now)) {
	141	id = crypto.randomUUID().slice(0, 8);
	142	startedAt = now;
	143	expiresAt = now + CAPTURE_MS;
	144	by = who;
	145	entries = [];
	146	full = false;
	147	timer = setTimeout(drop, CAPTURE_MS + 1_000);
	148	timer.unref();
	149	}
	150	return this.current(now)!;
	151	},
	152	stop: drop,
	153	current(now) {
	154	return live(now) ? { id, startedAt, expiresAt, by, entries, full } : null;
	155	},
	156	active: live,
	157	note(now, input) {
	158	if (!live(now) \|\| input.cls === "people" \|\| input.cls === "internal") return;
	159	if (new URL(input.request.url).pathname.startsWith("/settings/traffic")) return;
	160	if (entries.length >= CAPTURE_MAX) {
	161	full = true;
	162	return;
	163	}
	164	entries.push(describe(input, entries.length + 1, now));
	165	},
	166	};
	167	}