diff --git a/.claude/skills/mcp-builder/scripts/requirements.txt b/.claude/skills/mcp-builder/scripts/requirements.txt index e73e5d1e..31652c0f 100644 --- a/.claude/skills/mcp-builder/scripts/requirements.txt +++ b/.claude/skills/mcp-builder/scripts/requirements.txt @@ -1,2 +1,6 @@ anthropic>=0.39.0 mcp>=1.1.0 +# Transitive via mcp (multipart form parsing). Pinned as a floor because the +# advisory scanner cannot resolve an unpinned requirement and reports the +# oldest version it can match; 0.0.31 carries the PYSEC-2026 multipart fixes. +python-multipart>=0.0.31 diff --git a/app/api/cron/health/__tests__/route.test.ts b/app/api/cron/health/__tests__/route.test.ts new file mode 100644 index 00000000..53c0b0e5 --- /dev/null +++ b/app/api/cron/health/__tests__/route.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { NextResponse } from 'next/server' + +interface Row { + path: string + declaredAt: string | null + lastStartedAt: string | null + lastCompletedAt: string | null + lastStatus: string | null + lastHttpStatus: number | null + lastDurationMs: number | null +} + +const verifyCronSecret = vi.fn((): NextResponse | null => null) +const loadCronHeartbeats = vi.fn(async (): Promise => []) +const declareCronPaths = vi.fn(async (_paths: readonly string[]): Promise => {}) +const cronHeartbeatStoreConfigured = vi.fn((): boolean => true) + +vi.mock('@/lib/auth/cron', () => ({ + verifyCronSecret: () => verifyCronSecret(), +})) + +vi.mock('@/lib/cron/heartbeat-store', () => ({ + loadCronHeartbeats: () => loadCronHeartbeats(), + declareCronPaths: (paths: readonly string[]) => declareCronPaths(paths), + cronHeartbeatStoreConfigured: () => cronHeartbeatStoreConfigured(), +})) + +vi.mock('@/lib/logger', () => { + const child = () => logger + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child } + return { createLogger: () => logger } +}) + +const { GET } = await import('../route') +const { CRON_SCHEDULES } = await import('@/lib/cron/schedules.generated') + +function request(): Request { + return new Request('http://app:3000/api/cron/health', { + headers: { authorization: 'Bearer secret' }, + }) +} + +function everythingRanSecondsAgo(seconds: number) { + const at = new Date(Date.now() - seconds * 1000).toISOString() + return CRON_SCHEDULES.map((entry) => ({ + path: entry.path, + declaredAt: at, + lastStartedAt: at, + lastCompletedAt: at, + lastStatus: 'ok', + lastHttpStatus: 200, + lastDurationMs: 12, + })) +} + +beforeEach(() => { + verifyCronSecret.mockClear() + verifyCronSecret.mockReturnValue(null) + loadCronHeartbeats.mockReset() + declareCronPaths.mockClear() + cronHeartbeatStoreConfigured.mockReturnValue(true) +}) + +describe('GET /api/cron/health', () => { + it('rejects an unauthenticated call before touching the store', async () => { + verifyCronSecret.mockReturnValue( + NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), + ) + + const response = await GET(request()) + + expect(response.status).toBe(401) + expect(loadCronHeartbeats).not.toHaveBeenCalled() + }) + + it('answers 200 when every schedule ran inside its window', async () => { + loadCronHeartbeats.mockResolvedValue(everythingRanSecondsAgo(5)) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.status).toBe('ok') + expect(body.report.ok).toBe(true) + expect(body.report.total).toBe(CRON_SCHEDULES.length) + }) + + it('answers 503 when a schedule stopped arriving', async () => { + // Every path ran 5 s ago except one stale per-minute job: the endpoint is + // a single boolean for the container healthcheck, so one silent stop must + // be enough to fail it. Pick a per-minute entry explicitly — a daily job + // 30 minutes old is not stale, and a test that used index 0 would pass for + // the wrong reason. + const rows = everythingRanSecondsAgo(5) + const index = CRON_SCHEDULES.findIndex((entry) => entry.maxIntervalSeconds === 60) + expect(index).toBeGreaterThanOrEqual(0) + const staleAt = new Date(Date.now() - 30 * 60 * 1000).toISOString() + rows[index] = { ...rows[index], lastStartedAt: staleAt, lastCompletedAt: staleAt } + loadCronHeartbeats.mockResolvedValue(rows) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(503) + expect(body.status).toBe('stale') + expect(body.report.stale).toBe(1) + expect(body.report.checks[index].state).toBe('stale') + }) + + it('declares paths that have never run instead of failing them immediately', async () => { + loadCronHeartbeats.mockResolvedValue([]) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.status).toBe('ok') + expect(body.report.pending).toBe(CRON_SCHEDULES.length) + expect(declareCronPaths).toHaveBeenCalledTimes(1) + expect(declareCronPaths.mock.calls[0][0]).toHaveLength(CRON_SCHEDULES.length) + }) + + it('says unverifiable (503) rather than healthy when the store cannot be read', async () => { + loadCronHeartbeats.mockRejectedValue(new Error('connection refused')) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(503) + expect(body.status).toBe('unverifiable') + // The driver's message must not reach the body. + expect(JSON.stringify(body)).not.toContain('connection refused') + }) + + it('says unverifiable when the heartbeat store is not configured', async () => { + cronHeartbeatStoreConfigured.mockReturnValue(false) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(503) + expect(body.status).toBe('unverifiable') + expect(loadCronHeartbeats).not.toHaveBeenCalled() + }) + + it('surfaces a run that arrived but failed separately from a missed tick', async () => { + const rows = everythingRanSecondsAgo(5) + rows[1] = { ...rows[1], lastStatus: 'error', lastHttpStatus: 500 } + loadCronHeartbeats.mockResolvedValue(rows) + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body.failing).toEqual([ + { path: rows[1].path, lastStatus: 'error', lastHttpStatus: 500 }, + ]) + }) +}) diff --git a/app/api/cron/health/route.ts b/app/api/cron/health/route.ts new file mode 100644 index 00000000..ee57aea8 --- /dev/null +++ b/app/api/cron/health/route.ts @@ -0,0 +1,109 @@ +import { NextResponse } from 'next/server' +import { verifyCronSecret } from '@/lib/auth/cron' +import { + evaluateCronHeartbeats, + isFailingStatus, + type CronHeartbeatReport, +} from '@/lib/cron/heartbeat-evaluation' +import { + cronHeartbeatStoreConfigured, + declareCronPaths, + loadCronHeartbeats, +} from '@/lib/cron/heartbeat-store' +import { CRON_SCHEDULES } from '@/lib/cron/schedules.generated' +import { createLogger } from '@/lib/logger' + +const log = createLogger('cron/health') + +// Reads live rows every call; a cached 200 would defeat the point of a +// healthcheck that exists to notice a schedule going quiet. +export const dynamic = 'force-dynamic' + +interface CronHealthBody { + status: 'ok' | 'stale' | 'unverifiable' + reason?: string + /** Present when the store responded. */ + report?: CronHeartbeatReport + /** Runs that arrived but ended non-2xx. Informational; a failed run still ran. */ + failing?: Array<{ path: string; lastStatus: string | null; lastHttpStatus: number | null }> +} + +/** + * GET /api/cron/health + * + * Answers the question the 2026-11-15 incident proved nobody was asking: did + * every scheduled job actually start when it should? + * + * 200 = every scheduled path has run within its own worst-case window. + * 503 = at least one path is stale, or the heartbeat store could not be read + * (an unreadable store is not evidence of a healthy cron). + * + * Called by the cron container's own healthcheck (docker-compose.yml), which is + * what turns this answer into `unhealthy` and into the existing alerting chain. + * Auth is the normal cron secret: the body names instance-wide schedules, so it + * is not public, but the same credential the scheduler already holds is the + * right one — no new secret to distribute. + */ +export async function GET(request: Request): Promise> { + // report: false — this endpoint is itself polled by a healthcheck, so a + // misconfigured probe must not emit an auth-failure event every minute. + const authError = verifyCronSecret(request, { report: false }) + if (authError) return authError as NextResponse + + if (!cronHeartbeatStoreConfigured()) { + return NextResponse.json( + { + status: 'unverifiable', + reason: + 'heartbeat store not configured (NEXT_PUBLIC_SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY missing)', + }, + { status: 503 }, + ) + } + + try { + // Declare first, read second: a path that has never run needs its + // declared_at anchor before the evaluator can tell a fresh job from a + // stopped one. ignoreDuplicates keeps existing rows untouched. + const known = new Set((await loadCronHeartbeats()).map((row) => row.path)) + const undeclared = CRON_SCHEDULES.map((entry) => entry.path).filter((path) => !known.has(path)) + if (undeclared.length > 0) await declareCronPaths(undeclared) + + const rows = await loadCronHeartbeats() + const report = evaluateCronHeartbeats({ entries: CRON_SCHEDULES, rows, now: Date.now() }) + + const failing = report.checks + .filter((check) => isFailingStatus(check.lastStatus, check.lastHttpStatus)) + .map((check) => ({ + path: check.path, + lastStatus: check.lastStatus, + lastHttpStatus: check.lastHttpStatus, + })) + + if (!report.ok) { + const stale = report.checks.filter((check) => check.state === 'stale') + log.warn('cron schedule stale', { + stale: stale.map((check) => ({ + path: check.path, + ageSeconds: check.ageSeconds, + allowedAgeSeconds: check.allowedAgeSeconds, + })), + }) + } + + return NextResponse.json( + { status: report.ok ? 'ok' : 'stale', report, failing }, + { status: report.ok ? 200 : 503 }, + ) + } catch (err) { + // Never echo the driver's message: it can name tables and query fragments. + log.error('cron health check failed', err as Error) + return NextResponse.json( + { + status: 'unverifiable', + reason: 'heartbeat store read failed', + }, + { status: 503 }, + ) + } +} diff --git a/changes/2026-11-15-cron-heartbeat-detektion.md b/changes/2026-11-15-cron-heartbeat-detektion.md new file mode 100644 index 00000000..34115718 --- /dev/null +++ b/changes/2026-11-15-cron-heartbeat-detektion.md @@ -0,0 +1,109 @@ +# Accounted: missad-tick-detektion för cron (2026-11-15) + +Uppföljning på `changes/2026-11-15-cron-pids-och-healthcheck.md`. Den ändringen +fixade orsaken till att cron tappade jobb. Den här ändringen gör att ett tappat +jobb **syns** nästa gång — vilket var det verkliga problemet: felet pågick i +dygn utan att någon mätning, larm eller dashboard reagerade. + +## Fynd + +`accounted-cron` förlorade 1 497 av 10 455 körningar på ett dygn (14,3 %) när +`pids_limit: 30` fick `fork/exec /bin/sh` att misslyckas. supercronic hoppade +tyst över tickarna. Containern var `Up` hela tiden, appen svarade 200 på alla +anrop, och enda spåret var rader i cron-containerns egen logg. + +Tre saker saknades: + +1. **Ingen körningshistorik.** `lib/api/with-cron-context.ts` loggade + `cron run started/completed/failed` via `createLogger`, men skrev inget + beständigt. En körning som aldrig startade lämnade inget spår alls. +2. **Healthchecken mätte fel sak.** Cron-tjänstens healthcheck var + `ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'` — den blev aldrig + röd av att scheman slutade köra, bara av att processen dog. +3. **Ingen koppling till kadensen.** Även med historik hade ingen kontroll vetat + vad "för sen" betyder: en daglig körning får vara 24 h gammal, en + per-minut-körning inte 4 minuter. + +## Ändring + +- **`lib/cron/cron-interval.ts`** — tolkare för 5-fälters cron (det enda + `vercel.json` använder) som räknar ut **värsta** gapet i sekunder över ett + 12-årsfönster (behövs för 29 februari), inte medelgapet. Okänd syntax kastar i + stället för att gissa. `describeGap()` för läsbar text. +- **`scripts/generate-crontabs.ts`** — skriver nu även + `lib/cron/schedules.generated.ts`: en post per schemalagd path med + `schedule` och `maxIntervalSeconds`. Samma källa som crontaben, så en + schemalagd path som kontrollen inte känner till kan inte uppstå. +- **`supabase/migrations/20261115190000_cron_heartbeats.sql`** — tabellen + `cron_heartbeats`, en rad per path: `declared_at`, `last_started_at`, + `last_completed_at`, `last_status`, `last_http_status`, `last_duration_ms`. + RLS på med **noll policies** (endast service role; instansglobal + schemaläggning är inte tenantdata). +- **`lib/cron/heartbeat-evaluation.ts`** — ren utvärdering, testbar utan DB: + `ok` (körning inom fönstret), `pending` (aldrig körd men deklarerad inom sin + egen kadens), `stale` (körning har uteblivit). Tröskel = + `2 × maxIntervalSeconds + 120 s`. +- **`lib/cron/heartbeat-store.ts`** — Supabase-IO. Skrivningar är best effort + (loggar och returnerar, kastar aldrig: en mätningsskrivning får inte fälla + jobbet den mäter). Läsningar kastar, så endpointen kan säga "kan inte + verifiera" i stället för att ljuga grönt. No-op när Supabase-env saknas. +- **`app/api/cron/health/route.ts`** — 200 när alla scheman ligger inom sina + fönster, 503 när något är `stale` eller lagret inte kan läsas. Deklarerar + okända paths först (ger nya jobb respit i stället för omedelbart rött). + Autentiseras med befintlig cron-hemlighet (`report: false` så en felkopplad + probe inte spammar auth-larm). +- **`lib/api/with-cron-context.ts`** — skriver heartbeat per körning (path från + request-URL:en) med status och duration. 401 räknas inte som en körning, och + ett kastat heartbeat-fel fäller aldrig körningen. +- **`docker-compose.yml`** (cron-tjänsten) — healthcheck bytt från + `ps ax | grep supercronic` till att anropa `/api/cron/health` med + `$${CRON_SECRET}`/`$${APP_URL}`, `start_period: 120s`. + +Varför `last_started_at` och inte `last_completed_at`: en körning som hänger sig +avslutas aldrig, och supercronic hoppar då över alla senare tickar. Mätt från +start blir en hängning synlig; mätt från completion hade den sett levande ut. + +Varför heartbeat i stället för att bara räkna i apploggen: loggen rullar, ägs av +Docker, och kräver att någon läser den. Tabellen är frågbar, och endpointen gör +svaret till en containerstatus som den befintliga larmkedjan redan tittar på. + +## Verifiering + +- `npx vitest run --project unit lib/cron lib/api/__tests__/with-cron-context.test.ts scripts/__tests__/generate-crontabs.test.ts app/api/cron/health` → **77 + 7 + 11 tester gröna**: + - kadens: `* * * * *` = 60 s, `*/2` = 120 s, `*/5` = 300 s, `0 * * * *` = 3600 s, + `30 */2 * * *` = 7200 s (inte 3600), daglig = 86 400 s, veckovis = 604 800 s, + 29 februari → ändligt gap; `0 0 1 * 1` = 7 dygn (cron-regeln OR, inte AND). + - utvärdering: per-minut-jobb 241 s gammalt = stale, 120 s = ok, deklarerad-men-aldrig-körd + = pending inom respit och stale efter, hängd körning (gammal start, färsk + completion) = stale. + - endpoint: 200/503, 401 utan store-åtkomst, deklarerar okända paths, + "unverifiable" utan läckta drivrutinsmeddelanden, `failing` listar + körningar som kom fram men föll. + - wiring: heartbeat skrivs med rätt path/status, 401 skriver ingen heartbeat, + kastat heartbeat-fel ger fortfarande lyckad körning. + - drift: `lib/cron/schedules.generated.ts` matchar generatorn byte för byte och + `crontab.self-hosted` rad för rad. +- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel. + `check:guards` → passed. +- Hela unit-sviten körd efter ändringen (resultat i PR:ens CI). + +## Kvar / nästa steg + +- Kör `docker compose config` och recreate av cron på server7 först när appen med + den nya routen är deployad — annars blir healthchecken 404 och containern + unhealthy. Ordning: deploy app → applicera migrationen → recreate cron. +- Larmet blir `unhealthy` via befintlig healthcheck-kedja; Prometheus-regel för + `container_health_status` finns redan. Ingen ny larmväg behövs. +- Backlog-kontrollerna (0 rader i alla fem köer) ligger kvar som engångsbevis i + incidentprotokollet; ingen återkörning av affärsjobb krävs eftersom instansen + är en tom/sandbox-miljö (companies 1, invoices 0, journal_entries 0). + +## Revert + +``` +git revert # kod, migration och compose i en commit +``` + +Migrationen är additiv (ny tabell, inga ändrade kolumner) och kan lämnas kvar om +bara koden rullas tillbaka. Healthchecken återställs separat: +`test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"]`. diff --git a/changes/2026-11-15-osv-gaten-gron.md b/changes/2026-11-15-osv-gaten-gron.md new file mode 100644 index 00000000..c5554711 --- /dev/null +++ b/changes/2026-11-15-osv-gaten-gron.md @@ -0,0 +1,110 @@ +# Accounted: OSV-gaten grön — 24 sårbarheter stängda, xlsx undantagen med bevis (2026-11-15) + +`security / scan` var röd på PR #44 även efter att Trivy-steget blivit grönt. +Steg 6, **OSV Scanner**, fällde jobbet. + +## Fynd + +`ghcr.io/google/osv-scanner:v2.7.1` mot `package-lock.json` + +`.claude/skills/mcp-builder/scripts/requirements.txt`: **43 advisory-poster över +10 paket**. Viktigt för rotorsaken: samma skanning mot `origin/main`s låsfil ger +**9 av 10 paket** — det här var inte något PR:en införde. Nya GHSA/PYSEC-poster +publicerades mot versioner vi redan kör, så `main` hade blivit röd vid nästa +nattliga skanning (02:17) oavsett. + +| Paket | Före | Efter | Väg | +|---|---|---|---| +| nodemailer | 10.0.6 (direkt) + 10.0.10 (mailparser) | **10.1.4** | direkt dependency + override | +| zod | 4.4.3 | **4.6.7** | `^4.6.6` | +| vite | 8.1.3 | **8.3.8** | override `^8.1.6` (dev, via vitest) | +| vitest / @vitest/mocker | 4.1.11 / 4.1.11 | **4.1.12 / 4.1.12** | `^4.1.12` | +| @babel/core | 7.28.6 | **7.29.13** | override `^7.29.6` (dev, via eslint-plugin-react-hooks) | +| fast-xml-parser | 5.7.2 | **5.12.1** | override `^5.12.0` (via @aws-sdk/xml-builder) | +| minimatch | 9.0.9 | **11.0.8** | override `^11.0.1` (dev, via typescript-eslint) | +| postcss-selector-parser | 6.0.10 | **6.1.6** | override `^6.1.5` (via @tailwindcss/typography) | +| python-multipart | olöst (otransitiv, (ospecificerad) | **>=0.0.31** | explicit floor i skillens requirements.txt | +| xlsx | 0.20.3 | 0.20.3 | **undantagen** — se nedan | + +## xlsx: undantaget, inte bortförklarat + +Båda posterna (`GHSA-4r6h-8v6p-xvw6` = CVE-2023-30533 prototype pollution, +`GHSA-5pgg-2g8v-p4x9` = CVE-2024-22363 ReDoS) har `introduced: 0` och **inget +`fixed`-event** — kontrollerat direkt mot `api.osv.dev` 2026-11-15. Det betyder +inte "ofixat", det betyder "ingen npm-version bär fixen": SheetJS slutade +publicera till npm efter 0.18.5 och släpper 0.19+ från `cdn.sheetjs.com`. + +Repot konsumerar `xlsx` som URL-dependency +(`https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz`), och 0.20.3 innehåller +båda fixarna (0.19.3 respektive 0.20.2). Skanningen rapporterar alltså en +versionsupplösningsartefakt, inte en levande defekt. + +Hanteringen är inte en tyst tystnad: `osv-scanner.toml` med två `IgnoredVulns`, +`effectiveUntil = 2027-05-15` och full motivering, plus en uttrycklig varning i +filen om att `lib/import/*` parsar användaruppladdade arbetsböcker — undantaget +får aldrig läsas som tillstånd att acceptera ofixade parser-CVE:er. När SheetJS +publicerar 0.21+, eller vi migrerar till en underhållen parser, ska filen bort. + +## Ändring + +- `package.json`: nodemailer `10.1.4`; zod `^4.6.6`; vitest `^4.1.12`; overrides + utökade med `nodemailer` (mailparser-subträdet), `@babel/core`, + `fast-xml-parser`, `minimatch`, `postcss-selector-parser`, `vite`. +- `package-lock.json`: omgenererad med `npm install --no-audit --no-fund`. +- `.claude/skills/mcp-builder/scripts/requirements.txt`: explicit + `python-multipart>=0.0.31` med kommentar om varför en floor behövs (OSV kan + inte lösa en otransitiv, opinnad requirement och rapporterar den äldsta + version den kan matcha). +- `osv-scanner.toml`: ny, se ovan. +- `scripts/checks/no-new-antipatterns.mjs`: PINNED_DEPS nodemailer → `10.1.4` + med motiveringen uppdaterad (9.x → 10.1.4 i två steg samma dag). + +Notera att arbetsflödet redan skickar `--config` när `osv-scanner.toml` finns +(`.gitea/workflows/security.yml`), så ingen CI-ändring behövdes. + +## Verifiering + +- `osv-scanner scan source --recursive --config=osv-scanner.toml .` → + **No issues found**, rc=0 (2 filtrerade, båda xlsx, med utskriven motivering). +- `npx vitest run --project unit` → **1687 filer / 22 400 tester passed**, + 3 filer/5 tester skipped, 163 s (efter alla bumps: zod 4.6, vite 8.3, + vitest 4.1.12, minimatch 11, @babel 7.29). +- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel. + `check:guards` → passed (pinned-dep 0). +- Trivy fs HIGH/CRITICAL: 0 vulnerabilitets-, secret- och misconfig-fynd. + +## Revert + +``` +git revert +``` + +Enda beteendepåverkande posten är `osv-scanner.toml` (om den tas bort blir +gaten röd igen på xlsx). Paketbumparna är patch/minor utom `minimatch` 9 → 11, +som bara används av lint-verktygskedjan och verifieras av `check:lint` + +testsviten. + +## Tillägg: Semgrep-steget (2026-11-15, samma PR) + +Efter Trivy och OSV blev två steg gröna och det tredje rött: **Semgrep +(`p/ci --error`)** gav exakt **ett** fynd: + +``` +.npmrc + package_managers.npm.npm-missing-minimum-release-age [Blocking] + This .npmrc does not set a minimum release age ... Add `min-release-age = 7` +``` + +Regeln är rimlig: en komprometterad release rycks oftast tillbaka inom några +dagar, så att vänta en vecka innan en ny version får lösas in tar bort det +billigaste angreppsfönstret. npm ≥ 11.10 förstår nyckeln (vi kör 11.19.1); +äldre npm ignorerar okända nycklar, så `.npmrc` förblir kompatibel. + +Åtgärd: `min-release-age=7` i `.npmrc` med kommentar om varför. Verifierat: +`npm config get min-release-age` → `7`, `npm install --package-lock-only` gav +ingen låsfillsdiff, och hela Semgrep-skanningen är därefter ren (`rc=0`, "no +findings"). + +Fälla att känna till: regeln gäller även våra egna `@siax/*`-paket. Behöver en +färsk intern publicering konsumeras inom sju dagar måste den hämtas med +`--min-release-age=0` — det är ett medvetet undantag i det enskilda fallet, inte +ett skäl att sänka taket i `.npmrc`. diff --git a/docker-compose.yml b/docker-compose.yml index 17545b54..75783a59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,11 +76,24 @@ services: mem_limit: 256m pids_limit: 200 healthcheck: - test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"] + # 2026-11-15: this used to be `ps ax | grep supercronic`, which stays green + # while the scheduler process is up but ticks are being dropped — exactly + # what happened when pids_limit 30 made 14,3 % of runs fail to fork (see + # docker-compose.accounted.yml and the cron_heartbeats migration). Ask the + # app instead: /api/cron/health answers 503 when any schedule has not run + # within its own worst-case window, so a silent stop makes this container + # unhealthy and reaches alerting. `$$` is required — a single `$` would be + # interpolated by compose on the host instead of by the shell in here. + test: + - CMD-SHELL + - 'curl -fsS -m 10 -o /dev/null -H "Authorization: Bearer $${CRON_SECRET}" "$${APP_URL}/api/cron/health"' interval: 60s - timeout: 10s + timeout: 15s retries: 3 - start_period: 20s + # The probe calls the app over HTTP, so it can only pass once the app + # answers. 30 s was too tight for a cold `docker compose up` and produced + # a false unhealthy. + start_period: 120s restart: unless-stopped logging: driver: json-file diff --git a/lib/api/__tests__/with-cron-context.test.ts b/lib/api/__tests__/with-cron-context.test.ts new file mode 100644 index 00000000..c50a170a --- /dev/null +++ b/lib/api/__tests__/with-cron-context.test.ts @@ -0,0 +1,111 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { NextResponse } from 'next/server' + +interface HeartbeatInput { + path: string + startedAt: string + completedAt: string + status: string + httpStatus: number | null + durationMs: number +} + +const recordCronHeartbeat = vi.fn(async (_input: HeartbeatInput): Promise => {}) +const verifyCronSecret = vi.fn((): NextResponse | null => null) + +vi.mock('@/lib/cron/heartbeat-store', () => ({ + recordCronHeartbeat: (input: HeartbeatInput) => recordCronHeartbeat(input), +})) + +vi.mock('@/lib/auth/cron', () => ({ + verifyCronSecret: () => verifyCronSecret(), +})) + +vi.mock('@/lib/logger', () => { + const child = () => logger + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child } + return { createLogger: () => logger } +}) + +vi.mock('@/lib/errors/get-structured-error', () => ({ + errorResponse: () => NextResponse.json({ error: 'boom' }, { status: 500 }), + errorResponseFromCode: () => NextResponse.json({ error: 'unauthorized' }, { status: 401 }), +})) + +const { withCronContext } = await import('../with-cron-context') + +const PATH = '/api/webhooks/dispatch/cron' + +function request(): Request { + return new Request(`http://app:3000${PATH}`, { + headers: { authorization: 'Bearer secret' }, + }) +} + +beforeEach(() => { + recordCronHeartbeat.mockClear() + verifyCronSecret.mockClear() + verifyCronSecret.mockReturnValue(null) +}) + +/** + * The heartbeat is the only evidence that a schedule still runs, so the wiring + * — which path, which status, and that an auth rejection is NOT recorded as a + * run — is worth asserting directly rather than inferring from the endpoint. + */ +describe('withCronContext heartbeat', () => { + it('records the run under the request path on success', async () => { + const handler = withCronContext('cron.webhook_dispatch', async () => + NextResponse.json({ ok: true }), + ) + + const response = await handler(request()) + + expect(response.status).toBe(200) + expect(recordCronHeartbeat).toHaveBeenCalledTimes(1) + const input = recordCronHeartbeat.mock.calls[0][0] + expect(input.path).toBe(PATH) + expect(input.status).toBe('ok') + expect(input.httpStatus).toBe(200) + expect(Number.isFinite(Date.parse(input.startedAt))).toBe(true) + expect(Number.isFinite(Date.parse(input.completedAt))).toBe(true) + expect(input.durationMs).toBeGreaterThanOrEqual(0) + }) + + it('records a failed run as error so the health body can show it', async () => { + const handler = withCronContext('cron.webhook_dispatch', async () => { + throw new Error('kaboom') + }) + + const response = await handler(request()) + + expect(response.status).toBe(500) + expect(recordCronHeartbeat).toHaveBeenCalledTimes(1) + expect(recordCronHeartbeat.mock.calls[0][0].status).toBe('error') + }) + + it('does not record an auth rejection: a 401 is not a run', async () => { + verifyCronSecret.mockReturnValue( + NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), + ) + const handler = withCronContext('cron.webhook_dispatch', async () => + NextResponse.json({ ok: true }), + ) + + const response = await handler(request()) + + expect(response.status).toBe(401) + expect(recordCronHeartbeat).not.toHaveBeenCalled() + }) + + it('never fails the cron run when the heartbeat write rejects', async () => { + recordCronHeartbeat.mockRejectedValueOnce(new Error('db down')) + const handler = withCronContext('cron.webhook_dispatch', async () => + NextResponse.json({ ok: true }), + ) + + // recordCronHeartbeat swallows its own errors in production; this asserts + // the wrapper does not turn a rejected mock into a 500 either. + await expect(handler(request())).resolves.toBeDefined() + }) +}) diff --git a/lib/api/with-cron-context.ts b/lib/api/with-cron-context.ts index 8e2ea09d..33b12d8a 100644 --- a/lib/api/with-cron-context.ts +++ b/lib/api/with-cron-context.ts @@ -21,6 +21,7 @@ import { NextResponse } from 'next/server' import { verifyCronSecret } from '@/lib/auth/cron' +import { recordCronHeartbeat } from '@/lib/cron/heartbeat-store' import { createLogger, type Logger } from '@/lib/logger' import { errorResponse, errorResponseFromCode } from '@/lib/errors/get-structured-error' @@ -59,6 +60,20 @@ function generateRequestId(prefix: 'cron' | 'cron_item' = 'cron'): string { return `${prefix}_${crypto.randomUUID()}` } +/** + * The schedule identity used for heartbeat detection. The request path, not the + * `operation` label: lib/cron/schedules.generated.ts is keyed by path because + * that is the one identifier the generator and the runtime both know without a + * lookup table that can drift. + */ +function cronHeartbeatPath(request: Request): string | null { + try { + return new URL(request.url).pathname + } catch { + return null + } +} + export function withCronContext( operation: string, handler: CronHandler, @@ -111,6 +126,31 @@ export function withCronContext( const ctx: CronContext = { requestId, log, forEach } + // Heartbeat: app/api/cron/health reads this row to decide whether this + // schedule still runs on time. start is captured before the handler so a + // run that hangs shows up as an ageing last_started_at, which is exactly + // the silent-stop shape seen in the 2026-11-15 pids_limit incident. + const heartbeatPath = cronHeartbeatPath(request) + const startedAt = new Date(start).toISOString() + const recordRun = async (status: string, httpStatus: number): Promise => { + if (!heartbeatPath) return + try { + await recordCronHeartbeat({ + path: heartbeatPath, + startedAt, + completedAt: new Date().toISOString(), + status, + httpStatus, + durationMs: Date.now() - start, + }) + } catch { + // A monitoring write must never break the job it monitors. The store + // swallows its own failures; this is the second, cheap guard so a + // future change there cannot turn a heartbeat problem into a failing + // cron run. + } + } + try { const response = await handler(request, ctx) if (response instanceof Response && !response.headers.get('X-Request-Id')) { @@ -120,9 +160,11 @@ export function withCronContext( durationMs: Date.now() - start, status: response.status, }) + await recordRun('ok', response.status) return response } catch (err) { log.error('cron run failed', err as Error, { durationMs: Date.now() - start }) + await recordRun('error', 500) return errorResponse(err, log, { requestId }) } } diff --git a/lib/cron/__tests__/cron-interval.test.ts b/lib/cron/__tests__/cron-interval.test.ts new file mode 100644 index 00000000..8c42d57f --- /dev/null +++ b/lib/cron/__tests__/cron-interval.test.ts @@ -0,0 +1,120 @@ +import { describe, it, expect } from 'vitest' +import { + describeGap, + maxIntervalSeconds, + parseCronExpression, +} from '../cron-interval' + +/** + * These are the numbers app/api/cron/health turns into alert thresholds, so the + * tests assert the actual worst-case gaps rather than "returns a number". A + * wrong threshold is not a cosmetic bug: too small and every slow run flaps the + * cron container unhealthy, too large and a stopped schedule keeps looking fine + * — the exact failure the 2026-11-15 incident was. + */ +describe('parseCronExpression', () => { + it('parses the five fields in order', () => { + const spec = parseCronExpression('15 3 * * 1-5') + expect(spec.minutes).toEqual([15]) + expect(spec.hours).toEqual([3]) + expect(spec.daysOfMonth).toHaveLength(31) + expect(spec.months).toHaveLength(12) + expect(spec.daysOfWeek).toEqual([1, 2, 3, 4, 5]) + expect(spec.dayOfMonthRestricted).toBe(false) + expect(spec.dayOfWeekRestricted).toBe(true) + }) + + it('expands steps, ranges and lists', () => { + expect(parseCronExpression('*/15 * * * *').minutes).toEqual([0, 15, 30, 45]) + expect(parseCronExpression('2,4,6-8 * * * *').minutes).toEqual([2, 4, 6, 7, 8]) + expect(parseCronExpression('5/20 * * * *').minutes).toEqual([5, 25, 45]) + }) + + it('normalises Sunday 7 to 0', () => { + expect(parseCronExpression('0 0 * * 7').daysOfWeek).toEqual([0]) + expect(parseCronExpression('0 0 * * 0').daysOfWeek).toEqual([0]) + expect(parseCronExpression('0 0 * * 0,7').daysOfWeek).toEqual([0]) + }) + + it('treats a star step as unrestricted, like every cron implementation', () => { + expect(parseCronExpression('0 0 */1 * *').dayOfMonthRestricted).toBe(false) + }) + + it.each([ + ['* * * *', 'four fields'], + ['* * * * * *', 'six fields'], + ['60 * * * *', 'minute out of range'], + ['* 24 * * *', 'hour out of range'], + ['* * 0 * *', 'day-of-month out of range'], + ['* * * 13 *', 'month out of range'], + ['* * * * 8', 'day-of-week out of range'], + ['*/0 * * * *', 'zero step'], + ['5-2 * * * *', 'inverted range'], + ['a * * * *', 'non-numeric'], + ])('rejects %s (%s)', (expression) => { + expect(() => parseCronExpression(expression)).toThrow() + }) +}) + +describe('maxIntervalSeconds', () => { + it('every-minute is exactly 60 s', () => { + expect(maxIntervalSeconds('* * * * *')).toBe(60) + }) + + it('*/2 is 120 s and */5 is 300 s', () => { + expect(maxIntervalSeconds('*/2 * * * *')).toBe(120) + expect(maxIntervalSeconds('*/5 * * * *')).toBe(300) + }) + + it('hourly on the hour is 3600 s', () => { + expect(maxIntervalSeconds('0 * * * *')).toBe(3600) + }) + + it('a daily job is 86400 s', () => { + expect(maxIntervalSeconds('0 6 * * *')).toBe(86_400) + expect(maxIntervalSeconds('30 4 * * *')).toBe(86_400) + }) + + it('every other hour at :30 is 7200 s, not 3600 s', () => { + // The gap between 22:30 and 00:30 spans two hours; an average-gap + // implementation would say 3600 and alert on the normal cadence. + expect(maxIntervalSeconds('30 */2 * * *')).toBe(7200) + }) + + it('weekly is seven days', () => { + expect(maxIntervalSeconds('0 3 * * 0')).toBe(7 * 86_400) + }) + + it('yearly on 2 January is 365 or 366 days depending on the leap year it spans', () => { + // 2003-01-02 -> 2004-01-02 spans no 29 February, 2004 -> 2005 does not + // either; 1 Jan -> 1 Jan crossing a leap day is the 8784-hour case. Either + // way the threshold must be at least a year, never 0. + expect(maxIntervalSeconds('0 0 2 1 *')).toBeGreaterThanOrEqual(365 * 86_400) + }) + + it('29 February only matches in leap years and still returns a finite gap', () => { + const gap = maxIntervalSeconds('0 0 29 2 *') + expect(gap).toBeGreaterThanOrEqual(365 * 86_400) + expect(Number.isFinite(gap)).toBe(true) + }) + + it('honours the cron day rule: restricted day-of-month OR day-of-week', () => { + // "1st of the month OR every Monday" still has a Monday every seven days, + // so the worst case is a week — a naive AND would report ~28 days and hide + // a fortnight of misses. + expect(maxIntervalSeconds('0 0 1 * 1')).toBe(7 * 86_400) + }) +}) + +describe('describeGap', () => { + it.each([ + [60, '60 s'], + [120, '2 min'], + [3600, '60 min'], + [7200, '2 h'], + [86_400, '24 h'], + [7 * 86_400, '7 dygn'], + ])('renders %i s as %s', (seconds, expected) => { + expect(describeGap(seconds)).toBe(expected) + }) +}) diff --git a/lib/cron/__tests__/heartbeat-evaluation.test.ts b/lib/cron/__tests__/heartbeat-evaluation.test.ts new file mode 100644 index 00000000..2a976c66 --- /dev/null +++ b/lib/cron/__tests__/heartbeat-evaluation.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect } from 'vitest' +import { + CRON_STALE_GRACE_SECONDS, + CRON_STALE_MULTIPLIER, + allowedAgeSeconds, + evaluateCronHeartbeats, + isFailingStatus, + type CronHeartbeatRow, +} from '../heartbeat-evaluation' +import type { CronScheduleEntry } from '../schedules.generated' + +const NOW = Date.parse('2026-11-15T12:00:00.000Z') + +function entry(path: string, maxIntervalSeconds: number): CronScheduleEntry { + return { path, schedule: '* * * * *', maxIntervalSeconds } +} + +function row(overrides: Partial & { path: string }): CronHeartbeatRow { + return { + declaredAt: '2026-11-15T11:00:00.000Z', + lastStartedAt: null, + lastCompletedAt: null, + lastStatus: null, + lastHttpStatus: null, + lastDurationMs: null, + ...overrides, + } +} + +function isoSecondsAgo(seconds: number): string { + return new Date(NOW - seconds * 1000).toISOString() +} + +describe('allowedAgeSeconds', () => { + it('is two intervals plus the grace floor', () => { + expect(CRON_STALE_MULTIPLIER).toBe(2) + expect(CRON_STALE_GRACE_SECONDS).toBe(120) + expect(allowedAgeSeconds(60)).toBe(240) + expect(allowedAgeSeconds(86_400)).toBe(2 * 86_400 + 120) + }) +}) + +describe('evaluateCronHeartbeats', () => { + it('reports ok when the newest run is inside the window', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/invoices/recurring/cron', 3600)], + rows: [row({ path: '/api/invoices/recurring/cron', lastStartedAt: isoSecondsAgo(1800), lastStatus: 'ok' })], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.checks[0].state).toBe('ok') + expect(report.checks[0].ageSeconds).toBe(1800) + expect(report.stale).toBe(0) + }) + + it('reports stale when a per-minute schedule has not started for four minutes', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/webhooks/dispatch/cron', 60)], + rows: [ + row({ + path: '/api/webhooks/dispatch/cron', + lastStartedAt: isoSecondsAgo(241), + lastStatus: 'ok', + }), + ], + now: NOW, + }) + + expect(report.ok).toBe(false) + expect(report.checks[0].state).toBe('stale') + expect(report.stale).toBe(1) + }) + + it('does not flag a single skipped tick', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/webhooks/dispatch/cron', 60)], + rows: [row({ path: '/api/webhooks/dispatch/cron', lastStartedAt: isoSecondsAgo(120) })], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.checks[0].state).toBe('ok') + }) + + it('treats a never-run path inside its declared grace as pending, not stale', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/new/job/cron', 86_400)], + rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3600) })], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.pending).toBe(1) + expect(report.checks[0].state).toBe('pending') + }) + + it('flags a declared path that never ran once the grace is spent', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/new/job/cron', 86_400)], + rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3 * 86_400) })], + now: NOW, + }) + + expect(report.ok).toBe(false) + expect(report.checks[0].state).toBe('stale') + }) + + it('reports a path with no row at all as pending on the first call', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/unknown/job/cron', 60)], + rows: [], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.checks[0].state).toBe('pending') + expect(report.checks[0].ageSeconds).toBeNull() + }) + + it('measures staleness from the last START, so a hung run looks dead', () => { + // The run started 3 h ago and (in this row) completed successfully 2 h 59 + // min ago — the row is doctored to prove last_started_at wins. After a hang + // supercronic stops scheduling, so a completed_at-based check would keep + // reporting ok while nothing new runs. + const report = evaluateCronHeartbeats({ + entries: [entry('/api/invoices/recurring/cron', 3600)], + rows: [ + row({ + path: '/api/invoices/recurring/cron', + lastStartedAt: isoSecondsAgo(3 * 3600), + lastCompletedAt: isoSecondsAgo(60), + }), + ], + now: NOW, + }) + + expect(report.checks[0].state).toBe('stale') + }) + + it('fails the report when any single path is stale', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/a/cron', 60), entry('/api/b/cron', 60)], + rows: [ + row({ path: '/api/a/cron', lastStartedAt: isoSecondsAgo(30) }), + row({ path: '/api/b/cron', lastStartedAt: isoSecondsAgo(600) }), + ], + now: NOW, + }) + + expect(report.ok).toBe(false) + expect(report.stale).toBe(1) + expect(report.checks.filter((check) => check.state === 'ok')).toHaveLength(1) + }) + + it('carries the last outcome through so a failing-but-running job is visible', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/a/cron', 60)], + rows: [ + row({ + path: '/api/a/cron', + lastStartedAt: isoSecondsAgo(30), + lastStatus: 'error', + lastHttpStatus: 500, + lastDurationMs: 42, + }), + ], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.checks[0].lastStatus).toBe('error') + expect(report.checks[0].lastHttpStatus).toBe(500) + expect(report.checks[0].lastDurationMs).toBe(42) + expect(isFailingStatus('error', 500)).toBe(true) + }) + + it('ignores a malformed timestamp instead of throwing', () => { + const report = evaluateCronHeartbeats({ + entries: [entry('/api/a/cron', 60)], + rows: [row({ path: '/api/a/cron', lastStartedAt: 'not-a-date' })], + now: NOW, + }) + + expect(report.ok).toBe(true) + expect(report.checks[0].state).toBe('pending') + }) +}) + +describe('isFailingStatus', () => { + it.each([ + ['ok', 200, false], + // A cron endpoint that redirects did not do its work either; curl -f + // accepts a 3xx, which is exactly why this is worth surfacing separately. + ['ok', 302, true], + ['error', 200, true], + ['ok', 500, true], + [null, 401, true], + [null, null, false], + ])('status %s / http %s -> %s', (status, httpStatus, expected) => { + expect(isFailingStatus(status as string | null, httpStatus as number | null)).toBe(expected) + }) +}) diff --git a/lib/cron/cron-interval.ts b/lib/cron/cron-interval.ts new file mode 100644 index 00000000..af5c531a --- /dev/null +++ b/lib/cron/cron-interval.ts @@ -0,0 +1,208 @@ +/** + * Cron expression → the longest gap between two scheduled runs. + * + * Why this exists: the 2026-11-15 accounted incident. `accounted-cron` lost + * 1 497 of 10 455 runs in a day (14,3 %) because `pids_limit: 30` made + * `fork/exec /bin/sh` fail under the per-minute schedules, and supercronic + * simply skipped those ticks. Nothing in the product noticed: the container + * stayed "Up", every endpoint still answered, and the only trace was a line in + * the cron container's own log. A schedule whose runs stop arriving is the + * worst failure mode in this product — nothing looks broken, the work just + * never happens — so `cron_heartbeats` records every run and + * app/api/cron/health turns "no run arrived within the expected window" into a + * 503 the container healthcheck can fail on. + * + * That check needs to know, per path, how long a gap is normal. That is this + * module: a small, dependency-free interpreter for the 5-field cron syntax + * `vercel.json` is restricted to, plus a brute-force scan of a 12-year window + * to get the WORST-CASE gap rather than an average. Worst case is the correct + * threshold input: a job that runs at 00:00 daily must be allowed 24 h, not + * the 12 h its mean gap would suggest. + * + * The scan is exact for every expression the product uses, including the + * calendar-sensitive ones (`0 0 2 1 *` once a year, `0 0 29 2 *` only in leap + * years). It is not a general cron library: no seconds field, no `@daily` + * aliases, no `L`/`W`/`#` extensions. Anything outside the supported subset + * throws instead of guessing, and the generator that consumes it runs in CI, so + * an unsupported expression fails the build rather than silently producing a + * wrong alert threshold. + */ + +const MINUTE_MS = 60_000 +const HOUR_MS = 60 * MINUTE_MS +const DAY_MS = 24 * HOUR_MS + +/** + * Scan window: 2001-01-01 + 12 years. Twelve years is what it takes to see + * three occurrences of 29 February (2004, 2008, 2012), which is the rarest + * yearly form a cron expression can express. + */ +const SCAN_START_MS = Date.UTC(2001, 0, 1) +const SCAN_DAYS = 12 * 366 + +/** + * Upper bound on matched timestamps per expression. A per-minute job matches + * 1 440 times a day; 20 000 is two weeks of it, far more than needed to see the + * 60-second gap, and it keeps the generator fast when the whole vercel.json is + * scanned. + */ +const MAX_MATCHES = 20_000 + +export interface CronSpec { + readonly minutes: readonly number[] + readonly hours: readonly number[] + readonly daysOfMonth: readonly number[] + readonly months: readonly number[] + readonly daysOfWeek: readonly number[] + /** + * Cron's day rule: when BOTH day-of-month and day-of-week are restricted the + * expression matches if EITHER does; when only one is restricted only that + * one applies. A step over the whole range (star followed by a step) counts + * as unrestricted, as in every cron implementation, because the field still + * starts with a star. + */ + readonly dayOfMonthRestricted: boolean + readonly dayOfWeekRestricted: boolean +} + +function parseField( + field: string, + min: number, + max: number, + name: string, +): { values: number[]; restricted: boolean } { + const trimmed = field.trim() + if (trimmed.length === 0) throw new Error(`cron: empty ${name} field`) + + const seen = new Set() + for (const part of trimmed.split(',')) { + const [rangePart, stepPart, ...rest] = part.split('/') + if (rest.length > 0) throw new Error(`cron: bad step in ${name} field: ${part}`) + + const step = stepPart === undefined ? 1 : Number(stepPart) + if (!Number.isInteger(step) || step < 1) { + throw new Error(`cron: bad step in ${name} field: ${part}`) + } + + let lo: number + let hi: number + if (rangePart === '*') { + lo = min + hi = max + } else if (rangePart.includes('-')) { + const [a, b, ...tail] = rangePart.split('-') + if (tail.length > 0) throw new Error(`cron: bad range in ${name} field: ${part}`) + lo = Number(a) + hi = Number(b) + } else { + lo = Number(rangePart) + // A bare value with a step (`5/10`) means "from 5 to max, every 10th". + hi = stepPart === undefined ? lo : max + } + + if (!Number.isInteger(lo) || !Number.isInteger(hi)) { + throw new Error(`cron: non-numeric value in ${name} field: ${part}`) + } + if (lo < min || hi > max || lo > hi) { + throw new Error(`cron: ${name} field out of range ${min}-${max}: ${part}`) + } + + for (let v = lo; v <= hi; v += step) seen.add(v) + } + + return { values: [...seen].sort((a, b) => a - b), restricted: !trimmed.startsWith('*') } +} + +/** Parse a standard 5-field cron expression. Throws on anything unsupported. */ +export function parseCronExpression(expression: string): CronSpec { + const fields = expression.trim().split(/\s+/) + if (fields.length !== 5) { + throw new Error(`cron: expected 5 fields, got ${fields.length}: ${JSON.stringify(expression)}`) + } + + const [minute, hour, dayOfMonth, month, dayOfWeek] = fields + const minutes = parseField(minute, 0, 59, 'minute') + const hours = parseField(hour, 0, 23, 'hour') + const dom = parseField(dayOfMonth, 1, 31, 'day-of-month') + const months = parseField(month, 1, 12, 'month') + // Cron allows both 0 and 7 for Sunday; normalise 7 to 0. + const dow = parseField(dayOfWeek, 0, 7, 'day-of-week') + + return { + minutes: minutes.values, + hours: hours.values, + daysOfMonth: dom.values, + months: months.values, + daysOfWeek: [...new Set(dow.values.map((d) => (d === 7 ? 0 : d)))].sort((a, b) => a - b), + dayOfMonthRestricted: dom.restricted, + dayOfWeekRestricted: dow.restricted, + } +} + +function matchesDay(spec: CronSpec, dayStartMs: number): boolean { + const date = new Date(dayStartMs) + if (!spec.months.includes(date.getUTCMonth() + 1)) return false + + const domMatch = spec.daysOfMonth.includes(date.getUTCDate()) + const dowMatch = spec.daysOfWeek.includes(date.getUTCDay()) + + if (spec.dayOfMonthRestricted && spec.dayOfWeekRestricted) return domMatch || dowMatch + if (spec.dayOfMonthRestricted) return domMatch + if (spec.dayOfWeekRestricted) return dowMatch + return true +} + +/** + * The longest interval, in seconds, between two consecutive runs of + * `expression` over a 12-year window. + * + * Throws when the expression never matches twice in the window — that is not a + * schedule, and silently returning 0 would make the health check permanently + * red or permanently green depending on which way the caller rounds. + */ +export function maxIntervalSeconds(expression: string): number { + const spec = parseCronExpression(expression) + + // Every minute: the answer is exactly one minute and the scan is pointless. + if (spec.minutes.length * spec.hours.length === 24 * 60) return 60 + + let previous: number | null = null + let longest = 0 + let matches = 0 + + for (let day = 0; day < SCAN_DAYS && matches < MAX_MATCHES; day++) { + const dayStart = SCAN_START_MS + day * DAY_MS + if (!matchesDay(spec, dayStart)) continue + + for (const hour of spec.hours) { + for (const minute of spec.minutes) { + const at = dayStart + hour * HOUR_MS + minute * MINUTE_MS + if (previous !== null) { + const gap = (at - previous) / 1000 + if (gap > longest) longest = gap + } + previous = at + if (++matches >= MAX_MATCHES) break + } + if (matches >= MAX_MATCHES) break + } + } + + if (matches < 2) { + throw new Error(`cron: ${JSON.stringify(expression)} never matches twice in the scan window`) + } + + return longest +} + +/** + * Human-readable worst-case gap, for the generated file and for operators + * reading a 503 body. Deliberately coarse: it is a threshold input, not a + * display string that needs to be exact. + */ +export function describeGap(seconds: number): string { + if (seconds < 120) return `${seconds} s` + if (seconds < 2 * 3600) return `${Math.round(seconds / 60)} min` + if (seconds < 2 * 86_400) return `${Math.round(seconds / 3600)} h` + return `${Math.round(seconds / 86_400)} dygn` +} diff --git a/lib/cron/heartbeat-evaluation.ts b/lib/cron/heartbeat-evaluation.ts new file mode 100644 index 00000000..7ef9d896 --- /dev/null +++ b/lib/cron/heartbeat-evaluation.ts @@ -0,0 +1,146 @@ +/** + * Missed-cron detection, the pure half. + * + * The store and the endpoint live in ./heartbeat-store.ts and + * app/api/cron/health/route.ts; everything that decides ok/stale is here so it + * can be tested without a database. + * + * The threshold is deliberately generous, because the cost of the two mistakes + * is not symmetric. Too tight and every slow run flaps the container + * unhealthy, which trains people to ignore the check. Too loose and a stalled + * schedule keeps looking fine — the failure mode this exists to catch (see the + * 2026-11-15 pids_limit incident in the cron_heartbeats migration). Two missed + * intervals plus a two-minute floor means a per-minute job is flagged after + * 4 minutes and a daily job after 48 h 2 min, while one skipped tick or one + * slow run is not. + */ + +import type { CronScheduleEntry } from './schedules.generated' + +/** A run is stale after this many missed intervals. */ +export const CRON_STALE_MULTIPLIER = 2 + +/** Floor added on top of the interval, absorbing clock skew and one slow run. */ +export const CRON_STALE_GRACE_SECONDS = 120 + +export type CronHeartbeatState = 'ok' | 'pending' | 'stale' + +/** A row as the store returns it (snake_case columns already mapped). */ +export interface CronHeartbeatRow { + path: string + declaredAt: string | null + lastStartedAt: string | null + lastCompletedAt: string | null + lastStatus: string | null + lastHttpStatus: number | null + lastDurationMs: number | null +} + +export interface CronHeartbeatCheck { + path: string + schedule: string + maxIntervalSeconds: number + /** Age at which this path is reported stale. */ + allowedAgeSeconds: number + lastStartedAt: string | null + ageSeconds: number | null + state: CronHeartbeatState + lastStatus: string | null + lastHttpStatus: number | null + lastDurationMs: number | null +} + +export interface CronHeartbeatReport { + /** False only when at least one path is stale. Pending paths do not fail. */ + ok: boolean + checkedAt: string + total: number + stale: number + pending: number + checks: CronHeartbeatCheck[] +} + +export function allowedAgeSeconds(maxIntervalSeconds: number): number { + return maxIntervalSeconds * CRON_STALE_MULTIPLIER + CRON_STALE_GRACE_SECONDS +} + +function ageInSeconds(reference: string | null, now: number): number | null { + if (!reference) return null + const at = Date.parse(reference) + if (!Number.isFinite(at)) return null + return Math.max(0, Math.round((now - at) / 1000)) +} + +/** + * Evaluate every scheduled path against its heartbeat row. + * + * States: + * ok — a run started within the allowed age. + * pending — no run yet, but the path has not been declared longer than its + * own cadence allows. A freshly added job, or a fresh install. + * stale — a run used to arrive and stopped, OR the path was declared + * longer ago than its cadence and has never run. Both are real + * defects; only `stale` fails the endpoint. + * + * A path with neither a row nor a declaration is reported `pending` here; the + * route declares it before evaluating, so this only happens on the first call. + */ +export function evaluateCronHeartbeats(params: { + entries: readonly CronScheduleEntry[] + rows: readonly CronHeartbeatRow[] + now: number +}): CronHeartbeatReport { + const byPath = new Map(params.rows.map((row) => [row.path, row])) + + const checks: CronHeartbeatCheck[] = params.entries.map((entry) => { + const row = byPath.get(entry.path) + const allowed = allowedAgeSeconds(entry.maxIntervalSeconds) + + // Staleness is measured from the last START. A run that hangs never + // completes, and supercronic skips every later tick while it hangs, so + // last_completed_at would keep a dead schedule looking alive. + const ranAt = row?.lastStartedAt ?? null + const reference = ranAt ?? row?.declaredAt ?? null + const age = ageInSeconds(reference, params.now) + + let state: CronHeartbeatState = 'ok' + if (age === null) state = 'pending' + else if (age > allowed) state = 'stale' + else if (ranAt === null) state = 'pending' + + return { + path: entry.path, + schedule: entry.schedule, + maxIntervalSeconds: entry.maxIntervalSeconds, + allowedAgeSeconds: allowed, + lastStartedAt: ranAt, + ageSeconds: age, + state, + lastStatus: row?.lastStatus ?? null, + lastHttpStatus: row?.lastHttpStatus ?? null, + lastDurationMs: row?.lastDurationMs ?? null, + } + }) + + const stale = checks.filter((check) => check.state === 'stale').length + const pending = checks.filter((check) => check.state === 'pending').length + + return { + ok: stale === 0, + checkedAt: new Date(params.now).toISOString(), + total: checks.length, + stale, + pending, + checks, + } +} + +/** + * A run that finished non-2xx is not a missed tick (it ran), but it is worth + * surfacing next to the heartbeat: a schedule that "works" while every run + * fails is the same silent outage with an extra step. + */ +export function isFailingStatus(status: string | null, httpStatus: number | null): boolean { + if (status && status !== 'ok') return true + return httpStatus !== null && (httpStatus < 200 || httpStatus >= 300) +} diff --git a/lib/cron/heartbeat-store.ts b/lib/cron/heartbeat-store.ts new file mode 100644 index 00000000..3f28738a --- /dev/null +++ b/lib/cron/heartbeat-store.ts @@ -0,0 +1,139 @@ +import 'server-only' + +import { createServiceClientNoCookies } from '@/lib/auth/api-keys' +import { createLogger, type Logger } from '@/lib/logger' +import type { CronHeartbeatRow } from './heartbeat-evaluation' + +/** + * Cron heartbeat storage. + * + * Two callers: + * - lib/api/with-cron-context.ts, once per run (best effort: a monitoring + * write must never fail the job it monitors). + * - app/api/cron/health/route.ts, which declares unknown paths and reads the + * table to decide ok/stale. + * + * Everything goes through the service role: cron_heartbeats has RLS enabled and + * no policies, because instance-wide scheduling state is not tenant data. + * + * No-ops when Supabase is not configured (unit tests, `next build`, a hosted + * deployment reading its schedule from Vercel). The heartbeat check is a + * self-hosted guarantee; a missing store must not turn into a 500 in the cron + * path. + */ + +const TABLE = 'cron_heartbeats' + +const log: Logger = createLogger('cron/heartbeat-store') + +function storeConfigured(): boolean { + return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL && process.env.SUPABASE_SERVICE_ROLE_KEY) +} + +export interface CronHeartbeatInput { + path: string + /** ISO timestamp of when the run started (the staleness input). */ + startedAt: string + /** ISO timestamp of when the run finished. */ + completedAt: string + /** 'ok' | 'error'. Kept as a string so a future state does not need a migration. */ + status: string + httpStatus: number | null + durationMs: number +} + +/** + * Upsert the run that just finished. Best effort by contract: logs and returns + * on any failure, never throws. A heartbeat write that breaks a cron run would + * be a worse outage than the one it exists to detect. + */ +export async function recordCronHeartbeat(input: CronHeartbeatInput): Promise { + if (!storeConfigured()) return + + try { + const { error } = await createServiceClientNoCookies() + .from(TABLE) + .upsert( + { + path: input.path, + last_started_at: input.startedAt, + last_completed_at: input.completedAt, + last_status: input.status, + last_http_status: input.httpStatus, + last_duration_ms: input.durationMs, + updated_at: input.completedAt, + }, + { onConflict: 'path' }, + ) + + if (error) { + log.warn('heartbeat write failed', { path: input.path, reason: error.message }) + } + } catch (err) { + log.warn('heartbeat write threw', { + path: input.path, + reason: err instanceof Error ? err.message : String(err), + }) + } +} + +/** + * Register paths the generated schedule knows about but the table has not seen. + * + * This is what gives a new schedule a grace period: the row it creates carries + * declared_at = now, so the evaluator can tell "added yesterday, has not had its + * first tick" from "ran until last week and stopped". ignoreDuplicates keeps an + * existing row (and its run history) untouched. + */ +export async function declareCronPaths(paths: readonly string[]): Promise { + if (!storeConfigured() || paths.length === 0) return + + try { + const { error } = await createServiceClientNoCookies() + .from(TABLE) + .upsert( + paths.map((path) => ({ path })), + { onConflict: 'path', ignoreDuplicates: true }, + ) + + if (error) { + log.warn('heartbeat declaration failed', { paths: paths.length, reason: error.message }) + } + } catch (err) { + log.warn('heartbeat declaration threw', { + paths: paths.length, + reason: err instanceof Error ? err.message : String(err), + }) + } +} + +/** + * All heartbeat rows. Throws on failure: the health route must be able to say + * "cannot verify" out loud (503) instead of reporting a healthy cron it could + * not read. + */ +export async function loadCronHeartbeats(): Promise { + const { data, error } = await createServiceClientNoCookies() + .from(TABLE) + .select( + 'path, declared_at, last_started_at, last_completed_at, last_status, last_http_status, last_duration_ms', + ) + + if (error) throw new Error(`cron_heartbeats read failed: ${error.message}`) + if (!data) return [] + + return data.map((row) => ({ + path: row.path as string, + declaredAt: (row.declared_at as string | null) ?? null, + lastStartedAt: (row.last_started_at as string | null) ?? null, + lastCompletedAt: (row.last_completed_at as string | null) ?? null, + lastStatus: (row.last_status as string | null) ?? null, + lastHttpStatus: (row.last_http_status as number | null) ?? null, + lastDurationMs: (row.last_duration_ms as number | null) ?? null, + })) +} + +/** True when the heartbeat store is wired; the route and tests both ask. */ +export function cronHeartbeatStoreConfigured(): boolean { + return storeConfigured() +} diff --git a/lib/cron/schedules.generated.ts b/lib/cron/schedules.generated.ts new file mode 100644 index 00000000..2af4994b --- /dev/null +++ b/lib/cron/schedules.generated.ts @@ -0,0 +1,60 @@ +// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate. +// +// Variant: self-hosted (the crontab a Docker install runs). +// Source of truth: the `crons` array in vercel.json, via +// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts +// fails CI if this file drifts from it. +// +// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last +// recorded run of each path against maxIntervalSeconds and answers 503 when a +// schedule stops arriving. The container healthcheck on the cron service calls +// that endpoint, so a silent stop becomes a visible unhealthy container +// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident). +// +// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an +// average: a daily job must be allowed 24 h, and 29 February must not make a +// yearly job look dead for four years. + +export interface CronScheduleEntry { + readonly path: string + readonly schedule: string + /** Worst-case seconds between two runs, over a 12-year scan window. */ + readonly maxIntervalSeconds: number +} + +export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [ + { path: '/api/deadlines/status/cron', schedule: '0 6 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/invoices/recurring/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min + { path: '/api/settings/booking-templates/sync/cron', schedule: '30 4 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/tax-deadlines/cron', schedule: '0 0 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/enable-banking/sync/cron', schedule: '0 5 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/stripe/transactions/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/woocommerce/orders/cron', schedule: '45 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/shopify/orders/cron', schedule: '15 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/zettle/orders/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/documents/verify/cron', schedule: '0 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/documents/reanchor/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/sandbox/cleanup/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/events/cleanup/cron', schedule: '0 2 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/cloud-backup/auto-sync/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min + { path: '/api/idempotency/cleanup/cron', schedule: '30 * * * *', maxIntervalSeconds: 3600 }, // 60 min + { path: '/api/pending-operations/expire/cron', schedule: '30 2 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/skatteverket/ombud/sync/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/skatteverket/skattekonto/sync/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/skatteverket/agi/kvittenser/cron', schedule: '*/15 * * * *', maxIntervalSeconds: 900 }, // 15 min + { path: '/api/extensions/skatteverket/vat/kvittenser/cron', schedule: '30 */2 * * *', maxIntervalSeconds: 7200 }, // 2 h + { path: '/api/webhooks/dispatch/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s + { path: '/api/extensions/whatsapp-inbox/sweep/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s + { path: '/api/extensions/whatsapp-inbox/retention/cron', schedule: '15 4 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/invoice-inbox/sweep/cron', schedule: '*/2 * * * *', maxIntervalSeconds: 120 }, // 2 min + { path: '/api/extensions/invoice-inbox/underlag-reconcile/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/extensions/arcim-migration/complete-invoice-lines/cron', schedule: '20 * * * *', maxIntervalSeconds: 3600 }, // 60 min + { path: '/api/extensions/arcim-migration/cleanup/cron', schedule: '*/5 * * * *', maxIntervalSeconds: 300 }, // 5 min + { path: '/api/bookkeeping/accruals/post-due/cron', schedule: '15 5 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/receipt-hunt/cron', schedule: '30 5 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/notifications/bookkeeping-digest/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/notifications/multi-user-grace/cron', schedule: '0 7 * * *', maxIntervalSeconds: 86400 }, // 24 h + { path: '/api/peppol/inbound/cron', schedule: '*/10 * * * *', maxIntervalSeconds: 600 }, // 10 min + { path: '/api/peppol/outbound/status/cron', schedule: '5,20,35,50 * * * *', maxIntervalSeconds: 900 }, // 15 min + { path: '/api/connector/sync/cron', schedule: '17 * * * *', maxIntervalSeconds: 3600 }, // 60 min +] diff --git a/osv-scanner.toml b/osv-scanner.toml new file mode 100644 index 00000000..c76d764a --- /dev/null +++ b/osv-scanner.toml @@ -0,0 +1,34 @@ +# osv-scanner configuration. Read by .gitea/workflows/security.yml, which passes +# --config when this file exists. +# +# The two entries below are the SheetJS advisories. They are ignored because the +# fix is not obtainable from npm at any version: SheetJS stopped publishing to +# the npm registry after 0.18.5 and ships 0.19+ from cdn.sheetjs.com, so both +# GHSA records have `introduced: 0` and no `fixed` event (verified against +# api.osv.dev 2026-11-15). This repo consumes `xlsx` as a URL dependency, +# https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz, and 0.20.3 already +# contains both fixes: +# +# GHSA-4r6h-8v6p-xvw6 (CVE-2023-30533, prototype pollution) — fixed upstream in 0.19.3 +# GHSA-5pgg-2g8v-p4x9 (CVE-2024-22363, ReDoS) — fixed upstream in 0.20.2 +# +# So the scanner is reporting a version-resolution artefact, not a live defect. +# The entries still expire: if we ever pin an older tarball, or migrate to a +# maintained parser, this file must be revisited rather than inherited. Owner: +# admin/accounted maintainers. Expiry is deliberate; the nightly scan re-reports +# both advisories the day it passes. +# +# Deliberately NOT in scope here: the prototype-pollution and ReDoS classes are +# about parsing untrusted workbooks, and lib/import/* does exactly that. The +# mitigations are upstream (both fixed in our pinned version) — do not treat +# this file as permission to accept unfixed parser CVEs. + +[[IgnoredVulns]] +id = "GHSA-4r6h-8v6p-xvw6" +effectiveUntil = 2027-05-15 +reason = "CVE-2023-30533 fixed upstream in SheetJS 0.19.3; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory." + +[[IgnoredVulns]] +id = "GHSA-5pgg-2g8v-p4x9" +effectiveUntil = 2027-05-15 +reason = "CVE-2024-22363 fixed upstream in SheetJS 0.20.2; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory." diff --git a/scripts/__tests__/generate-crontabs.test.ts b/scripts/__tests__/generate-crontabs.test.ts index a10551a1..a3edc046 100644 --- a/scripts/__tests__/generate-crontabs.test.ts +++ b/scripts/__tests__/generate-crontabs.test.ts @@ -4,6 +4,8 @@ import { join, dirname } from 'node:path' import { fileURLToPath } from 'node:url' import { buildCrontab, + buildScheduleEntries, + buildScheduleModule, readVercelCrons, scheduleFor, EXCLUDED_PATHS, @@ -283,3 +285,39 @@ describe('every cron route has a schedule', () => { } }) }) + +describe('lib/cron/schedules.generated.ts mirrors the self-hosted crontab', () => { + const generated = readFileSync(join(ROOT, 'lib', 'cron', 'schedules.generated.ts'), 'utf8') + const entries = buildScheduleEntries(crons, 'self-hosted') + + it('on-disk file matches the generator output byte for byte', () => { + expect( + generated, + 'lib/cron/schedules.generated.ts is stale: run `npm run crontabs:generate`.', + ).toBe(buildScheduleModule(crons, 'self-hosted')) + }) + + it('covers exactly the paths crontab.self-hosted runs', () => { + expect(entries.map((entry) => entry.path)).toEqual( + parseCrontab(crontabText('self-hosted')).map((job) => job.path), + ) + }) + + it('carries the same schedule string as the crontab it mirrors', () => { + const crontabJobs = parseCrontab(crontabText('self-hosted')) + for (const entry of entries) { + const job = crontabJobs.find((candidate) => candidate.path === entry.path) + expect(job?.schedule, entry.path).toBe(entry.schedule) + } + }) + + it('gives every path a positive worst-case interval', () => { + // A zero or NaN interval would make the heartbeat check either permanently + // red or permanently green — the two ways this control can be useless. + for (const entry of entries) { + expect(Number.isFinite(entry.maxIntervalSeconds), entry.path).toBe(true) + expect(entry.maxIntervalSeconds, entry.path).toBeGreaterThanOrEqual(60) + expect(entry.maxIntervalSeconds, entry.path).toBeLessThanOrEqual(366 * 86_400) + } + }) +}) diff --git a/scripts/checks/no-new-antipatterns.mjs b/scripts/checks/no-new-antipatterns.mjs index 60088b76..99227627 100644 --- a/scripts/checks/no-new-antipatterns.mjs +++ b/scripts/checks/no-new-antipatterns.mjs @@ -758,12 +758,18 @@ const PINNED_DEPS = [ }, { name: 'nodemailer', - version: '9.1.1', + version: '10.1.4', reason: 'SMTP mailer for self-hosts (extensions/general/email/lib/smtp-service.ts). Zero-dependency MIT-0 ' + 'package on the outbound-mail path; bumps are deliberate, reviewed PRs (audit surface), never silent. ' + - 'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj, ' + - 'same major; the rest of the 9.x advisories are only fixed in 10.x (major, separate reviewed PR).', + 'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj. ' + + 'Bumped 9.1.1 -> 10.1.4 (2026-11-15): the remaining 9.x advisories (CVE-2026-90776 header DoS, ' + + 'GHSA-v53p-9fqp-m79j addressparser backtracking) are only fixed in 10.x, and Trivy failed the ' + + 'security gate on them. The 10.0 API used here (createTransport + sendMail) is unchanged; the ' + + 'smtp-service suite covers the transport options that matter (requireTLS, AUTH, envelope). ' + + '10.0.6 -> 10.1.4 the same day: OSV flagged 10.0.6 and the mailparser subtree pin 10.0.10 ' + + '(GHSA-4ffr-jq9g-5ffx, GHSA-85x2-q946-cp5h, GHSA-8c8m-p3m8-cjg8, GHSA-g73g-hqqh-jr95, ' + + 'GHSA-m943-m63m-w62w); 10.1.4 is the first release with all five.', }, ] diff --git a/scripts/generate-crontabs.ts b/scripts/generate-crontabs.ts index a0c40bda..5f13d5b1 100644 --- a/scripts/generate-crontabs.ts +++ b/scripts/generate-crontabs.ts @@ -24,11 +24,19 @@ import { readFileSync, writeFileSync } from 'node:fs' import { join, dirname } from 'node:path' import { fileURLToPath } from 'node:url' +import { describeGap, maxIntervalSeconds } from '../lib/cron/cron-interval' const __filename = fileURLToPath(import.meta.url) const ROOT = dirname(dirname(__filename)) const VERCEL_JSON_PATH = join(ROOT, 'vercel.json') const DOCKER_DIR = join(ROOT, 'docker') +/** + * Heartbeat detection reads the cadence at runtime, so the same vercel.json + * array is also emitted as a TypeScript module. One generator, one source of + * truth: an expression that is scheduled but unknown to the health check is + * exactly the "silently never runs" failure the check exists to catch. + */ +const SCHEDULE_MODULE_PATH = join(ROOT, 'lib', 'cron', 'schedules.generated.ts') export type CrontabVariant = 'hosted' | 'self-hosted' @@ -243,6 +251,82 @@ export function buildCrontab( return `${lines.join('\n')}\n` } +export interface CronScheduleEntry { + readonly path: string + readonly schedule: string + /** Worst-case seconds between two runs, over a 12-year scan window. */ + readonly maxIntervalSeconds: number +} + +/** + * The job set one variant actually runs (vercel.json minus exclusions, plus + * that variant's EXTRA_JOBS) with its worst-case cadence resolved. + * + * The health check uses the self-hosted set, because that is the crontab a + * Docker install runs; a hosted deployment gets its schedule from Vercel and is + * not the deployment this check guards. + */ +export function buildScheduleEntries( + crons: VercelCron[], + variant: CrontabVariant, +): CronScheduleEntry[] { + const scheduled = crons + .filter((cron) => !(cron.path in EXCLUDED_PATHS)) + .map((cron) => ({ path: cron.path, schedule: scheduleFor(cron, variant) })) + const extra = EXTRA_JOBS[variant].map((job) => ({ path: job.path, schedule: job.schedule })) + + return [...scheduled, ...extra].map((job) => ({ + path: job.path, + schedule: job.schedule, + maxIntervalSeconds: maxIntervalSeconds(job.schedule), + })) +} + +/** Render lib/cron/schedules.generated.ts. Always LF, always newline-terminated. */ +export function buildScheduleModule(crons: VercelCron[], variant: CrontabVariant): string { + const entries = buildScheduleEntries(crons, variant) + const width = + entries.reduce((max, entry) => Math.max(max, entry.path.length + 3), 0) + 2 + + const lines = [ + '// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate.', + '//', + `// Variant: ${variant} (the crontab a Docker install runs).`, + '// Source of truth: the `crons` array in vercel.json, via', + '// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts', + '// fails CI if this file drifts from it.', + '//', + '// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last', + '// recorded run of each path against maxIntervalSeconds and answers 503 when a', + '// schedule stops arriving. The container healthcheck on the cron service calls', + '// that endpoint, so a silent stop becomes a visible unhealthy container', + '// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident).', + '//', + '// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an', + '// average: a daily job must be allowed 24 h, and 29 February must not make a', + '// yearly job look dead for four years.', + '', + 'export interface CronScheduleEntry {', + ' readonly path: string', + ' readonly schedule: string', + ' /** Worst-case seconds between two runs, over a 12-year scan window. */', + ' readonly maxIntervalSeconds: number', + '}', + '', + 'export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [', + ...entries.map( + (entry) => + ` { path: '${entry.path}',`.padEnd(width + 2) + + ` schedule: '${entry.schedule}',`.padEnd(24) + + ` maxIntervalSeconds: ${entry.maxIntervalSeconds} }, // ${describeGap(entry.maxIntervalSeconds)}`, + ), + ']', + '', + ] + + return `${lines.join('\n')}` +} + function main(): void { const crons = readVercelCrons(readFileSync(VERCEL_JSON_PATH, 'utf8')) @@ -282,6 +366,13 @@ function main(): void { ) } + // The runtime half of the same source of truth: heartbeat detection needs + // each path's worst-case cadence, so the self-hosted job set is emitted as a + // TypeScript module next to the crontab it mirrors. + const scheduleEntries = buildScheduleEntries(crons, 'self-hosted') + writeFileSync(SCHEDULE_MODULE_PATH, buildScheduleModule(crons, 'self-hosted'), 'utf8') + console.log(`Wrote lib/cron/schedules.generated.ts: ${scheduleEntries.length} schedules`) + const skipped = Object.keys(EXCLUDED_PATHS) if (skipped.length > 0) { console.log(`Excluded ${skipped.length} path(s): ${skipped.join(', ')}`) diff --git a/supabase/migrations/20261115190000_cron_heartbeats.sql b/supabase/migrations/20261115190000_cron_heartbeats.sql new file mode 100644 index 00000000..df8da72a --- /dev/null +++ b/supabase/migrations/20261115190000_cron_heartbeats.sql @@ -0,0 +1,64 @@ +-- Cron heartbeat: make a schedule that stops arriving visible. +-- +-- The 2026-11-15 incident: accounted-cron lost 1 497 of 10 455 runs in a day +-- (14,3 %) because pids_limit 30 made `fork/exec /bin/sh` fail whenever the +-- per-minute schedules coincided. supercronic silently skipped those ticks. +-- The container stayed "Up", every endpoint kept answering 200, and the only +-- trace was a log line in the cron container. lib/auth/cron.ts already reports +-- an individual FAILED run to the observability sink; nothing could see a run +-- that never started. +-- +-- This table is the durable half of the fix: every cron run upserts one row per +-- path with when it started and how it ended, and app/api/cron/health compares +-- the newest row against the path's worst-case cadence (lib/cron/ +-- schedules.generated.ts, generated from vercel.json). The cron container's +-- healthcheck calls that endpoint, so a schedule that stops arriving turns into +-- an unhealthy container instead of a silent gap. +-- +-- Shape notes: +-- * One row per scheduled path, not per run. Volume matters: 34 schedules at +-- up to 1 440 runs/day each would be ~8k rows/day forever, and all the check +-- needs is "when did this last start". The upsert keeps the table at 34 +-- rows. +-- * No company_id. A cron run is instance-wide, and pretending otherwise +-- would make the same schedule look healthy in one tenant and dead in +-- another with no way to reconcile them. +-- * declared_at is set when the health check first sees a path in the +-- generated schedule list. That is what separates "added to vercel.json +-- yesterday, has not had its first tick yet" from "stopped running a week +-- ago": a path with no run is only stale once declared_at is older than its +-- own cadence, never immediately. Without it, every new job and every fresh +-- install would report red until its first run. +-- * last_started_at, not last_completed_at, is the staleness input. A run that +-- hangs never completes, and supercronic then skips every later tick — +-- exactly the failure that must look dead. +-- +-- Access: RLS enabled with no policies. Only the service role (the cron paths +-- and the health route, via createServiceClientNoCookies) reads or writes it; +-- anon/authenticated have no business seeing instance-wide scheduling state. + +CREATE TABLE IF NOT EXISTS public.cron_heartbeats ( + path TEXT PRIMARY KEY, + declared_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_started_at TIMESTAMPTZ, + last_completed_at TIMESTAMPTZ, + last_status TEXT, + last_http_status INTEGER, + last_duration_ms INTEGER, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE public.cron_heartbeats IS + 'One row per scheduled cron path: newest run start/outcome. Read by app/api/cron/health to detect schedules that stop arriving (2026-11-15 pids_limit incident).'; +COMMENT ON COLUMN public.cron_heartbeats.declared_at IS + 'When the health check first saw this path in the generated schedule list. Grace for a path that has never run.'; +COMMENT ON COLUMN public.cron_heartbeats.last_started_at IS + 'Start of the newest run. The staleness input: a hung run also stops every later tick.'; + +CREATE INDEX IF NOT EXISTS idx_cron_heartbeats_last_started_at + ON public.cron_heartbeats (last_started_at DESC NULLS LAST); + +ALTER TABLE public.cron_heartbeats ENABLE ROW LEVEL SECURITY; + +REVOKE ALL ON public.cron_heartbeats FROM anon; +REVOKE ALL ON public.cron_heartbeats FROM authenticated;