feat(cron): missad-tick-detektion — heartbeat per schema, 503 när ett jobb slutar komma #45

Merged
admin merged 6 commits from feat/cron-heartbeat-20261115 into main 2026-11-15 19:45:55 +00:00
19 changed files with 1773 additions and 6 deletions
@@ -1,2 +1,6 @@
anthropic>=0.39.0
mcp>=1.1.0
# Transitive via mcp (multipart form parsing). Pinned as a floor because the
# advisory scanner cannot resolve an unpinned requirement and reports the
# oldest version it can match; 0.0.31 carries the PYSEC-2026 multipart fixes.
python-multipart>=0.0.31
+160
View File
@@ -0,0 +1,160 @@
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { NextResponse } from 'next/server'
interface Row {
path: string
declaredAt: string | null
lastStartedAt: string | null
lastCompletedAt: string | null
lastStatus: string | null
lastHttpStatus: number | null
lastDurationMs: number | null
}
const verifyCronSecret = vi.fn((): NextResponse | null => null)
const loadCronHeartbeats = vi.fn(async (): Promise<Row[]> => [])
const declareCronPaths = vi.fn(async (_paths: readonly string[]): Promise<void> => {})
const cronHeartbeatStoreConfigured = vi.fn((): boolean => true)
vi.mock('@/lib/auth/cron', () => ({
verifyCronSecret: () => verifyCronSecret(),
}))
vi.mock('@/lib/cron/heartbeat-store', () => ({
loadCronHeartbeats: () => loadCronHeartbeats(),
declareCronPaths: (paths: readonly string[]) => declareCronPaths(paths),
cronHeartbeatStoreConfigured: () => cronHeartbeatStoreConfigured(),
}))
vi.mock('@/lib/logger', () => {
const child = () => logger
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child }
return { createLogger: () => logger }
})
const { GET } = await import('../route')
const { CRON_SCHEDULES } = await import('@/lib/cron/schedules.generated')
function request(): Request {
return new Request('http://app:3000/api/cron/health', {
headers: { authorization: 'Bearer secret' },
})
}
function everythingRanSecondsAgo(seconds: number) {
const at = new Date(Date.now() - seconds * 1000).toISOString()
return CRON_SCHEDULES.map((entry) => ({
path: entry.path,
declaredAt: at,
lastStartedAt: at,
lastCompletedAt: at,
lastStatus: 'ok',
lastHttpStatus: 200,
lastDurationMs: 12,
}))
}
beforeEach(() => {
verifyCronSecret.mockClear()
verifyCronSecret.mockReturnValue(null)
loadCronHeartbeats.mockReset()
declareCronPaths.mockClear()
cronHeartbeatStoreConfigured.mockReturnValue(true)
})
describe('GET /api/cron/health', () => {
it('rejects an unauthenticated call before touching the store', async () => {
verifyCronSecret.mockReturnValue(
NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
)
const response = await GET(request())
expect(response.status).toBe(401)
expect(loadCronHeartbeats).not.toHaveBeenCalled()
})
it('answers 200 when every schedule ran inside its window', async () => {
loadCronHeartbeats.mockResolvedValue(everythingRanSecondsAgo(5))
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(200)
expect(body.status).toBe('ok')
expect(body.report.ok).toBe(true)
expect(body.report.total).toBe(CRON_SCHEDULES.length)
})
it('answers 503 when a schedule stopped arriving', async () => {
// Every path ran 5 s ago except one stale per-minute job: the endpoint is
// a single boolean for the container healthcheck, so one silent stop must
// be enough to fail it. Pick a per-minute entry explicitly — a daily job
// 30 minutes old is not stale, and a test that used index 0 would pass for
// the wrong reason.
const rows = everythingRanSecondsAgo(5)
const index = CRON_SCHEDULES.findIndex((entry) => entry.maxIntervalSeconds === 60)
expect(index).toBeGreaterThanOrEqual(0)
const staleAt = new Date(Date.now() - 30 * 60 * 1000).toISOString()
rows[index] = { ...rows[index], lastStartedAt: staleAt, lastCompletedAt: staleAt }
loadCronHeartbeats.mockResolvedValue(rows)
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(503)
expect(body.status).toBe('stale')
expect(body.report.stale).toBe(1)
expect(body.report.checks[index].state).toBe('stale')
})
it('declares paths that have never run instead of failing them immediately', async () => {
loadCronHeartbeats.mockResolvedValue([])
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(200)
expect(body.status).toBe('ok')
expect(body.report.pending).toBe(CRON_SCHEDULES.length)
expect(declareCronPaths).toHaveBeenCalledTimes(1)
expect(declareCronPaths.mock.calls[0][0]).toHaveLength(CRON_SCHEDULES.length)
})
it('says unverifiable (503) rather than healthy when the store cannot be read', async () => {
loadCronHeartbeats.mockRejectedValue(new Error('connection refused'))
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(503)
expect(body.status).toBe('unverifiable')
// The driver's message must not reach the body.
expect(JSON.stringify(body)).not.toContain('connection refused')
})
it('says unverifiable when the heartbeat store is not configured', async () => {
cronHeartbeatStoreConfigured.mockReturnValue(false)
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(503)
expect(body.status).toBe('unverifiable')
expect(loadCronHeartbeats).not.toHaveBeenCalled()
})
it('surfaces a run that arrived but failed separately from a missed tick', async () => {
const rows = everythingRanSecondsAgo(5)
rows[1] = { ...rows[1], lastStatus: 'error', lastHttpStatus: 500 }
loadCronHeartbeats.mockResolvedValue(rows)
const response = await GET(request())
const body = await response.json()
expect(response.status).toBe(200)
expect(body.failing).toEqual([
{ path: rows[1].path, lastStatus: 'error', lastHttpStatus: 500 },
])
})
})
+109
View File
@@ -0,0 +1,109 @@
import { NextResponse } from 'next/server'
import { verifyCronSecret } from '@/lib/auth/cron'
import {
evaluateCronHeartbeats,
isFailingStatus,
type CronHeartbeatReport,
} from '@/lib/cron/heartbeat-evaluation'
import {
cronHeartbeatStoreConfigured,
declareCronPaths,
loadCronHeartbeats,
} from '@/lib/cron/heartbeat-store'
import { CRON_SCHEDULES } from '@/lib/cron/schedules.generated'
import { createLogger } from '@/lib/logger'
const log = createLogger('cron/health')
// Reads live rows every call; a cached 200 would defeat the point of a
// healthcheck that exists to notice a schedule going quiet.
export const dynamic = 'force-dynamic'
interface CronHealthBody {
status: 'ok' | 'stale' | 'unverifiable'
reason?: string
/** Present when the store responded. */
report?: CronHeartbeatReport
/** Runs that arrived but ended non-2xx. Informational; a failed run still ran. */
failing?: Array<{ path: string; lastStatus: string | null; lastHttpStatus: number | null }>
}
/**
* GET /api/cron/health
*
* Answers the question the 2026-11-15 incident proved nobody was asking: did
* every scheduled job actually start when it should?
*
* 200 = every scheduled path has run within its own worst-case window.
* 503 = at least one path is stale, or the heartbeat store could not be read
* (an unreadable store is not evidence of a healthy cron).
*
* Called by the cron container's own healthcheck (docker-compose.yml), which is
* what turns this answer into `unhealthy` and into the existing alerting chain.
* Auth is the normal cron secret: the body names instance-wide schedules, so it
* is not public, but the same credential the scheduler already holds is the
* right one — no new secret to distribute.
*/
export async function GET(request: Request): Promise<NextResponse<CronHealthBody>> {
// report: false — this endpoint is itself polled by a healthcheck, so a
// misconfigured probe must not emit an auth-failure event every minute.
const authError = verifyCronSecret(request, { report: false })
if (authError) return authError as NextResponse<CronHealthBody>
if (!cronHeartbeatStoreConfigured()) {
return NextResponse.json<CronHealthBody>(
{
status: 'unverifiable',
reason:
'heartbeat store not configured (NEXT_PUBLIC_SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY missing)',
},
{ status: 503 },
)
}
try {
// Declare first, read second: a path that has never run needs its
// declared_at anchor before the evaluator can tell a fresh job from a
// stopped one. ignoreDuplicates keeps existing rows untouched.
const known = new Set((await loadCronHeartbeats()).map((row) => row.path))
const undeclared = CRON_SCHEDULES.map((entry) => entry.path).filter((path) => !known.has(path))
if (undeclared.length > 0) await declareCronPaths(undeclared)
const rows = await loadCronHeartbeats()
const report = evaluateCronHeartbeats({ entries: CRON_SCHEDULES, rows, now: Date.now() })
const failing = report.checks
.filter((check) => isFailingStatus(check.lastStatus, check.lastHttpStatus))
.map((check) => ({
path: check.path,
lastStatus: check.lastStatus,
lastHttpStatus: check.lastHttpStatus,
}))
if (!report.ok) {
const stale = report.checks.filter((check) => check.state === 'stale')
log.warn('cron schedule stale', {
stale: stale.map((check) => ({
path: check.path,
ageSeconds: check.ageSeconds,
allowedAgeSeconds: check.allowedAgeSeconds,
})),
})
}
return NextResponse.json<CronHealthBody>(
{ status: report.ok ? 'ok' : 'stale', report, failing },
{ status: report.ok ? 200 : 503 },
)
} catch (err) {
// Never echo the driver's message: it can name tables and query fragments.
log.error('cron health check failed', err as Error)
return NextResponse.json<CronHealthBody>(
{
status: 'unverifiable',
reason: 'heartbeat store read failed',
},
{ status: 503 },
)
}
}
@@ -0,0 +1,109 @@
# Accounted: missad-tick-detektion för cron (2026-11-15)
Uppföljning på `changes/2026-11-15-cron-pids-och-healthcheck.md`. Den ändringen
fixade orsaken till att cron tappade jobb. Den här ändringen gör att ett tappat
jobb **syns** nästa gång — vilket var det verkliga problemet: felet pågick i
dygn utan att någon mätning, larm eller dashboard reagerade.
## Fynd
`accounted-cron` förlorade 1 497 av 10 455 körningar på ett dygn (14,3 %) när
`pids_limit: 30` fick `fork/exec /bin/sh` att misslyckas. supercronic hoppade
tyst över tickarna. Containern var `Up` hela tiden, appen svarade 200 på alla
anrop, och enda spåret var rader i cron-containerns egen logg.
Tre saker saknades:
1. **Ingen körningshistorik.** `lib/api/with-cron-context.ts` loggade
`cron run started/completed/failed` via `createLogger`, men skrev inget
beständigt. En körning som aldrig startade lämnade inget spår alls.
2. **Healthchecken mätte fel sak.** Cron-tjänstens healthcheck var
`ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'` — den blev aldrig
röd av att scheman slutade köra, bara av att processen dog.
3. **Ingen koppling till kadensen.** Även med historik hade ingen kontroll vetat
vad "för sen" betyder: en daglig körning får vara 24 h gammal, en
per-minut-körning inte 4 minuter.
## Ändring
- **`lib/cron/cron-interval.ts`** — tolkare för 5-fälters cron (det enda
`vercel.json` använder) som räknar ut **värsta** gapet i sekunder över ett
12-årsfönster (behövs för 29 februari), inte medelgapet. Okänd syntax kastar i
stället för att gissa. `describeGap()` för läsbar text.
- **`scripts/generate-crontabs.ts`** — skriver nu även
`lib/cron/schedules.generated.ts`: en post per schemalagd path med
`schedule` och `maxIntervalSeconds`. Samma källa som crontaben, så en
schemalagd path som kontrollen inte känner till kan inte uppstå.
- **`supabase/migrations/20261115190000_cron_heartbeats.sql`** — tabellen
`cron_heartbeats`, en rad per path: `declared_at`, `last_started_at`,
`last_completed_at`, `last_status`, `last_http_status`, `last_duration_ms`.
RLS på med **noll policies** (endast service role; instansglobal
schemaläggning är inte tenantdata).
- **`lib/cron/heartbeat-evaluation.ts`** — ren utvärdering, testbar utan DB:
`ok` (körning inom fönstret), `pending` (aldrig körd men deklarerad inom sin
egen kadens), `stale` (körning har uteblivit). Tröskel =
`2 × maxIntervalSeconds + 120 s`.
- **`lib/cron/heartbeat-store.ts`** — Supabase-IO. Skrivningar är best effort
(loggar och returnerar, kastar aldrig: en mätningsskrivning får inte fälla
jobbet den mäter). Läsningar kastar, så endpointen kan säga "kan inte
verifiera" i stället för att ljuga grönt. No-op när Supabase-env saknas.
- **`app/api/cron/health/route.ts`** — 200 när alla scheman ligger inom sina
fönster, 503 när något är `stale` eller lagret inte kan läsas. Deklarerar
okända paths först (ger nya jobb respit i stället för omedelbart rött).
Autentiseras med befintlig cron-hemlighet (`report: false` så en felkopplad
probe inte spammar auth-larm).
- **`lib/api/with-cron-context.ts`** — skriver heartbeat per körning (path från
request-URL:en) med status och duration. 401 räknas inte som en körning, och
ett kastat heartbeat-fel fäller aldrig körningen.
- **`docker-compose.yml`** (cron-tjänsten) — healthcheck bytt från
`ps ax | grep supercronic` till att anropa `/api/cron/health` med
`$${CRON_SECRET}`/`$${APP_URL}`, `start_period: 120s`.
Varför `last_started_at` och inte `last_completed_at`: en körning som hänger sig
avslutas aldrig, och supercronic hoppar då över alla senare tickar. Mätt från
start blir en hängning synlig; mätt från completion hade den sett levande ut.
Varför heartbeat i stället för att bara räkna i apploggen: loggen rullar, ägs av
Docker, och kräver att någon läser den. Tabellen är frågbar, och endpointen gör
svaret till en containerstatus som den befintliga larmkedjan redan tittar på.
## Verifiering
- `npx vitest run --project unit lib/cron lib/api/__tests__/with-cron-context.test.ts scripts/__tests__/generate-crontabs.test.ts app/api/cron/health` → **77 + 7 + 11 tester gröna**:
- kadens: `* * * * *` = 60 s, `*/2` = 120 s, `*/5` = 300 s, `0 * * * *` = 3600 s,
`30 */2 * * *` = 7200 s (inte 3600), daglig = 86 400 s, veckovis = 604 800 s,
29 februari → ändligt gap; `0 0 1 * 1` = 7 dygn (cron-regeln OR, inte AND).
- utvärdering: per-minut-jobb 241 s gammalt = stale, 120 s = ok, deklarerad-men-aldrig-körd
= pending inom respit och stale efter, hängd körning (gammal start, färsk
completion) = stale.
- endpoint: 200/503, 401 utan store-åtkomst, deklarerar okända paths,
"unverifiable" utan läckta drivrutinsmeddelanden, `failing` listar
körningar som kom fram men föll.
- wiring: heartbeat skrivs med rätt path/status, 401 skriver ingen heartbeat,
kastat heartbeat-fel ger fortfarande lyckad körning.
- drift: `lib/cron/schedules.generated.ts` matchar generatorn byte för byte och
`crontab.self-hosted` rad för rad.
- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel.
`check:guards` → passed.
- Hela unit-sviten körd efter ändringen (resultat i PR:ens CI).
## Kvar / nästa steg
- Kör `docker compose config` och recreate av cron på server7 först när appen med
den nya routen är deployad — annars blir healthchecken 404 och containern
unhealthy. Ordning: deploy app → applicera migrationen → recreate cron.
- Larmet blir `unhealthy` via befintlig healthcheck-kedja; Prometheus-regel för
`container_health_status` finns redan. Ingen ny larmväg behövs.
- Backlog-kontrollerna (0 rader i alla fem köer) ligger kvar som engångsbevis i
incidentprotokollet; ingen återkörning av affärsjobb krävs eftersom instansen
är en tom/sandbox-miljö (companies 1, invoices 0, journal_entries 0).
## Revert
```
git revert <commit> # kod, migration och compose i en commit
```
Migrationen är additiv (ny tabell, inga ändrade kolumner) och kan lämnas kvar om
bara koden rullas tillbaka. Healthchecken återställs separat:
`test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"]`.
+110
View File
@@ -0,0 +1,110 @@
# Accounted: OSV-gaten grön — 24 sårbarheter stängda, xlsx undantagen med bevis (2026-11-15)
`security / scan` var röd på PR #44 även efter att Trivy-steget blivit grönt.
Steg 6, **OSV Scanner**, fällde jobbet.
## Fynd
`ghcr.io/google/osv-scanner:v2.7.1` mot `package-lock.json` +
`.claude/skills/mcp-builder/scripts/requirements.txt`: **43 advisory-poster över
10 paket**. Viktigt för rotorsaken: samma skanning mot `origin/main`s låsfil ger
**9 av 10 paket** — det här var inte något PR:en införde. Nya GHSA/PYSEC-poster
publicerades mot versioner vi redan kör, så `main` hade blivit röd vid nästa
nattliga skanning (02:17) oavsett.
| Paket | Före | Efter | Väg |
|---|---|---|---|
| nodemailer | 10.0.6 (direkt) + 10.0.10 (mailparser) | **10.1.4** | direkt dependency + override |
| zod | 4.4.3 | **4.6.7** | `^4.6.6` |
| vite | 8.1.3 | **8.3.8** | override `^8.1.6` (dev, via vitest) |
| vitest / @vitest/mocker | 4.1.11 / 4.1.11 | **4.1.12 / 4.1.12** | `^4.1.12` |
| @babel/core | 7.28.6 | **7.29.13** | override `^7.29.6` (dev, via eslint-plugin-react-hooks) |
| fast-xml-parser | 5.7.2 | **5.12.1** | override `^5.12.0` (via @aws-sdk/xml-builder) |
| minimatch | 9.0.9 | **11.0.8** | override `^11.0.1` (dev, via typescript-eslint) |
| postcss-selector-parser | 6.0.10 | **6.1.6** | override `^6.1.5` (via @tailwindcss/typography) |
| python-multipart | olöst (otransitiv, (ospecificerad) | **>=0.0.31** | explicit floor i skillens requirements.txt |
| xlsx | 0.20.3 | 0.20.3 | **undantagen** — se nedan |
## xlsx: undantaget, inte bortförklarat
Båda posterna (`GHSA-4r6h-8v6p-xvw6` = CVE-2023-30533 prototype pollution,
`GHSA-5pgg-2g8v-p4x9` = CVE-2024-22363 ReDoS) har `introduced: 0` och **inget
`fixed`-event** — kontrollerat direkt mot `api.osv.dev` 2026-11-15. Det betyder
inte "ofixat", det betyder "ingen npm-version bär fixen": SheetJS slutade
publicera till npm efter 0.18.5 och släpper 0.19+ från `cdn.sheetjs.com`.
Repot konsumerar `xlsx` som URL-dependency
(`https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz`), och 0.20.3 innehåller
båda fixarna (0.19.3 respektive 0.20.2). Skanningen rapporterar alltså en
versionsupplösningsartefakt, inte en levande defekt.
Hanteringen är inte en tyst tystnad: `osv-scanner.toml` med två `IgnoredVulns`,
`effectiveUntil = 2027-05-15` och full motivering, plus en uttrycklig varning i
filen om att `lib/import/*` parsar användaruppladdade arbetsböcker — undantaget
får aldrig läsas som tillstånd att acceptera ofixade parser-CVE:er. När SheetJS
publicerar 0.21+, eller vi migrerar till en underhållen parser, ska filen bort.
## Ändring
- `package.json`: nodemailer `10.1.4`; zod `^4.6.6`; vitest `^4.1.12`; overrides
utökade med `nodemailer` (mailparser-subträdet), `@babel/core`,
`fast-xml-parser`, `minimatch`, `postcss-selector-parser`, `vite`.
- `package-lock.json`: omgenererad med `npm install --no-audit --no-fund`.
- `.claude/skills/mcp-builder/scripts/requirements.txt`: explicit
`python-multipart>=0.0.31` med kommentar om varför en floor behövs (OSV kan
inte lösa en otransitiv, opinnad requirement och rapporterar den äldsta
version den kan matcha).
- `osv-scanner.toml`: ny, se ovan.
- `scripts/checks/no-new-antipatterns.mjs`: PINNED_DEPS nodemailer → `10.1.4`
med motiveringen uppdaterad (9.x → 10.1.4 i två steg samma dag).
Notera att arbetsflödet redan skickar `--config` när `osv-scanner.toml` finns
(`.gitea/workflows/security.yml`), så ingen CI-ändring behövdes.
## Verifiering
- `osv-scanner scan source --recursive --config=osv-scanner.toml .` →
**No issues found**, rc=0 (2 filtrerade, båda xlsx, med utskriven motivering).
- `npx vitest run --project unit` → **1687 filer / 22 400 tester passed**,
3 filer/5 tester skipped, 163 s (efter alla bumps: zod 4.6, vite 8.3,
vitest 4.1.12, minimatch 11, @babel 7.29).
- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel.
`check:guards` → passed (pinned-dep 0).
- Trivy fs HIGH/CRITICAL: 0 vulnerabilitets-, secret- och misconfig-fynd.
## Revert
```
git revert <commit>
```
Enda beteendepåverkande posten är `osv-scanner.toml` (om den tas bort blir
gaten röd igen på xlsx). Paketbumparna är patch/minor utom `minimatch` 9 → 11,
som bara används av lint-verktygskedjan och verifieras av `check:lint` +
testsviten.
## Tillägg: Semgrep-steget (2026-11-15, samma PR)
Efter Trivy och OSV blev två steg gröna och det tredje rött: **Semgrep
(`p/ci --error`)** gav exakt **ett** fynd:
```
.npmrc
package_managers.npm.npm-missing-minimum-release-age [Blocking]
This .npmrc does not set a minimum release age ... Add `min-release-age = 7`
```
Regeln är rimlig: en komprometterad release rycks oftast tillbaka inom några
dagar, så att vänta en vecka innan en ny version får lösas in tar bort det
billigaste angreppsfönstret. npm ≥ 11.10 förstår nyckeln (vi kör 11.19.1);
äldre npm ignorerar okända nycklar, så `.npmrc` förblir kompatibel.
Åtgärd: `min-release-age=7` i `.npmrc` med kommentar om varför. Verifierat:
`npm config get min-release-age` → `7`, `npm install --package-lock-only` gav
ingen låsfillsdiff, och hela Semgrep-skanningen är därefter ren (`rc=0`, "no
findings").
Fälla att känna till: regeln gäller även våra egna `@siax/*`-paket. Behöver en
färsk intern publicering konsumeras inom sju dagar måste den hämtas med
`--min-release-age=0` — det är ett medvetet undantag i det enskilda fallet, inte
ett skäl att sänka taket i `.npmrc`.
+16 -3
View File
@@ -76,11 +76,24 @@ services:
mem_limit: 256m
pids_limit: 200
healthcheck:
test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"]
# 2026-11-15: this used to be `ps ax | grep supercronic`, which stays green
# while the scheduler process is up but ticks are being dropped — exactly
# what happened when pids_limit 30 made 14,3 % of runs fail to fork (see
# docker-compose.accounted.yml and the cron_heartbeats migration). Ask the
# app instead: /api/cron/health answers 503 when any schedule has not run
# within its own worst-case window, so a silent stop makes this container
# unhealthy and reaches alerting. `$$` is required — a single `$` would be
# interpolated by compose on the host instead of by the shell in here.
test:
- CMD-SHELL
- 'curl -fsS -m 10 -o /dev/null -H "Authorization: Bearer $${CRON_SECRET}" "$${APP_URL}/api/cron/health"'
interval: 60s
timeout: 10s
timeout: 15s
retries: 3
start_period: 20s
# The probe calls the app over HTTP, so it can only pass once the app
# answers. 30 s was too tight for a cold `docker compose up` and produced
# a false unhealthy.
start_period: 120s
restart: unless-stopped
logging:
driver: json-file
+111
View File
@@ -0,0 +1,111 @@
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { NextResponse } from 'next/server'
interface HeartbeatInput {
path: string
startedAt: string
completedAt: string
status: string
httpStatus: number | null
durationMs: number
}
const recordCronHeartbeat = vi.fn(async (_input: HeartbeatInput): Promise<void> => {})
const verifyCronSecret = vi.fn((): NextResponse | null => null)
vi.mock('@/lib/cron/heartbeat-store', () => ({
recordCronHeartbeat: (input: HeartbeatInput) => recordCronHeartbeat(input),
}))
vi.mock('@/lib/auth/cron', () => ({
verifyCronSecret: () => verifyCronSecret(),
}))
vi.mock('@/lib/logger', () => {
const child = () => logger
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child }
return { createLogger: () => logger }
})
vi.mock('@/lib/errors/get-structured-error', () => ({
errorResponse: () => NextResponse.json({ error: 'boom' }, { status: 500 }),
errorResponseFromCode: () => NextResponse.json({ error: 'unauthorized' }, { status: 401 }),
}))
const { withCronContext } = await import('../with-cron-context')
const PATH = '/api/webhooks/dispatch/cron'
function request(): Request {
return new Request(`http://app:3000${PATH}`, {
headers: { authorization: 'Bearer secret' },
})
}
beforeEach(() => {
recordCronHeartbeat.mockClear()
verifyCronSecret.mockClear()
verifyCronSecret.mockReturnValue(null)
})
/**
* The heartbeat is the only evidence that a schedule still runs, so the wiring
* — which path, which status, and that an auth rejection is NOT recorded as a
* run — is worth asserting directly rather than inferring from the endpoint.
*/
describe('withCronContext heartbeat', () => {
it('records the run under the request path on success', async () => {
const handler = withCronContext('cron.webhook_dispatch', async () =>
NextResponse.json({ ok: true }),
)
const response = await handler(request())
expect(response.status).toBe(200)
expect(recordCronHeartbeat).toHaveBeenCalledTimes(1)
const input = recordCronHeartbeat.mock.calls[0][0]
expect(input.path).toBe(PATH)
expect(input.status).toBe('ok')
expect(input.httpStatus).toBe(200)
expect(Number.isFinite(Date.parse(input.startedAt))).toBe(true)
expect(Number.isFinite(Date.parse(input.completedAt))).toBe(true)
expect(input.durationMs).toBeGreaterThanOrEqual(0)
})
it('records a failed run as error so the health body can show it', async () => {
const handler = withCronContext('cron.webhook_dispatch', async () => {
throw new Error('kaboom')
})
const response = await handler(request())
expect(response.status).toBe(500)
expect(recordCronHeartbeat).toHaveBeenCalledTimes(1)
expect(recordCronHeartbeat.mock.calls[0][0].status).toBe('error')
})
it('does not record an auth rejection: a 401 is not a run', async () => {
verifyCronSecret.mockReturnValue(
NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
)
const handler = withCronContext('cron.webhook_dispatch', async () =>
NextResponse.json({ ok: true }),
)
const response = await handler(request())
expect(response.status).toBe(401)
expect(recordCronHeartbeat).not.toHaveBeenCalled()
})
it('never fails the cron run when the heartbeat write rejects', async () => {
recordCronHeartbeat.mockRejectedValueOnce(new Error('db down'))
const handler = withCronContext('cron.webhook_dispatch', async () =>
NextResponse.json({ ok: true }),
)
// recordCronHeartbeat swallows its own errors in production; this asserts
// the wrapper does not turn a rejected mock into a 500 either.
await expect(handler(request())).resolves.toBeDefined()
})
})
+42
View File
@@ -21,6 +21,7 @@
import { NextResponse } from 'next/server'
import { verifyCronSecret } from '@/lib/auth/cron'
import { recordCronHeartbeat } from '@/lib/cron/heartbeat-store'
import { createLogger, type Logger } from '@/lib/logger'
import { errorResponse, errorResponseFromCode } from '@/lib/errors/get-structured-error'
@@ -59,6 +60,20 @@ function generateRequestId(prefix: 'cron' | 'cron_item' = 'cron'): string {
return `${prefix}_${crypto.randomUUID()}`
}
/**
* The schedule identity used for heartbeat detection. The request path, not the
* `operation` label: lib/cron/schedules.generated.ts is keyed by path because
* that is the one identifier the generator and the runtime both know without a
* lookup table that can drift.
*/
function cronHeartbeatPath(request: Request): string | null {
try {
return new URL(request.url).pathname
} catch {
return null
}
}
export function withCronContext(
operation: string,
handler: CronHandler,
@@ -111,6 +126,31 @@ export function withCronContext(
const ctx: CronContext = { requestId, log, forEach }
// Heartbeat: app/api/cron/health reads this row to decide whether this
// schedule still runs on time. start is captured before the handler so a
// run that hangs shows up as an ageing last_started_at, which is exactly
// the silent-stop shape seen in the 2026-11-15 pids_limit incident.
const heartbeatPath = cronHeartbeatPath(request)
const startedAt = new Date(start).toISOString()
const recordRun = async (status: string, httpStatus: number): Promise<void> => {
if (!heartbeatPath) return
try {
await recordCronHeartbeat({
path: heartbeatPath,
startedAt,
completedAt: new Date().toISOString(),
status,
httpStatus,
durationMs: Date.now() - start,
})
} catch {
// A monitoring write must never break the job it monitors. The store
// swallows its own failures; this is the second, cheap guard so a
// future change there cannot turn a heartbeat problem into a failing
// cron run.
}
}
try {
const response = await handler(request, ctx)
if (response instanceof Response && !response.headers.get('X-Request-Id')) {
@@ -120,9 +160,11 @@ export function withCronContext(
durationMs: Date.now() - start,
status: response.status,
})
await recordRun('ok', response.status)
return response
} catch (err) {
log.error('cron run failed', err as Error, { durationMs: Date.now() - start })
await recordRun('error', 500)
return errorResponse(err, log, { requestId })
}
}
+120
View File
@@ -0,0 +1,120 @@
import { describe, it, expect } from 'vitest'
import {
describeGap,
maxIntervalSeconds,
parseCronExpression,
} from '../cron-interval'
/**
* These are the numbers app/api/cron/health turns into alert thresholds, so the
* tests assert the actual worst-case gaps rather than "returns a number". A
* wrong threshold is not a cosmetic bug: too small and every slow run flaps the
* cron container unhealthy, too large and a stopped schedule keeps looking fine
* — the exact failure the 2026-11-15 incident was.
*/
describe('parseCronExpression', () => {
it('parses the five fields in order', () => {
const spec = parseCronExpression('15 3 * * 1-5')
expect(spec.minutes).toEqual([15])
expect(spec.hours).toEqual([3])
expect(spec.daysOfMonth).toHaveLength(31)
expect(spec.months).toHaveLength(12)
expect(spec.daysOfWeek).toEqual([1, 2, 3, 4, 5])
expect(spec.dayOfMonthRestricted).toBe(false)
expect(spec.dayOfWeekRestricted).toBe(true)
})
it('expands steps, ranges and lists', () => {
expect(parseCronExpression('*/15 * * * *').minutes).toEqual([0, 15, 30, 45])
expect(parseCronExpression('2,4,6-8 * * * *').minutes).toEqual([2, 4, 6, 7, 8])
expect(parseCronExpression('5/20 * * * *').minutes).toEqual([5, 25, 45])
})
it('normalises Sunday 7 to 0', () => {
expect(parseCronExpression('0 0 * * 7').daysOfWeek).toEqual([0])
expect(parseCronExpression('0 0 * * 0').daysOfWeek).toEqual([0])
expect(parseCronExpression('0 0 * * 0,7').daysOfWeek).toEqual([0])
})
it('treats a star step as unrestricted, like every cron implementation', () => {
expect(parseCronExpression('0 0 */1 * *').dayOfMonthRestricted).toBe(false)
})
it.each([
['* * * *', 'four fields'],
['* * * * * *', 'six fields'],
['60 * * * *', 'minute out of range'],
['* 24 * * *', 'hour out of range'],
['* * 0 * *', 'day-of-month out of range'],
['* * * 13 *', 'month out of range'],
['* * * * 8', 'day-of-week out of range'],
['*/0 * * * *', 'zero step'],
['5-2 * * * *', 'inverted range'],
['a * * * *', 'non-numeric'],
])('rejects %s (%s)', (expression) => {
expect(() => parseCronExpression(expression)).toThrow()
})
})
describe('maxIntervalSeconds', () => {
it('every-minute is exactly 60 s', () => {
expect(maxIntervalSeconds('* * * * *')).toBe(60)
})
it('*/2 is 120 s and */5 is 300 s', () => {
expect(maxIntervalSeconds('*/2 * * * *')).toBe(120)
expect(maxIntervalSeconds('*/5 * * * *')).toBe(300)
})
it('hourly on the hour is 3600 s', () => {
expect(maxIntervalSeconds('0 * * * *')).toBe(3600)
})
it('a daily job is 86400 s', () => {
expect(maxIntervalSeconds('0 6 * * *')).toBe(86_400)
expect(maxIntervalSeconds('30 4 * * *')).toBe(86_400)
})
it('every other hour at :30 is 7200 s, not 3600 s', () => {
// The gap between 22:30 and 00:30 spans two hours; an average-gap
// implementation would say 3600 and alert on the normal cadence.
expect(maxIntervalSeconds('30 */2 * * *')).toBe(7200)
})
it('weekly is seven days', () => {
expect(maxIntervalSeconds('0 3 * * 0')).toBe(7 * 86_400)
})
it('yearly on 2 January is 365 or 366 days depending on the leap year it spans', () => {
// 2003-01-02 -> 2004-01-02 spans no 29 February, 2004 -> 2005 does not
// either; 1 Jan -> 1 Jan crossing a leap day is the 8784-hour case. Either
// way the threshold must be at least a year, never 0.
expect(maxIntervalSeconds('0 0 2 1 *')).toBeGreaterThanOrEqual(365 * 86_400)
})
it('29 February only matches in leap years and still returns a finite gap', () => {
const gap = maxIntervalSeconds('0 0 29 2 *')
expect(gap).toBeGreaterThanOrEqual(365 * 86_400)
expect(Number.isFinite(gap)).toBe(true)
})
it('honours the cron day rule: restricted day-of-month OR day-of-week', () => {
// "1st of the month OR every Monday" still has a Monday every seven days,
// so the worst case is a week — a naive AND would report ~28 days and hide
// a fortnight of misses.
expect(maxIntervalSeconds('0 0 1 * 1')).toBe(7 * 86_400)
})
})
describe('describeGap', () => {
it.each([
[60, '60 s'],
[120, '2 min'],
[3600, '60 min'],
[7200, '2 h'],
[86_400, '24 h'],
[7 * 86_400, '7 dygn'],
])('renders %i s as %s', (seconds, expected) => {
expect(describeGap(seconds)).toBe(expected)
})
})
@@ -0,0 +1,203 @@
import { describe, it, expect } from 'vitest'
import {
CRON_STALE_GRACE_SECONDS,
CRON_STALE_MULTIPLIER,
allowedAgeSeconds,
evaluateCronHeartbeats,
isFailingStatus,
type CronHeartbeatRow,
} from '../heartbeat-evaluation'
import type { CronScheduleEntry } from '../schedules.generated'
const NOW = Date.parse('2026-11-15T12:00:00.000Z')
function entry(path: string, maxIntervalSeconds: number): CronScheduleEntry {
return { path, schedule: '* * * * *', maxIntervalSeconds }
}
function row(overrides: Partial<CronHeartbeatRow> & { path: string }): CronHeartbeatRow {
return {
declaredAt: '2026-11-15T11:00:00.000Z',
lastStartedAt: null,
lastCompletedAt: null,
lastStatus: null,
lastHttpStatus: null,
lastDurationMs: null,
...overrides,
}
}
function isoSecondsAgo(seconds: number): string {
return new Date(NOW - seconds * 1000).toISOString()
}
describe('allowedAgeSeconds', () => {
it('is two intervals plus the grace floor', () => {
expect(CRON_STALE_MULTIPLIER).toBe(2)
expect(CRON_STALE_GRACE_SECONDS).toBe(120)
expect(allowedAgeSeconds(60)).toBe(240)
expect(allowedAgeSeconds(86_400)).toBe(2 * 86_400 + 120)
})
})
describe('evaluateCronHeartbeats', () => {
it('reports ok when the newest run is inside the window', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/invoices/recurring/cron', 3600)],
rows: [row({ path: '/api/invoices/recurring/cron', lastStartedAt: isoSecondsAgo(1800), lastStatus: 'ok' })],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.checks[0].state).toBe('ok')
expect(report.checks[0].ageSeconds).toBe(1800)
expect(report.stale).toBe(0)
})
it('reports stale when a per-minute schedule has not started for four minutes', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/webhooks/dispatch/cron', 60)],
rows: [
row({
path: '/api/webhooks/dispatch/cron',
lastStartedAt: isoSecondsAgo(241),
lastStatus: 'ok',
}),
],
now: NOW,
})
expect(report.ok).toBe(false)
expect(report.checks[0].state).toBe('stale')
expect(report.stale).toBe(1)
})
it('does not flag a single skipped tick', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/webhooks/dispatch/cron', 60)],
rows: [row({ path: '/api/webhooks/dispatch/cron', lastStartedAt: isoSecondsAgo(120) })],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.checks[0].state).toBe('ok')
})
it('treats a never-run path inside its declared grace as pending, not stale', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/new/job/cron', 86_400)],
rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3600) })],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.pending).toBe(1)
expect(report.checks[0].state).toBe('pending')
})
it('flags a declared path that never ran once the grace is spent', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/new/job/cron', 86_400)],
rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3 * 86_400) })],
now: NOW,
})
expect(report.ok).toBe(false)
expect(report.checks[0].state).toBe('stale')
})
it('reports a path with no row at all as pending on the first call', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/unknown/job/cron', 60)],
rows: [],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.checks[0].state).toBe('pending')
expect(report.checks[0].ageSeconds).toBeNull()
})
it('measures staleness from the last START, so a hung run looks dead', () => {
// The run started 3 h ago and (in this row) completed successfully 2 h 59
// min ago — the row is doctored to prove last_started_at wins. After a hang
// supercronic stops scheduling, so a completed_at-based check would keep
// reporting ok while nothing new runs.
const report = evaluateCronHeartbeats({
entries: [entry('/api/invoices/recurring/cron', 3600)],
rows: [
row({
path: '/api/invoices/recurring/cron',
lastStartedAt: isoSecondsAgo(3 * 3600),
lastCompletedAt: isoSecondsAgo(60),
}),
],
now: NOW,
})
expect(report.checks[0].state).toBe('stale')
})
it('fails the report when any single path is stale', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/a/cron', 60), entry('/api/b/cron', 60)],
rows: [
row({ path: '/api/a/cron', lastStartedAt: isoSecondsAgo(30) }),
row({ path: '/api/b/cron', lastStartedAt: isoSecondsAgo(600) }),
],
now: NOW,
})
expect(report.ok).toBe(false)
expect(report.stale).toBe(1)
expect(report.checks.filter((check) => check.state === 'ok')).toHaveLength(1)
})
it('carries the last outcome through so a failing-but-running job is visible', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/a/cron', 60)],
rows: [
row({
path: '/api/a/cron',
lastStartedAt: isoSecondsAgo(30),
lastStatus: 'error',
lastHttpStatus: 500,
lastDurationMs: 42,
}),
],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.checks[0].lastStatus).toBe('error')
expect(report.checks[0].lastHttpStatus).toBe(500)
expect(report.checks[0].lastDurationMs).toBe(42)
expect(isFailingStatus('error', 500)).toBe(true)
})
it('ignores a malformed timestamp instead of throwing', () => {
const report = evaluateCronHeartbeats({
entries: [entry('/api/a/cron', 60)],
rows: [row({ path: '/api/a/cron', lastStartedAt: 'not-a-date' })],
now: NOW,
})
expect(report.ok).toBe(true)
expect(report.checks[0].state).toBe('pending')
})
})
describe('isFailingStatus', () => {
it.each([
['ok', 200, false],
// A cron endpoint that redirects did not do its work either; curl -f
// accepts a 3xx, which is exactly why this is worth surfacing separately.
['ok', 302, true],
['error', 200, true],
['ok', 500, true],
[null, 401, true],
[null, null, false],
])('status %s / http %s -> %s', (status, httpStatus, expected) => {
expect(isFailingStatus(status as string | null, httpStatus as number | null)).toBe(expected)
})
})
+208
View File
@@ -0,0 +1,208 @@
/**
* Cron expression → the longest gap between two scheduled runs.
*
* Why this exists: the 2026-11-15 accounted incident. `accounted-cron` lost
* 1 497 of 10 455 runs in a day (14,3 %) because `pids_limit: 30` made
* `fork/exec /bin/sh` fail under the per-minute schedules, and supercronic
* simply skipped those ticks. Nothing in the product noticed: the container
* stayed "Up", every endpoint still answered, and the only trace was a line in
* the cron container's own log. A schedule whose runs stop arriving is the
* worst failure mode in this product — nothing looks broken, the work just
* never happens — so `cron_heartbeats` records every run and
* app/api/cron/health turns "no run arrived within the expected window" into a
* 503 the container healthcheck can fail on.
*
* That check needs to know, per path, how long a gap is normal. That is this
* module: a small, dependency-free interpreter for the 5-field cron syntax
* `vercel.json` is restricted to, plus a brute-force scan of a 12-year window
* to get the WORST-CASE gap rather than an average. Worst case is the correct
* threshold input: a job that runs at 00:00 daily must be allowed 24 h, not
* the 12 h its mean gap would suggest.
*
* The scan is exact for every expression the product uses, including the
* calendar-sensitive ones (`0 0 2 1 *` once a year, `0 0 29 2 *` only in leap
* years). It is not a general cron library: no seconds field, no `@daily`
* aliases, no `L`/`W`/`#` extensions. Anything outside the supported subset
* throws instead of guessing, and the generator that consumes it runs in CI, so
* an unsupported expression fails the build rather than silently producing a
* wrong alert threshold.
*/
const MINUTE_MS = 60_000
const HOUR_MS = 60 * MINUTE_MS
const DAY_MS = 24 * HOUR_MS
/**
* Scan window: 2001-01-01 + 12 years. Twelve years is what it takes to see
* three occurrences of 29 February (2004, 2008, 2012), which is the rarest
* yearly form a cron expression can express.
*/
const SCAN_START_MS = Date.UTC(2001, 0, 1)
const SCAN_DAYS = 12 * 366
/**
* Upper bound on matched timestamps per expression. A per-minute job matches
* 1 440 times a day; 20 000 is two weeks of it, far more than needed to see the
* 60-second gap, and it keeps the generator fast when the whole vercel.json is
* scanned.
*/
const MAX_MATCHES = 20_000
export interface CronSpec {
readonly minutes: readonly number[]
readonly hours: readonly number[]
readonly daysOfMonth: readonly number[]
readonly months: readonly number[]
readonly daysOfWeek: readonly number[]
/**
* Cron's day rule: when BOTH day-of-month and day-of-week are restricted the
* expression matches if EITHER does; when only one is restricted only that
* one applies. A step over the whole range (star followed by a step) counts
* as unrestricted, as in every cron implementation, because the field still
* starts with a star.
*/
readonly dayOfMonthRestricted: boolean
readonly dayOfWeekRestricted: boolean
}
function parseField(
field: string,
min: number,
max: number,
name: string,
): { values: number[]; restricted: boolean } {
const trimmed = field.trim()
if (trimmed.length === 0) throw new Error(`cron: empty ${name} field`)
const seen = new Set<number>()
for (const part of trimmed.split(',')) {
const [rangePart, stepPart, ...rest] = part.split('/')
if (rest.length > 0) throw new Error(`cron: bad step in ${name} field: ${part}`)
const step = stepPart === undefined ? 1 : Number(stepPart)
if (!Number.isInteger(step) || step < 1) {
throw new Error(`cron: bad step in ${name} field: ${part}`)
}
let lo: number
let hi: number
if (rangePart === '*') {
lo = min
hi = max
} else if (rangePart.includes('-')) {
const [a, b, ...tail] = rangePart.split('-')
if (tail.length > 0) throw new Error(`cron: bad range in ${name} field: ${part}`)
lo = Number(a)
hi = Number(b)
} else {
lo = Number(rangePart)
// A bare value with a step (`5/10`) means "from 5 to max, every 10th".
hi = stepPart === undefined ? lo : max
}
if (!Number.isInteger(lo) || !Number.isInteger(hi)) {
throw new Error(`cron: non-numeric value in ${name} field: ${part}`)
}
if (lo < min || hi > max || lo > hi) {
throw new Error(`cron: ${name} field out of range ${min}-${max}: ${part}`)
}
for (let v = lo; v <= hi; v += step) seen.add(v)
}
return { values: [...seen].sort((a, b) => a - b), restricted: !trimmed.startsWith('*') }
}
/** Parse a standard 5-field cron expression. Throws on anything unsupported. */
export function parseCronExpression(expression: string): CronSpec {
const fields = expression.trim().split(/\s+/)
if (fields.length !== 5) {
throw new Error(`cron: expected 5 fields, got ${fields.length}: ${JSON.stringify(expression)}`)
}
const [minute, hour, dayOfMonth, month, dayOfWeek] = fields
const minutes = parseField(minute, 0, 59, 'minute')
const hours = parseField(hour, 0, 23, 'hour')
const dom = parseField(dayOfMonth, 1, 31, 'day-of-month')
const months = parseField(month, 1, 12, 'month')
// Cron allows both 0 and 7 for Sunday; normalise 7 to 0.
const dow = parseField(dayOfWeek, 0, 7, 'day-of-week')
return {
minutes: minutes.values,
hours: hours.values,
daysOfMonth: dom.values,
months: months.values,
daysOfWeek: [...new Set(dow.values.map((d) => (d === 7 ? 0 : d)))].sort((a, b) => a - b),
dayOfMonthRestricted: dom.restricted,
dayOfWeekRestricted: dow.restricted,
}
}
function matchesDay(spec: CronSpec, dayStartMs: number): boolean {
const date = new Date(dayStartMs)
if (!spec.months.includes(date.getUTCMonth() + 1)) return false
const domMatch = spec.daysOfMonth.includes(date.getUTCDate())
const dowMatch = spec.daysOfWeek.includes(date.getUTCDay())
if (spec.dayOfMonthRestricted && spec.dayOfWeekRestricted) return domMatch || dowMatch
if (spec.dayOfMonthRestricted) return domMatch
if (spec.dayOfWeekRestricted) return dowMatch
return true
}
/**
* The longest interval, in seconds, between two consecutive runs of
* `expression` over a 12-year window.
*
* Throws when the expression never matches twice in the window — that is not a
* schedule, and silently returning 0 would make the health check permanently
* red or permanently green depending on which way the caller rounds.
*/
export function maxIntervalSeconds(expression: string): number {
const spec = parseCronExpression(expression)
// Every minute: the answer is exactly one minute and the scan is pointless.
if (spec.minutes.length * spec.hours.length === 24 * 60) return 60
let previous: number | null = null
let longest = 0
let matches = 0
for (let day = 0; day < SCAN_DAYS && matches < MAX_MATCHES; day++) {
const dayStart = SCAN_START_MS + day * DAY_MS
if (!matchesDay(spec, dayStart)) continue
for (const hour of spec.hours) {
for (const minute of spec.minutes) {
const at = dayStart + hour * HOUR_MS + minute * MINUTE_MS
if (previous !== null) {
const gap = (at - previous) / 1000
if (gap > longest) longest = gap
}
previous = at
if (++matches >= MAX_MATCHES) break
}
if (matches >= MAX_MATCHES) break
}
}
if (matches < 2) {
throw new Error(`cron: ${JSON.stringify(expression)} never matches twice in the scan window`)
}
return longest
}
/**
* Human-readable worst-case gap, for the generated file and for operators
* reading a 503 body. Deliberately coarse: it is a threshold input, not a
* display string that needs to be exact.
*/
export function describeGap(seconds: number): string {
if (seconds < 120) return `${seconds} s`
if (seconds < 2 * 3600) return `${Math.round(seconds / 60)} min`
if (seconds < 2 * 86_400) return `${Math.round(seconds / 3600)} h`
return `${Math.round(seconds / 86_400)} dygn`
}
+146
View File
@@ -0,0 +1,146 @@
/**
* Missed-cron detection, the pure half.
*
* The store and the endpoint live in ./heartbeat-store.ts and
* app/api/cron/health/route.ts; everything that decides ok/stale is here so it
* can be tested without a database.
*
* The threshold is deliberately generous, because the cost of the two mistakes
* is not symmetric. Too tight and every slow run flaps the container
* unhealthy, which trains people to ignore the check. Too loose and a stalled
* schedule keeps looking fine — the failure mode this exists to catch (see the
* 2026-11-15 pids_limit incident in the cron_heartbeats migration). Two missed
* intervals plus a two-minute floor means a per-minute job is flagged after
* 4 minutes and a daily job after 48 h 2 min, while one skipped tick or one
* slow run is not.
*/
import type { CronScheduleEntry } from './schedules.generated'
/** A run is stale after this many missed intervals. */
export const CRON_STALE_MULTIPLIER = 2
/** Floor added on top of the interval, absorbing clock skew and one slow run. */
export const CRON_STALE_GRACE_SECONDS = 120
export type CronHeartbeatState = 'ok' | 'pending' | 'stale'
/** A row as the store returns it (snake_case columns already mapped). */
export interface CronHeartbeatRow {
path: string
declaredAt: string | null
lastStartedAt: string | null
lastCompletedAt: string | null
lastStatus: string | null
lastHttpStatus: number | null
lastDurationMs: number | null
}
export interface CronHeartbeatCheck {
path: string
schedule: string
maxIntervalSeconds: number
/** Age at which this path is reported stale. */
allowedAgeSeconds: number
lastStartedAt: string | null
ageSeconds: number | null
state: CronHeartbeatState
lastStatus: string | null
lastHttpStatus: number | null
lastDurationMs: number | null
}
export interface CronHeartbeatReport {
/** False only when at least one path is stale. Pending paths do not fail. */
ok: boolean
checkedAt: string
total: number
stale: number
pending: number
checks: CronHeartbeatCheck[]
}
export function allowedAgeSeconds(maxIntervalSeconds: number): number {
return maxIntervalSeconds * CRON_STALE_MULTIPLIER + CRON_STALE_GRACE_SECONDS
}
function ageInSeconds(reference: string | null, now: number): number | null {
if (!reference) return null
const at = Date.parse(reference)
if (!Number.isFinite(at)) return null
return Math.max(0, Math.round((now - at) / 1000))
}
/**
* Evaluate every scheduled path against its heartbeat row.
*
* States:
* ok — a run started within the allowed age.
* pending — no run yet, but the path has not been declared longer than its
* own cadence allows. A freshly added job, or a fresh install.
* stale — a run used to arrive and stopped, OR the path was declared
* longer ago than its cadence and has never run. Both are real
* defects; only `stale` fails the endpoint.
*
* A path with neither a row nor a declaration is reported `pending` here; the
* route declares it before evaluating, so this only happens on the first call.
*/
export function evaluateCronHeartbeats(params: {
entries: readonly CronScheduleEntry[]
rows: readonly CronHeartbeatRow[]
now: number
}): CronHeartbeatReport {
const byPath = new Map(params.rows.map((row) => [row.path, row]))
const checks: CronHeartbeatCheck[] = params.entries.map((entry) => {
const row = byPath.get(entry.path)
const allowed = allowedAgeSeconds(entry.maxIntervalSeconds)
// Staleness is measured from the last START. A run that hangs never
// completes, and supercronic skips every later tick while it hangs, so
// last_completed_at would keep a dead schedule looking alive.
const ranAt = row?.lastStartedAt ?? null
const reference = ranAt ?? row?.declaredAt ?? null
const age = ageInSeconds(reference, params.now)
let state: CronHeartbeatState = 'ok'
if (age === null) state = 'pending'
else if (age > allowed) state = 'stale'
else if (ranAt === null) state = 'pending'
return {
path: entry.path,
schedule: entry.schedule,
maxIntervalSeconds: entry.maxIntervalSeconds,
allowedAgeSeconds: allowed,
lastStartedAt: ranAt,
ageSeconds: age,
state,
lastStatus: row?.lastStatus ?? null,
lastHttpStatus: row?.lastHttpStatus ?? null,
lastDurationMs: row?.lastDurationMs ?? null,
}
})
const stale = checks.filter((check) => check.state === 'stale').length
const pending = checks.filter((check) => check.state === 'pending').length
return {
ok: stale === 0,
checkedAt: new Date(params.now).toISOString(),
total: checks.length,
stale,
pending,
checks,
}
}
/**
* A run that finished non-2xx is not a missed tick (it ran), but it is worth
* surfacing next to the heartbeat: a schedule that "works" while every run
* fails is the same silent outage with an extra step.
*/
export function isFailingStatus(status: string | null, httpStatus: number | null): boolean {
if (status && status !== 'ok') return true
return httpStatus !== null && (httpStatus < 200 || httpStatus >= 300)
}
+139
View File
@@ -0,0 +1,139 @@
import 'server-only'
import { createServiceClientNoCookies } from '@/lib/auth/api-keys'
import { createLogger, type Logger } from '@/lib/logger'
import type { CronHeartbeatRow } from './heartbeat-evaluation'
/**
* Cron heartbeat storage.
*
* Two callers:
* - lib/api/with-cron-context.ts, once per run (best effort: a monitoring
* write must never fail the job it monitors).
* - app/api/cron/health/route.ts, which declares unknown paths and reads the
* table to decide ok/stale.
*
* Everything goes through the service role: cron_heartbeats has RLS enabled and
* no policies, because instance-wide scheduling state is not tenant data.
*
* No-ops when Supabase is not configured (unit tests, `next build`, a hosted
* deployment reading its schedule from Vercel). The heartbeat check is a
* self-hosted guarantee; a missing store must not turn into a 500 in the cron
* path.
*/
const TABLE = 'cron_heartbeats'
const log: Logger = createLogger('cron/heartbeat-store')
function storeConfigured(): boolean {
return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL && process.env.SUPABASE_SERVICE_ROLE_KEY)
}
export interface CronHeartbeatInput {
path: string
/** ISO timestamp of when the run started (the staleness input). */
startedAt: string
/** ISO timestamp of when the run finished. */
completedAt: string
/** 'ok' | 'error'. Kept as a string so a future state does not need a migration. */
status: string
httpStatus: number | null
durationMs: number
}
/**
* Upsert the run that just finished. Best effort by contract: logs and returns
* on any failure, never throws. A heartbeat write that breaks a cron run would
* be a worse outage than the one it exists to detect.
*/
export async function recordCronHeartbeat(input: CronHeartbeatInput): Promise<void> {
if (!storeConfigured()) return
try {
const { error } = await createServiceClientNoCookies()
.from(TABLE)
.upsert(
{
path: input.path,
last_started_at: input.startedAt,
last_completed_at: input.completedAt,
last_status: input.status,
last_http_status: input.httpStatus,
last_duration_ms: input.durationMs,
updated_at: input.completedAt,
},
{ onConflict: 'path' },
)
if (error) {
log.warn('heartbeat write failed', { path: input.path, reason: error.message })
}
} catch (err) {
log.warn('heartbeat write threw', {
path: input.path,
reason: err instanceof Error ? err.message : String(err),
})
}
}
/**
* Register paths the generated schedule knows about but the table has not seen.
*
* This is what gives a new schedule a grace period: the row it creates carries
* declared_at = now, so the evaluator can tell "added yesterday, has not had its
* first tick" from "ran until last week and stopped". ignoreDuplicates keeps an
* existing row (and its run history) untouched.
*/
export async function declareCronPaths(paths: readonly string[]): Promise<void> {
if (!storeConfigured() || paths.length === 0) return
try {
const { error } = await createServiceClientNoCookies()
.from(TABLE)
.upsert(
paths.map((path) => ({ path })),
{ onConflict: 'path', ignoreDuplicates: true },
)
if (error) {
log.warn('heartbeat declaration failed', { paths: paths.length, reason: error.message })
}
} catch (err) {
log.warn('heartbeat declaration threw', {
paths: paths.length,
reason: err instanceof Error ? err.message : String(err),
})
}
}
/**
* All heartbeat rows. Throws on failure: the health route must be able to say
* "cannot verify" out loud (503) instead of reporting a healthy cron it could
* not read.
*/
export async function loadCronHeartbeats(): Promise<CronHeartbeatRow[]> {
const { data, error } = await createServiceClientNoCookies()
.from(TABLE)
.select(
'path, declared_at, last_started_at, last_completed_at, last_status, last_http_status, last_duration_ms',
)
if (error) throw new Error(`cron_heartbeats read failed: ${error.message}`)
if (!data) return []
return data.map((row) => ({
path: row.path as string,
declaredAt: (row.declared_at as string | null) ?? null,
lastStartedAt: (row.last_started_at as string | null) ?? null,
lastCompletedAt: (row.last_completed_at as string | null) ?? null,
lastStatus: (row.last_status as string | null) ?? null,
lastHttpStatus: (row.last_http_status as number | null) ?? null,
lastDurationMs: (row.last_duration_ms as number | null) ?? null,
}))
}
/** True when the heartbeat store is wired; the route and tests both ask. */
export function cronHeartbeatStoreConfigured(): boolean {
return storeConfigured()
}
+60
View File
@@ -0,0 +1,60 @@
// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate.
//
// Variant: self-hosted (the crontab a Docker install runs).
// Source of truth: the `crons` array in vercel.json, via
// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts
// fails CI if this file drifts from it.
//
// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last
// recorded run of each path against maxIntervalSeconds and answers 503 when a
// schedule stops arriving. The container healthcheck on the cron service calls
// that endpoint, so a silent stop becomes a visible unhealthy container
// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident).
//
// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an
// average: a daily job must be allowed 24 h, and 29 February must not make a
// yearly job look dead for four years.
export interface CronScheduleEntry {
readonly path: string
readonly schedule: string
/** Worst-case seconds between two runs, over a 12-year scan window. */
readonly maxIntervalSeconds: number
}
export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [
{ path: '/api/deadlines/status/cron', schedule: '0 6 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/invoices/recurring/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min
{ path: '/api/settings/booking-templates/sync/cron', schedule: '30 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/tax-deadlines/cron', schedule: '0 0 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/enable-banking/sync/cron', schedule: '0 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/stripe/transactions/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/woocommerce/orders/cron', schedule: '45 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/shopify/orders/cron', schedule: '15 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/zettle/orders/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/documents/verify/cron', schedule: '0 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/documents/reanchor/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/sandbox/cleanup/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/events/cleanup/cron', schedule: '0 2 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/cloud-backup/auto-sync/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min
{ path: '/api/idempotency/cleanup/cron', schedule: '30 * * * *', maxIntervalSeconds: 3600 }, // 60 min
{ path: '/api/pending-operations/expire/cron', schedule: '30 2 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/skatteverket/ombud/sync/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/skatteverket/skattekonto/sync/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/skatteverket/agi/kvittenser/cron', schedule: '*/15 * * * *', maxIntervalSeconds: 900 }, // 15 min
{ path: '/api/extensions/skatteverket/vat/kvittenser/cron', schedule: '30 */2 * * *', maxIntervalSeconds: 7200 }, // 2 h
{ path: '/api/webhooks/dispatch/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s
{ path: '/api/extensions/whatsapp-inbox/sweep/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s
{ path: '/api/extensions/whatsapp-inbox/retention/cron', schedule: '15 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/invoice-inbox/sweep/cron', schedule: '*/2 * * * *', maxIntervalSeconds: 120 }, // 2 min
{ path: '/api/extensions/invoice-inbox/underlag-reconcile/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/extensions/arcim-migration/complete-invoice-lines/cron', schedule: '20 * * * *', maxIntervalSeconds: 3600 }, // 60 min
{ path: '/api/extensions/arcim-migration/cleanup/cron', schedule: '*/5 * * * *', maxIntervalSeconds: 300 }, // 5 min
{ path: '/api/bookkeeping/accruals/post-due/cron', schedule: '15 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/receipt-hunt/cron', schedule: '30 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/notifications/bookkeeping-digest/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/notifications/multi-user-grace/cron', schedule: '0 7 * * *', maxIntervalSeconds: 86400 }, // 24 h
{ path: '/api/peppol/inbound/cron', schedule: '*/10 * * * *', maxIntervalSeconds: 600 }, // 10 min
{ path: '/api/peppol/outbound/status/cron', schedule: '5,20,35,50 * * * *', maxIntervalSeconds: 900 }, // 15 min
{ path: '/api/connector/sync/cron', schedule: '17 * * * *', maxIntervalSeconds: 3600 }, // 60 min
]
+34
View File
@@ -0,0 +1,34 @@
# osv-scanner configuration. Read by .gitea/workflows/security.yml, which passes
# --config when this file exists.
#
# The two entries below are the SheetJS advisories. They are ignored because the
# fix is not obtainable from npm at any version: SheetJS stopped publishing to
# the npm registry after 0.18.5 and ships 0.19+ from cdn.sheetjs.com, so both
# GHSA records have `introduced: 0` and no `fixed` event (verified against
# api.osv.dev 2026-11-15). This repo consumes `xlsx` as a URL dependency,
# https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz, and 0.20.3 already
# contains both fixes:
#
# GHSA-4r6h-8v6p-xvw6 (CVE-2023-30533, prototype pollution) — fixed upstream in 0.19.3
# GHSA-5pgg-2g8v-p4x9 (CVE-2024-22363, ReDoS) — fixed upstream in 0.20.2
#
# So the scanner is reporting a version-resolution artefact, not a live defect.
# The entries still expire: if we ever pin an older tarball, or migrate to a
# maintained parser, this file must be revisited rather than inherited. Owner:
# admin/accounted maintainers. Expiry is deliberate; the nightly scan re-reports
# both advisories the day it passes.
#
# Deliberately NOT in scope here: the prototype-pollution and ReDoS classes are
# about parsing untrusted workbooks, and lib/import/* does exactly that. The
# mitigations are upstream (both fixed in our pinned version) — do not treat
# this file as permission to accept unfixed parser CVEs.
[[IgnoredVulns]]
id = "GHSA-4r6h-8v6p-xvw6"
effectiveUntil = 2027-05-15
reason = "CVE-2023-30533 fixed upstream in SheetJS 0.19.3; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory."
[[IgnoredVulns]]
id = "GHSA-5pgg-2g8v-p4x9"
effectiveUntil = 2027-05-15
reason = "CVE-2024-22363 fixed upstream in SheetJS 0.20.2; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory."
@@ -4,6 +4,8 @@ import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
import {
buildCrontab,
buildScheduleEntries,
buildScheduleModule,
readVercelCrons,
scheduleFor,
EXCLUDED_PATHS,
@@ -283,3 +285,39 @@ describe('every cron route has a schedule', () => {
}
})
})
describe('lib/cron/schedules.generated.ts mirrors the self-hosted crontab', () => {
const generated = readFileSync(join(ROOT, 'lib', 'cron', 'schedules.generated.ts'), 'utf8')
const entries = buildScheduleEntries(crons, 'self-hosted')
it('on-disk file matches the generator output byte for byte', () => {
expect(
generated,
'lib/cron/schedules.generated.ts is stale: run `npm run crontabs:generate`.',
).toBe(buildScheduleModule(crons, 'self-hosted'))
})
it('covers exactly the paths crontab.self-hosted runs', () => {
expect(entries.map((entry) => entry.path)).toEqual(
parseCrontab(crontabText('self-hosted')).map((job) => job.path),
)
})
it('carries the same schedule string as the crontab it mirrors', () => {
const crontabJobs = parseCrontab(crontabText('self-hosted'))
for (const entry of entries) {
const job = crontabJobs.find((candidate) => candidate.path === entry.path)
expect(job?.schedule, entry.path).toBe(entry.schedule)
}
})
it('gives every path a positive worst-case interval', () => {
// A zero or NaN interval would make the heartbeat check either permanently
// red or permanently green — the two ways this control can be useless.
for (const entry of entries) {
expect(Number.isFinite(entry.maxIntervalSeconds), entry.path).toBe(true)
expect(entry.maxIntervalSeconds, entry.path).toBeGreaterThanOrEqual(60)
expect(entry.maxIntervalSeconds, entry.path).toBeLessThanOrEqual(366 * 86_400)
}
})
})
+9 -3
View File
@@ -758,12 +758,18 @@ const PINNED_DEPS = [
},
{
name: 'nodemailer',
version: '9.1.1',
version: '10.1.4',
reason:
'SMTP mailer for self-hosts (extensions/general/email/lib/smtp-service.ts). Zero-dependency MIT-0 ' +
'package on the outbound-mail path; bumps are deliberate, reviewed PRs (audit surface), never silent. ' +
'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj, ' +
'same major; the rest of the 9.x advisories are only fixed in 10.x (major, separate reviewed PR).',
'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj. ' +
'Bumped 9.1.1 -> 10.1.4 (2026-11-15): the remaining 9.x advisories (CVE-2026-90776 header DoS, ' +
'GHSA-v53p-9fqp-m79j addressparser backtracking) are only fixed in 10.x, and Trivy failed the ' +
'security gate on them. The 10.0 API used here (createTransport + sendMail) is unchanged; the ' +
'smtp-service suite covers the transport options that matter (requireTLS, AUTH, envelope). ' +
'10.0.6 -> 10.1.4 the same day: OSV flagged 10.0.6 and the mailparser subtree pin 10.0.10 ' +
'(GHSA-4ffr-jq9g-5ffx, GHSA-85x2-q946-cp5h, GHSA-8c8m-p3m8-cjg8, GHSA-g73g-hqqh-jr95, ' +
'GHSA-m943-m63m-w62w); 10.1.4 is the first release with all five.',
},
]
+91
View File
@@ -24,11 +24,19 @@
import { readFileSync, writeFileSync } from 'node:fs'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describeGap, maxIntervalSeconds } from '../lib/cron/cron-interval'
const __filename = fileURLToPath(import.meta.url)
const ROOT = dirname(dirname(__filename))
const VERCEL_JSON_PATH = join(ROOT, 'vercel.json')
const DOCKER_DIR = join(ROOT, 'docker')
/**
* Heartbeat detection reads the cadence at runtime, so the same vercel.json
* array is also emitted as a TypeScript module. One generator, one source of
* truth: an expression that is scheduled but unknown to the health check is
* exactly the "silently never runs" failure the check exists to catch.
*/
const SCHEDULE_MODULE_PATH = join(ROOT, 'lib', 'cron', 'schedules.generated.ts')
export type CrontabVariant = 'hosted' | 'self-hosted'
@@ -243,6 +251,82 @@ export function buildCrontab(
return `${lines.join('\n')}\n`
}
export interface CronScheduleEntry {
readonly path: string
readonly schedule: string
/** Worst-case seconds between two runs, over a 12-year scan window. */
readonly maxIntervalSeconds: number
}
/**
* The job set one variant actually runs (vercel.json minus exclusions, plus
* that variant's EXTRA_JOBS) with its worst-case cadence resolved.
*
* The health check uses the self-hosted set, because that is the crontab a
* Docker install runs; a hosted deployment gets its schedule from Vercel and is
* not the deployment this check guards.
*/
export function buildScheduleEntries(
crons: VercelCron[],
variant: CrontabVariant,
): CronScheduleEntry[] {
const scheduled = crons
.filter((cron) => !(cron.path in EXCLUDED_PATHS))
.map((cron) => ({ path: cron.path, schedule: scheduleFor(cron, variant) }))
const extra = EXTRA_JOBS[variant].map((job) => ({ path: job.path, schedule: job.schedule }))
return [...scheduled, ...extra].map((job) => ({
path: job.path,
schedule: job.schedule,
maxIntervalSeconds: maxIntervalSeconds(job.schedule),
}))
}
/** Render lib/cron/schedules.generated.ts. Always LF, always newline-terminated. */
export function buildScheduleModule(crons: VercelCron[], variant: CrontabVariant): string {
const entries = buildScheduleEntries(crons, variant)
const width =
entries.reduce((max, entry) => Math.max(max, entry.path.length + 3), 0) + 2
const lines = [
'// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate.',
'//',
`// Variant: ${variant} (the crontab a Docker install runs).`,
'// Source of truth: the `crons` array in vercel.json, via',
'// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts',
'// fails CI if this file drifts from it.',
'//',
'// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last',
'// recorded run of each path against maxIntervalSeconds and answers 503 when a',
'// schedule stops arriving. The container healthcheck on the cron service calls',
'// that endpoint, so a silent stop becomes a visible unhealthy container',
'// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident).',
'//',
'// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an',
'// average: a daily job must be allowed 24 h, and 29 February must not make a',
'// yearly job look dead for four years.',
'',
'export interface CronScheduleEntry {',
' readonly path: string',
' readonly schedule: string',
' /** Worst-case seconds between two runs, over a 12-year scan window. */',
' readonly maxIntervalSeconds: number',
'}',
'',
'export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [',
...entries.map(
(entry) =>
` { path: '${entry.path}',`.padEnd(width + 2) +
` schedule: '${entry.schedule}',`.padEnd(24) +
` maxIntervalSeconds: ${entry.maxIntervalSeconds} }, // ${describeGap(entry.maxIntervalSeconds)}`,
),
']',
'',
]
return `${lines.join('\n')}`
}
function main(): void {
const crons = readVercelCrons(readFileSync(VERCEL_JSON_PATH, 'utf8'))
@@ -282,6 +366,13 @@ function main(): void {
)
}
// The runtime half of the same source of truth: heartbeat detection needs
// each path's worst-case cadence, so the self-hosted job set is emitted as a
// TypeScript module next to the crontab it mirrors.
const scheduleEntries = buildScheduleEntries(crons, 'self-hosted')
writeFileSync(SCHEDULE_MODULE_PATH, buildScheduleModule(crons, 'self-hosted'), 'utf8')
console.log(`Wrote lib/cron/schedules.generated.ts: ${scheduleEntries.length} schedules`)
const skipped = Object.keys(EXCLUDED_PATHS)
if (skipped.length > 0) {
console.log(`Excluded ${skipped.length} path(s): ${skipped.join(', ')}`)
@@ -0,0 +1,64 @@
-- Cron heartbeat: make a schedule that stops arriving visible.
--
-- The 2026-11-15 incident: accounted-cron lost 1 497 of 10 455 runs in a day
-- (14,3 %) because pids_limit 30 made `fork/exec /bin/sh` fail whenever the
-- per-minute schedules coincided. supercronic silently skipped those ticks.
-- The container stayed "Up", every endpoint kept answering 200, and the only
-- trace was a log line in the cron container. lib/auth/cron.ts already reports
-- an individual FAILED run to the observability sink; nothing could see a run
-- that never started.
--
-- This table is the durable half of the fix: every cron run upserts one row per
-- path with when it started and how it ended, and app/api/cron/health compares
-- the newest row against the path's worst-case cadence (lib/cron/
-- schedules.generated.ts, generated from vercel.json). The cron container's
-- healthcheck calls that endpoint, so a schedule that stops arriving turns into
-- an unhealthy container instead of a silent gap.
--
-- Shape notes:
-- * One row per scheduled path, not per run. Volume matters: 34 schedules at
-- up to 1 440 runs/day each would be ~8k rows/day forever, and all the check
-- needs is "when did this last start". The upsert keeps the table at 34
-- rows.
-- * No company_id. A cron run is instance-wide, and pretending otherwise
-- would make the same schedule look healthy in one tenant and dead in
-- another with no way to reconcile them.
-- * declared_at is set when the health check first sees a path in the
-- generated schedule list. That is what separates "added to vercel.json
-- yesterday, has not had its first tick yet" from "stopped running a week
-- ago": a path with no run is only stale once declared_at is older than its
-- own cadence, never immediately. Without it, every new job and every fresh
-- install would report red until its first run.
-- * last_started_at, not last_completed_at, is the staleness input. A run that
-- hangs never completes, and supercronic then skips every later tick —
-- exactly the failure that must look dead.
--
-- Access: RLS enabled with no policies. Only the service role (the cron paths
-- and the health route, via createServiceClientNoCookies) reads or writes it;
-- anon/authenticated have no business seeing instance-wide scheduling state.
CREATE TABLE IF NOT EXISTS public.cron_heartbeats (
path TEXT PRIMARY KEY,
declared_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_started_at TIMESTAMPTZ,
last_completed_at TIMESTAMPTZ,
last_status TEXT,
last_http_status INTEGER,
last_duration_ms INTEGER,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
COMMENT ON TABLE public.cron_heartbeats IS
'One row per scheduled cron path: newest run start/outcome. Read by app/api/cron/health to detect schedules that stop arriving (2026-11-15 pids_limit incident).';
COMMENT ON COLUMN public.cron_heartbeats.declared_at IS
'When the health check first saw this path in the generated schedule list. Grace for a path that has never run.';
COMMENT ON COLUMN public.cron_heartbeats.last_started_at IS
'Start of the newest run. The staleness input: a hung run also stops every later tick.';
CREATE INDEX IF NOT EXISTS idx_cron_heartbeats_last_started_at
ON public.cron_heartbeats (last_started_at DESC NULLS LAST);
ALTER TABLE public.cron_heartbeats ENABLE ROW LEVEL SECURITY;
REVOKE ALL ON public.cron_heartbeats FROM anon;
REVOKE ALL ON public.cron_heartbeats FROM authenticated;