feat(cron): missad-tick-detektion — heartbeat per schema, 503 när ett jobb slutar komma #45
@@ -1,2 +1,6 @@
|
||||
anthropic>=0.39.0
|
||||
mcp>=1.1.0
|
||||
# Transitive via mcp (multipart form parsing). Pinned as a floor because the
|
||||
# advisory scanner cannot resolve an unpinned requirement and reports the
|
||||
# oldest version it can match; 0.0.31 carries the PYSEC-2026 multipart fixes.
|
||||
python-multipart>=0.0.31
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { NextResponse } from 'next/server'
|
||||
|
||||
interface Row {
|
||||
path: string
|
||||
declaredAt: string | null
|
||||
lastStartedAt: string | null
|
||||
lastCompletedAt: string | null
|
||||
lastStatus: string | null
|
||||
lastHttpStatus: number | null
|
||||
lastDurationMs: number | null
|
||||
}
|
||||
|
||||
const verifyCronSecret = vi.fn((): NextResponse | null => null)
|
||||
const loadCronHeartbeats = vi.fn(async (): Promise<Row[]> => [])
|
||||
const declareCronPaths = vi.fn(async (_paths: readonly string[]): Promise<void> => {})
|
||||
const cronHeartbeatStoreConfigured = vi.fn((): boolean => true)
|
||||
|
||||
vi.mock('@/lib/auth/cron', () => ({
|
||||
verifyCronSecret: () => verifyCronSecret(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/cron/heartbeat-store', () => ({
|
||||
loadCronHeartbeats: () => loadCronHeartbeats(),
|
||||
declareCronPaths: (paths: readonly string[]) => declareCronPaths(paths),
|
||||
cronHeartbeatStoreConfigured: () => cronHeartbeatStoreConfigured(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/logger', () => {
|
||||
const child = () => logger
|
||||
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child }
|
||||
return { createLogger: () => logger }
|
||||
})
|
||||
|
||||
const { GET } = await import('../route')
|
||||
const { CRON_SCHEDULES } = await import('@/lib/cron/schedules.generated')
|
||||
|
||||
function request(): Request {
|
||||
return new Request('http://app:3000/api/cron/health', {
|
||||
headers: { authorization: 'Bearer secret' },
|
||||
})
|
||||
}
|
||||
|
||||
function everythingRanSecondsAgo(seconds: number) {
|
||||
const at = new Date(Date.now() - seconds * 1000).toISOString()
|
||||
return CRON_SCHEDULES.map((entry) => ({
|
||||
path: entry.path,
|
||||
declaredAt: at,
|
||||
lastStartedAt: at,
|
||||
lastCompletedAt: at,
|
||||
lastStatus: 'ok',
|
||||
lastHttpStatus: 200,
|
||||
lastDurationMs: 12,
|
||||
}))
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
verifyCronSecret.mockClear()
|
||||
verifyCronSecret.mockReturnValue(null)
|
||||
loadCronHeartbeats.mockReset()
|
||||
declareCronPaths.mockClear()
|
||||
cronHeartbeatStoreConfigured.mockReturnValue(true)
|
||||
})
|
||||
|
||||
describe('GET /api/cron/health', () => {
|
||||
it('rejects an unauthenticated call before touching the store', async () => {
|
||||
verifyCronSecret.mockReturnValue(
|
||||
NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
|
||||
)
|
||||
|
||||
const response = await GET(request())
|
||||
|
||||
expect(response.status).toBe(401)
|
||||
expect(loadCronHeartbeats).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('answers 200 when every schedule ran inside its window', async () => {
|
||||
loadCronHeartbeats.mockResolvedValue(everythingRanSecondsAgo(5))
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(body.status).toBe('ok')
|
||||
expect(body.report.ok).toBe(true)
|
||||
expect(body.report.total).toBe(CRON_SCHEDULES.length)
|
||||
})
|
||||
|
||||
it('answers 503 when a schedule stopped arriving', async () => {
|
||||
// Every path ran 5 s ago except one stale per-minute job: the endpoint is
|
||||
// a single boolean for the container healthcheck, so one silent stop must
|
||||
// be enough to fail it. Pick a per-minute entry explicitly — a daily job
|
||||
// 30 minutes old is not stale, and a test that used index 0 would pass for
|
||||
// the wrong reason.
|
||||
const rows = everythingRanSecondsAgo(5)
|
||||
const index = CRON_SCHEDULES.findIndex((entry) => entry.maxIntervalSeconds === 60)
|
||||
expect(index).toBeGreaterThanOrEqual(0)
|
||||
const staleAt = new Date(Date.now() - 30 * 60 * 1000).toISOString()
|
||||
rows[index] = { ...rows[index], lastStartedAt: staleAt, lastCompletedAt: staleAt }
|
||||
loadCronHeartbeats.mockResolvedValue(rows)
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(body.status).toBe('stale')
|
||||
expect(body.report.stale).toBe(1)
|
||||
expect(body.report.checks[index].state).toBe('stale')
|
||||
})
|
||||
|
||||
it('declares paths that have never run instead of failing them immediately', async () => {
|
||||
loadCronHeartbeats.mockResolvedValue([])
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(body.status).toBe('ok')
|
||||
expect(body.report.pending).toBe(CRON_SCHEDULES.length)
|
||||
expect(declareCronPaths).toHaveBeenCalledTimes(1)
|
||||
expect(declareCronPaths.mock.calls[0][0]).toHaveLength(CRON_SCHEDULES.length)
|
||||
})
|
||||
|
||||
it('says unverifiable (503) rather than healthy when the store cannot be read', async () => {
|
||||
loadCronHeartbeats.mockRejectedValue(new Error('connection refused'))
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(body.status).toBe('unverifiable')
|
||||
// The driver's message must not reach the body.
|
||||
expect(JSON.stringify(body)).not.toContain('connection refused')
|
||||
})
|
||||
|
||||
it('says unverifiable when the heartbeat store is not configured', async () => {
|
||||
cronHeartbeatStoreConfigured.mockReturnValue(false)
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(503)
|
||||
expect(body.status).toBe('unverifiable')
|
||||
expect(loadCronHeartbeats).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('surfaces a run that arrived but failed separately from a missed tick', async () => {
|
||||
const rows = everythingRanSecondsAgo(5)
|
||||
rows[1] = { ...rows[1], lastStatus: 'error', lastHttpStatus: 500 }
|
||||
loadCronHeartbeats.mockResolvedValue(rows)
|
||||
|
||||
const response = await GET(request())
|
||||
const body = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(body.failing).toEqual([
|
||||
{ path: rows[1].path, lastStatus: 'error', lastHttpStatus: 500 },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,109 @@
|
||||
import { NextResponse } from 'next/server'
|
||||
import { verifyCronSecret } from '@/lib/auth/cron'
|
||||
import {
|
||||
evaluateCronHeartbeats,
|
||||
isFailingStatus,
|
||||
type CronHeartbeatReport,
|
||||
} from '@/lib/cron/heartbeat-evaluation'
|
||||
import {
|
||||
cronHeartbeatStoreConfigured,
|
||||
declareCronPaths,
|
||||
loadCronHeartbeats,
|
||||
} from '@/lib/cron/heartbeat-store'
|
||||
import { CRON_SCHEDULES } from '@/lib/cron/schedules.generated'
|
||||
import { createLogger } from '@/lib/logger'
|
||||
|
||||
const log = createLogger('cron/health')
|
||||
|
||||
// Reads live rows every call; a cached 200 would defeat the point of a
|
||||
// healthcheck that exists to notice a schedule going quiet.
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
interface CronHealthBody {
|
||||
status: 'ok' | 'stale' | 'unverifiable'
|
||||
reason?: string
|
||||
/** Present when the store responded. */
|
||||
report?: CronHeartbeatReport
|
||||
/** Runs that arrived but ended non-2xx. Informational; a failed run still ran. */
|
||||
failing?: Array<{ path: string; lastStatus: string | null; lastHttpStatus: number | null }>
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/cron/health
|
||||
*
|
||||
* Answers the question the 2026-11-15 incident proved nobody was asking: did
|
||||
* every scheduled job actually start when it should?
|
||||
*
|
||||
* 200 = every scheduled path has run within its own worst-case window.
|
||||
* 503 = at least one path is stale, or the heartbeat store could not be read
|
||||
* (an unreadable store is not evidence of a healthy cron).
|
||||
*
|
||||
* Called by the cron container's own healthcheck (docker-compose.yml), which is
|
||||
* what turns this answer into `unhealthy` and into the existing alerting chain.
|
||||
* Auth is the normal cron secret: the body names instance-wide schedules, so it
|
||||
* is not public, but the same credential the scheduler already holds is the
|
||||
* right one — no new secret to distribute.
|
||||
*/
|
||||
export async function GET(request: Request): Promise<NextResponse<CronHealthBody>> {
|
||||
// report: false — this endpoint is itself polled by a healthcheck, so a
|
||||
// misconfigured probe must not emit an auth-failure event every minute.
|
||||
const authError = verifyCronSecret(request, { report: false })
|
||||
if (authError) return authError as NextResponse<CronHealthBody>
|
||||
|
||||
if (!cronHeartbeatStoreConfigured()) {
|
||||
return NextResponse.json<CronHealthBody>(
|
||||
{
|
||||
status: 'unverifiable',
|
||||
reason:
|
||||
'heartbeat store not configured (NEXT_PUBLIC_SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY missing)',
|
||||
},
|
||||
{ status: 503 },
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
// Declare first, read second: a path that has never run needs its
|
||||
// declared_at anchor before the evaluator can tell a fresh job from a
|
||||
// stopped one. ignoreDuplicates keeps existing rows untouched.
|
||||
const known = new Set((await loadCronHeartbeats()).map((row) => row.path))
|
||||
const undeclared = CRON_SCHEDULES.map((entry) => entry.path).filter((path) => !known.has(path))
|
||||
if (undeclared.length > 0) await declareCronPaths(undeclared)
|
||||
|
||||
const rows = await loadCronHeartbeats()
|
||||
const report = evaluateCronHeartbeats({ entries: CRON_SCHEDULES, rows, now: Date.now() })
|
||||
|
||||
const failing = report.checks
|
||||
.filter((check) => isFailingStatus(check.lastStatus, check.lastHttpStatus))
|
||||
.map((check) => ({
|
||||
path: check.path,
|
||||
lastStatus: check.lastStatus,
|
||||
lastHttpStatus: check.lastHttpStatus,
|
||||
}))
|
||||
|
||||
if (!report.ok) {
|
||||
const stale = report.checks.filter((check) => check.state === 'stale')
|
||||
log.warn('cron schedule stale', {
|
||||
stale: stale.map((check) => ({
|
||||
path: check.path,
|
||||
ageSeconds: check.ageSeconds,
|
||||
allowedAgeSeconds: check.allowedAgeSeconds,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
return NextResponse.json<CronHealthBody>(
|
||||
{ status: report.ok ? 'ok' : 'stale', report, failing },
|
||||
{ status: report.ok ? 200 : 503 },
|
||||
)
|
||||
} catch (err) {
|
||||
// Never echo the driver's message: it can name tables and query fragments.
|
||||
log.error('cron health check failed', err as Error)
|
||||
return NextResponse.json<CronHealthBody>(
|
||||
{
|
||||
status: 'unverifiable',
|
||||
reason: 'heartbeat store read failed',
|
||||
},
|
||||
{ status: 503 },
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Accounted: missad-tick-detektion för cron (2026-11-15)
|
||||
|
||||
Uppföljning på `changes/2026-11-15-cron-pids-och-healthcheck.md`. Den ändringen
|
||||
fixade orsaken till att cron tappade jobb. Den här ändringen gör att ett tappat
|
||||
jobb **syns** nästa gång — vilket var det verkliga problemet: felet pågick i
|
||||
dygn utan att någon mätning, larm eller dashboard reagerade.
|
||||
|
||||
## Fynd
|
||||
|
||||
`accounted-cron` förlorade 1 497 av 10 455 körningar på ett dygn (14,3 %) när
|
||||
`pids_limit: 30` fick `fork/exec /bin/sh` att misslyckas. supercronic hoppade
|
||||
tyst över tickarna. Containern var `Up` hela tiden, appen svarade 200 på alla
|
||||
anrop, och enda spåret var rader i cron-containerns egen logg.
|
||||
|
||||
Tre saker saknades:
|
||||
|
||||
1. **Ingen körningshistorik.** `lib/api/with-cron-context.ts` loggade
|
||||
`cron run started/completed/failed` via `createLogger`, men skrev inget
|
||||
beständigt. En körning som aldrig startade lämnade inget spår alls.
|
||||
2. **Healthchecken mätte fel sak.** Cron-tjänstens healthcheck var
|
||||
`ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'` — den blev aldrig
|
||||
röd av att scheman slutade köra, bara av att processen dog.
|
||||
3. **Ingen koppling till kadensen.** Även med historik hade ingen kontroll vetat
|
||||
vad "för sen" betyder: en daglig körning får vara 24 h gammal, en
|
||||
per-minut-körning inte 4 minuter.
|
||||
|
||||
## Ändring
|
||||
|
||||
- **`lib/cron/cron-interval.ts`** — tolkare för 5-fälters cron (det enda
|
||||
`vercel.json` använder) som räknar ut **värsta** gapet i sekunder över ett
|
||||
12-årsfönster (behövs för 29 februari), inte medelgapet. Okänd syntax kastar i
|
||||
stället för att gissa. `describeGap()` för läsbar text.
|
||||
- **`scripts/generate-crontabs.ts`** — skriver nu även
|
||||
`lib/cron/schedules.generated.ts`: en post per schemalagd path med
|
||||
`schedule` och `maxIntervalSeconds`. Samma källa som crontaben, så en
|
||||
schemalagd path som kontrollen inte känner till kan inte uppstå.
|
||||
- **`supabase/migrations/20261115190000_cron_heartbeats.sql`** — tabellen
|
||||
`cron_heartbeats`, en rad per path: `declared_at`, `last_started_at`,
|
||||
`last_completed_at`, `last_status`, `last_http_status`, `last_duration_ms`.
|
||||
RLS på med **noll policies** (endast service role; instansglobal
|
||||
schemaläggning är inte tenantdata).
|
||||
- **`lib/cron/heartbeat-evaluation.ts`** — ren utvärdering, testbar utan DB:
|
||||
`ok` (körning inom fönstret), `pending` (aldrig körd men deklarerad inom sin
|
||||
egen kadens), `stale` (körning har uteblivit). Tröskel =
|
||||
`2 × maxIntervalSeconds + 120 s`.
|
||||
- **`lib/cron/heartbeat-store.ts`** — Supabase-IO. Skrivningar är best effort
|
||||
(loggar och returnerar, kastar aldrig: en mätningsskrivning får inte fälla
|
||||
jobbet den mäter). Läsningar kastar, så endpointen kan säga "kan inte
|
||||
verifiera" i stället för att ljuga grönt. No-op när Supabase-env saknas.
|
||||
- **`app/api/cron/health/route.ts`** — 200 när alla scheman ligger inom sina
|
||||
fönster, 503 när något är `stale` eller lagret inte kan läsas. Deklarerar
|
||||
okända paths först (ger nya jobb respit i stället för omedelbart rött).
|
||||
Autentiseras med befintlig cron-hemlighet (`report: false` så en felkopplad
|
||||
probe inte spammar auth-larm).
|
||||
- **`lib/api/with-cron-context.ts`** — skriver heartbeat per körning (path från
|
||||
request-URL:en) med status och duration. 401 räknas inte som en körning, och
|
||||
ett kastat heartbeat-fel fäller aldrig körningen.
|
||||
- **`docker-compose.yml`** (cron-tjänsten) — healthcheck bytt från
|
||||
`ps ax | grep supercronic` till att anropa `/api/cron/health` med
|
||||
`$${CRON_SECRET}`/`$${APP_URL}`, `start_period: 120s`.
|
||||
|
||||
Varför `last_started_at` och inte `last_completed_at`: en körning som hänger sig
|
||||
avslutas aldrig, och supercronic hoppar då över alla senare tickar. Mätt från
|
||||
start blir en hängning synlig; mätt från completion hade den sett levande ut.
|
||||
|
||||
Varför heartbeat i stället för att bara räkna i apploggen: loggen rullar, ägs av
|
||||
Docker, och kräver att någon läser den. Tabellen är frågbar, och endpointen gör
|
||||
svaret till en containerstatus som den befintliga larmkedjan redan tittar på.
|
||||
|
||||
## Verifiering
|
||||
|
||||
- `npx vitest run --project unit lib/cron lib/api/__tests__/with-cron-context.test.ts scripts/__tests__/generate-crontabs.test.ts app/api/cron/health` → **77 + 7 + 11 tester gröna**:
|
||||
- kadens: `* * * * *` = 60 s, `*/2` = 120 s, `*/5` = 300 s, `0 * * * *` = 3600 s,
|
||||
`30 */2 * * *` = 7200 s (inte 3600), daglig = 86 400 s, veckovis = 604 800 s,
|
||||
29 februari → ändligt gap; `0 0 1 * 1` = 7 dygn (cron-regeln OR, inte AND).
|
||||
- utvärdering: per-minut-jobb 241 s gammalt = stale, 120 s = ok, deklarerad-men-aldrig-körd
|
||||
= pending inom respit och stale efter, hängd körning (gammal start, färsk
|
||||
completion) = stale.
|
||||
- endpoint: 200/503, 401 utan store-åtkomst, deklarerar okända paths,
|
||||
"unverifiable" utan läckta drivrutinsmeddelanden, `failing` listar
|
||||
körningar som kom fram men föll.
|
||||
- wiring: heartbeat skrivs med rätt path/status, 401 skriver ingen heartbeat,
|
||||
kastat heartbeat-fel ger fortfarande lyckad körning.
|
||||
- drift: `lib/cron/schedules.generated.ts` matchar generatorn byte för byte och
|
||||
`crontab.self-hosted` rad för rad.
|
||||
- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel.
|
||||
`check:guards` → passed.
|
||||
- Hela unit-sviten körd efter ändringen (resultat i PR:ens CI).
|
||||
|
||||
## Kvar / nästa steg
|
||||
|
||||
- Kör `docker compose config` och recreate av cron på server7 först när appen med
|
||||
den nya routen är deployad — annars blir healthchecken 404 och containern
|
||||
unhealthy. Ordning: deploy app → applicera migrationen → recreate cron.
|
||||
- Larmet blir `unhealthy` via befintlig healthcheck-kedja; Prometheus-regel för
|
||||
`container_health_status` finns redan. Ingen ny larmväg behövs.
|
||||
- Backlog-kontrollerna (0 rader i alla fem köer) ligger kvar som engångsbevis i
|
||||
incidentprotokollet; ingen återkörning av affärsjobb krävs eftersom instansen
|
||||
är en tom/sandbox-miljö (companies 1, invoices 0, journal_entries 0).
|
||||
|
||||
## Revert
|
||||
|
||||
```
|
||||
git revert <commit> # kod, migration och compose i en commit
|
||||
```
|
||||
|
||||
Migrationen är additiv (ny tabell, inga ändrade kolumner) och kan lämnas kvar om
|
||||
bara koden rullas tillbaka. Healthchecken återställs separat:
|
||||
`test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"]`.
|
||||
@@ -0,0 +1,110 @@
|
||||
# Accounted: OSV-gaten grön — 24 sårbarheter stängda, xlsx undantagen med bevis (2026-11-15)
|
||||
|
||||
`security / scan` var röd på PR #44 även efter att Trivy-steget blivit grönt.
|
||||
Steg 6, **OSV Scanner**, fällde jobbet.
|
||||
|
||||
## Fynd
|
||||
|
||||
`ghcr.io/google/osv-scanner:v2.7.1` mot `package-lock.json` +
|
||||
`.claude/skills/mcp-builder/scripts/requirements.txt`: **43 advisory-poster över
|
||||
10 paket**. Viktigt för rotorsaken: samma skanning mot `origin/main`s låsfil ger
|
||||
**9 av 10 paket** — det här var inte något PR:en införde. Nya GHSA/PYSEC-poster
|
||||
publicerades mot versioner vi redan kör, så `main` hade blivit röd vid nästa
|
||||
nattliga skanning (02:17) oavsett.
|
||||
|
||||
| Paket | Före | Efter | Väg |
|
||||
|---|---|---|---|
|
||||
| nodemailer | 10.0.6 (direkt) + 10.0.10 (mailparser) | **10.1.4** | direkt dependency + override |
|
||||
| zod | 4.4.3 | **4.6.7** | `^4.6.6` |
|
||||
| vite | 8.1.3 | **8.3.8** | override `^8.1.6` (dev, via vitest) |
|
||||
| vitest / @vitest/mocker | 4.1.11 / 4.1.11 | **4.1.12 / 4.1.12** | `^4.1.12` |
|
||||
| @babel/core | 7.28.6 | **7.29.13** | override `^7.29.6` (dev, via eslint-plugin-react-hooks) |
|
||||
| fast-xml-parser | 5.7.2 | **5.12.1** | override `^5.12.0` (via @aws-sdk/xml-builder) |
|
||||
| minimatch | 9.0.9 | **11.0.8** | override `^11.0.1` (dev, via typescript-eslint) |
|
||||
| postcss-selector-parser | 6.0.10 | **6.1.6** | override `^6.1.5` (via @tailwindcss/typography) |
|
||||
| python-multipart | olöst (otransitiv, (ospecificerad) | **>=0.0.31** | explicit floor i skillens requirements.txt |
|
||||
| xlsx | 0.20.3 | 0.20.3 | **undantagen** — se nedan |
|
||||
|
||||
## xlsx: undantaget, inte bortförklarat
|
||||
|
||||
Båda posterna (`GHSA-4r6h-8v6p-xvw6` = CVE-2023-30533 prototype pollution,
|
||||
`GHSA-5pgg-2g8v-p4x9` = CVE-2024-22363 ReDoS) har `introduced: 0` och **inget
|
||||
`fixed`-event** — kontrollerat direkt mot `api.osv.dev` 2026-11-15. Det betyder
|
||||
inte "ofixat", det betyder "ingen npm-version bär fixen": SheetJS slutade
|
||||
publicera till npm efter 0.18.5 och släpper 0.19+ från `cdn.sheetjs.com`.
|
||||
|
||||
Repot konsumerar `xlsx` som URL-dependency
|
||||
(`https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz`), och 0.20.3 innehåller
|
||||
båda fixarna (0.19.3 respektive 0.20.2). Skanningen rapporterar alltså en
|
||||
versionsupplösningsartefakt, inte en levande defekt.
|
||||
|
||||
Hanteringen är inte en tyst tystnad: `osv-scanner.toml` med två `IgnoredVulns`,
|
||||
`effectiveUntil = 2027-05-15` och full motivering, plus en uttrycklig varning i
|
||||
filen om att `lib/import/*` parsar användaruppladdade arbetsböcker — undantaget
|
||||
får aldrig läsas som tillstånd att acceptera ofixade parser-CVE:er. När SheetJS
|
||||
publicerar 0.21+, eller vi migrerar till en underhållen parser, ska filen bort.
|
||||
|
||||
## Ändring
|
||||
|
||||
- `package.json`: nodemailer `10.1.4`; zod `^4.6.6`; vitest `^4.1.12`; overrides
|
||||
utökade med `nodemailer` (mailparser-subträdet), `@babel/core`,
|
||||
`fast-xml-parser`, `minimatch`, `postcss-selector-parser`, `vite`.
|
||||
- `package-lock.json`: omgenererad med `npm install --no-audit --no-fund`.
|
||||
- `.claude/skills/mcp-builder/scripts/requirements.txt`: explicit
|
||||
`python-multipart>=0.0.31` med kommentar om varför en floor behövs (OSV kan
|
||||
inte lösa en otransitiv, opinnad requirement och rapporterar den äldsta
|
||||
version den kan matcha).
|
||||
- `osv-scanner.toml`: ny, se ovan.
|
||||
- `scripts/checks/no-new-antipatterns.mjs`: PINNED_DEPS nodemailer → `10.1.4`
|
||||
med motiveringen uppdaterad (9.x → 10.1.4 i två steg samma dag).
|
||||
|
||||
Notera att arbetsflödet redan skickar `--config` när `osv-scanner.toml` finns
|
||||
(`.gitea/workflows/security.yml`), så ingen CI-ändring behövdes.
|
||||
|
||||
## Verifiering
|
||||
|
||||
- `osv-scanner scan source --recursive --config=osv-scanner.toml .` →
|
||||
**No issues found**, rc=0 (2 filtrerade, båda xlsx, med utskriven motivering).
|
||||
- `npx vitest run --project unit` → **1687 filer / 22 400 tester passed**,
|
||||
3 filer/5 tester skipped, 163 s (efter alla bumps: zod 4.6, vite 8.3,
|
||||
vitest 4.1.12, minimatch 11, @babel 7.29).
|
||||
- `npm run check:types` → OK (533 fel, baseline 533). `check:lint` → 0 fel.
|
||||
`check:guards` → passed (pinned-dep 0).
|
||||
- Trivy fs HIGH/CRITICAL: 0 vulnerabilitets-, secret- och misconfig-fynd.
|
||||
|
||||
## Revert
|
||||
|
||||
```
|
||||
git revert <commit>
|
||||
```
|
||||
|
||||
Enda beteendepåverkande posten är `osv-scanner.toml` (om den tas bort blir
|
||||
gaten röd igen på xlsx). Paketbumparna är patch/minor utom `minimatch` 9 → 11,
|
||||
som bara används av lint-verktygskedjan och verifieras av `check:lint` +
|
||||
testsviten.
|
||||
|
||||
## Tillägg: Semgrep-steget (2026-11-15, samma PR)
|
||||
|
||||
Efter Trivy och OSV blev två steg gröna och det tredje rött: **Semgrep
|
||||
(`p/ci --error`)** gav exakt **ett** fynd:
|
||||
|
||||
```
|
||||
.npmrc
|
||||
package_managers.npm.npm-missing-minimum-release-age [Blocking]
|
||||
This .npmrc does not set a minimum release age ... Add `min-release-age = 7`
|
||||
```
|
||||
|
||||
Regeln är rimlig: en komprometterad release rycks oftast tillbaka inom några
|
||||
dagar, så att vänta en vecka innan en ny version får lösas in tar bort det
|
||||
billigaste angreppsfönstret. npm ≥ 11.10 förstår nyckeln (vi kör 11.19.1);
|
||||
äldre npm ignorerar okända nycklar, så `.npmrc` förblir kompatibel.
|
||||
|
||||
Åtgärd: `min-release-age=7` i `.npmrc` med kommentar om varför. Verifierat:
|
||||
`npm config get min-release-age` → `7`, `npm install --package-lock-only` gav
|
||||
ingen låsfillsdiff, och hela Semgrep-skanningen är därefter ren (`rc=0`, "no
|
||||
findings").
|
||||
|
||||
Fälla att känna till: regeln gäller även våra egna `@siax/*`-paket. Behöver en
|
||||
färsk intern publicering konsumeras inom sju dagar måste den hämtas med
|
||||
`--min-release-age=0` — det är ett medvetet undantag i det enskilda fallet, inte
|
||||
ett skäl att sänka taket i `.npmrc`.
|
||||
+16
-3
@@ -76,11 +76,24 @@ services:
|
||||
mem_limit: 256m
|
||||
pids_limit: 200
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "ps ax | grep -q '[s]upercronic /etc/supercronic/crontab'"]
|
||||
# 2026-11-15: this used to be `ps ax | grep supercronic`, which stays green
|
||||
# while the scheduler process is up but ticks are being dropped — exactly
|
||||
# what happened when pids_limit 30 made 14,3 % of runs fail to fork (see
|
||||
# docker-compose.accounted.yml and the cron_heartbeats migration). Ask the
|
||||
# app instead: /api/cron/health answers 503 when any schedule has not run
|
||||
# within its own worst-case window, so a silent stop makes this container
|
||||
# unhealthy and reaches alerting. `$$` is required — a single `$` would be
|
||||
# interpolated by compose on the host instead of by the shell in here.
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- 'curl -fsS -m 10 -o /dev/null -H "Authorization: Bearer $${CRON_SECRET}" "$${APP_URL}/api/cron/health"'
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
timeout: 15s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
# The probe calls the app over HTTP, so it can only pass once the app
|
||||
# answers. 30 s was too tight for a cold `docker compose up` and produced
|
||||
# a false unhealthy.
|
||||
start_period: 120s
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { NextResponse } from 'next/server'
|
||||
|
||||
interface HeartbeatInput {
|
||||
path: string
|
||||
startedAt: string
|
||||
completedAt: string
|
||||
status: string
|
||||
httpStatus: number | null
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
const recordCronHeartbeat = vi.fn(async (_input: HeartbeatInput): Promise<void> => {})
|
||||
const verifyCronSecret = vi.fn((): NextResponse | null => null)
|
||||
|
||||
vi.mock('@/lib/cron/heartbeat-store', () => ({
|
||||
recordCronHeartbeat: (input: HeartbeatInput) => recordCronHeartbeat(input),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/auth/cron', () => ({
|
||||
verifyCronSecret: () => verifyCronSecret(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/logger', () => {
|
||||
const child = () => logger
|
||||
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), child }
|
||||
return { createLogger: () => logger }
|
||||
})
|
||||
|
||||
vi.mock('@/lib/errors/get-structured-error', () => ({
|
||||
errorResponse: () => NextResponse.json({ error: 'boom' }, { status: 500 }),
|
||||
errorResponseFromCode: () => NextResponse.json({ error: 'unauthorized' }, { status: 401 }),
|
||||
}))
|
||||
|
||||
const { withCronContext } = await import('../with-cron-context')
|
||||
|
||||
const PATH = '/api/webhooks/dispatch/cron'
|
||||
|
||||
function request(): Request {
|
||||
return new Request(`http://app:3000${PATH}`, {
|
||||
headers: { authorization: 'Bearer secret' },
|
||||
})
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
recordCronHeartbeat.mockClear()
|
||||
verifyCronSecret.mockClear()
|
||||
verifyCronSecret.mockReturnValue(null)
|
||||
})
|
||||
|
||||
/**
|
||||
* The heartbeat is the only evidence that a schedule still runs, so the wiring
|
||||
* — which path, which status, and that an auth rejection is NOT recorded as a
|
||||
* run — is worth asserting directly rather than inferring from the endpoint.
|
||||
*/
|
||||
describe('withCronContext heartbeat', () => {
|
||||
it('records the run under the request path on success', async () => {
|
||||
const handler = withCronContext('cron.webhook_dispatch', async () =>
|
||||
NextResponse.json({ ok: true }),
|
||||
)
|
||||
|
||||
const response = await handler(request())
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(recordCronHeartbeat).toHaveBeenCalledTimes(1)
|
||||
const input = recordCronHeartbeat.mock.calls[0][0]
|
||||
expect(input.path).toBe(PATH)
|
||||
expect(input.status).toBe('ok')
|
||||
expect(input.httpStatus).toBe(200)
|
||||
expect(Number.isFinite(Date.parse(input.startedAt))).toBe(true)
|
||||
expect(Number.isFinite(Date.parse(input.completedAt))).toBe(true)
|
||||
expect(input.durationMs).toBeGreaterThanOrEqual(0)
|
||||
})
|
||||
|
||||
it('records a failed run as error so the health body can show it', async () => {
|
||||
const handler = withCronContext('cron.webhook_dispatch', async () => {
|
||||
throw new Error('kaboom')
|
||||
})
|
||||
|
||||
const response = await handler(request())
|
||||
|
||||
expect(response.status).toBe(500)
|
||||
expect(recordCronHeartbeat).toHaveBeenCalledTimes(1)
|
||||
expect(recordCronHeartbeat.mock.calls[0][0].status).toBe('error')
|
||||
})
|
||||
|
||||
it('does not record an auth rejection: a 401 is not a run', async () => {
|
||||
verifyCronSecret.mockReturnValue(
|
||||
NextResponse.json({ error: 'Unauthorized' }, { status: 401 }),
|
||||
)
|
||||
const handler = withCronContext('cron.webhook_dispatch', async () =>
|
||||
NextResponse.json({ ok: true }),
|
||||
)
|
||||
|
||||
const response = await handler(request())
|
||||
|
||||
expect(response.status).toBe(401)
|
||||
expect(recordCronHeartbeat).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('never fails the cron run when the heartbeat write rejects', async () => {
|
||||
recordCronHeartbeat.mockRejectedValueOnce(new Error('db down'))
|
||||
const handler = withCronContext('cron.webhook_dispatch', async () =>
|
||||
NextResponse.json({ ok: true }),
|
||||
)
|
||||
|
||||
// recordCronHeartbeat swallows its own errors in production; this asserts
|
||||
// the wrapper does not turn a rejected mock into a 500 either.
|
||||
await expect(handler(request())).resolves.toBeDefined()
|
||||
})
|
||||
})
|
||||
@@ -21,6 +21,7 @@
|
||||
|
||||
import { NextResponse } from 'next/server'
|
||||
import { verifyCronSecret } from '@/lib/auth/cron'
|
||||
import { recordCronHeartbeat } from '@/lib/cron/heartbeat-store'
|
||||
import { createLogger, type Logger } from '@/lib/logger'
|
||||
import { errorResponse, errorResponseFromCode } from '@/lib/errors/get-structured-error'
|
||||
|
||||
@@ -59,6 +60,20 @@ function generateRequestId(prefix: 'cron' | 'cron_item' = 'cron'): string {
|
||||
return `${prefix}_${crypto.randomUUID()}`
|
||||
}
|
||||
|
||||
/**
|
||||
* The schedule identity used for heartbeat detection. The request path, not the
|
||||
* `operation` label: lib/cron/schedules.generated.ts is keyed by path because
|
||||
* that is the one identifier the generator and the runtime both know without a
|
||||
* lookup table that can drift.
|
||||
*/
|
||||
function cronHeartbeatPath(request: Request): string | null {
|
||||
try {
|
||||
return new URL(request.url).pathname
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function withCronContext(
|
||||
operation: string,
|
||||
handler: CronHandler,
|
||||
@@ -111,6 +126,31 @@ export function withCronContext(
|
||||
|
||||
const ctx: CronContext = { requestId, log, forEach }
|
||||
|
||||
// Heartbeat: app/api/cron/health reads this row to decide whether this
|
||||
// schedule still runs on time. start is captured before the handler so a
|
||||
// run that hangs shows up as an ageing last_started_at, which is exactly
|
||||
// the silent-stop shape seen in the 2026-11-15 pids_limit incident.
|
||||
const heartbeatPath = cronHeartbeatPath(request)
|
||||
const startedAt = new Date(start).toISOString()
|
||||
const recordRun = async (status: string, httpStatus: number): Promise<void> => {
|
||||
if (!heartbeatPath) return
|
||||
try {
|
||||
await recordCronHeartbeat({
|
||||
path: heartbeatPath,
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
status,
|
||||
httpStatus,
|
||||
durationMs: Date.now() - start,
|
||||
})
|
||||
} catch {
|
||||
// A monitoring write must never break the job it monitors. The store
|
||||
// swallows its own failures; this is the second, cheap guard so a
|
||||
// future change there cannot turn a heartbeat problem into a failing
|
||||
// cron run.
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await handler(request, ctx)
|
||||
if (response instanceof Response && !response.headers.get('X-Request-Id')) {
|
||||
@@ -120,9 +160,11 @@ export function withCronContext(
|
||||
durationMs: Date.now() - start,
|
||||
status: response.status,
|
||||
})
|
||||
await recordRun('ok', response.status)
|
||||
return response
|
||||
} catch (err) {
|
||||
log.error('cron run failed', err as Error, { durationMs: Date.now() - start })
|
||||
await recordRun('error', 500)
|
||||
return errorResponse(err, log, { requestId })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
describeGap,
|
||||
maxIntervalSeconds,
|
||||
parseCronExpression,
|
||||
} from '../cron-interval'
|
||||
|
||||
/**
|
||||
* These are the numbers app/api/cron/health turns into alert thresholds, so the
|
||||
* tests assert the actual worst-case gaps rather than "returns a number". A
|
||||
* wrong threshold is not a cosmetic bug: too small and every slow run flaps the
|
||||
* cron container unhealthy, too large and a stopped schedule keeps looking fine
|
||||
* — the exact failure the 2026-11-15 incident was.
|
||||
*/
|
||||
describe('parseCronExpression', () => {
|
||||
it('parses the five fields in order', () => {
|
||||
const spec = parseCronExpression('15 3 * * 1-5')
|
||||
expect(spec.minutes).toEqual([15])
|
||||
expect(spec.hours).toEqual([3])
|
||||
expect(spec.daysOfMonth).toHaveLength(31)
|
||||
expect(spec.months).toHaveLength(12)
|
||||
expect(spec.daysOfWeek).toEqual([1, 2, 3, 4, 5])
|
||||
expect(spec.dayOfMonthRestricted).toBe(false)
|
||||
expect(spec.dayOfWeekRestricted).toBe(true)
|
||||
})
|
||||
|
||||
it('expands steps, ranges and lists', () => {
|
||||
expect(parseCronExpression('*/15 * * * *').minutes).toEqual([0, 15, 30, 45])
|
||||
expect(parseCronExpression('2,4,6-8 * * * *').minutes).toEqual([2, 4, 6, 7, 8])
|
||||
expect(parseCronExpression('5/20 * * * *').minutes).toEqual([5, 25, 45])
|
||||
})
|
||||
|
||||
it('normalises Sunday 7 to 0', () => {
|
||||
expect(parseCronExpression('0 0 * * 7').daysOfWeek).toEqual([0])
|
||||
expect(parseCronExpression('0 0 * * 0').daysOfWeek).toEqual([0])
|
||||
expect(parseCronExpression('0 0 * * 0,7').daysOfWeek).toEqual([0])
|
||||
})
|
||||
|
||||
it('treats a star step as unrestricted, like every cron implementation', () => {
|
||||
expect(parseCronExpression('0 0 */1 * *').dayOfMonthRestricted).toBe(false)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['* * * *', 'four fields'],
|
||||
['* * * * * *', 'six fields'],
|
||||
['60 * * * *', 'minute out of range'],
|
||||
['* 24 * * *', 'hour out of range'],
|
||||
['* * 0 * *', 'day-of-month out of range'],
|
||||
['* * * 13 *', 'month out of range'],
|
||||
['* * * * 8', 'day-of-week out of range'],
|
||||
['*/0 * * * *', 'zero step'],
|
||||
['5-2 * * * *', 'inverted range'],
|
||||
['a * * * *', 'non-numeric'],
|
||||
])('rejects %s (%s)', (expression) => {
|
||||
expect(() => parseCronExpression(expression)).toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('maxIntervalSeconds', () => {
|
||||
it('every-minute is exactly 60 s', () => {
|
||||
expect(maxIntervalSeconds('* * * * *')).toBe(60)
|
||||
})
|
||||
|
||||
it('*/2 is 120 s and */5 is 300 s', () => {
|
||||
expect(maxIntervalSeconds('*/2 * * * *')).toBe(120)
|
||||
expect(maxIntervalSeconds('*/5 * * * *')).toBe(300)
|
||||
})
|
||||
|
||||
it('hourly on the hour is 3600 s', () => {
|
||||
expect(maxIntervalSeconds('0 * * * *')).toBe(3600)
|
||||
})
|
||||
|
||||
it('a daily job is 86400 s', () => {
|
||||
expect(maxIntervalSeconds('0 6 * * *')).toBe(86_400)
|
||||
expect(maxIntervalSeconds('30 4 * * *')).toBe(86_400)
|
||||
})
|
||||
|
||||
it('every other hour at :30 is 7200 s, not 3600 s', () => {
|
||||
// The gap between 22:30 and 00:30 spans two hours; an average-gap
|
||||
// implementation would say 3600 and alert on the normal cadence.
|
||||
expect(maxIntervalSeconds('30 */2 * * *')).toBe(7200)
|
||||
})
|
||||
|
||||
it('weekly is seven days', () => {
|
||||
expect(maxIntervalSeconds('0 3 * * 0')).toBe(7 * 86_400)
|
||||
})
|
||||
|
||||
it('yearly on 2 January is 365 or 366 days depending on the leap year it spans', () => {
|
||||
// 2003-01-02 -> 2004-01-02 spans no 29 February, 2004 -> 2005 does not
|
||||
// either; 1 Jan -> 1 Jan crossing a leap day is the 8784-hour case. Either
|
||||
// way the threshold must be at least a year, never 0.
|
||||
expect(maxIntervalSeconds('0 0 2 1 *')).toBeGreaterThanOrEqual(365 * 86_400)
|
||||
})
|
||||
|
||||
it('29 February only matches in leap years and still returns a finite gap', () => {
|
||||
const gap = maxIntervalSeconds('0 0 29 2 *')
|
||||
expect(gap).toBeGreaterThanOrEqual(365 * 86_400)
|
||||
expect(Number.isFinite(gap)).toBe(true)
|
||||
})
|
||||
|
||||
it('honours the cron day rule: restricted day-of-month OR day-of-week', () => {
|
||||
// "1st of the month OR every Monday" still has a Monday every seven days,
|
||||
// so the worst case is a week — a naive AND would report ~28 days and hide
|
||||
// a fortnight of misses.
|
||||
expect(maxIntervalSeconds('0 0 1 * 1')).toBe(7 * 86_400)
|
||||
})
|
||||
})
|
||||
|
||||
describe('describeGap', () => {
|
||||
it.each([
|
||||
[60, '60 s'],
|
||||
[120, '2 min'],
|
||||
[3600, '60 min'],
|
||||
[7200, '2 h'],
|
||||
[86_400, '24 h'],
|
||||
[7 * 86_400, '7 dygn'],
|
||||
])('renders %i s as %s', (seconds, expected) => {
|
||||
expect(describeGap(seconds)).toBe(expected)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,203 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
CRON_STALE_GRACE_SECONDS,
|
||||
CRON_STALE_MULTIPLIER,
|
||||
allowedAgeSeconds,
|
||||
evaluateCronHeartbeats,
|
||||
isFailingStatus,
|
||||
type CronHeartbeatRow,
|
||||
} from '../heartbeat-evaluation'
|
||||
import type { CronScheduleEntry } from '../schedules.generated'
|
||||
|
||||
const NOW = Date.parse('2026-11-15T12:00:00.000Z')
|
||||
|
||||
function entry(path: string, maxIntervalSeconds: number): CronScheduleEntry {
|
||||
return { path, schedule: '* * * * *', maxIntervalSeconds }
|
||||
}
|
||||
|
||||
function row(overrides: Partial<CronHeartbeatRow> & { path: string }): CronHeartbeatRow {
|
||||
return {
|
||||
declaredAt: '2026-11-15T11:00:00.000Z',
|
||||
lastStartedAt: null,
|
||||
lastCompletedAt: null,
|
||||
lastStatus: null,
|
||||
lastHttpStatus: null,
|
||||
lastDurationMs: null,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function isoSecondsAgo(seconds: number): string {
|
||||
return new Date(NOW - seconds * 1000).toISOString()
|
||||
}
|
||||
|
||||
describe('allowedAgeSeconds', () => {
|
||||
it('is two intervals plus the grace floor', () => {
|
||||
expect(CRON_STALE_MULTIPLIER).toBe(2)
|
||||
expect(CRON_STALE_GRACE_SECONDS).toBe(120)
|
||||
expect(allowedAgeSeconds(60)).toBe(240)
|
||||
expect(allowedAgeSeconds(86_400)).toBe(2 * 86_400 + 120)
|
||||
})
|
||||
})
|
||||
|
||||
describe('evaluateCronHeartbeats', () => {
|
||||
it('reports ok when the newest run is inside the window', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/invoices/recurring/cron', 3600)],
|
||||
rows: [row({ path: '/api/invoices/recurring/cron', lastStartedAt: isoSecondsAgo(1800), lastStatus: 'ok' })],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.checks[0].state).toBe('ok')
|
||||
expect(report.checks[0].ageSeconds).toBe(1800)
|
||||
expect(report.stale).toBe(0)
|
||||
})
|
||||
|
||||
it('reports stale when a per-minute schedule has not started for four minutes', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/webhooks/dispatch/cron', 60)],
|
||||
rows: [
|
||||
row({
|
||||
path: '/api/webhooks/dispatch/cron',
|
||||
lastStartedAt: isoSecondsAgo(241),
|
||||
lastStatus: 'ok',
|
||||
}),
|
||||
],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(false)
|
||||
expect(report.checks[0].state).toBe('stale')
|
||||
expect(report.stale).toBe(1)
|
||||
})
|
||||
|
||||
it('does not flag a single skipped tick', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/webhooks/dispatch/cron', 60)],
|
||||
rows: [row({ path: '/api/webhooks/dispatch/cron', lastStartedAt: isoSecondsAgo(120) })],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.checks[0].state).toBe('ok')
|
||||
})
|
||||
|
||||
it('treats a never-run path inside its declared grace as pending, not stale', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/new/job/cron', 86_400)],
|
||||
rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3600) })],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.pending).toBe(1)
|
||||
expect(report.checks[0].state).toBe('pending')
|
||||
})
|
||||
|
||||
it('flags a declared path that never ran once the grace is spent', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/new/job/cron', 86_400)],
|
||||
rows: [row({ path: '/api/new/job/cron', declaredAt: isoSecondsAgo(3 * 86_400) })],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(false)
|
||||
expect(report.checks[0].state).toBe('stale')
|
||||
})
|
||||
|
||||
it('reports a path with no row at all as pending on the first call', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/unknown/job/cron', 60)],
|
||||
rows: [],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.checks[0].state).toBe('pending')
|
||||
expect(report.checks[0].ageSeconds).toBeNull()
|
||||
})
|
||||
|
||||
it('measures staleness from the last START, so a hung run looks dead', () => {
|
||||
// The run started 3 h ago and (in this row) completed successfully 2 h 59
|
||||
// min ago — the row is doctored to prove last_started_at wins. After a hang
|
||||
// supercronic stops scheduling, so a completed_at-based check would keep
|
||||
// reporting ok while nothing new runs.
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/invoices/recurring/cron', 3600)],
|
||||
rows: [
|
||||
row({
|
||||
path: '/api/invoices/recurring/cron',
|
||||
lastStartedAt: isoSecondsAgo(3 * 3600),
|
||||
lastCompletedAt: isoSecondsAgo(60),
|
||||
}),
|
||||
],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.checks[0].state).toBe('stale')
|
||||
})
|
||||
|
||||
it('fails the report when any single path is stale', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/a/cron', 60), entry('/api/b/cron', 60)],
|
||||
rows: [
|
||||
row({ path: '/api/a/cron', lastStartedAt: isoSecondsAgo(30) }),
|
||||
row({ path: '/api/b/cron', lastStartedAt: isoSecondsAgo(600) }),
|
||||
],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(false)
|
||||
expect(report.stale).toBe(1)
|
||||
expect(report.checks.filter((check) => check.state === 'ok')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('carries the last outcome through so a failing-but-running job is visible', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/a/cron', 60)],
|
||||
rows: [
|
||||
row({
|
||||
path: '/api/a/cron',
|
||||
lastStartedAt: isoSecondsAgo(30),
|
||||
lastStatus: 'error',
|
||||
lastHttpStatus: 500,
|
||||
lastDurationMs: 42,
|
||||
}),
|
||||
],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.checks[0].lastStatus).toBe('error')
|
||||
expect(report.checks[0].lastHttpStatus).toBe(500)
|
||||
expect(report.checks[0].lastDurationMs).toBe(42)
|
||||
expect(isFailingStatus('error', 500)).toBe(true)
|
||||
})
|
||||
|
||||
it('ignores a malformed timestamp instead of throwing', () => {
|
||||
const report = evaluateCronHeartbeats({
|
||||
entries: [entry('/api/a/cron', 60)],
|
||||
rows: [row({ path: '/api/a/cron', lastStartedAt: 'not-a-date' })],
|
||||
now: NOW,
|
||||
})
|
||||
|
||||
expect(report.ok).toBe(true)
|
||||
expect(report.checks[0].state).toBe('pending')
|
||||
})
|
||||
})
|
||||
|
||||
describe('isFailingStatus', () => {
|
||||
it.each([
|
||||
['ok', 200, false],
|
||||
// A cron endpoint that redirects did not do its work either; curl -f
|
||||
// accepts a 3xx, which is exactly why this is worth surfacing separately.
|
||||
['ok', 302, true],
|
||||
['error', 200, true],
|
||||
['ok', 500, true],
|
||||
[null, 401, true],
|
||||
[null, null, false],
|
||||
])('status %s / http %s -> %s', (status, httpStatus, expected) => {
|
||||
expect(isFailingStatus(status as string | null, httpStatus as number | null)).toBe(expected)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* Cron expression → the longest gap between two scheduled runs.
|
||||
*
|
||||
* Why this exists: the 2026-11-15 accounted incident. `accounted-cron` lost
|
||||
* 1 497 of 10 455 runs in a day (14,3 %) because `pids_limit: 30` made
|
||||
* `fork/exec /bin/sh` fail under the per-minute schedules, and supercronic
|
||||
* simply skipped those ticks. Nothing in the product noticed: the container
|
||||
* stayed "Up", every endpoint still answered, and the only trace was a line in
|
||||
* the cron container's own log. A schedule whose runs stop arriving is the
|
||||
* worst failure mode in this product — nothing looks broken, the work just
|
||||
* never happens — so `cron_heartbeats` records every run and
|
||||
* app/api/cron/health turns "no run arrived within the expected window" into a
|
||||
* 503 the container healthcheck can fail on.
|
||||
*
|
||||
* That check needs to know, per path, how long a gap is normal. That is this
|
||||
* module: a small, dependency-free interpreter for the 5-field cron syntax
|
||||
* `vercel.json` is restricted to, plus a brute-force scan of a 12-year window
|
||||
* to get the WORST-CASE gap rather than an average. Worst case is the correct
|
||||
* threshold input: a job that runs at 00:00 daily must be allowed 24 h, not
|
||||
* the 12 h its mean gap would suggest.
|
||||
*
|
||||
* The scan is exact for every expression the product uses, including the
|
||||
* calendar-sensitive ones (`0 0 2 1 *` once a year, `0 0 29 2 *` only in leap
|
||||
* years). It is not a general cron library: no seconds field, no `@daily`
|
||||
* aliases, no `L`/`W`/`#` extensions. Anything outside the supported subset
|
||||
* throws instead of guessing, and the generator that consumes it runs in CI, so
|
||||
* an unsupported expression fails the build rather than silently producing a
|
||||
* wrong alert threshold.
|
||||
*/
|
||||
|
||||
const MINUTE_MS = 60_000
|
||||
const HOUR_MS = 60 * MINUTE_MS
|
||||
const DAY_MS = 24 * HOUR_MS
|
||||
|
||||
/**
|
||||
* Scan window: 2001-01-01 + 12 years. Twelve years is what it takes to see
|
||||
* three occurrences of 29 February (2004, 2008, 2012), which is the rarest
|
||||
* yearly form a cron expression can express.
|
||||
*/
|
||||
const SCAN_START_MS = Date.UTC(2001, 0, 1)
|
||||
const SCAN_DAYS = 12 * 366
|
||||
|
||||
/**
|
||||
* Upper bound on matched timestamps per expression. A per-minute job matches
|
||||
* 1 440 times a day; 20 000 is two weeks of it, far more than needed to see the
|
||||
* 60-second gap, and it keeps the generator fast when the whole vercel.json is
|
||||
* scanned.
|
||||
*/
|
||||
const MAX_MATCHES = 20_000
|
||||
|
||||
export interface CronSpec {
|
||||
readonly minutes: readonly number[]
|
||||
readonly hours: readonly number[]
|
||||
readonly daysOfMonth: readonly number[]
|
||||
readonly months: readonly number[]
|
||||
readonly daysOfWeek: readonly number[]
|
||||
/**
|
||||
* Cron's day rule: when BOTH day-of-month and day-of-week are restricted the
|
||||
* expression matches if EITHER does; when only one is restricted only that
|
||||
* one applies. A step over the whole range (star followed by a step) counts
|
||||
* as unrestricted, as in every cron implementation, because the field still
|
||||
* starts with a star.
|
||||
*/
|
||||
readonly dayOfMonthRestricted: boolean
|
||||
readonly dayOfWeekRestricted: boolean
|
||||
}
|
||||
|
||||
function parseField(
|
||||
field: string,
|
||||
min: number,
|
||||
max: number,
|
||||
name: string,
|
||||
): { values: number[]; restricted: boolean } {
|
||||
const trimmed = field.trim()
|
||||
if (trimmed.length === 0) throw new Error(`cron: empty ${name} field`)
|
||||
|
||||
const seen = new Set<number>()
|
||||
for (const part of trimmed.split(',')) {
|
||||
const [rangePart, stepPart, ...rest] = part.split('/')
|
||||
if (rest.length > 0) throw new Error(`cron: bad step in ${name} field: ${part}`)
|
||||
|
||||
const step = stepPart === undefined ? 1 : Number(stepPart)
|
||||
if (!Number.isInteger(step) || step < 1) {
|
||||
throw new Error(`cron: bad step in ${name} field: ${part}`)
|
||||
}
|
||||
|
||||
let lo: number
|
||||
let hi: number
|
||||
if (rangePart === '*') {
|
||||
lo = min
|
||||
hi = max
|
||||
} else if (rangePart.includes('-')) {
|
||||
const [a, b, ...tail] = rangePart.split('-')
|
||||
if (tail.length > 0) throw new Error(`cron: bad range in ${name} field: ${part}`)
|
||||
lo = Number(a)
|
||||
hi = Number(b)
|
||||
} else {
|
||||
lo = Number(rangePart)
|
||||
// A bare value with a step (`5/10`) means "from 5 to max, every 10th".
|
||||
hi = stepPart === undefined ? lo : max
|
||||
}
|
||||
|
||||
if (!Number.isInteger(lo) || !Number.isInteger(hi)) {
|
||||
throw new Error(`cron: non-numeric value in ${name} field: ${part}`)
|
||||
}
|
||||
if (lo < min || hi > max || lo > hi) {
|
||||
throw new Error(`cron: ${name} field out of range ${min}-${max}: ${part}`)
|
||||
}
|
||||
|
||||
for (let v = lo; v <= hi; v += step) seen.add(v)
|
||||
}
|
||||
|
||||
return { values: [...seen].sort((a, b) => a - b), restricted: !trimmed.startsWith('*') }
|
||||
}
|
||||
|
||||
/** Parse a standard 5-field cron expression. Throws on anything unsupported. */
|
||||
export function parseCronExpression(expression: string): CronSpec {
|
||||
const fields = expression.trim().split(/\s+/)
|
||||
if (fields.length !== 5) {
|
||||
throw new Error(`cron: expected 5 fields, got ${fields.length}: ${JSON.stringify(expression)}`)
|
||||
}
|
||||
|
||||
const [minute, hour, dayOfMonth, month, dayOfWeek] = fields
|
||||
const minutes = parseField(minute, 0, 59, 'minute')
|
||||
const hours = parseField(hour, 0, 23, 'hour')
|
||||
const dom = parseField(dayOfMonth, 1, 31, 'day-of-month')
|
||||
const months = parseField(month, 1, 12, 'month')
|
||||
// Cron allows both 0 and 7 for Sunday; normalise 7 to 0.
|
||||
const dow = parseField(dayOfWeek, 0, 7, 'day-of-week')
|
||||
|
||||
return {
|
||||
minutes: minutes.values,
|
||||
hours: hours.values,
|
||||
daysOfMonth: dom.values,
|
||||
months: months.values,
|
||||
daysOfWeek: [...new Set(dow.values.map((d) => (d === 7 ? 0 : d)))].sort((a, b) => a - b),
|
||||
dayOfMonthRestricted: dom.restricted,
|
||||
dayOfWeekRestricted: dow.restricted,
|
||||
}
|
||||
}
|
||||
|
||||
function matchesDay(spec: CronSpec, dayStartMs: number): boolean {
|
||||
const date = new Date(dayStartMs)
|
||||
if (!spec.months.includes(date.getUTCMonth() + 1)) return false
|
||||
|
||||
const domMatch = spec.daysOfMonth.includes(date.getUTCDate())
|
||||
const dowMatch = spec.daysOfWeek.includes(date.getUTCDay())
|
||||
|
||||
if (spec.dayOfMonthRestricted && spec.dayOfWeekRestricted) return domMatch || dowMatch
|
||||
if (spec.dayOfMonthRestricted) return domMatch
|
||||
if (spec.dayOfWeekRestricted) return dowMatch
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* The longest interval, in seconds, between two consecutive runs of
|
||||
* `expression` over a 12-year window.
|
||||
*
|
||||
* Throws when the expression never matches twice in the window — that is not a
|
||||
* schedule, and silently returning 0 would make the health check permanently
|
||||
* red or permanently green depending on which way the caller rounds.
|
||||
*/
|
||||
export function maxIntervalSeconds(expression: string): number {
|
||||
const spec = parseCronExpression(expression)
|
||||
|
||||
// Every minute: the answer is exactly one minute and the scan is pointless.
|
||||
if (spec.minutes.length * spec.hours.length === 24 * 60) return 60
|
||||
|
||||
let previous: number | null = null
|
||||
let longest = 0
|
||||
let matches = 0
|
||||
|
||||
for (let day = 0; day < SCAN_DAYS && matches < MAX_MATCHES; day++) {
|
||||
const dayStart = SCAN_START_MS + day * DAY_MS
|
||||
if (!matchesDay(spec, dayStart)) continue
|
||||
|
||||
for (const hour of spec.hours) {
|
||||
for (const minute of spec.minutes) {
|
||||
const at = dayStart + hour * HOUR_MS + minute * MINUTE_MS
|
||||
if (previous !== null) {
|
||||
const gap = (at - previous) / 1000
|
||||
if (gap > longest) longest = gap
|
||||
}
|
||||
previous = at
|
||||
if (++matches >= MAX_MATCHES) break
|
||||
}
|
||||
if (matches >= MAX_MATCHES) break
|
||||
}
|
||||
}
|
||||
|
||||
if (matches < 2) {
|
||||
throw new Error(`cron: ${JSON.stringify(expression)} never matches twice in the scan window`)
|
||||
}
|
||||
|
||||
return longest
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-readable worst-case gap, for the generated file and for operators
|
||||
* reading a 503 body. Deliberately coarse: it is a threshold input, not a
|
||||
* display string that needs to be exact.
|
||||
*/
|
||||
export function describeGap(seconds: number): string {
|
||||
if (seconds < 120) return `${seconds} s`
|
||||
if (seconds < 2 * 3600) return `${Math.round(seconds / 60)} min`
|
||||
if (seconds < 2 * 86_400) return `${Math.round(seconds / 3600)} h`
|
||||
return `${Math.round(seconds / 86_400)} dygn`
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Missed-cron detection, the pure half.
|
||||
*
|
||||
* The store and the endpoint live in ./heartbeat-store.ts and
|
||||
* app/api/cron/health/route.ts; everything that decides ok/stale is here so it
|
||||
* can be tested without a database.
|
||||
*
|
||||
* The threshold is deliberately generous, because the cost of the two mistakes
|
||||
* is not symmetric. Too tight and every slow run flaps the container
|
||||
* unhealthy, which trains people to ignore the check. Too loose and a stalled
|
||||
* schedule keeps looking fine — the failure mode this exists to catch (see the
|
||||
* 2026-11-15 pids_limit incident in the cron_heartbeats migration). Two missed
|
||||
* intervals plus a two-minute floor means a per-minute job is flagged after
|
||||
* 4 minutes and a daily job after 48 h 2 min, while one skipped tick or one
|
||||
* slow run is not.
|
||||
*/
|
||||
|
||||
import type { CronScheduleEntry } from './schedules.generated'
|
||||
|
||||
/** A run is stale after this many missed intervals. */
|
||||
export const CRON_STALE_MULTIPLIER = 2
|
||||
|
||||
/** Floor added on top of the interval, absorbing clock skew and one slow run. */
|
||||
export const CRON_STALE_GRACE_SECONDS = 120
|
||||
|
||||
export type CronHeartbeatState = 'ok' | 'pending' | 'stale'
|
||||
|
||||
/** A row as the store returns it (snake_case columns already mapped). */
|
||||
export interface CronHeartbeatRow {
|
||||
path: string
|
||||
declaredAt: string | null
|
||||
lastStartedAt: string | null
|
||||
lastCompletedAt: string | null
|
||||
lastStatus: string | null
|
||||
lastHttpStatus: number | null
|
||||
lastDurationMs: number | null
|
||||
}
|
||||
|
||||
export interface CronHeartbeatCheck {
|
||||
path: string
|
||||
schedule: string
|
||||
maxIntervalSeconds: number
|
||||
/** Age at which this path is reported stale. */
|
||||
allowedAgeSeconds: number
|
||||
lastStartedAt: string | null
|
||||
ageSeconds: number | null
|
||||
state: CronHeartbeatState
|
||||
lastStatus: string | null
|
||||
lastHttpStatus: number | null
|
||||
lastDurationMs: number | null
|
||||
}
|
||||
|
||||
export interface CronHeartbeatReport {
|
||||
/** False only when at least one path is stale. Pending paths do not fail. */
|
||||
ok: boolean
|
||||
checkedAt: string
|
||||
total: number
|
||||
stale: number
|
||||
pending: number
|
||||
checks: CronHeartbeatCheck[]
|
||||
}
|
||||
|
||||
export function allowedAgeSeconds(maxIntervalSeconds: number): number {
|
||||
return maxIntervalSeconds * CRON_STALE_MULTIPLIER + CRON_STALE_GRACE_SECONDS
|
||||
}
|
||||
|
||||
function ageInSeconds(reference: string | null, now: number): number | null {
|
||||
if (!reference) return null
|
||||
const at = Date.parse(reference)
|
||||
if (!Number.isFinite(at)) return null
|
||||
return Math.max(0, Math.round((now - at) / 1000))
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate every scheduled path against its heartbeat row.
|
||||
*
|
||||
* States:
|
||||
* ok — a run started within the allowed age.
|
||||
* pending — no run yet, but the path has not been declared longer than its
|
||||
* own cadence allows. A freshly added job, or a fresh install.
|
||||
* stale — a run used to arrive and stopped, OR the path was declared
|
||||
* longer ago than its cadence and has never run. Both are real
|
||||
* defects; only `stale` fails the endpoint.
|
||||
*
|
||||
* A path with neither a row nor a declaration is reported `pending` here; the
|
||||
* route declares it before evaluating, so this only happens on the first call.
|
||||
*/
|
||||
export function evaluateCronHeartbeats(params: {
|
||||
entries: readonly CronScheduleEntry[]
|
||||
rows: readonly CronHeartbeatRow[]
|
||||
now: number
|
||||
}): CronHeartbeatReport {
|
||||
const byPath = new Map(params.rows.map((row) => [row.path, row]))
|
||||
|
||||
const checks: CronHeartbeatCheck[] = params.entries.map((entry) => {
|
||||
const row = byPath.get(entry.path)
|
||||
const allowed = allowedAgeSeconds(entry.maxIntervalSeconds)
|
||||
|
||||
// Staleness is measured from the last START. A run that hangs never
|
||||
// completes, and supercronic skips every later tick while it hangs, so
|
||||
// last_completed_at would keep a dead schedule looking alive.
|
||||
const ranAt = row?.lastStartedAt ?? null
|
||||
const reference = ranAt ?? row?.declaredAt ?? null
|
||||
const age = ageInSeconds(reference, params.now)
|
||||
|
||||
let state: CronHeartbeatState = 'ok'
|
||||
if (age === null) state = 'pending'
|
||||
else if (age > allowed) state = 'stale'
|
||||
else if (ranAt === null) state = 'pending'
|
||||
|
||||
return {
|
||||
path: entry.path,
|
||||
schedule: entry.schedule,
|
||||
maxIntervalSeconds: entry.maxIntervalSeconds,
|
||||
allowedAgeSeconds: allowed,
|
||||
lastStartedAt: ranAt,
|
||||
ageSeconds: age,
|
||||
state,
|
||||
lastStatus: row?.lastStatus ?? null,
|
||||
lastHttpStatus: row?.lastHttpStatus ?? null,
|
||||
lastDurationMs: row?.lastDurationMs ?? null,
|
||||
}
|
||||
})
|
||||
|
||||
const stale = checks.filter((check) => check.state === 'stale').length
|
||||
const pending = checks.filter((check) => check.state === 'pending').length
|
||||
|
||||
return {
|
||||
ok: stale === 0,
|
||||
checkedAt: new Date(params.now).toISOString(),
|
||||
total: checks.length,
|
||||
stale,
|
||||
pending,
|
||||
checks,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A run that finished non-2xx is not a missed tick (it ran), but it is worth
|
||||
* surfacing next to the heartbeat: a schedule that "works" while every run
|
||||
* fails is the same silent outage with an extra step.
|
||||
*/
|
||||
export function isFailingStatus(status: string | null, httpStatus: number | null): boolean {
|
||||
if (status && status !== 'ok') return true
|
||||
return httpStatus !== null && (httpStatus < 200 || httpStatus >= 300)
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import 'server-only'
|
||||
|
||||
import { createServiceClientNoCookies } from '@/lib/auth/api-keys'
|
||||
import { createLogger, type Logger } from '@/lib/logger'
|
||||
import type { CronHeartbeatRow } from './heartbeat-evaluation'
|
||||
|
||||
/**
|
||||
* Cron heartbeat storage.
|
||||
*
|
||||
* Two callers:
|
||||
* - lib/api/with-cron-context.ts, once per run (best effort: a monitoring
|
||||
* write must never fail the job it monitors).
|
||||
* - app/api/cron/health/route.ts, which declares unknown paths and reads the
|
||||
* table to decide ok/stale.
|
||||
*
|
||||
* Everything goes through the service role: cron_heartbeats has RLS enabled and
|
||||
* no policies, because instance-wide scheduling state is not tenant data.
|
||||
*
|
||||
* No-ops when Supabase is not configured (unit tests, `next build`, a hosted
|
||||
* deployment reading its schedule from Vercel). The heartbeat check is a
|
||||
* self-hosted guarantee; a missing store must not turn into a 500 in the cron
|
||||
* path.
|
||||
*/
|
||||
|
||||
const TABLE = 'cron_heartbeats'
|
||||
|
||||
const log: Logger = createLogger('cron/heartbeat-store')
|
||||
|
||||
function storeConfigured(): boolean {
|
||||
return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL && process.env.SUPABASE_SERVICE_ROLE_KEY)
|
||||
}
|
||||
|
||||
export interface CronHeartbeatInput {
|
||||
path: string
|
||||
/** ISO timestamp of when the run started (the staleness input). */
|
||||
startedAt: string
|
||||
/** ISO timestamp of when the run finished. */
|
||||
completedAt: string
|
||||
/** 'ok' | 'error'. Kept as a string so a future state does not need a migration. */
|
||||
status: string
|
||||
httpStatus: number | null
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert the run that just finished. Best effort by contract: logs and returns
|
||||
* on any failure, never throws. A heartbeat write that breaks a cron run would
|
||||
* be a worse outage than the one it exists to detect.
|
||||
*/
|
||||
export async function recordCronHeartbeat(input: CronHeartbeatInput): Promise<void> {
|
||||
if (!storeConfigured()) return
|
||||
|
||||
try {
|
||||
const { error } = await createServiceClientNoCookies()
|
||||
.from(TABLE)
|
||||
.upsert(
|
||||
{
|
||||
path: input.path,
|
||||
last_started_at: input.startedAt,
|
||||
last_completed_at: input.completedAt,
|
||||
last_status: input.status,
|
||||
last_http_status: input.httpStatus,
|
||||
last_duration_ms: input.durationMs,
|
||||
updated_at: input.completedAt,
|
||||
},
|
||||
{ onConflict: 'path' },
|
||||
)
|
||||
|
||||
if (error) {
|
||||
log.warn('heartbeat write failed', { path: input.path, reason: error.message })
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn('heartbeat write threw', {
|
||||
path: input.path,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register paths the generated schedule knows about but the table has not seen.
|
||||
*
|
||||
* This is what gives a new schedule a grace period: the row it creates carries
|
||||
* declared_at = now, so the evaluator can tell "added yesterday, has not had its
|
||||
* first tick" from "ran until last week and stopped". ignoreDuplicates keeps an
|
||||
* existing row (and its run history) untouched.
|
||||
*/
|
||||
export async function declareCronPaths(paths: readonly string[]): Promise<void> {
|
||||
if (!storeConfigured() || paths.length === 0) return
|
||||
|
||||
try {
|
||||
const { error } = await createServiceClientNoCookies()
|
||||
.from(TABLE)
|
||||
.upsert(
|
||||
paths.map((path) => ({ path })),
|
||||
{ onConflict: 'path', ignoreDuplicates: true },
|
||||
)
|
||||
|
||||
if (error) {
|
||||
log.warn('heartbeat declaration failed', { paths: paths.length, reason: error.message })
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn('heartbeat declaration threw', {
|
||||
paths: paths.length,
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* All heartbeat rows. Throws on failure: the health route must be able to say
|
||||
* "cannot verify" out loud (503) instead of reporting a healthy cron it could
|
||||
* not read.
|
||||
*/
|
||||
export async function loadCronHeartbeats(): Promise<CronHeartbeatRow[]> {
|
||||
const { data, error } = await createServiceClientNoCookies()
|
||||
.from(TABLE)
|
||||
.select(
|
||||
'path, declared_at, last_started_at, last_completed_at, last_status, last_http_status, last_duration_ms',
|
||||
)
|
||||
|
||||
if (error) throw new Error(`cron_heartbeats read failed: ${error.message}`)
|
||||
if (!data) return []
|
||||
|
||||
return data.map((row) => ({
|
||||
path: row.path as string,
|
||||
declaredAt: (row.declared_at as string | null) ?? null,
|
||||
lastStartedAt: (row.last_started_at as string | null) ?? null,
|
||||
lastCompletedAt: (row.last_completed_at as string | null) ?? null,
|
||||
lastStatus: (row.last_status as string | null) ?? null,
|
||||
lastHttpStatus: (row.last_http_status as number | null) ?? null,
|
||||
lastDurationMs: (row.last_duration_ms as number | null) ?? null,
|
||||
}))
|
||||
}
|
||||
|
||||
/** True when the heartbeat store is wired; the route and tests both ask. */
|
||||
export function cronHeartbeatStoreConfigured(): boolean {
|
||||
return storeConfigured()
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate.
|
||||
//
|
||||
// Variant: self-hosted (the crontab a Docker install runs).
|
||||
// Source of truth: the `crons` array in vercel.json, via
|
||||
// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts
|
||||
// fails CI if this file drifts from it.
|
||||
//
|
||||
// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last
|
||||
// recorded run of each path against maxIntervalSeconds and answers 503 when a
|
||||
// schedule stops arriving. The container healthcheck on the cron service calls
|
||||
// that endpoint, so a silent stop becomes a visible unhealthy container
|
||||
// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident).
|
||||
//
|
||||
// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an
|
||||
// average: a daily job must be allowed 24 h, and 29 February must not make a
|
||||
// yearly job look dead for four years.
|
||||
|
||||
export interface CronScheduleEntry {
|
||||
readonly path: string
|
||||
readonly schedule: string
|
||||
/** Worst-case seconds between two runs, over a 12-year scan window. */
|
||||
readonly maxIntervalSeconds: number
|
||||
}
|
||||
|
||||
export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [
|
||||
{ path: '/api/deadlines/status/cron', schedule: '0 6 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/invoices/recurring/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min
|
||||
{ path: '/api/settings/booking-templates/sync/cron', schedule: '30 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/tax-deadlines/cron', schedule: '0 0 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/enable-banking/sync/cron', schedule: '0 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/stripe/transactions/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/woocommerce/orders/cron', schedule: '45 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/shopify/orders/cron', schedule: '15 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/zettle/orders/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/documents/verify/cron', schedule: '0 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/documents/reanchor/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/sandbox/cleanup/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/events/cleanup/cron', schedule: '0 2 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/cloud-backup/auto-sync/cron', schedule: '0 * * * *', maxIntervalSeconds: 3600 }, // 60 min
|
||||
{ path: '/api/idempotency/cleanup/cron', schedule: '30 * * * *', maxIntervalSeconds: 3600 }, // 60 min
|
||||
{ path: '/api/pending-operations/expire/cron', schedule: '30 2 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/skatteverket/ombud/sync/cron', schedule: '30 3 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/skatteverket/skattekonto/sync/cron', schedule: '0 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/skatteverket/agi/kvittenser/cron', schedule: '*/15 * * * *', maxIntervalSeconds: 900 }, // 15 min
|
||||
{ path: '/api/extensions/skatteverket/vat/kvittenser/cron', schedule: '30 */2 * * *', maxIntervalSeconds: 7200 }, // 2 h
|
||||
{ path: '/api/webhooks/dispatch/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s
|
||||
{ path: '/api/extensions/whatsapp-inbox/sweep/cron', schedule: '* * * * *', maxIntervalSeconds: 60 }, // 60 s
|
||||
{ path: '/api/extensions/whatsapp-inbox/retention/cron', schedule: '15 4 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/invoice-inbox/sweep/cron', schedule: '*/2 * * * *', maxIntervalSeconds: 120 }, // 2 min
|
||||
{ path: '/api/extensions/invoice-inbox/underlag-reconcile/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/extensions/arcim-migration/complete-invoice-lines/cron', schedule: '20 * * * *', maxIntervalSeconds: 3600 }, // 60 min
|
||||
{ path: '/api/extensions/arcim-migration/cleanup/cron', schedule: '*/5 * * * *', maxIntervalSeconds: 300 }, // 5 min
|
||||
{ path: '/api/bookkeeping/accruals/post-due/cron', schedule: '15 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/receipt-hunt/cron', schedule: '30 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/notifications/bookkeeping-digest/cron', schedule: '45 5 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/notifications/multi-user-grace/cron', schedule: '0 7 * * *', maxIntervalSeconds: 86400 }, // 24 h
|
||||
{ path: '/api/peppol/inbound/cron', schedule: '*/10 * * * *', maxIntervalSeconds: 600 }, // 10 min
|
||||
{ path: '/api/peppol/outbound/status/cron', schedule: '5,20,35,50 * * * *', maxIntervalSeconds: 900 }, // 15 min
|
||||
{ path: '/api/connector/sync/cron', schedule: '17 * * * *', maxIntervalSeconds: 3600 }, // 60 min
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
# osv-scanner configuration. Read by .gitea/workflows/security.yml, which passes
|
||||
# --config when this file exists.
|
||||
#
|
||||
# The two entries below are the SheetJS advisories. They are ignored because the
|
||||
# fix is not obtainable from npm at any version: SheetJS stopped publishing to
|
||||
# the npm registry after 0.18.5 and ships 0.19+ from cdn.sheetjs.com, so both
|
||||
# GHSA records have `introduced: 0` and no `fixed` event (verified against
|
||||
# api.osv.dev 2026-11-15). This repo consumes `xlsx` as a URL dependency,
|
||||
# https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz, and 0.20.3 already
|
||||
# contains both fixes:
|
||||
#
|
||||
# GHSA-4r6h-8v6p-xvw6 (CVE-2023-30533, prototype pollution) — fixed upstream in 0.19.3
|
||||
# GHSA-5pgg-2g8v-p4x9 (CVE-2024-22363, ReDoS) — fixed upstream in 0.20.2
|
||||
#
|
||||
# So the scanner is reporting a version-resolution artefact, not a live defect.
|
||||
# The entries still expire: if we ever pin an older tarball, or migrate to a
|
||||
# maintained parser, this file must be revisited rather than inherited. Owner:
|
||||
# admin/accounted maintainers. Expiry is deliberate; the nightly scan re-reports
|
||||
# both advisories the day it passes.
|
||||
#
|
||||
# Deliberately NOT in scope here: the prototype-pollution and ReDoS classes are
|
||||
# about parsing untrusted workbooks, and lib/import/* does exactly that. The
|
||||
# mitigations are upstream (both fixed in our pinned version) — do not treat
|
||||
# this file as permission to accept unfixed parser CVEs.
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-4r6h-8v6p-xvw6"
|
||||
effectiveUntil = 2027-05-15
|
||||
reason = "CVE-2023-30533 fixed upstream in SheetJS 0.19.3; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory."
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-5pgg-2g8v-p4x9"
|
||||
effectiveUntil = 2027-05-15
|
||||
reason = "CVE-2024-22363 fixed upstream in SheetJS 0.20.2; we pin 0.20.3 from cdn.sheetjs.com. No npm release carries the fix, so no version can satisfy the advisory."
|
||||
@@ -4,6 +4,8 @@ import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
buildCrontab,
|
||||
buildScheduleEntries,
|
||||
buildScheduleModule,
|
||||
readVercelCrons,
|
||||
scheduleFor,
|
||||
EXCLUDED_PATHS,
|
||||
@@ -283,3 +285,39 @@ describe('every cron route has a schedule', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('lib/cron/schedules.generated.ts mirrors the self-hosted crontab', () => {
|
||||
const generated = readFileSync(join(ROOT, 'lib', 'cron', 'schedules.generated.ts'), 'utf8')
|
||||
const entries = buildScheduleEntries(crons, 'self-hosted')
|
||||
|
||||
it('on-disk file matches the generator output byte for byte', () => {
|
||||
expect(
|
||||
generated,
|
||||
'lib/cron/schedules.generated.ts is stale: run `npm run crontabs:generate`.',
|
||||
).toBe(buildScheduleModule(crons, 'self-hosted'))
|
||||
})
|
||||
|
||||
it('covers exactly the paths crontab.self-hosted runs', () => {
|
||||
expect(entries.map((entry) => entry.path)).toEqual(
|
||||
parseCrontab(crontabText('self-hosted')).map((job) => job.path),
|
||||
)
|
||||
})
|
||||
|
||||
it('carries the same schedule string as the crontab it mirrors', () => {
|
||||
const crontabJobs = parseCrontab(crontabText('self-hosted'))
|
||||
for (const entry of entries) {
|
||||
const job = crontabJobs.find((candidate) => candidate.path === entry.path)
|
||||
expect(job?.schedule, entry.path).toBe(entry.schedule)
|
||||
}
|
||||
})
|
||||
|
||||
it('gives every path a positive worst-case interval', () => {
|
||||
// A zero or NaN interval would make the heartbeat check either permanently
|
||||
// red or permanently green — the two ways this control can be useless.
|
||||
for (const entry of entries) {
|
||||
expect(Number.isFinite(entry.maxIntervalSeconds), entry.path).toBe(true)
|
||||
expect(entry.maxIntervalSeconds, entry.path).toBeGreaterThanOrEqual(60)
|
||||
expect(entry.maxIntervalSeconds, entry.path).toBeLessThanOrEqual(366 * 86_400)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -758,12 +758,18 @@ const PINNED_DEPS = [
|
||||
},
|
||||
{
|
||||
name: 'nodemailer',
|
||||
version: '9.1.1',
|
||||
version: '10.1.4',
|
||||
reason:
|
||||
'SMTP mailer for self-hosts (extensions/general/email/lib/smtp-service.ts). Zero-dependency MIT-0 ' +
|
||||
'package on the outbound-mail path; bumps are deliberate, reviewed PRs (audit surface), never silent. ' +
|
||||
'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj, ' +
|
||||
'same major; the rest of the 9.x advisories are only fixed in 10.x (major, separate reviewed PR).',
|
||||
'Bumped 9.0.5 -> 9.1.1 (2026-09-30) for HIGH advisories GHSA-2x7j-588g-ccc2 / GHSA-8m3c-c648-2xjj. ' +
|
||||
'Bumped 9.1.1 -> 10.1.4 (2026-11-15): the remaining 9.x advisories (CVE-2026-90776 header DoS, ' +
|
||||
'GHSA-v53p-9fqp-m79j addressparser backtracking) are only fixed in 10.x, and Trivy failed the ' +
|
||||
'security gate on them. The 10.0 API used here (createTransport + sendMail) is unchanged; the ' +
|
||||
'smtp-service suite covers the transport options that matter (requireTLS, AUTH, envelope). ' +
|
||||
'10.0.6 -> 10.1.4 the same day: OSV flagged 10.0.6 and the mailparser subtree pin 10.0.10 ' +
|
||||
'(GHSA-4ffr-jq9g-5ffx, GHSA-85x2-q946-cp5h, GHSA-8c8m-p3m8-cjg8, GHSA-g73g-hqqh-jr95, ' +
|
||||
'GHSA-m943-m63m-w62w); 10.1.4 is the first release with all five.',
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -24,11 +24,19 @@
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describeGap, maxIntervalSeconds } from '../lib/cron/cron-interval'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const ROOT = dirname(dirname(__filename))
|
||||
const VERCEL_JSON_PATH = join(ROOT, 'vercel.json')
|
||||
const DOCKER_DIR = join(ROOT, 'docker')
|
||||
/**
|
||||
* Heartbeat detection reads the cadence at runtime, so the same vercel.json
|
||||
* array is also emitted as a TypeScript module. One generator, one source of
|
||||
* truth: an expression that is scheduled but unknown to the health check is
|
||||
* exactly the "silently never runs" failure the check exists to catch.
|
||||
*/
|
||||
const SCHEDULE_MODULE_PATH = join(ROOT, 'lib', 'cron', 'schedules.generated.ts')
|
||||
|
||||
export type CrontabVariant = 'hosted' | 'self-hosted'
|
||||
|
||||
@@ -243,6 +251,82 @@ export function buildCrontab(
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
|
||||
export interface CronScheduleEntry {
|
||||
readonly path: string
|
||||
readonly schedule: string
|
||||
/** Worst-case seconds between two runs, over a 12-year scan window. */
|
||||
readonly maxIntervalSeconds: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The job set one variant actually runs (vercel.json minus exclusions, plus
|
||||
* that variant's EXTRA_JOBS) with its worst-case cadence resolved.
|
||||
*
|
||||
* The health check uses the self-hosted set, because that is the crontab a
|
||||
* Docker install runs; a hosted deployment gets its schedule from Vercel and is
|
||||
* not the deployment this check guards.
|
||||
*/
|
||||
export function buildScheduleEntries(
|
||||
crons: VercelCron[],
|
||||
variant: CrontabVariant,
|
||||
): CronScheduleEntry[] {
|
||||
const scheduled = crons
|
||||
.filter((cron) => !(cron.path in EXCLUDED_PATHS))
|
||||
.map((cron) => ({ path: cron.path, schedule: scheduleFor(cron, variant) }))
|
||||
const extra = EXTRA_JOBS[variant].map((job) => ({ path: job.path, schedule: job.schedule }))
|
||||
|
||||
return [...scheduled, ...extra].map((job) => ({
|
||||
path: job.path,
|
||||
schedule: job.schedule,
|
||||
maxIntervalSeconds: maxIntervalSeconds(job.schedule),
|
||||
}))
|
||||
}
|
||||
|
||||
/** Render lib/cron/schedules.generated.ts. Always LF, always newline-terminated. */
|
||||
export function buildScheduleModule(crons: VercelCron[], variant: CrontabVariant): string {
|
||||
const entries = buildScheduleEntries(crons, variant)
|
||||
const width =
|
||||
entries.reduce((max, entry) => Math.max(max, entry.path.length + 3), 0) + 2
|
||||
|
||||
const lines = [
|
||||
'// AUTO-GENERATED: do not edit. Run `npm run crontabs:generate` to regenerate.',
|
||||
'//',
|
||||
`// Variant: ${variant} (the crontab a Docker install runs).`,
|
||||
'// Source of truth: the `crons` array in vercel.json, via',
|
||||
'// scripts/generate-crontabs.ts. scripts/__tests__/generate-crontabs.test.ts',
|
||||
'// fails CI if this file drifts from it.',
|
||||
'//',
|
||||
'// Consumed by lib/cron/heartbeats.ts: app/api/cron/health compares the last',
|
||||
'// recorded run of each path against maxIntervalSeconds and answers 503 when a',
|
||||
'// schedule stops arriving. The container healthcheck on the cron service calls',
|
||||
'// that endpoint, so a silent stop becomes a visible unhealthy container',
|
||||
'// instead of a log line nobody reads (see the 2026-11-15 pids_limit incident).',
|
||||
'//',
|
||||
'// maxIntervalSeconds is the WORST-CASE gap over a 12-year scan, not an',
|
||||
'// average: a daily job must be allowed 24 h, and 29 February must not make a',
|
||||
'// yearly job look dead for four years.',
|
||||
'',
|
||||
'export interface CronScheduleEntry {',
|
||||
' readonly path: string',
|
||||
' readonly schedule: string',
|
||||
' /** Worst-case seconds between two runs, over a 12-year scan window. */',
|
||||
' readonly maxIntervalSeconds: number',
|
||||
'}',
|
||||
'',
|
||||
'export const CRON_SCHEDULES: readonly CronScheduleEntry[] = [',
|
||||
...entries.map(
|
||||
(entry) =>
|
||||
` { path: '${entry.path}',`.padEnd(width + 2) +
|
||||
` schedule: '${entry.schedule}',`.padEnd(24) +
|
||||
` maxIntervalSeconds: ${entry.maxIntervalSeconds} }, // ${describeGap(entry.maxIntervalSeconds)}`,
|
||||
),
|
||||
']',
|
||||
'',
|
||||
]
|
||||
|
||||
return `${lines.join('\n')}`
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const crons = readVercelCrons(readFileSync(VERCEL_JSON_PATH, 'utf8'))
|
||||
|
||||
@@ -282,6 +366,13 @@ function main(): void {
|
||||
)
|
||||
}
|
||||
|
||||
// The runtime half of the same source of truth: heartbeat detection needs
|
||||
// each path's worst-case cadence, so the self-hosted job set is emitted as a
|
||||
// TypeScript module next to the crontab it mirrors.
|
||||
const scheduleEntries = buildScheduleEntries(crons, 'self-hosted')
|
||||
writeFileSync(SCHEDULE_MODULE_PATH, buildScheduleModule(crons, 'self-hosted'), 'utf8')
|
||||
console.log(`Wrote lib/cron/schedules.generated.ts: ${scheduleEntries.length} schedules`)
|
||||
|
||||
const skipped = Object.keys(EXCLUDED_PATHS)
|
||||
if (skipped.length > 0) {
|
||||
console.log(`Excluded ${skipped.length} path(s): ${skipped.join(', ')}`)
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
-- Cron heartbeat: make a schedule that stops arriving visible.
|
||||
--
|
||||
-- The 2026-11-15 incident: accounted-cron lost 1 497 of 10 455 runs in a day
|
||||
-- (14,3 %) because pids_limit 30 made `fork/exec /bin/sh` fail whenever the
|
||||
-- per-minute schedules coincided. supercronic silently skipped those ticks.
|
||||
-- The container stayed "Up", every endpoint kept answering 200, and the only
|
||||
-- trace was a log line in the cron container. lib/auth/cron.ts already reports
|
||||
-- an individual FAILED run to the observability sink; nothing could see a run
|
||||
-- that never started.
|
||||
--
|
||||
-- This table is the durable half of the fix: every cron run upserts one row per
|
||||
-- path with when it started and how it ended, and app/api/cron/health compares
|
||||
-- the newest row against the path's worst-case cadence (lib/cron/
|
||||
-- schedules.generated.ts, generated from vercel.json). The cron container's
|
||||
-- healthcheck calls that endpoint, so a schedule that stops arriving turns into
|
||||
-- an unhealthy container instead of a silent gap.
|
||||
--
|
||||
-- Shape notes:
|
||||
-- * One row per scheduled path, not per run. Volume matters: 34 schedules at
|
||||
-- up to 1 440 runs/day each would be ~8k rows/day forever, and all the check
|
||||
-- needs is "when did this last start". The upsert keeps the table at 34
|
||||
-- rows.
|
||||
-- * No company_id. A cron run is instance-wide, and pretending otherwise
|
||||
-- would make the same schedule look healthy in one tenant and dead in
|
||||
-- another with no way to reconcile them.
|
||||
-- * declared_at is set when the health check first sees a path in the
|
||||
-- generated schedule list. That is what separates "added to vercel.json
|
||||
-- yesterday, has not had its first tick yet" from "stopped running a week
|
||||
-- ago": a path with no run is only stale once declared_at is older than its
|
||||
-- own cadence, never immediately. Without it, every new job and every fresh
|
||||
-- install would report red until its first run.
|
||||
-- * last_started_at, not last_completed_at, is the staleness input. A run that
|
||||
-- hangs never completes, and supercronic then skips every later tick —
|
||||
-- exactly the failure that must look dead.
|
||||
--
|
||||
-- Access: RLS enabled with no policies. Only the service role (the cron paths
|
||||
-- and the health route, via createServiceClientNoCookies) reads or writes it;
|
||||
-- anon/authenticated have no business seeing instance-wide scheduling state.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.cron_heartbeats (
|
||||
path TEXT PRIMARY KEY,
|
||||
declared_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_started_at TIMESTAMPTZ,
|
||||
last_completed_at TIMESTAMPTZ,
|
||||
last_status TEXT,
|
||||
last_http_status INTEGER,
|
||||
last_duration_ms INTEGER,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
COMMENT ON TABLE public.cron_heartbeats IS
|
||||
'One row per scheduled cron path: newest run start/outcome. Read by app/api/cron/health to detect schedules that stop arriving (2026-11-15 pids_limit incident).';
|
||||
COMMENT ON COLUMN public.cron_heartbeats.declared_at IS
|
||||
'When the health check first saw this path in the generated schedule list. Grace for a path that has never run.';
|
||||
COMMENT ON COLUMN public.cron_heartbeats.last_started_at IS
|
||||
'Start of the newest run. The staleness input: a hung run also stops every later tick.';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_heartbeats_last_started_at
|
||||
ON public.cron_heartbeats (last_started_at DESC NULLS LAST);
|
||||
|
||||
ALTER TABLE public.cron_heartbeats ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
REVOKE ALL ON public.cron_heartbeats FROM anon;
|
||||
REVOKE ALL ON public.cron_heartbeats FROM authenticated;
|
||||
Reference in New Issue
Block a user