diff --git a/DECISIONS.md b/DECISIONS.md index e317c5aa..87e8aa98 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -1147,3 +1147,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-21] Provider underlag import (arcim-migration /import-documents) is now time-budgeted + resumable (200 s slice, stable provider-id order, `cursor`/`nextCursor`/`partial`, UI loops with "x av y") and opts out of AI extraction (`extractionOwner: 'none'`, stamped `skipped:opted_out`): Fabian's 113-file Fortnox import ran every file's Sonnet extraction inline inside one request, hit the hosted 300 s function limit after ~17 files twice (09:02 and 10:49 UTC) and the UI showed the generic "underlagen kunde inte importeras". The files are linked to posted verifikat on arrival, so the extraction bought nothing and cost a model call per file; chunking rather than raising maxDuration because a 2,000-file archive would still not fit. Opt-out is per-call (not "skip whenever journal_entry_id is set") to keep the diff scoped. [2026-08-21] MCP signed Storage URLs (gnubok_create_document_upload upload_url, gnubok_get_document_content signed_url, audit-package download_url) are served through a same-origin proxy, /api/storage/[...path] → .supabase.co/storage/v1/object/{sign,upload/sign}/documents/..., instead of a Next rewrite: Claude Desktop's sandbox only reaches the MCP host (app.accounted.se) and blocked the PUT to supabase.co (Fabian, 2026-08-21). A route handler reads NEXT_PUBLIC_SUPABASE_URL at runtime (rewrites bake at build, which breaks the Docker image), is deliberately NOT withRouteContext (the signed token is the only credential, validated by Storage per object path; the proxy forwards only signed documents-bucket paths to our own host) and is a no-op rewrite when NEXT_PUBLIC_APP_URL is unset so a self-host never gets a localhost link. [2026-08-21] RIP-4 cascade step 3 (UI): the AI booking proposal is surfaced INSIDE the existing QuickReviewDialog rather than a new inline-row card, so it reuses that dialog's proven, deterministic, balanced commit path (POST /api/transactions/[id]/categorize) instead of a parallel one. components/transactions/AiCategorizeProposal.tsx fetches POST /api/agent/categorize on dialog open (keyed on tx.id so it remounts per transaction), pre-fills accountOverride + vatTreatment via handleAccountChange (class-2 VAT clearing preserved), and shows the confidence band (säker/trolig/välj konto) + "Varför" + the candidate alternatives (click to re-apply). Falls back SILENTLY to the deterministic defaults on error, and shows a soft note on 503 (ai_unconfigured) — the dialog always works without AI. NO silent auto-posting (founder call, avoids the storno-on-undo mess): "säker" = pre-filled, one-tap Bokför via the dialog's existing button; true hands-off auto-book waits for calibration. i18n: strings inline Swedish for now (assistant surface), lift to messages/{sv,en}.json before final merge. Confidence bands (0.8/0.5) are placeholders until calibration. Needs founder visual sign-off before merge ([[project_nav_ia_redesign]]). +[2026-08-21] RIP-4 step 4 = calibration. lib/agent/categorize/calibration.ts is the engine: isotonic regression via pool-adjacent-violators (distribution-free, monotonic) over (confidence, was_correct) samples → a calibrator that turns raw selector confidence into a probability that actually means what it says; plus reliabilityByBucket/ECE and bandFor(). bandFor NEVER returns 'auto' without a fitted calibrator (no silent booking on an unproven score) and never auto-books above an amount cap (default 2000 kr) — so "säker" stays honest until proven. Measurement loop: migration 20260821100000 categorize_calibration_samples (append-only, company-scoped RLS, confidence CHECK [0,1]); POST /api/agent/categorize/outcome logs one sample (proposed vs booked account → was_correct) fire-and-forget from QuickReviewDialog on a successful book (sandbox skipped to keep the corpus clean); AiCategorizeProposal surfaces the proposal metadata via onProposal. scripts/fit-categorize-calibration.ts (READ-ONLY) prints the reliability diagram + ECE + fitted calibrator once data exists — run it in a few weeks, then store the calibrator/thresholds where bandFor reads them and only THEN consider enabling auto-book. Fitting needs >=200 real samples so nothing calibrates today; the loop just starts collecting. Migration applies on merge (auto-apply-on-merge active) — not applied manually. diff --git a/app/api/agent/categorize/outcome/__tests__/route.test.ts b/app/api/agent/categorize/outcome/__tests__/route.test.ts new file mode 100644 index 00000000..4438de97 --- /dev/null +++ b/app/api/agent/categorize/outcome/__tests__/route.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { NextResponse } from 'next/server' +import { createMockRequest } from '@/tests/helpers' + +const requireAuthMock = vi.fn() +vi.mock('@/lib/auth/require-auth', () => ({ requireAuth: () => requireAuthMock() })) +vi.mock('@/lib/company/context', () => ({ getActiveCompanyId: vi.fn().mockResolvedValue('company-1') })) +const guardSandbox = vi.fn() +vi.mock('@/lib/sandbox/guard', () => ({ guardSandbox: () => guardSandbox() })) + +import { POST } from '../route' + +const inserts: Record[] = [] +function makeSupabase(membership: unknown = { user_id: 'user-1' }) { + return { + from(table: string) { + const chain = { + select: () => chain, + eq: () => chain, + maybeSingle: async () => ({ data: membership }), + insert: async (payload: Record) => { + if (table === 'categorize_calibration_samples') inserts.push(payload) + return { error: null } + }, + } + return chain + }, + } +} +const supabase = makeSupabase() + +const body = (o: Record = {}) => ({ confidence: 0.9, booked_account: '5410', ...o }) + +beforeEach(() => { + vi.clearAllMocks() + inserts.length = 0 + requireAuthMock.mockResolvedValue({ user: { id: 'user-1' }, supabase, error: null }) + guardSandbox.mockResolvedValue(null) +}) + +describe('POST /api/agent/categorize/outcome', () => { + it('401 when unauthenticated', async () => { + requireAuthMock.mockResolvedValue({ user: null, supabase, error: NextResponse.json({ error: 'x' }, { status: 401 }) }) + expect((await POST(createMockRequest('/x', { method: 'POST', body: body() }))).status).toBe(401) + }) + + it('400 on an invalid body', async () => { + expect((await POST(createMockRequest('/x', { method: 'POST', body: { confidence: 2 } }))).status).toBe(400) + expect((await POST(createMockRequest('/x', { method: 'POST', body: { booked_account: '5410' } }))).status).toBe(400) + }) + + it('403 without company membership', async () => { + requireAuthMock.mockResolvedValue({ user: { id: 'user-1' }, supabase: makeSupabase(null), error: null }) + expect((await POST(createMockRequest('/x', { method: 'POST', body: body() }))).status).toBe(403) + }) + + it('skips sandbox bookings (204, no sample)', async () => { + guardSandbox.mockResolvedValue(NextResponse.json({ error: 'sandbox' }, { status: 403 })) + const res = await POST(createMockRequest('/x', { method: 'POST', body: body() })) + expect(res.status).toBe(204) + expect(inserts).toHaveLength(0) + }) + + it('logs was_correct=true when the proposed account was booked', async () => { + const res = await POST( + createMockRequest('/x', { + method: 'POST', + body: body({ proposed_account: '5410', booked_account: '5410', confidence: 0.86, source: 'counterparty_template', amount: 499 }), + }), + ) + expect(res.status).toBe(204) + expect(inserts).toHaveLength(1) + expect(inserts[0]).toMatchObject({ + company_id: 'company-1', + confidence: 0.86, + proposed_account: '5410', + booked_account: '5410', + was_correct: true, + source: 'counterparty_template', + amount: 499, + }) + }) + + it('logs was_correct=false when the user booked a different account', async () => { + await POST(createMockRequest('/x', { method: 'POST', body: body({ proposed_account: '5410', booked_account: '6110' }) })) + expect(inserts[0].was_correct).toBe(false) + }) + + it('logs was_correct=false when there was no proposed account', async () => { + await POST(createMockRequest('/x', { method: 'POST', body: body({ proposed_account: null, booked_account: '5410' }) })) + expect(inserts[0].was_correct).toBe(false) + }) +}) diff --git a/app/api/agent/categorize/outcome/route.ts b/app/api/agent/categorize/outcome/route.ts new file mode 100644 index 00000000..f9fdacbd --- /dev/null +++ b/app/api/agent/categorize/outcome/route.ts @@ -0,0 +1,81 @@ +import { NextResponse } from 'next/server' +import { z } from 'zod' +import { requireAuth } from '@/lib/auth/require-auth' +import { getActiveCompanyId } from '@/lib/company/context' +import { guardSandbox } from '@/lib/sandbox/guard' + +/** + * POST /api/agent/categorize/outcome: log one calibration sample. + * + * Called (fire-and-forget) after a user books an AI proposal: it records the + * confidence the model reported and whether the proposed account was the one + * actually booked. That corpus is what lib/agent/categorize/calibration.ts + * later fits an isotonic calibrator on, so "säker" can be made to mean ~right. + * + * Telemetry only: it never posts anything and is gated on auth + membership. + * Sandbox bookings run on seed data, so they are silently skipped (a 204) to + * keep the corpus clean. + */ + +const Schema = z.object({ + company_id: z.string().uuid().optional(), + confidence: z.number().min(0).max(1), + proposed_account: z.string().max(20).nullable().optional(), + booked_account: z.string().min(1).max(20), + agreement: z.number().min(0).max(1).nullable().optional(), + model_confidence: z.enum(['high', 'medium', 'low']).nullable().optional(), + source: z.string().max(40).nullable().optional(), + amount: z.number().nullable().optional(), +}) + +const noContent = () => new Response(null, { status: 204 }) + +export async function POST(request: Request): Promise { + const { user, supabase, error } = await requireAuth() + if (error) return error + + let body: unknown + try { + body = await request.json() + } catch { + return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 }) + } + const parsed = Schema.safeParse(body) + if (!parsed.success) return NextResponse.json({ error: 'Invalid body' }, { status: 400 }) + + const companyId = parsed.data.company_id ?? (await getActiveCompanyId(supabase, user.id)) + if (!companyId) return noContent() + + const { data: membership } = await supabase + .from('company_members') + .select('user_id') + .eq('company_id', companyId) + .eq('user_id', user.id) + .maybeSingle() + if (!membership) return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + + // Sandbox bookings are seed data: don't pollute the calibration corpus. + const blocked = await guardSandbox(supabase, companyId) + if (blocked) return noContent() + + const proposed = parsed.data.proposed_account ?? null + + // Best-effort: a failed telemetry insert must never surface to the user. + try { + await supabase.from('categorize_calibration_samples').insert({ + company_id: companyId, + confidence: parsed.data.confidence, + agreement: parsed.data.agreement ?? null, + model_confidence: parsed.data.model_confidence ?? null, + source: parsed.data.source ?? null, + proposed_account: proposed, + booked_account: parsed.data.booked_account, + was_correct: proposed !== null && proposed === parsed.data.booked_account, + amount: parsed.data.amount ?? null, + }) + } catch { + // swallow + } + + return noContent() +} diff --git a/components/transactions/AiCategorizeProposal.tsx b/components/transactions/AiCategorizeProposal.tsx index 9df12239..60eed82e 100644 --- a/components/transactions/AiCategorizeProposal.tsx +++ b/components/transactions/AiCategorizeProposal.tsx @@ -34,11 +34,22 @@ interface ProposalDto { reverseCharge: boolean confidence: number agreement: number + modelConfidence: 'high' | 'medium' | 'low' + fromCandidate: boolean reasoning: string choice: { kind: 'candidate' | 'category' | 'needs_review' } candidates: CandidateDto[] } +/** What the dialog needs to log a calibration sample when the user books. */ +export interface AiProposalMeta { + account: string + confidence: number + agreement: number + modelConfidence: 'high' | 'medium' | 'low' + source: string +} + type State = | { status: 'loading' } | { status: 'error' } @@ -51,6 +62,8 @@ interface Props { open: boolean /** Apply an account + VAT to the dialog fields. */ onApply: (account: string, vat: VatTreatment | 'none') => void + /** Surface the proposal metadata so the dialog can log a calibration sample on book. */ + onProposal?: (meta: AiProposalMeta) => void } type Band = 'sure' | 'likely' | 'review' @@ -63,7 +76,7 @@ function bandOf(p: ProposalDto): Band { const BAND_LABEL: Record = { sure: 'Säker', likely: 'Trolig', review: 'Välj konto' } -export default function AiCategorizeProposal({ transactionId, open, onApply }: Props) { +export default function AiCategorizeProposal({ transactionId, open, onApply, onProposal }: Props) { const [state, setState] = useState({ status: 'loading' }) // Apply the pick to the dialog exactly once per fetch, so the user's later // manual edits are never clobbered by a re-render. @@ -98,15 +111,34 @@ export default function AiCategorizeProposal({ transactionId, open, onApply }: P } }, [open, transactionId]) - // Pre-fill the dialog once, when a confident proposal with a real account arrives. + const reportedRef = useRef(false) + + // When a proposal with a real account arrives: surface its metadata to the + // dialog (for calibration logging on book) and pre-fill the fields. useEffect(() => { if (state.status !== 'ready') return const p = state.proposal - if (!p.account || appliedRef.current === p.account) return - if (bandOf(p) === 'review') return + if (!p.account) return + + if (!reportedRef.current) { + reportedRef.current = true + const source = p.fromCandidate + ? (p.candidates.find((c) => c.account === p.account)?.source ?? 'candidate') + : 'category' + onProposal?.({ + account: p.account, + confidence: p.confidence, + agreement: p.agreement, + modelConfidence: p.modelConfidence, + source, + }) + } + + // Pre-fill only when it's not the low "review" band, and only once. + if (appliedRef.current === p.account || bandOf(p) === 'review') return appliedRef.current = p.account onApply(p.account, p.vatTreatment ?? 'none') - }, [state, onApply]) + }, [state, onApply, onProposal]) if (state.status === 'loading') { return ( diff --git a/components/transactions/QuickReviewDialog.tsx b/components/transactions/QuickReviewDialog.tsx index 9f248e48..204e1ec6 100644 --- a/components/transactions/QuickReviewDialog.tsx +++ b/components/transactions/QuickReviewDialog.tsx @@ -27,7 +27,7 @@ import InboxDocumentPicker from '@/components/bookkeeping/InboxDocumentPicker' import type { UploadedFile } from '@/components/bookkeeping/DocumentUploadZone' import type { AvailableInboxDoc } from '@/components/bookkeeping/InboxDocumentPicker' import VatTreatmentSelect from './VatTreatmentSelect' -import AiCategorizeProposal from './AiCategorizeProposal' +import AiCategorizeProposal, { type AiProposalMeta } from './AiCategorizeProposal' import { VAT_TREATMENT_OPTIONS } from './transaction-types' import type { TransactionWithInvoice } from './transaction-types' import type { TransactionCategory, VatTreatment, BASAccount, EntityType, LinePatternEntry } from '@/types' @@ -91,6 +91,9 @@ export default function QuickReviewDialog({ const [accountOverride, setAccountOverride] = useState(defaultAccount ?? '') const [vatTreatment, setVatTreatment] = useState(defaultVat) const [accounts, setAccounts] = useState([]) + // The AI proposal shown this session, kept so we can log a calibration sample + // (proposed vs actually booked) once the user confirms. + const [aiProposal, setAiProposal] = useState(null) const [isProcessing, setIsProcessing] = useState(false) const [error, setError] = useState(null) const [uploadedFiles, setUploadedFiles] = useState([]) @@ -291,6 +294,24 @@ export default function QuickReviewDialog({ Object.keys(cleanedDims).length > 0 ? cleanedDims : undefined, ) + // Calibration telemetry: what the model proposed vs what was actually + // booked. Best-effort and fire-and-forget — never blocks the booking. + if (journalEntryId && aiProposal) { + void fetch('/api/agent/categorize/outcome', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + confidence: aiProposal.confidence, + agreement: aiProposal.agreement, + model_confidence: aiProposal.modelConfidence, + source: aiProposal.source, + proposed_account: aiProposal.account, + booked_account: override ?? catDefault, + amount: Math.abs(sekAmount), + }), + }).catch(() => {}) + } + // Attach the uploaded underlag to the verifikat the booking just created. // BFL 5 kap 7 § requires the verifikation to reference its underlag and // BFL 7 kap requires that underlag to be archived with it; the verifikat @@ -447,6 +468,7 @@ export default function QuickReviewDialog({ key={tx.id} transactionId={tx.id} open={open} + onProposal={setAiProposal} onApply={(account, vat) => { handleAccountChange(account) // handleAccountChange clears VAT for class-2 accounts; for the diff --git a/lib/agent/categorize/__tests__/calibration.test.ts b/lib/agent/categorize/__tests__/calibration.test.ts new file mode 100644 index 00000000..9b67840b --- /dev/null +++ b/lib/agent/categorize/__tests__/calibration.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect } from 'vitest' +import { + reliabilityByBucket, + expectedCalibrationError, + fitIsotonic, + calibrate, + bandFor, + type Sample, +} from '../calibration' + +// Build N samples at a given confidence with a given true accuracy. +function samplesAt(confidence: number, accuracy: number, n: number): Sample[] { + const correct = Math.round(accuracy * n) + return Array.from({ length: n }, (_, i) => ({ confidence, correct: i < correct })) +} + +describe('reliabilityByBucket', () => { + it('reports empirical accuracy per confidence bucket', () => { + const samples = [...samplesAt(0.95, 0.7, 100), ...samplesAt(0.55, 0.5, 100)] + const buckets = reliabilityByBucket(samples, 10) + const high = buckets.find((b) => b.lo === 0.9)! + const mid = buckets.find((b) => b.lo === 0.5)! + expect(high.n).toBe(100) + expect(high.accuracy).toBeCloseTo(0.7, 2) + expect(mid.accuracy).toBeCloseTo(0.5, 2) + }) +}) + +describe('expectedCalibrationError', () => { + it('is ~0 for a well-calibrated corpus', () => { + const samples = [...samplesAt(0.9, 0.9, 100), ...samplesAt(0.5, 0.5, 100)] + expect(expectedCalibrationError(samples)).toBeLessThan(0.02) + }) + it('is large for an overconfident corpus', () => { + // Model says 0.95 but is only right 60% of the time. + const samples = samplesAt(0.95, 0.6, 200) + expect(expectedCalibrationError(samples)).toBeGreaterThan(0.3) + }) +}) + +describe('fitIsotonic', () => { + it('returns null below the minimum sample count (stay uncalibrated)', () => { + expect(fitIsotonic(samplesAt(0.9, 0.9, 50), 200)).toBeNull() + }) + + it('learns to pull an overconfident score down', () => { + // 0.95 raw but only 60% correct → calibrated should be ~0.6, not ~0.95. + const samples = [...samplesAt(0.95, 0.6, 300), ...samplesAt(0.5, 0.5, 300)] + const cal = fitIsotonic(samples, 200)! + expect(cal.fittedOn).toBe(600) + expect(calibrate(0.95, cal)).toBeLessThan(0.7) + expect(calibrate(0.95, cal)).toBeGreaterThan(0.5) + }) + + it('is monotonic non-decreasing (PAV guarantee)', () => { + // Deliberately non-monotonic raw→accuracy; PAV must pool the violation. + const samples = [ + ...samplesAt(0.4, 0.8, 200), // low conf but high accuracy + ...samplesAt(0.7, 0.5, 200), // higher conf but lower accuracy (violation) + ...samplesAt(0.9, 0.9, 200), + ] + const cal = fitIsotonic(samples, 200)! + const grid = [0.3, 0.5, 0.7, 0.9] + const vals = grid.map((r) => calibrate(r, cal)) + for (let i = 1; i < vals.length; i++) { + expect(vals[i]).toBeGreaterThanOrEqual(vals[i - 1]) + } + }) + + it('lifts an underconfident score up', () => { + // 0.55 raw but 90% correct → calibrated should be well above 0.55. + const samples = [...samplesAt(0.55, 0.9, 300), ...samplesAt(0.2, 0.2, 300)] + const cal = fitIsotonic(samples, 200)! + expect(calibrate(0.55, cal)).toBeGreaterThan(0.8) + }) +}) + +describe('bandFor', () => { + it('never auto-books without a calibrator, even at high raw confidence', () => { + expect(bandFor(0.99, null, { amount: 100 })).toBe('suggest') + expect(bandFor(0.6, null)).toBe('review') + }) + + it('auto-books a small amount at high calibrated probability with a calibrator', () => { + const cal = fitIsotonic(samplesAt(0.96, 0.99, 400), 200)! + expect(bandFor(0.96, cal, { amount: 499 })).toBe('auto') + }) + + it('never auto-books a large amount, however confident', () => { + const cal = fitIsotonic(samplesAt(0.96, 0.99, 400), 200)! + expect(bandFor(0.96, cal, { amount: 50000 })).toBe('suggest') + }) + + it('drops to review when the calibrated probability is low even if raw was high', () => { + // raw 0.95 but the calibrator learned it is really ~0.6 → not auto, not even suggest at 0.7. + const cal = fitIsotonic([...samplesAt(0.95, 0.6, 300), ...samplesAt(0.5, 0.5, 300)], 200)! + expect(bandFor(0.95, cal, { amount: 100 })).toBe('review') + }) + + it('honours custom thresholds', () => { + const cal = fitIsotonic(samplesAt(0.8, 0.85, 400), 200)! + expect(bandFor(0.8, cal, { amount: 100, autoThreshold: 0.8 })).toBe('auto') + }) +}) diff --git a/lib/agent/categorize/calibration.ts b/lib/agent/categorize/calibration.ts new file mode 100644 index 00000000..fbbc8ade --- /dev/null +++ b/lib/agent/categorize/calibration.ts @@ -0,0 +1,175 @@ +/** + * Calibration for the auto-booking cascade (step 4). + * + * The Tier-2 selector emits a raw combined confidence in [0,1], but raw model/ + * ensemble scores are NOT calibrated: "0.9" does not mean "right 90% of the + * time" until you fit it against real outcomes. This module turns a corpus of + * (confidence, was_correct) samples — collected as users book or edit the + * proposals — into a monotonic calibrator (isotonic regression), and decides + * the auto-book / suggest / review band from the CALIBRATED probability. + * + * Until a calibrator is fitted (not enough data yet), `bandFor` runs in + * uncalibrated mode: it never returns 'auto' (no silent booking on an + * unproven score) and uses conservative raw thresholds for suggest/review. + * This is what keeps "säker" honest before the data exists. + * + * Pure functions only: no I/O. The samples come from the caller. + */ + +export interface Sample { + /** Raw combined confidence the selector reported, in [0,1]. */ + confidence: number + /** True when the proposed account was the one actually booked (unedited). */ + correct: boolean +} + +/** A fitted, monotonic non-decreasing mapping from raw confidence to calibrated probability. */ +export interface Calibrator { + /** Sorted ascending; each point maps a raw confidence to its calibrated probability. */ + points: { raw: number; calibrated: number }[] + /** How many samples it was fitted on (for trust / staleness checks). */ + fittedOn: number +} + +function clamp01(x: number): number { + return x < 0 ? 0 : x > 1 ? 1 : x +} + +/** + * Reliability diagram: bucket samples by confidence and report empirical + * accuracy per bucket. The gap between meanConfidence and accuracy is the + * miscalibration; the classic "overconfident" model shows accuracy < confidence. + */ +export function reliabilityByBucket( + samples: Sample[], + bins = 10, +): { lo: number; hi: number; n: number; meanConfidence: number; accuracy: number }[] { + const out: { lo: number; hi: number; n: number; meanConfidence: number; accuracy: number }[] = [] + for (let b = 0; b < bins; b++) { + const lo = b / bins + const hi = (b + 1) / bins + const inBucket = samples.filter( + (s) => s.confidence >= lo && (b === bins - 1 ? s.confidence <= hi : s.confidence < hi), + ) + const n = inBucket.length + const meanConfidence = n ? inBucket.reduce((a, s) => a + s.confidence, 0) / n : 0 + const accuracy = n ? inBucket.filter((s) => s.correct).length / n : 0 + out.push({ lo, hi, n, meanConfidence, accuracy }) + } + return out +} + +/** + * Expected Calibration Error: the sample-weighted average |confidence - accuracy| + * across buckets. 0 = perfectly calibrated. Empty buckets contribute nothing. + */ +export function expectedCalibrationError(samples: Sample[], bins = 10): number { + if (samples.length === 0) return 0 + const buckets = reliabilityByBucket(samples, bins) + let err = 0 + for (const bkt of buckets) { + if (bkt.n === 0) continue + err += (bkt.n / samples.length) * Math.abs(bkt.meanConfidence - bkt.accuracy) + } + return err +} + +/** + * Fit an isotonic (monotonic non-decreasing) calibrator via pool-adjacent- + * violators (PAV). This is the standard, distribution-free way to calibrate a + * ranking score: it never assumes a parametric shape, only that higher raw + * confidence should not map to lower true accuracy. + * + * Returns null when there is too little data to trust (below `minSamples`): + * the caller then stays in uncalibrated mode. + */ +export function fitIsotonic(samples: Sample[], minSamples = 200): Calibrator | null { + if (samples.length < minSamples) return null + + // Sort by raw confidence; y = 1 for correct, 0 for wrong. + const sorted = [...samples].sort((a, b) => a.confidence - b.confidence) + + // PAV over blocks of (sum, count) → each block's mean is monotonic non-decreasing. + interface Block { + x: number // representative raw confidence (max in block, so it's a step boundary) + sum: number + count: number + } + const blocks: Block[] = [] + for (const s of sorted) { + blocks.push({ x: s.confidence, sum: s.correct ? 1 : 0, count: 1 }) + // Merge while the last block violates monotonicity (its mean < previous mean). + while ( + blocks.length >= 2 && + blocks[blocks.length - 1].sum / blocks[blocks.length - 1].count < + blocks[blocks.length - 2].sum / blocks[blocks.length - 2].count + ) { + const b = blocks.pop()! + const a = blocks.pop()! + blocks.push({ x: Math.max(a.x, b.x), sum: a.sum + b.sum, count: a.count + b.count }) + } + } + + const points = blocks.map((b) => ({ raw: b.x, calibrated: clamp01(b.sum / b.count) })) + return { points, fittedOn: samples.length } +} + +/** + * Map a raw confidence to its calibrated probability using a fitted calibrator. + * Step function: the calibrated value of the first block whose boundary is ≥ raw + * (clamped to the ends). Monotonic by construction. + */ +export function calibrate(raw: number, calibrator: Calibrator): number { + const r = clamp01(raw) + const { points } = calibrator + if (points.length === 0) return r + for (const p of points) { + if (r <= p.raw) return p.calibrated + } + return points[points.length - 1].calibrated +} + +export type Band = 'auto' | 'suggest' | 'review' + +export interface BandOptions { + /** The transaction's absolute amount (SEK); large/unusual never auto-books. */ + amount?: number + /** Above this the item is never auto-booked regardless of confidence (default 2000 kr). */ + autoBookAmountCap?: number + /** Calibrated probability required to auto-book (default 0.95). */ + autoThreshold?: number + /** Calibrated probability required to pre-fill as a suggestion (default 0.70). */ + suggestThreshold?: number +} + +/** + * Decide the band from a raw confidence, calibrating first when a calibrator is + * available. Without a calibrator (not enough data), 'auto' is never returned: + * an unproven score must not silently book. + */ +export function bandFor( + rawConfidence: number, + calibrator: Calibrator | null, + opts: BandOptions = {}, +): Band { + const { + amount, + autoBookAmountCap = 2000, + autoThreshold = 0.95, + suggestThreshold = 0.7, + } = opts + + const p = calibrator ? calibrate(rawConfidence, calibrator) : clamp01(rawConfidence) + + // Auto-book only with a real calibrator, a high calibrated probability, and a + // small/routine amount. Any of those missing → at most a suggestion. + if ( + calibrator && + p >= autoThreshold && + (amount === undefined || Math.abs(amount) <= autoBookAmountCap) + ) { + return 'auto' + } + if (p >= suggestThreshold) return 'suggest' + return 'review' +} diff --git a/lib/reports/full-archive-export.ts b/lib/reports/full-archive-export.ts index d6d27011..c73ba27b 100644 --- a/lib/reports/full-archive-export.ts +++ b/lib/reports/full-archive-export.ts @@ -1024,6 +1024,7 @@ export const ARCHIVE_EXCLUDED_TABLES: Record = { booking_template_usage: 'usage telemetry', calendar_feeds: 'feed tokens (secrets)', capability_grants: 'entitlement state', + categorize_calibration_samples: 'auto-booking confidence telemetry, not räkenskapsinformation', chat_messages: 'AI assistant state, not räkenskapsinformation', chat_sessions: 'AI assistant state, not räkenskapsinformation', company_capability_config: 'entitlement state', diff --git a/scripts/fit-categorize-calibration.ts b/scripts/fit-categorize-calibration.ts new file mode 100644 index 00000000..c2ddbd59 --- /dev/null +++ b/scripts/fit-categorize-calibration.ts @@ -0,0 +1,91 @@ +/** + * Fit and report the auto-booking confidence calibration (RIP-4 step 4). + * + * READ-ONLY. Reads categorize_calibration_samples, prints the reliability + * diagram + expected calibration error, fits an isotonic calibrator, and shows + * what the auto-book / suggest / review bands would look like on the calibrated + * probability. Run this once real outcomes have accumulated (>= a few hundred); + * it changes nothing on its own. + * + * npx tsx scripts/fit-categorize-calibration.ts + * + * Note: .env.local points at production; this only SELECTs, so it is safe, but + * it is still the prod corpus you are reading. + */ +import { createClient } from '@supabase/supabase-js' +import { + reliabilityByBucket, + expectedCalibrationError, + fitIsotonic, + calibrate, + bandFor, + type Sample, +} from '@/lib/agent/categorize/calibration' + +const url = process.env.NEXT_PUBLIC_SUPABASE_URL +const key = process.env.SUPABASE_SERVICE_ROLE_KEY +if (!url || !key) { + console.error('Missing NEXT_PUBLIC_SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in .env.local') + process.exit(1) +} +const supabase = createClient(url, key) + +async function main() { + const rows: { confidence: number; was_correct: boolean }[] = [] + const PAGE = 1000 + for (let from = 0; ; from += PAGE) { + const { data, error } = await supabase + .from('categorize_calibration_samples') + .select('confidence, was_correct') + .order('created_at', { ascending: false }) + .range(from, from + PAGE - 1) + if (error) throw error + if (!data || data.length === 0) break + rows.push(...(data as { confidence: number; was_correct: boolean }[])) + if (data.length < PAGE) break + } + + const samples: Sample[] = rows.map((r) => ({ confidence: Number(r.confidence), correct: r.was_correct })) + console.log(`\nSamples: ${samples.length}`) + if (samples.length === 0) { + console.log('No calibration samples yet. Let people book AI proposals first.') + return + } + + const overall = samples.filter((s) => s.correct).length / samples.length + console.log(`Overall accuracy (proposal booked unedited): ${(overall * 100).toFixed(1)}%`) + console.log(`Expected calibration error (ECE): ${expectedCalibrationError(samples).toFixed(4)}\n`) + + console.log('Reliability diagram (raw confidence bucket → empirical accuracy):') + for (const b of reliabilityByBucket(samples)) { + if (b.n === 0) continue + const bar = '#'.repeat(Math.round(b.accuracy * 20)) + console.log( + ` ${b.lo.toFixed(1)}-${b.hi.toFixed(1)} n=${String(b.n).padStart(5)} ` + + `conf=${b.meanConfidence.toFixed(2)} acc=${b.accuracy.toFixed(2)} ${bar}`, + ) + } + + const cal = fitIsotonic(samples) + if (!cal) { + console.log(`\nNot enough data to fit a calibrator yet (need >= 200). Bands stay uncalibrated (no auto-book).`) + return + } + + console.log(`\nFitted isotonic calibrator on ${cal.fittedOn} samples.`) + console.log('Raw → calibrated (and the band for a small, routine amount):') + for (const raw of [0.3, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99]) { + const p = calibrate(raw, cal) + const band = bandFor(raw, cal, { amount: 499 }) + console.log(` ${raw.toFixed(2)} → ${p.toFixed(2)} ${band}`) + } + console.log( + `\nNext: store this calibrator (or its thresholds) where bandFor reads it, ` + + `then enable auto-book for the top band once the empirical accuracy there is acceptable.`, + ) +} + +main().catch((e) => { + console.error(e) + process.exit(1) +}) diff --git a/supabase/migrations/20260821100000_categorize_calibration_samples.sql b/supabase/migrations/20260821100000_categorize_calibration_samples.sql new file mode 100644 index 00000000..603ba56b --- /dev/null +++ b/supabase/migrations/20260821100000_categorize_calibration_samples.sql @@ -0,0 +1,61 @@ +-- Migration: categorize_calibration_samples — the measurement loop for the +-- auto-booking cascade's confidence (RIP-4 step 4). +-- +-- The Tier-2 selector emits a raw combined confidence, but a raw score is not +-- calibrated until it is measured against reality. Every time a user books (or +-- edits) an AI proposal, we log one sample: the confidence the model reported +-- and whether the proposed account was the one actually booked. Fitting an +-- isotonic calibrator (lib/agent/categorize/calibration.ts) over these turns +-- "0.9" into a probability that really means 90%. +-- +-- Append-only: a calibration corpus you can edit is a calibration corpus you +-- can lie to. No UPDATE/DELETE policy. Company-scoped for RLS + attribution; +-- the fit job reads across companies with the service role (the model's +-- calibration is a property of the model, not one tenant). + +CREATE TABLE public.categorize_calibration_samples ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + company_id uuid NOT NULL REFERENCES public.companies(id) ON DELETE CASCADE, + + -- The raw combined confidence the selector reported, in [0,1]. + confidence numeric NOT NULL CHECK (confidence >= 0 AND confidence <= 1), + -- Self-consistency agreement fraction and the model's stated confidence, + -- kept for later analysis of which signal calibrates best. + agreement numeric, + model_confidence text, + + -- Where the proposal came from ('counterparty_template' | 'mapping_rule' | + -- 'history' | 'pattern' | 'category'), for per-source reliability. + source text, + + -- The label: proposed vs what was actually booked. + proposed_account text, + booked_account text NOT NULL, + was_correct boolean NOT NULL, + + -- The transaction's absolute amount, so the auto-book amount cap can be + -- tuned against real outcomes. + amount numeric, + + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Fit job reads recent samples, newest first. +CREATE INDEX idx_calib_samples_company_created + ON public.categorize_calibration_samples (company_id, created_at DESC); + +ALTER TABLE public.categorize_calibration_samples ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "categorize_calibration_samples_select" + ON public.categorize_calibration_samples + FOR SELECT + USING (company_id IN (SELECT public.user_company_ids())); + +CREATE POLICY "categorize_calibration_samples_insert" + ON public.categorize_calibration_samples + FOR INSERT + WITH CHECK (company_id IN (SELECT public.user_company_ids())); + +-- No UPDATE / DELETE policies: the corpus is append-only. + +NOTIFY pgrst, 'reload schema'; diff --git a/tests/pg/categorize-calibration-samples.pg.test.ts b/tests/pg/categorize-calibration-samples.pg.test.ts new file mode 100644 index 00000000..af879138 --- /dev/null +++ b/tests/pg/categorize-calibration-samples.pg.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect } from 'vitest' +import { getPool, withUserContext } from './setup' +import { seedCompany } from './fixtures' + +// categorize_calibration_samples (20260821100000): RLS via user_company_ids() +// on SELECT + INSERT only (append-only: no UPDATE/DELETE policy), the +// confidence CHECK [0,1], and company scoping on both read and write. + +async function insertSample( + client: { query: (q: string, p: unknown[]) => Promise<{ rows: unknown[]; rowCount: number | null }> }, + companyId: string, + confidence = 0.9, +) { + return client.query( + `INSERT INTO public.categorize_calibration_samples + (company_id, confidence, booked_account, was_correct) + VALUES ($1, $2, '5410', true) RETURNING id`, + [companyId, confidence], + ) +} + +describe('categorize_calibration_samples', () => { + it('lets a member insert and read their own company samples', async () => { + const { userId, companyId } = await seedCompany() + await withUserContext(userId, async (client) => { + await insertSample(client, companyId) + const { rows } = await client.query( + `SELECT company_id, was_correct FROM public.categorize_calibration_samples WHERE company_id = $1`, + [companyId], + ) + expect(rows.length).toBe(1) + }) + }) + + it('hides another company samples (RLS SELECT)', async () => { + const a = await seedCompany() + const b = await seedCompany() + await insertSample({ query: (q, p) => getPool().query(q, p) } as never, a.companyId) + await withUserContext(b.userId, async (client) => { + const { rows } = await client.query( + `SELECT id FROM public.categorize_calibration_samples WHERE company_id = $1`, + [a.companyId], + ) + expect(rows.length).toBe(0) + }) + }) + + it('refuses inserting a sample for another company (RLS WITH CHECK)', async () => { + const a = await seedCompany() + const b = await seedCompany() + await withUserContext(a.userId, async (client) => { + await expect(insertSample(client, b.companyId)).rejects.toThrow() + }) + }) + + it('is append-only: UPDATE and DELETE affect zero rows', async () => { + const { userId, companyId } = await seedCompany() + await withUserContext(userId, async (client) => { + const { rows } = await insertSample(client, companyId) + const id = (rows[0] as { id: string }).id + const upd = await client.query( + `UPDATE public.categorize_calibration_samples SET was_correct = false WHERE id = $1`, + [id], + ) + expect(upd.rowCount).toBe(0) + const del = await client.query( + `DELETE FROM public.categorize_calibration_samples WHERE id = $1`, + [id], + ) + expect(del.rowCount).toBe(0) + }) + }) + + it('enforces the confidence CHECK [0,1]', async () => { + const { companyId } = await seedCompany() + await expect( + getPool().query( + `INSERT INTO public.categorize_calibration_samples (company_id, confidence, booked_account, was_correct) + VALUES ($1, 2, '5410', true)`, + [companyId], + ), + ).rejects.toThrow() + }) +})