diff --git a/.env.example b/.env.example index ff354b05..efe15884 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,17 @@ NEXT_PUBLIC_APP_URL=http://localhost:3000 # Any non-empty random string for local dev: openssl rand -hex 16 CRON_SECRET=generate-a-random-secret +# Receipt hunt (nightly matcher, 05:30 UTC): comma-separated company ids the +# hunt may stage proposals for. Unset means it runs for nobody, so enabling it +# is always a deliberate act rather than a side effect of deploying. +# Model used to resolve bank descriptors to merchants and to decide which mail +# is the receipt for which purchase. Falls back to BEDROCK_MODEL_ID. +RECEIPT_HUNT_MODEL_ID= +# Floor for accepting the model's pairing. Every proposal is human-reviewed, so +# this trades recall against review effort, not against correctness. +RECEIPT_HUNT_MIN_CONFIDENCE= +RECEIPT_HUNT_COMPANY_IDS= + # Hosted session security defaults: 30 minutes idle, 12 hours absolute, # with a warning 2 minutes before expiry. Set a timeout to 0 to disable that # limit. Self-hosted deployments default both limits to 0 unless overridden. diff --git a/DECISIONS.md b/DECISIONS.md index 0336d6be..8c3b52f4 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -850,3 +850,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-10] Transactions inbox fetches ALL pending rows merged into the single transactions state array (tracked window boundary via pagedCountRef/pagedThroughDate) instead of a parallel pendingTransactions state: ~20 setTransactions mutation call sites (book/ignore/edit/delete) would each need dual updates and would drift; the merged array keeps mutations one code path, at the cost of a date-boundary filter for the history view. [2026-08-09] /migrate streaming is opt-in via Accept: application/x-ndjson instead of replacing the JSON contract: the wizard is the only caller today but a hard cutover would break open pre-deploy tabs and the route's locked error-status tests; mid-stream failures re-send the structured envelope as a terminal error event because the 200 is already committed once the stream opens. [2026-08-09] Regeluppdat + docs-freshness scans (#1417) built as local loop skills with due-date self-gating, not cloud crons: cloud routines were retired 2026-07-20, and session crons die at 7 days, so weekly/monthly cadence is achieved by loop-ignite running each loop when its run marker says it is due. loop-regeluppdat files tickets only (no auto-fix PRs): regulatory changes touch money math and compliance logic, which .claude/loops.md forbids loops from changing. Docs check diffs the live .md mirror routes against repo-built markdown (exact, canonicalised both sides) instead of diffing the gnubok-website checkout, so it also catches deployed-but-stale and route-404 states. +[2026-08-10] Receipt hunt cron keeps searchMail=false: the mailbox leg stays manual until its time budget is proven. A sweep of one 172-message mailbox took over 600s while the route's maxDuration is 300, so enabling it nightly would time out mid-run. Flip both this flag and RECEIPT_HUNT_COMPANY_IDS together once the per-company budget is measured. diff --git a/app/(dashboard)/settings/mail/page.tsx b/app/(dashboard)/settings/mail/page.tsx new file mode 100644 index 00000000..478829d7 --- /dev/null +++ b/app/(dashboard)/settings/mail/page.tsx @@ -0,0 +1,5 @@ +import { MailSettingsContent } from '@/components/settings/sections/MailSettingsContent' + +export default function MailSettingsPage() { + return +} diff --git a/app/api/receipt-hunt/cron/__tests__/route.test.ts b/app/api/receipt-hunt/cron/__tests__/route.test.ts new file mode 100644 index 00000000..5c538c3c --- /dev/null +++ b/app/api/receipt-hunt/cron/__tests__/route.test.ts @@ -0,0 +1,118 @@ +/** + * The cron shell around the hunt: authorization, the allowlist kill-switch, + * aggregation, and per-company isolation. The ranking itself is covered by + * lib/receipt-hunt/__tests__/select.test.ts. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { NextResponse } from 'next/server' + +vi.mock('@/lib/auth/cron', () => ({ + verifyCronSecret: vi.fn(() => null), +})) + +vi.mock('@/lib/init', () => ({ + ensureInitialized: vi.fn(), +})) + +vi.mock('@/lib/supabase/server', () => ({ + createServiceClient: vi.fn(() => ({})), +})) + +const mockHuntCompany = vi.fn() +vi.mock('@/lib/receipt-hunt/hunt', async () => { + // resolveAllowlist is pure and part of what this route's behaviour depends + // on, so it stays real; only the database-touching half is faked. + const actual = await vi.importActual( + '@/lib/receipt-hunt/hunt', + ) + return { ...actual, huntCompany: (...args: unknown[]) => mockHuntCompany(...args) } +}) + +import { verifyCronSecret } from '@/lib/auth/cron' +import { createServiceClient } from '@/lib/supabase/server' +import { GET } from '../route' + +const ORIGINAL_ALLOWLIST = process.env.RECEIPT_HUNT_COMPANY_IDS + +function request(): Request { + return new Request('https://app.accounted.se/api/receipt-hunt/cron') +} + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(verifyCronSecret).mockReturnValue(null) + mockHuntCompany.mockReset() +}) + +afterEach(() => { + if (ORIGINAL_ALLOWLIST === undefined) delete process.env.RECEIPT_HUNT_COMPANY_IDS + else process.env.RECEIPT_HUNT_COMPANY_IDS = ORIGINAL_ALLOWLIST +}) + +describe('GET /api/receipt-hunt/cron', () => { + it('rejects an unauthorized caller without touching the database', async () => { + vi.mocked(verifyCronSecret).mockReturnValueOnce( + NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), + ) + process.env.RECEIPT_HUNT_COMPANY_IDS = 'co-1' + + const response = await GET(request()) + + expect(response.status).toBe(401) + expect(vi.mocked(createServiceClient)).not.toHaveBeenCalled() + expect(mockHuntCompany).not.toHaveBeenCalled() + }) + + it('hunts nobody when the allowlist is unset', async () => { + delete process.env.RECEIPT_HUNT_COMPANY_IDS + + const response = await GET(request()) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body).toMatchObject({ success: true, skipped: true, total: 0 }) + expect(mockHuntCompany).not.toHaveBeenCalled() + // Fail-safe matters more than the response shape: an empty variable must + // never be read as "every company". + expect(vi.mocked(createServiceClient)).not.toHaveBeenCalled() + }) + + it('hunts nobody when the allowlist is blank or comma-only', async () => { + process.env.RECEIPT_HUNT_COMPANY_IDS = ' , ,, ' + + const body = await (await GET(request())).json() + + expect(body.skipped).toBe(true) + expect(mockHuntCompany).not.toHaveBeenCalled() + }) + + it('runs each allowlisted company and aggregates what was proposed', async () => { + process.env.RECEIPT_HUNT_COMPANY_IDS = 'co-1, co-2' + mockHuntCompany + .mockResolvedValueOnce({ companyId: 'co-1', candidates: 9, poolSize: 4, proposed: 3 }) + .mockResolvedValueOnce({ companyId: 'co-2', candidates: 2, poolSize: 0, proposed: 0 }) + + const body = await (await GET(request())).json() + + expect(mockHuntCompany).toHaveBeenCalledTimes(2) + expect(body).toMatchObject({ success: true, total: 2, succeeded: 2, failed: 0, proposed: 3 }) + expect(body.results).toHaveLength(2) + // Every proposal from one night shares a run id so the run can be read back. + expect(body.runId).toEqual(expect.any(String)) + const [, companyId, runId] = mockHuntCompany.mock.calls[0] + expect(companyId).toBe('co-1') + expect(runId).toBe(body.runId) + }) + + it('lets one company fail without stopping the rest', async () => { + process.env.RECEIPT_HUNT_COMPANY_IDS = 'co-1,co-2' + mockHuntCompany + .mockRejectedValueOnce(new Error('boom')) + .mockResolvedValueOnce({ companyId: 'co-2', candidates: 5, poolSize: 2, proposed: 1 }) + + const body = await (await GET(request())).json() + + expect(body).toMatchObject({ success: true, total: 2, succeeded: 1, failed: 1, proposed: 1 }) + expect(body.failures).toHaveLength(1) + }) +}) diff --git a/app/api/receipt-hunt/cron/route.ts b/app/api/receipt-hunt/cron/route.ts new file mode 100644 index 00000000..85220208 --- /dev/null +++ b/app/api/receipt-hunt/cron/route.ts @@ -0,0 +1,93 @@ +import { NextResponse } from 'next/server' +import { ensureInitialized } from '@/lib/init' +import { withCronContext } from '@/lib/api/with-cron-context' +import { createServiceClient } from '@/lib/supabase/server' +import { huntCompany, resolveAllowlist } from '@/lib/receipt-hunt/hunt' + +ensureInitialized() + +/** + * GET /api/receipt-hunt/cron, daily 05:30 UTC. + * + * Pairs unbooked card purchases that have no receipt with the unconsumed + * underlag the company already holds, and stages each pairing for approval. + * Runs after the 05:00 bank sync so the night's new transactions are swept the + * same morning. + * + * Writes nothing to the journal: every pairing becomes an + * `attach_document_to_transaction` pending operation, and the document only + * reaches a verifikat later, when the user books the transaction. + * + * Scoped to `RECEIPT_HUNT_COMPANY_IDS` while the feature is piloted. Unset + * means the hunt runs for nobody, so a deploy cannot silently start staging + * proposals in every company at once. + */ +export const GET = withCronContext('cron.receipt_hunt', async (_request, ctx) => { + const companyIds = resolveAllowlist(process.env.RECEIPT_HUNT_COMPANY_IDS) + if (companyIds.length === 0) { + ctx.log.info('receipt hunt skipped: no companies allowlisted') + return NextResponse.json({ + success: true, + skipped: true, + reason: 'RECEIPT_HUNT_COMPANY_IDS is empty', + total: 0, + }) + } + + const supabase = createServiceClient() + const runId = crypto.randomUUID() + + ctx.log.info('receipt hunt starting', { companyCount: companyIds.length, runId }) + + const results: Array<{ + companyId: string + candidates: number + poolSize: number + proposed: number + }> = [] + + const summary = await ctx.forEach('company', companyIds, async (companyId, itemCtx) => { + try { + const result = await huntCompany(supabase, companyId, runId) + results.push({ + companyId: result.companyId, + candidates: result.candidates, + poolSize: result.poolSize, + proposed: result.proposed, + }) + if (result.skippedNoOwner) { + itemCtx.log.warn('receipt hunt found proposals but the company has no members', { + companyId, + }) + } + } catch (error) { + itemCtx.log.error('receipt hunt failed for company', error as Error, { companyId }) + throw error + } + }) + + const proposed = results.reduce((sum, r) => sum + r.proposed, 0) + ctx.log.info('receipt hunt summary', { + runId, + total: summary.total, + succeeded: summary.succeeded, + failed: summary.failed, + proposed, + }) + + return NextResponse.json({ + success: true, + runId, + total: summary.total, + succeeded: summary.succeeded, + failed: summary.failed, + failures: summary.failures, + proposed, + results, + }) +}) + +export const POST = GET + +/** Companies are hunted sequentially and each does a handful of reads. */ +export const maxDuration = 300 diff --git a/components/auth/GoogleAuthButton.tsx b/components/auth/GoogleAuthButton.tsx index 544d5c49..f640c65c 100644 --- a/components/auth/GoogleAuthButton.tsx +++ b/components/auth/GoogleAuthButton.tsx @@ -6,30 +6,8 @@ import { createClient } from '@/lib/supabase/client' import { Button } from '@/components/ui/button' import { Loader2 } from 'lucide-react' import { getErrorMessage, type ErrorLocale } from '@/lib/errors/get-error-message' +import { GoogleMark } from '@/components/ui/provider-marks' -/** Official Google "G" mark, drawn inline (no external assets on auth pages). */ -function GoogleMark() { - return ( - - - - - - - ) -} /** * "Continue with Google" for the login and register pages. diff --git a/components/extensions/general/MailConnectionsPanel.tsx b/components/extensions/general/MailConnectionsPanel.tsx new file mode 100644 index 00000000..03f02688 --- /dev/null +++ b/components/extensions/general/MailConnectionsPanel.tsx @@ -0,0 +1,151 @@ +'use client' + +import { useCallback, useEffect, useState } from 'react' +import { useTranslations } from 'next-intl' +import { Loader2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Badge } from '@/components/ui/badge' +import { + SettingsGroup, + SettingsRow, + SettingsRowEnd, + SettingsRowNote, +} from '@/components/settings/SettingsRows' +import { ConfirmDialog } from '@/components/ui/confirm-dialog' +import { GoogleMark, MicrosoftMark } from '@/components/ui/provider-marks' +import { formatDateLong } from '@/lib/utils' + +interface MailConnection { + id: string + provider: 'gmail' | 'microsoft' + emailAddress: string + scopeLabel: string | null + status: 'active' | 'needs_reconsent' | 'revoked' + lastSearchedAt: string | null + lastErrorCode: string | null +} + +const BASE = '/api/extensions/ext/mail' + +export function MailConnectionsPanel() { + const t = useTranslations('mail') + const [connections, setConnections] = useState([]) + const [configured, setConfigured] = useState(true) + const [loading, setLoading] = useState(true) + const [connecting, setConnecting] = useState(false) + const [pendingDisconnect, setPendingDisconnect] = useState(null) + + const load = useCallback(async () => { + try { + const response = await fetch(`${BASE}/connections`) + if (!response.ok) return + const body = (await response.json()) as { + data: { connections: MailConnection[]; configured: boolean } + } + setConnections(body.data.connections) + setConfigured(body.data.configured) + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + void load() + }, [load]) + + async function connect() { + setConnecting(true) + try { + // The consent screen must open from the user's own gesture, so the tab is + // opened first and its location set once the URL is known: opening it + // after the await is what popup blockers stop. + const tab = window.open('', '_blank') + const response = await fetch(`${BASE}/oauth/start`, { method: 'POST' }) + if (!response.ok) { + tab?.close() + return + } + const body = (await response.json()) as { url: string } + if (tab) tab.location.href = body.url + else window.location.href = body.url + } finally { + setConnecting(false) + } + } + + async function disconnect(connection: MailConnection) { + await fetch(`${BASE}/connections?id=${encodeURIComponent(connection.id)}`, { method: 'DELETE' }) + setPendingDisconnect(null) + void load() + } + + if (loading) return null + + return ( +
+ + {connections.length === 0 ? ( + + {t('none')} + + ) : ( + connections.map((connection) => ( + + {connection.provider === 'gmail' ? ( + + ) : ( + + )} + {connection.provider === 'gmail' ? 'Gmail' : 'Microsoft 365'} + + } + > + {connection.emailAddress} + {connection.status === 'needs_reconsent' ? ( + {t('needs_reconsent')} + ) : null} + + {connection.lastSearchedAt ? ( + + {t('last_searched', { date: formatDateLong(connection.lastSearchedAt, 'sv') })} + + ) : null} + + + + )) + )} + + +
+ + {!configured ? {t('not_configured')} : null} +
+ +

{t('promise')}

+ + !open && setPendingDisconnect(null)} + title={t('disconnect_title')} + description={t('disconnect_body', { address: pendingDisconnect?.emailAddress ?? '' })} + confirmLabel={t('disconnect')} + onConfirm={async () => { + if (pendingDisconnect) await disconnect(pendingDisconnect) + }} + /> +
+ ) +} diff --git a/components/settings/sections/MailSettingsContent.tsx b/components/settings/sections/MailSettingsContent.tsx new file mode 100644 index 00000000..656667b0 --- /dev/null +++ b/components/settings/sections/MailSettingsContent.tsx @@ -0,0 +1,17 @@ +'use client' + +import { useTranslations } from 'next-intl' +import { MailConnectionsPanel } from '@/components/extensions/general/MailConnectionsPanel' +import { SettingsSectionHeader } from '@/components/settings/SettingsRows' + +export function MailSettingsContent() { + const tNav = useTranslations('settings_nav') + const tIntro = useTranslations('settings_intro') + + return ( +
+ + +
+ ) +} diff --git a/components/settings/sections/index.ts b/components/settings/sections/index.ts index ecabf775..13753f8d 100644 --- a/components/settings/sections/index.ts +++ b/components/settings/sections/index.ts @@ -50,6 +50,10 @@ const WhatsAppSettingsContent = dynamic(() => import('./WhatsAppSettingsContent').then((module) => ({ default: module.WhatsAppSettingsContent })), { loading: SettingsLoadingSkeleton }, ) +const MailSettingsContent = dynamic(() => + import('./MailSettingsContent').then((module) => ({ default: module.MailSettingsContent })), + { loading: SettingsLoadingSkeleton }, +) /** * Single source of truth mapping a settings section id to the component that @@ -71,6 +75,7 @@ export const SETTINGS_SECTIONS: Record = { api: ApiSettingsContent, billing: BillingSettingsContent, whatsapp: WhatsAppSettingsContent, + mail: MailSettingsContent, } export type SettingsSectionId = keyof typeof SETTINGS_SECTIONS diff --git a/components/settings/useSettingsNavItems.ts b/components/settings/useSettingsNavItems.ts index 1d800d20..1f509ed5 100644 --- a/components/settings/useSettingsNavItems.ts +++ b/components/settings/useSettingsNavItems.ts @@ -42,6 +42,7 @@ export function useSettingsNavItems(): { items: SettingsNavItem[]; groups: Setti const hasBankingExtension = ENABLED_EXTENSION_IDS.has('enable-banking') const hasMcpExtension = ENABLED_EXTENSION_IDS.has('mcp-server') const hasWhatsAppExtension = ENABLED_EXTENSION_IDS.has('whatsapp-inbox') + const hasMailExtension = ENABLED_EXTENSION_IDS.has('mail') // Företagsprofil (TIC-snapshot) lives under Företag; Skatteverket under Skatt; // assistentens minne + kunskap under Assistenten; säkerhetsbackup under @@ -60,6 +61,7 @@ export function useSettingsNavItems(): { items: SettingsNavItem[]; groups: Setti { id: 'templates', href: '/settings/templates', label: t('templates'), group: 'sales', show: hasCompany }, { id: 'banking', href: '/settings/banking', label: t('banking'), group: 'tools', show: hasCompany && !isSandbox && hasBankingExtension }, { id: 'whatsapp', href: '/settings/whatsapp', label: t('whatsapp'), group: 'tools', show: hasCompany && !isSandbox && hasWhatsAppExtension }, + { id: 'mail', href: '/settings/mail', label: t('mail'), group: 'tools', show: hasCompany && !isSandbox && hasMailExtension }, { id: 'assistant', href: '/settings/assistant', label: t('assistant'), group: 'tools', show: hasCompany && identity.isVerified }, { id: 'api', href: '/settings/api', label: t('api'), group: 'tools', show: hasCompany && hasMcpExtension }, ] diff --git a/components/ui/provider-marks.tsx b/components/ui/provider-marks.tsx new file mode 100644 index 00000000..01b6afe7 --- /dev/null +++ b/components/ui/provider-marks.tsx @@ -0,0 +1,48 @@ +/** + * Third-party brand marks, drawn inline. + * + * Inline rather than an asset so auth pages and settings never wait on a + * network request for an icon, and so no external host is contacted before a + * user has agreed to anything. + * + * These are the only coloured glyphs in an otherwise achromatic interface, and + * deliberately so: a provider's own mark is how someone recognises which + * account they are about to connect, and Google's brand terms require its mark + * to be used unaltered rather than tinted to match a palette. + */ + +/** Official Google "G". */ +export function GoogleMark({ className = 'h-4 w-4' }: { className?: string }) { + return ( + + + + + + + ) +} + +/** Microsoft's four-square mark, for the Graph connector when it lands. */ +export function MicrosoftMark({ className = 'h-4 w-4' }: { className?: string }) { + return ( + + + + + + + ) +} diff --git a/docker/crontab.hosted b/docker/crontab.hosted index fa9c587d..5380a28a 100644 --- a/docker/crontab.hosted +++ b/docker/crontab.hosted @@ -44,3 +44,4 @@ * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/sweep/cron 15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron +30 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/receipt-hunt/cron diff --git a/docker/crontab.self-hosted b/docker/crontab.self-hosted index 9f47218f..357c4fc4 100644 --- a/docker/crontab.self-hosted +++ b/docker/crontab.self-hosted @@ -44,3 +44,4 @@ * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/sweep/cron 15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron +30 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/receipt-hunt/cron diff --git a/extensions.config.json b/extensions.config.json index fc6bd25f..07fc986b 100644 --- a/extensions.config.json +++ b/extensions.config.json @@ -1 +1 @@ -{"$schema":"./extensions.schema.json","extensions":["enable-banking","email","arcim-migration","tic","mcp-server","cloud-backup","skatteverket","invoice-inbox","document-extraction","stripe","whatsapp-inbox","woocommerce","shopify"]} +{"$schema":"./extensions.schema.json","extensions":["enable-banking","email","arcim-migration","tic","mcp-server","cloud-backup","skatteverket","invoice-inbox","document-extraction","stripe","whatsapp-inbox","woocommerce","shopify","mail"]} \ No newline at end of file diff --git a/extensions/general/mail/index.ts b/extensions/general/mail/index.ts new file mode 100644 index 00000000..c7bb8446 --- /dev/null +++ b/extensions/general/mail/index.ts @@ -0,0 +1,159 @@ +import { NextResponse } from 'next/server' +import type { Extension } from '@/lib/extensions/types' +import { registerMailSearchService } from '@/lib/mail-search/service' +import { createServiceClientNoCookies } from '@/lib/auth/api-keys' +import { GmailSearchService } from './lib/search-service' +import { createOAuthState, verifyOAuthState } from './lib/crypto' +import { + buildAuthorizationUrl, + exchangeCodeForTokens, + getGoogleOAuthEnv, + isGoogleMailConfigured, +} from './lib/google-oauth' +import { disconnect, listConnections, saveConnection } from './lib/connections' +import { resolveCallbackOrigin } from './lib/callback-origin' + +// Registered as soon as the extension loads, so the receipt hunt can search +// mail without core ever importing from @/extensions. +registerMailSearchService(new GmailSearchService()) + +function jsonError(message: string, status = 500): Response { + return NextResponse.json({ error: message }, { status }) +} + +/** How far back a newly connected mailbox may be searched, in days. */ +const BACKFILL_CHOICES = new Set([30, 90, 365]) + +export const mailExtension: Extension = { + id: 'mail', + name: 'Brevlådor', + version: '0.1.0', + sector: 'general', + + settingsPanel: { + label: 'Brevlådor', + path: '/settings/mail', + }, + + apiRoutes: [ + // Start the consent flow. Returns the URL rather than redirecting so the + // caller can open it in a deliberate, user-gesture tab. + { + method: 'POST', + path: '/oauth/start', + handler: async (request, ctx) => { + if (!ctx) return jsonError('Missing context', 500) + if (!isGoogleMailConfigured()) return jsonError('provider_not_configured', 400) + try { + const url = new URL(request.url) + const origin = resolveCallbackOrigin(url.origin) + const state = createOAuthState(ctx.userId, ctx.companyId) + const env = getGoogleOAuthEnv(origin) + return NextResponse.json({ url: buildAuthorizationUrl(env, state) }) + } catch (err) { + ctx.log.error('mail oauth start failed', err) + return jsonError(err instanceof Error ? err.message : 'Could not start OAuth', 500) + } + }, + }, + + // Google redirects here after consent. Registered in the Google console as + // an authorised redirect URI: the `mail` slug and this path are pinned and + // must never be renamed without re-registering. + { + method: 'GET', + path: '/oauth/callback', + skipAuth: true, + handler: async (request) => { + const url = new URL(request.url) + const code = url.searchParams.get('code') + const state = url.searchParams.get('state') + const error = url.searchParams.get('error') + const settingsUrl = `${resolveCallbackOrigin(url.origin)}/settings/mail` + + // The user declining is a normal outcome, not an error to shout about. + if (error) return NextResponse.redirect(`${settingsUrl}?mail=denied`) + if (!code || !state) return NextResponse.redirect(`${settingsUrl}?mail=invalid`) + + const verified = verifyOAuthState(state) + if (!verified) return NextResponse.redirect(`${settingsUrl}?mail=expired`) + + try { + const origin = resolveCallbackOrigin(url.origin) + const env = getGoogleOAuthEnv(origin) + const tokens = await exchangeCodeForTokens(env, code) + if (!tokens.refreshToken) { + return NextResponse.redirect(`${settingsUrl}?mail=no_refresh_token`) + } + if (!tokens.email) { + // Without the address we cannot tell two grants apart, and the + // unique key depends on it. + return NextResponse.redirect(`${settingsUrl}?mail=no_address`) + } + + await saveConnection(createServiceClientNoCookies(), { + companyId: verified.companyId, + userId: verified.userId, + provider: 'gmail', + emailAddress: tokens.email, + refreshToken: tokens.refreshToken, + accessToken: tokens.accessToken, + expiresAt: tokens.expiresAt, + scopes: tokens.scopes, + backfillFrom: null, + }) + return NextResponse.redirect(`${settingsUrl}?mail=connected`) + } catch { + return NextResponse.redirect(`${settingsUrl}?mail=failed`) + } + }, + }, + + // What this company has connected. Safe projection only: never tokens. + { + method: 'GET', + path: '/connections', + handler: async (_request, ctx) => { + if (!ctx) return jsonError('Missing context', 500) + const connections = await listConnections(createServiceClientNoCookies(), ctx.companyId) + return NextResponse.json({ data: { connections, configured: isGoogleMailConfigured() } }) + }, + }, + + { + method: 'DELETE', + path: '/connections', + handler: async (request, ctx) => { + if (!ctx) return jsonError('Missing context', 500) + const id = new URL(request.url).searchParams.get('id') + if (!id) return jsonError('missing_id', 400) + await disconnect(createServiceClientNoCookies(), ctx.companyId, id, ctx.userId) + return NextResponse.json({ data: { disconnected: true } }) + }, + }, + + // How far back a mailbox may be searched once, chosen by the user at + // connect time. Bounded to the offered choices so an arbitrary date cannot + // widen the grant's reach by hand. + { + method: 'POST', + path: '/connections/backfill', + handler: async (request, ctx) => { + if (!ctx) return jsonError('Missing context', 500) + const body = (await request.json().catch(() => ({}))) as { id?: string; days?: number } + if (!body.id || !body.days || !BACKFILL_CHOICES.has(body.days)) { + return jsonError('invalid_request', 400) + } + const from = new Date() + from.setDate(from.getDate() - body.days) + const supabase = createServiceClientNoCookies() + await supabase + .from('mail_connections') + .update({ backfill_from: from.toISOString().slice(0, 10) }) + .eq('id', body.id) + .eq('company_id', ctx.companyId) + return NextResponse.json({ data: { backfill_from: from.toISOString().slice(0, 10) } }) + }, + }, + ], +} diff --git a/extensions/general/mail/lib/__tests__/disconnect-audit.test.ts b/extensions/general/mail/lib/__tests__/disconnect-audit.test.ts new file mode 100644 index 00000000..78c41e24 --- /dev/null +++ b/extensions/general/mail/lib/__tests__/disconnect-audit.test.ts @@ -0,0 +1,99 @@ +/** + * Disconnecting a mailbox is a control change over how underlag reaches the + * books, so it has to be reconstructable (BFNAR 2013:2 kap 8). What must NOT + * happen is the audit entry preserving the credential the delete existed to + * destroy. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { disconnect } from '../connections' + +function mockSupabase( + existing: Record | null, + deleteError: { message: string } | null = null, +) { + const inserted: Array> = [] + const deleted: string[] = [] + const client = { + from(table: string) { + const chain: Record = {} + for (const m of ['select']) chain[m] = vi.fn(() => chain) + let eqCalls = 0 + chain.eq = vi.fn(() => { + eqCalls++ + // The delete chain resolves after its second .eq(); the select chain + // ends in .maybeSingle() instead. + return chain.deleting && eqCalls >= 2 + ? Promise.resolve({ error: deleteError }) + : chain + }) + chain.maybeSingle = vi.fn(() => Promise.resolve({ data: existing, error: null })) + chain.delete = vi.fn(() => { + deleted.push(table) + ;(chain as Record).deleting = true + eqCalls = 0 + return chain + }) + chain.insert = vi.fn((row: Record) => { + if (table === 'audit_log') inserted.push(row) + return Promise.resolve({ error: null }) + }) + return chain + }, + } + return { client: client as never, inserted, deleted } +} + +beforeEach(() => vi.clearAllMocks()) + +describe('disconnect', () => { + it('records who disconnected which mailbox, and when', async () => { + const { client, inserted, deleted } = mockSupabase({ + email_address: 'ekonomi@nordvik.se', + provider: 'gmail', + }) + await disconnect(client, 'co-1', 'conn-1', 'user-1') + + expect(deleted).toContain('mail_connections') + expect(inserted).toHaveLength(1) + expect(inserted[0]).toMatchObject({ + user_id: 'user-1', + company_id: 'co-1', + action: 'DELETE', + table_name: 'mail_connections', + record_id: 'conn-1', + }) + expect(String(inserted[0].description)).toContain('ekonomi@nordvik.se') + }) + + it('never copies the credential into the audit trail', async () => { + // The whole point of disconnecting is that the refresh token is gone. + // A write_audit_log trigger would have carried it into a second table. + const { client, inserted } = mockSupabase({ + email_address: 'ekonomi@nordvik.se', + provider: 'gmail', + encrypted_refresh_token: 'SECRET', + }) + await disconnect(client, 'co-1', 'conn-1', 'user-1') + + const blob = JSON.stringify(inserted[0]) + expect(blob).not.toContain('SECRET') + expect(blob).not.toContain('encrypted_refresh_token') + }) + + it('does not claim a disconnect that the database refused', async () => { + // An audit entry saying the mailbox was disconnected, while the credential + // is still live, is worse than no entry at all. + const { client, inserted } = mockSupabase( + { email_address: 'ekonomi@nordvik.se', provider: 'gmail' }, + { message: 'permission denied' }, + ) + await expect(disconnect(client, 'co-1', 'conn-1', 'user-1')).rejects.toThrow('permission denied') + expect(inserted).toEqual([]) + }) + + it('writes nothing when there was no such connection', async () => { + const { client, inserted } = mockSupabase(null) + await disconnect(client, 'co-1', 'missing', 'user-1') + expect(inserted).toEqual([]) + }) +}) diff --git a/extensions/general/mail/lib/__tests__/gmail-client.test.ts b/extensions/general/mail/lib/__tests__/gmail-client.test.ts new file mode 100644 index 00000000..ab036854 --- /dev/null +++ b/extensions/general/mail/lib/__tests__/gmail-client.test.ts @@ -0,0 +1,102 @@ +/** + * Reading a message well enough to know whether it carries an underlag. + * + * The case that matters is the one a provkörning caught: asking Gmail for + * `format=metadata` returns headers and no `payload.parts`, so every message + * looks attachment-free and the hunt can never file anything. These tests pin + * the format and the MIME walk. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { getMessageSummary } from '../gmail-client' + +const mockFetch = vi.fn() +vi.stubGlobal('fetch', (...args: unknown[]) => mockFetch(...args)) + +function respond(message: Record) { + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve(message), + text: () => Promise.resolve(''), + }) +} + +const HEADERS = [ + { name: 'Subject', value: 'Faktura-20251070' }, + { name: 'From', value: 'info@tic.io' }, +] + +beforeEach(() => vi.clearAllMocks()) + +describe('getMessageSummary', () => { + it('asks for the full message, because metadata omits the parts tree', async () => { + respond({ id: 'm1', payload: { headers: HEADERS } }) + await getMessageSummary('token', 'm1', 'conn-1', 'invoice@arcim.io') + + const url = String(mockFetch.mock.calls[0][0]) + expect(url).toContain('format=full') + // The bug this replaces: metadata returns no payload.parts at all. + expect(url).not.toContain('format=metadata') + }) + + it('finds a PDF nested inside a forwarded message', async () => { + // Forwarding is how most of these receipts arrive, and it buries the + // attachment two levels down inside a message/rfc822 part. + respond({ + id: 'm1', + payload: { + headers: HEADERS, + mimeType: 'multipart/mixed', + parts: [ + { mimeType: 'text/plain', body: { size: 12 } }, + { + mimeType: 'message/rfc822', + parts: [ + { + mimeType: 'multipart/mixed', + parts: [ + { mimeType: 'text/html', body: { size: 900 } }, + { + mimeType: 'application/pdf', + filename: 'faktura.pdf', + body: { attachmentId: 'att-deep', size: 51200 }, + }, + ], + }, + ], + }, + ], + }, + }) + + const candidate = await getMessageSummary('token', 'm1', 'conn-1', 'invoice@arcim.io') + expect(candidate.attachmentIds).toEqual(['att-deep']) + expect(candidate.bodyIsReceipt).toBe(false) + }) + + it('does not mistake an inline logo for an underlag', async () => { + respond({ + id: 'm1', + payload: { + headers: HEADERS, + parts: [ + { mimeType: 'image/png', filename: 'logo.png', body: { attachmentId: 'logo', size: 400 } }, + ], + }, + }) + + const candidate = await getMessageSummary('token', 'm1', 'conn-1', 'invoice@arcim.io') + expect(candidate.attachmentIds).toEqual([]) + expect(candidate.bodyIsReceipt).toBe(true) + }) + + it('reports a genuinely attachment-free mail as a body receipt', async () => { + respond({ + id: 'm1', + payload: { headers: HEADERS, mimeType: 'text/html', body: { size: 4000 } }, + }) + + const candidate = await getMessageSummary('token', 'm1', 'conn-1', 'invoice@arcim.io') + expect(candidate.bodyIsReceipt).toBe(true) + expect(candidate.subject).toBe('Faktura-20251070') + }) +}) diff --git a/extensions/general/mail/lib/__tests__/gmail-query.test.ts b/extensions/general/mail/lib/__tests__/gmail-query.test.ts new file mode 100644 index 00000000..50cc8061 --- /dev/null +++ b/extensions/general/mail/lib/__tests__/gmail-query.test.ts @@ -0,0 +1,135 @@ +/** + * The Gmail query is where recall is won or lost, so it is pure and tested + * without a mailbox. Cases are drawn from the real bank descriptors and + * confirmed receipt pairs observed in production. + */ +import { describe, it, expect } from 'vitest' +import { + DAYS_AFTER, + DAYS_BEFORE, + amountTerms, + buildGmailQuery, + looksLikeReceipt, + merchantTerms, +} from '../gmail-query' + +describe('merchantTerms', () => { + it('drops legal forms and other tokens that would match the whole mailbox', () => { + expect(merchantTerms('circle k')).toEqual(['circle']) + expect(merchantTerms('ryde sweden')).toEqual(['ryde']) + expect(merchantTerms('')).toEqual([]) + expect(merchantTerms(null)).toEqual([]) + }) + + it('keeps at most three terms so the query stays selective', () => { + expect(merchantTerms('alviks kott och fisk stockholm sodermalm').length).toBeLessThanOrEqual(3) + }) +}) + +describe('amountTerms', () => { + it('offers both Swedish and international decimal forms', () => { + // A Swedish receipt writes 438,75; a SaaS invoice writes 438.75. + expect(amountTerms(438.75)).toEqual(expect.arrayContaining(['438.75', '438,75'])) + }) + + it('adds the bare integer when the amount is whole', () => { + expect(amountTerms(425)).toEqual(expect.arrayContaining(['425.00', '425,00', '425'])) + }) + + it('ignores the sign, since bank outflows are negative', () => { + expect(amountTerms(-425)).toEqual(amountTerms(425)) + }) +}) + +describe('buildGmailQuery', () => { + const base = { merchant: 'circle', amount: 438.75, currency: 'SEK', date: '2026-05-02' } + + it('brackets the purchase with an asymmetric window', () => { + const q = buildGmailQuery(base) + // Receipts arrive at or after the purchase; the bank may post it late. + expect(DAYS_AFTER).toBeGreaterThan(DAYS_BEFORE) + expect(q).toContain('after:2026/04/28') + expect(q).toContain('before:2026/05/13') + }) + + it('leads with the amount, then ORs the merchant', () => { + // Amount first because it is the one signal that does not drift: banks post + // late and receipts get forwarded, so dates move, but the charged figure + // does not. Still an OR and never an AND: a receipt billed in USD contains + // no SEK amount, and a bank descriptor often names nothing in the receipt. + const q = buildGmailQuery(base) + expect(q).toMatch(/\("438\.75" OR .*"circle"/) + }) + + it('still searches on amount alone when the merchant is unusable', () => { + const q = buildGmailQuery({ ...base, merchant: 'ab' }) + expect(q).toContain('"438.75"') + expect(q).not.toContain('"ab"') + }) + + it('does not require an attachment, because many receipts are the body', () => { + expect(buildGmailQuery(base)).not.toContain('has:attachment') + }) +}) + +describe('looksLikeReceipt', () => { + it('keeps plausible receipts', () => { + expect(looksLikeReceipt('Ditt kvitto från Circle K', 'no-reply@circlek.se')).toBe(true) + expect(looksLikeReceipt('Your receipt from Anthropic', 'billing@anthropic.com')).toBe(true) + }) + + it('drops the obvious non-receipts before anything expensive reads them', () => { + expect(looksLikeReceipt('Nyhetsbrev maj', 'news@example.com')).toBe(false) + expect(looksLikeReceipt('Calendar invite: standup', 'cal@example.com')).toBe(false) + }) +}) + +describe('buildAuthorizationUrl', () => { + it('carries the CSRF state, which the callback refuses to proceed without', async () => { + const { buildAuthorizationUrl } = await import('../google-oauth') + const url = buildAuthorizationUrl( + { clientId: 'cid', clientSecret: 'sec', redirectUri: 'https://app.accounted.se/cb' }, + 'signed-state', + ) + const params = new URL(url).searchParams + expect(params.get('state')).toBe('signed-state') + // Read-only, and offline so a refresh token is actually issued. + expect(params.get('scope')).toContain('gmail.readonly') + expect(params.get('scope')).not.toContain('gmail.modify') + expect(params.get('scope')).not.toContain('gmail.send') + expect(params.get('access_type')).toBe('offline') + // Not incremental: include_granted_scopes would let Google fold scopes this + // app was granted elsewhere into the token issued here, so a mailbox grant + // could carry more authority than the consent screen showed. + expect(params.get('include_granted_scopes')).toBeNull() + }) +}) + +/** + * Regression from the first provkörning against a real ledger: the same seven + * unrelated messages came back for every purchase, because the bank's + * description is not a merchant name and its month and person tokens match most + * of a mailbox. + */ +describe('merchant terms drawn from a real bank description', () => { + it('does not search for the month in a salary row', () => { + expect(merchantTerms('Lön Juli Jakob Överföring via internet')).not.toContain('Juli') + }) + + it('drops the payment rail, which every row on the statement shares', () => { + const terms = merchantTerms('Kontor Sting apr BG 0000059142596 Bg-bet. via internet') + for (const noise of ['apr', 'via', 'internet']) { + expect(terms.map((t) => t.toLowerCase())).not.toContain(noise) + } + }) + + it('still keeps what actually names the supplier', () => { + // This row matched a real Sting invoice; the fix must not cost that. + const terms = merchantTerms('Kontor Sting apr BG 0000059142596 Bg-bet. via internet') + expect(terms.map((t) => t.toLowerCase())).toContain('sting') + }) + + it('leaves an ordinary card purchase alone', () => { + expect(merchantTerms('Elgiganten Aktiebolag')).toContain('Elgiganten') + }) +}) diff --git a/extensions/general/mail/lib/callback-origin.ts b/extensions/general/mail/lib/callback-origin.ts new file mode 100644 index 00000000..00defad6 --- /dev/null +++ b/extensions/general/mail/lib/callback-origin.ts @@ -0,0 +1,29 @@ +/** + * Resolve the origin used to build OAuth redirect URIs. + * + * Both OAuth legs must send the same redirect_uri. Pinning it to the + * deployment's canonical app URL also prevents an old domain alias or preview + * host from generating a callback that is not registered with the provider. + * Self-hosted deployments without NEXT_PUBLIC_APP_URL fall back to the + * request origin. + */ +export function resolveCallbackOrigin(requestOrigin: string): string { + const appUrl = process.env.NEXT_PUBLIC_APP_URL + if (appUrl && appUrl.trim().length > 0) { + try { + // Normalizes trailing slashes and strips paths so the provider receives + // the same bare origin on the authorization and token-exchange legs. + const configuredUrl = new URL(appUrl) + if ( + configuredUrl.protocol !== 'http:' && + configuredUrl.protocol !== 'https:' + ) { + return requestOrigin + } + return configuredUrl.origin + } catch { + return requestOrigin + } + } + return requestOrigin +} diff --git a/extensions/general/mail/lib/connections.ts b/extensions/general/mail/lib/connections.ts new file mode 100644 index 00000000..71666c99 --- /dev/null +++ b/extensions/general/mail/lib/connections.ts @@ -0,0 +1,242 @@ +/** + * Reading and maintaining mailbox grants. + * + * Every function here uses the service-role client: `mail_connections` has RLS + * enabled with no policies precisely so a live refresh token can never be + * selected by a browser session. + */ +import { createLogger } from '@/lib/logger' +import type { SupabaseClient } from '@supabase/supabase-js' +import { decryptToken, encryptToken } from './crypto' +import { + MailTokenRefreshError, + getGoogleOAuthEnv, + refreshAccessToken, +} from './google-oauth' + +const log = createLogger('mail-connections') + +export interface MailConnectionRow { + id: string + company_id: string + provider: 'gmail' | 'microsoft' + email_address: string + encrypted_refresh_token: string + encrypted_access_token: string | null + access_token_expires_at: string | null + scope_label: string | null + status: 'active' | 'needs_reconsent' | 'revoked' +} + +/** Safe projection for anything that answers a browser. Never includes tokens. */ +export interface MailConnectionSummary { + id: string + provider: 'gmail' | 'microsoft' + emailAddress: string + scopeLabel: string | null + status: 'active' | 'needs_reconsent' | 'revoked' + lastSearchedAt: string | null + lastErrorCode: string | null +} + +export async function listConnections( + supabase: SupabaseClient, + companyId: string, +): Promise { + const { data } = await supabase + .from('mail_connections') + .select('id, provider, email_address, scope_label, status, last_searched_at, last_error_code') + .eq('company_id', companyId) + .order('created_at', { ascending: true }) + + return ((data ?? []) as Array>).map((row) => ({ + id: row.id as string, + provider: row.provider as 'gmail' | 'microsoft', + emailAddress: row.email_address as string, + scopeLabel: (row.scope_label as string | null) ?? null, + status: row.status as MailConnectionSummary['status'], + lastSearchedAt: (row.last_searched_at as string | null) ?? null, + lastErrorCode: (row.last_error_code as string | null) ?? null, + })) +} + +export async function listActiveConnections( + supabase: SupabaseClient, + companyId: string, +): Promise { + const { data } = await supabase + .from('mail_connections') + .select( + 'id, company_id, provider, email_address, encrypted_refresh_token, encrypted_access_token, access_token_expires_at, scope_label, status', + ) + .eq('company_id', companyId) + .eq('status', 'active') + return (data ?? []) as MailConnectionRow[] +} + +/** + * Upsert on (company, provider, address) so reconnecting the same mailbox + * refreshes the grant instead of creating a twin that gets searched twice. + */ +export async function saveConnection( + supabase: SupabaseClient, + params: { + companyId: string + userId: string + provider: 'gmail' | 'microsoft' + emailAddress: string + refreshToken: string + accessToken: string + expiresAt: Date + scopes: string[] + backfillFrom: string | null + }, +): Promise { + const { error } = await supabase.from('mail_connections').upsert( + { + company_id: params.companyId, + provider: params.provider, + // Lowercased here rather than by an expression index, so the upsert's + // ON CONFLICT target matches the index exactly (Postgres 42P10 otherwise). + email_address: params.emailAddress.trim().toLowerCase(), + connected_by: params.userId, + encrypted_refresh_token: encryptToken(params.refreshToken), + encrypted_access_token: encryptToken(params.accessToken), + access_token_expires_at: params.expiresAt.toISOString(), + scopes: params.scopes, + backfill_from: params.backfillFrom, + status: 'active', + last_error_code: null, + last_error_at: null, + }, + { onConflict: 'company_id,provider,email_address' }, + ) + if (error) throw new Error(`Failed to save mail connection: ${error.message}`) +} + +async function markNeedsReconsent( + supabase: SupabaseClient, + connectionId: string, + code: string, +): Promise { + await supabase + .from('mail_connections') + .update({ status: 'needs_reconsent', last_error_code: code, last_error_at: new Date().toISOString() }) + .eq('id', connectionId) +} + +/** + * A usable access token for one connection, refreshing when it has expired. + * + * Returns null rather than throwing when the grant is dead: one revoked + * mailbox must shrink the hunt, never abort it. + */ +export async function getAccessToken( + supabase: SupabaseClient, + connection: MailConnectionRow, + origin: string, +): Promise { + const expiresAt = connection.access_token_expires_at + ? new Date(connection.access_token_expires_at) + : null + // 60s of slack so a token cannot expire mid-request. + if (connection.encrypted_access_token && expiresAt && expiresAt.getTime() - 60_000 > Date.now()) { + try { + return decryptToken(connection.encrypted_access_token) + } catch { + // Fall through to a refresh: an undecryptable token means the key + // rotated, which a refresh repairs. + } + } + + try { + const env = getGoogleOAuthEnv(origin) + const refreshToken = decryptToken(connection.encrypted_refresh_token) + const refreshed = await refreshAccessToken(env, refreshToken) + await supabase + .from('mail_connections') + .update({ + encrypted_access_token: encryptToken(refreshed.accessToken), + access_token_expires_at: refreshed.expiresAt.toISOString(), + }) + .eq('id', connection.id) + return refreshed.accessToken + } catch (error) { + if (error instanceof MailTokenRefreshError && error.permanent) { + await markNeedsReconsent(supabase, connection.id, 'invalid_grant') + } + return null + } +} + +export async function touchSearched( + supabase: SupabaseClient, + connectionId: string, +): Promise { + await supabase + .from('mail_connections') + .update({ last_searched_at: new Date().toISOString() }) + .eq('id', connectionId) +} + +export async function disconnect( + supabase: SupabaseClient, + companyId: string, + connectionId: string, + userId: string, +): Promise { + // Read the address before the row goes, so the audit entry can name the + // mailbox that stopped being searched. + const { data: existing } = await supabase + .from('mail_connections') + .select('email_address, provider') + .eq('id', connectionId) + .eq('company_id', companyId) + .maybeSingle() + + // Hard delete: the point of disconnecting is that the token is gone. Receipts + // already approved stay, because they belong to the bookkeeping now. + const { error: deleteError } = await supabase + .from('mail_connections') + .delete() + .eq('id', connectionId) + .eq('company_id', companyId) + // A failed delete must not leave an audit entry claiming the mailbox was + // disconnected when the credential is still live. + if (deleteError) throw new Error(deleteError.message) + if (!existing) return + + const row = existing as { email_address: string; provider: string } + + // BFNAR 2013:2 kap 8 (behandlingshistorik): which mailboxes feed underlag into + // the books is a control over how räkenskapsinformation is produced, so + // switching one off has to be reconstructable years later. + // + // Written by hand rather than by the write_audit_log trigger the accounting + // tables use. That trigger copies the whole row into audit_log, which here + // would mean copying an encrypted refresh token into a second table and + // keeping it after the point of the delete was to destroy it. The sibling + // credential tables (shopify_connections) omit the trigger for the same + // reason. Only the safe columns are recorded. + const { error: auditError } = await supabase.from('audit_log').insert({ + user_id: userId, + company_id: companyId, + action: 'DELETE', + table_name: 'mail_connections', + record_id: connectionId, + description: `Brevlåda frånkopplad: ${row.email_address} (${row.provider})`, + old_state: { email_address: row.email_address, provider: row.provider }, + new_state: null, + }) + // Deliberately not rolled back into one transaction. The two statements can + // only diverge one way now: the credential is destroyed and the note about it + // is missing. Recreating the credential to keep them in step would be worse + // than a missing note, so the gap is surfaced loudly instead of hidden. + if (auditError) { + log.error('mailbox disconnected but the audit entry failed to write', { + connectionId, + companyId, + error: auditError.message, + }) + } +} diff --git a/extensions/general/mail/lib/crypto.ts b/extensions/general/mail/lib/crypto.ts new file mode 100644 index 00000000..6299fccc --- /dev/null +++ b/extensions/general/mail/lib/crypto.ts @@ -0,0 +1,79 @@ +import crypto from 'crypto' + +/** + * AES-256-GCM for mailbox refresh tokens. + * + * A mail grant is the hottest credential this product holds: it reads someone's + * correspondence, not just their backups. So the key is its own env var by + * preference (MAIL_TOKEN_ENCRYPTION_KEY, 32 bytes hex) and can be rotated + * without touching the database password, following the Skatteverket + * token-store rather than cloud-backup's service-role derivation. + * + * The derived fallback exists so local development and self-hosted deployments + * work before anyone sets the variable; it is the same trust boundary as the + * database itself, and the purpose string keeps it distinct from every other + * derivation in the codebase. + */ + +const ALGORITHM = 'aes-256-gcm' + +function getKey(): Buffer { + const dedicated = process.env.MAIL_TOKEN_ENCRYPTION_KEY + if (dedicated && dedicated.trim().length > 0) { + const buf = Buffer.from(dedicated.trim(), 'hex') + if (buf.length !== 32) { + throw new Error('MAIL_TOKEN_ENCRYPTION_KEY must be 32 bytes of hex (openssl rand -hex 32)') + } + return buf + } + const secret = process.env.SUPABASE_SERVICE_ROLE_KEY + if (!secret) throw new Error('MAIL_TOKEN_ENCRYPTION_KEY or SUPABASE_SERVICE_ROLE_KEY is required') + return crypto.createHash('sha256').update('mail-connections:v1:' + secret).digest() +} + +export function encryptToken(plaintext: string): string { + const key = getKey() + const iv = crypto.randomBytes(12) + const cipher = crypto.createCipheriv(ALGORITHM, key, iv) + const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]) + const tag = cipher.getAuthTag() + return Buffer.concat([iv, tag, encrypted]).toString('base64url') +} + +export function decryptToken(ciphertext: string): string { + const key = getKey() + const combined = Buffer.from(ciphertext, 'base64url') + const iv = combined.subarray(0, 12) + const tag = combined.subarray(12, 28) + const encrypted = combined.subarray(28) + const decipher = crypto.createDecipheriv(ALGORITHM, key, iv) + decipher.setAuthTag(tag) + return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8') +} + +/** + * Short-lived signed state for OAuth CSRF protection. Stateless and + * self-expiring, so a callback needs no database round-trip to be trusted. + */ +const STATE_TTL_MS = 10 * 60 * 1000 + +interface StatePayload { + u: string + c: string + e: number +} + +export function createOAuthState(userId: string, companyId: string): string { + const payload: StatePayload = { u: userId, c: companyId, e: Date.now() + STATE_TTL_MS } + return encryptToken(JSON.stringify(payload)) +} + +export function verifyOAuthState(state: string): { userId: string; companyId: string } | null { + try { + const payload = JSON.parse(decryptToken(state)) as StatePayload + if (Date.now() > payload.e) return null + return { userId: payload.u, companyId: payload.c } + } catch { + return null + } +} diff --git a/extensions/general/mail/lib/gmail-client.ts b/extensions/general/mail/lib/gmail-client.ts new file mode 100644 index 00000000..9e6a2de4 --- /dev/null +++ b/extensions/general/mail/lib/gmail-client.ts @@ -0,0 +1,203 @@ +/** + * The thin slice of the Gmail API the hunt needs: search, read headers, fetch + * an attachment. Nothing here writes, because the granted scope cannot. + */ +import type { MailCandidate } from '@/lib/mail-search/service' + +const API = 'https://gmail.googleapis.com/gmail/v1/users/me' + +/** + * Deadline on every Gmail call. + * + * Mailboxes are searched with Promise.all, so one stalled request would hold + * the whole company's hunt open until the platform killed the run. A timeout + * turns that into one mailbox missing from tonight's sweep. + */ +export const GMAIL_TIMEOUT_MS = 15_000 + +/** Hits to consider per mailbox per purchase. */ +export const MAX_RESULTS = 8 + +interface GmailHeader { + name: string + value: string +} + +interface GmailPart { + filename?: string + mimeType?: string + body?: { attachmentId?: string; size?: number; data?: string } + parts?: GmailPart[] +} + +/** Longest body worth carrying: a receipt states its total near the top. */ +const MAX_BODY_CHARS = 2500 + +/** + * The readable text of a mail. + * + * Already on the wire (format=full is required to see the parts tree at all), + * so this costs nothing extra, and it is where the two facts a forwarded + * receipt hides live: the original sender and the original date, both written + * into the "Vidarebefordrat meddelande" header that Gmail's 200-character + * snippet cuts off. + */ +function collectBodyText(part: GmailPart | undefined, out: string[]): void { + if (!part) return + const type = part.mimeType ?? '' + if ((type === 'text/plain' || type === 'text/html') && part.body?.data) { + out.push(Buffer.from(part.body.data, 'base64url').toString('utf8')) + } + for (const child of part.parts ?? []) collectBodyText(child, out) +} + +function readableBody(msg: GmailMessage): string { + const chunks: string[] = [] + collectBodyText(msg.payload, chunks) + return chunks + .join('\n') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ |‌|͏/gi, ' ') + .replace(/&/gi, '&') + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_BODY_CHARS) +} + +interface GmailMessage { + id: string + internalDate?: string + snippet?: string + payload?: { + headers?: GmailHeader[] + filename?: string + mimeType?: string + body?: { attachmentId?: string; size?: number } + parts?: GmailPart[] + } +} + +function header(msg: GmailMessage, name: string): string | null { + const found = msg.payload?.headers?.find((h) => h.name.toLowerCase() === name.toLowerCase()) + return found?.value ?? null +} + +/** Attachments anywhere in the MIME tree, ignoring inline images. */ +function collectAttachments( + part: GmailPart | undefined, + out: Array<{ id: string; filename: string }>, +): void { + if (!part) return + const id = part.body?.attachmentId + const named = part.filename && part.filename.length > 0 + const isDocument = + named && + !/^image\/(png|gif)$/i.test(part.mimeType ?? '') // inline logos, not receipts + if (id && isDocument) out.push({ id, filename: part.filename as string }) + for (const child of part.parts ?? []) collectAttachments(child, out) +} + +async function gmailFetch(accessToken: string, path: string): Promise { + const response = await fetch(`${API}${path}`, { + headers: { Authorization: `Bearer ${accessToken}` }, + signal: AbortSignal.timeout(GMAIL_TIMEOUT_MS), + }) + if (!response.ok) { + const text = await response.text().catch(() => '') + throw new Error(`Gmail ${response.status}: ${text.slice(0, 200)}`) + } + return (await response.json()) as T +} + +export async function searchMessageIds( + accessToken: string, + query: string, + maxResults: number = MAX_RESULTS, +): Promise { + const params = new URLSearchParams({ q: query, maxResults: String(maxResults) }) + const data = await gmailFetch<{ messages?: Array<{ id: string }> }>( + accessToken, + `/messages?${params.toString()}`, + ) + return (data.messages ?? []).map((m) => m.id) +} + +/** + * Subject, sender, date and which parts are attachments. + * + * `format=full` rather than `format=metadata`: metadata returns headers only + * and omits `payload.parts` entirely, so every message came back looking like + * it had no attachments and the hunt could never file anything. Gmail offers no + * format that returns the MIME structure without the body, so the body does + * come down the wire here. It is read for nothing and stored nowhere: only + * attachment bytes are ever persisted, and only after a match. + */ +export async function getMessageSummary( + accessToken: string, + messageId: string, + connectionId: string, + mailbox: string, +): Promise { + const msg = await gmailFetch(accessToken, `/messages/${messageId}?format=full`) + const attachments: Array<{ id: string; filename: string }> = [] + collectAttachments(msg.payload, attachments) + + return { + connectionId, + mailbox, + provider: 'gmail', + messageId: msg.id, + subject: header(msg, 'Subject'), + from: header(msg, 'From'), + receivedAt: msg.internalDate + ? new Date(Number(msg.internalDate)).toISOString() + : header(msg, 'Date'), + // A receipt with no attachment is usually the mail body itself; the caller + // decides whether to render it. + attachmentIds: attachments.map((a) => a.id), + attachmentNames: attachments.map((a) => a.filename), + snippet: msg.snippet ?? null, + bodyText: readableBody(msg), + bodyIsReceipt: attachments.length === 0, + } +} + +export async function fetchAttachmentBytes( + accessToken: string, + messageId: string, + attachmentId: string, +): Promise { + const data = await gmailFetch<{ data?: string; size?: number }>( + accessToken, + `/messages/${messageId}/attachments/${attachmentId}`, + ) + if (!data.data) return null + return Buffer.from(data.data, 'base64url') +} + +/** + * Filename and MIME type live on the message, not on the attachment response, + * so they are read back from the parts tree. + */ +export async function describeAttachment( + accessToken: string, + messageId: string, + attachmentId: string, +): Promise<{ filename: string; mimeType: string } | null> { + const msg = await gmailFetch(accessToken, `/messages/${messageId}?format=full`) + let found: { filename: string; mimeType: string } | null = null + const walk = (part: GmailPart | undefined): void => { + if (!part || found) return + if (part.body?.attachmentId === attachmentId) { + found = { + filename: part.filename || 'underlag.pdf', + mimeType: part.mimeType || 'application/octet-stream', + } + return + } + for (const child of part.parts ?? []) walk(child) + } + walk(msg.payload) + return found +} diff --git a/extensions/general/mail/lib/gmail-query.ts b/extensions/general/mail/lib/gmail-query.ts new file mode 100644 index 00000000..42a6bcac --- /dev/null +++ b/extensions/general/mail/lib/gmail-query.ts @@ -0,0 +1,160 @@ +/** + * Turning a bank purchase into a Gmail search. + * + * Pure, because this is where recall is won or lost and it must be testable + * without a mailbox. The search runs provider-side and only the hits come back: + * we never sync or index a mailbox, which is what keeps the feature inside + * Google's Limited Use terms and inside GDPR data minimisation. + */ +import type { MailSearchQuery } from '@/lib/mail-search/service' + +/** + * How far around the purchase date to look. + * + * Asymmetric on purpose: a receipt is normally emailed at or just after the + * purchase, while the bank may post the charge a few days late, so the mail can + * legitimately predate the transaction date. Card settlement is the reason for + * the tail, and Pleo Fetch uses the same shape (-3 / +10). + */ +export const DAYS_BEFORE = 3 +export const DAYS_AFTER = 10 + +/** + * Merchant tokens too generic to search on alone. + * + * The bank's description is not a merchant name: it is whatever the payer typed + * plus the rail it went over. Month names and rail words are in here because a + * provkörning on a real ledger showed "Lön Juli Jakob Överföring via internet" + * searching for `"Juli"`, which matches most of a mailbox and returned the same + * seven unrelated messages for every purchase. + */ +const STOPWORDS = new Set([ + 'ab', 'hb', 'kb', 'inc', 'llc', 'ltd', 'gmbh', 'oy', 'pbc', 'plc', 'corp', 'co', + 'the', 'and', 'och', 'group', 'sweden', 'sverige', 'international', 'kortkop', + 'kortköp', 'uttag', 'betalning', 'payment', 'store', 'shop', 'www', 'com', + // Payment rails and the bank's own boilerplate. + 'överföring', 'overforing', 'internet', 'via', 'bankgiro', 'plusgiro', 'autogiro', + 'bgbet', 'bg-bet', 'insättning', 'insattning', 'inbetalning', 'utbetalning', + 'europabetalning', 'swish', 'faktura', 'invoice', 'kortköputtag', 'kortkoputtag', + // Months, Swedish and English, full and abbreviated. + 'januari', 'februari', 'mars', 'april', 'maj', 'juni', 'juli', 'augusti', + 'september', 'oktober', 'november', 'december', + 'january', 'february', 'march', 'may', 'june', 'july', 'august', 'october', + 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'okt', 'oct', + 'nov', 'dec', +]) + +function isoDay(d: Date): string { + return d.toISOString().slice(0, 10) +} + +function shiftDays(iso: string, days: number): string { + const d = new Date(iso) + d.setDate(d.getDate() + days) + return isoDay(d) +} + +/** + * Pick the tokens worth searching for. Short and generic tokens are dropped: + * a query for "ab" returns the whole mailbox and costs a page of results for + * nothing. + */ +export function merchantTerms(merchant: string | null): string[] { + if (!merchant) return [] + return merchant + .split(/\s+/) + .map((t) => t.trim()) + .filter((t) => t.length >= 3 && !STOPWORDS.has(t.toLowerCase())) + .slice(0, 3) +} + +/** + * Format the amount the way a receipt would write it, so the number itself + * becomes a search term. Swedish receipts write 1 234,50; most SaaS invoices + * write 1234.50. Both forms are offered. + */ +export function amountTerms(amount: number): string[] { + const abs = Math.abs(amount) + const twoDp = abs.toFixed(2) + const terms = new Set([twoDp, twoDp.replace('.', ',')]) + if (Number.isInteger(abs)) terms.add(String(abs)) + + // Swedish invoices group thousands with a space: 15 000,00, not 15000,00. + // Measured against a real mailbox, the Sting office invoice was findable as + // "15 000,00" and "15 000" and by nothing else: every ungrouped form + // returned zero. Cheap to add and it is pure recall. + const group = (v: string) => v.replace(/\B(?=(\d{3})+(?!\d))/g, ' ') + const [whole] = twoDp.split('.') + if (whole.length > 3) { + terms.add(`${group(whole)},${twoDp.split('.')[1]}`) + terms.add(group(whole)) + } + return [...terms] +} + +/** + * Build the Gmail `q` for one purchase. + * + * Structure: a date window, then merchant OR amount. It is deliberately an OR: + * requiring both misses the common cases where the bank's merchant string does + * not appear in the receipt at all (a reseller, a parent company, a brand alias + * like Anthropic billing as Claude), and the amount alone is a strong filter + * inside a two-week window. + * + * `has:attachment` is NOT required: plenty of receipts are the mail body itself. + */ +export function buildGmailQuery(query: MailSearchQuery): string { + const parts: string[] = [] + + // Off by default now. Receipts reach these mailboxes by being forwarded, and + // a forward carries the forwarding date, so windowing on the purchase date + // hides the very mail we want. The caller re-establishes precision by having + // the model judge the hits instead. + if (query.useDateWindow !== false) { + const after = shiftDays(query.date, -DAYS_BEFORE - 1) // Gmail's after: is exclusive + const before = shiftDays(query.date, DAYS_AFTER + 1) + parts.push(`after:${after.replace(/-/g, '/')}`, `before:${before.replace(/-/g, '/')}`) + } + + // Merchant OR amount, never one instead of the other. + // + // The amount is the single strongest signal a reconciliation has: dates drift + // because banks post late and mail gets forwarded, but an amount does not + // drift. Gmail indexes text inside PDF attachments, so a Swedish receipt is + // often findable by its total alone. It is an OR rather than an AND because + // neither signal survives every case: a receipt billed in USD never contains + // the SEK figure the bank charged, and a bank descriptor frequently names + // nothing that appears in the receipt. + const aliases = (query.aliases ?? []).map((a) => a.trim()).filter((a) => a.length >= 2) + const names = aliases.length > 0 ? aliases : merchantTerms(query.merchant) + const alternatives = [ + ...amountTerms(query.amount).map((a) => `"${a}"`), + ...names.map((t) => `"${t}"`), + ] + + if (alternatives.length > 0) { + parts.push(`(${alternatives.join(' OR ')})`) + } + + if (query.requireAttachment) parts.push('has:attachment') + + // Calendar invitations and the user's own outbound mail are never receipts. + parts.push('-in:chats') + + return parts.join(' ') +} + +/** + * Cheap pre-filter on a hit before spending a model call on it. + * + * Gmail's OR query is broad by design, so most hits are not receipts. Anything + * that looks like a newsletter or a calendar notice is dropped here rather than + * being read. + */ +const OBVIOUS_NON_RECEIPT = /\b(nyhetsbrev|newsletter|unsubscribe|prenumerera|kalender|calendar invite|inbjudan)\b/i + +export function looksLikeReceipt(subject: string | null, from: string | null): boolean { + const haystack = `${subject ?? ''} ${from ?? ''}` + if (OBVIOUS_NON_RECEIPT.test(haystack)) return false + return true +} diff --git a/extensions/general/mail/lib/google-oauth.ts b/extensions/general/mail/lib/google-oauth.ts new file mode 100644 index 00000000..c14c9bc8 --- /dev/null +++ b/extensions/general/mail/lib/google-oauth.ts @@ -0,0 +1,189 @@ +/** + * Gmail OAuth, read-only. + * + * The scope is `gmail.readonly` and nothing else. That is enough to search and + * to download attachment bytes (verified against Google's method-scope table), + * and it structurally cannot send, modify or delete: the promise made in the + * consent screen is enforced by the grant, not by our code being careful. + * + * Consequence worth remembering: because we never hold a send scope, the agent + * can prepare a forward for the user but can never send one itself. + */ +export const GMAIL_READONLY_SCOPE = 'https://www.googleapis.com/auth/gmail.readonly' + +const AUTH_ENDPOINT = 'https://accounts.google.com/o/oauth2/v2/auth' +/** + * Deadline on the token endpoint. + * + * A refresh happens inside every mailbox search, and searches run with + * Promise.all, so a stalled token endpoint would hold the whole company's hunt + * open. On timeout the connection simply yields nothing this run. + */ +export const TOKEN_TIMEOUT_MS = 15_000 + +const TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token' + +export interface GoogleOAuthEnv { + clientId: string + clientSecret: string + redirectUri: string +} + +/** + * Deliberately distinct from cloud-backup's GOOGLE_CLIENT_ID: that is a + * different OAuth client, in a different project, owned by a different founder, + * and sharing the pair would let one integration's credential rotation break + * the other. + */ +export function isGoogleMailConfigured(): boolean { + return Boolean(process.env.GOOGLE_MAIL_CLIENT_ID && process.env.GOOGLE_MAIL_CLIENT_SECRET) +} + +export function getGoogleOAuthEnv(origin: string): GoogleOAuthEnv { + const clientId = process.env.GOOGLE_MAIL_CLIENT_ID + const clientSecret = process.env.GOOGLE_MAIL_CLIENT_SECRET + if (!clientId || !clientSecret) { + throw new Error('Gmail is not configured: set GOOGLE_MAIL_CLIENT_ID and GOOGLE_MAIL_CLIENT_SECRET') + } + return { + clientId, + clientSecret, + // Must match the string registered in the Google console exactly; the + // extension slug `mail` is pinned for that reason. + redirectUri: `${origin}/api/extensions/ext/mail/oauth/callback`, + } +} + +export function buildAuthorizationUrl(env: GoogleOAuthEnv, state: string): string { + const params = new URLSearchParams({ + client_id: env.clientId, + redirect_uri: env.redirectUri, + response_type: 'code', + scope: `openid email ${GMAIL_READONLY_SCOPE}`, + // Signed, self-expiring CSRF token. The callback refuses anything without + // it, so omitting this breaks the flow as well as the protection. + state, + // offline + consent is what returns a refresh token at all; without it a + // grant dies in an hour and the nightly hunt silently stops. + access_type: 'offline', + // No include_granted_scopes: it lets Google add scopes this app was granted + // elsewhere to the token it returns here, so a mailbox grant could quietly + // carry more authority than the consent screen showed. + prompt: 'consent', + }) + return `${AUTH_ENDPOINT}?${params.toString()}` +} + +export interface GoogleTokens { + accessToken: string + refreshToken: string | null + expiresAt: Date + scopes: string[] + email: string | null +} + +/** Raised when a grant is dead rather than the request being unlucky. */ +export class MailTokenRefreshError extends Error { + constructor( + message: string, + readonly permanent: boolean, + ) { + super(message) + this.name = 'MailTokenRefreshError' + } +} + +function decodeIdTokenEmail(idToken: string | undefined): string | null { + if (!idToken) return null + try { + const payload = idToken.split('.')[1] + const json = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')) as { + email?: string + } + return json.email ?? null + } catch { + return null + } +} + +export async function exchangeCodeForTokens( + env: GoogleOAuthEnv, + code: string, +): Promise { + const response = await fetch(TOKEN_ENDPOINT, { + method: 'POST', + signal: AbortSignal.timeout(TOKEN_TIMEOUT_MS), + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + code, + client_id: env.clientId, + client_secret: env.clientSecret, + redirect_uri: env.redirectUri, + grant_type: 'authorization_code', + }), + }) + const body = (await response.json()) as { + access_token?: string + refresh_token?: string + expires_in?: number + scope?: string + id_token?: string + error?: string + error_description?: string + } + if (!response.ok || !body.access_token) { + throw new Error(body.error_description || body.error || 'Token exchange failed') + } + if (!body.refresh_token) { + // Google withholds it when the user has an older grant for this client. + // Say so plainly: the fix is to revoke at myaccount.google.com and retry, + // and a connection without one is useless the moment the hour is up. + throw new Error( + 'Google returned no refresh token. Remove the previous access for this app at myaccount.google.com/permissions and connect again.', + ) + } + return { + accessToken: body.access_token, + refreshToken: body.refresh_token, + expiresAt: new Date(Date.now() + (body.expires_in ?? 3600) * 1000), + scopes: (body.scope ?? '').split(' ').filter(Boolean), + email: decodeIdTokenEmail(body.id_token), + } +} + +export async function refreshAccessToken( + env: GoogleOAuthEnv, + refreshToken: string, +): Promise<{ accessToken: string; expiresAt: Date }> { + const response = await fetch(TOKEN_ENDPOINT, { + method: 'POST', + signal: AbortSignal.timeout(TOKEN_TIMEOUT_MS), + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + refresh_token: refreshToken, + client_id: env.clientId, + client_secret: env.clientSecret, + grant_type: 'refresh_token', + }), + }) + const body = (await response.json()) as { + access_token?: string + expires_in?: number + error?: string + error_description?: string + } + if (!response.ok || !body.access_token) { + // invalid_grant means revoked, expired or password-changed: retrying every + // night would just burn quota, so it is flagged permanent and the + // connection is parked as needs_reconsent. + const permanent = body.error === 'invalid_grant' + throw new MailTokenRefreshError( + body.error_description || body.error || 'Token refresh failed', + permanent, + ) + } + return { + accessToken: body.access_token, + expiresAt: new Date(Date.now() + (body.expires_in ?? 3600) * 1000), + } +} diff --git a/extensions/general/mail/lib/search-service.ts b/extensions/general/mail/lib/search-service.ts new file mode 100644 index 00000000..c177df4f --- /dev/null +++ b/extensions/general/mail/lib/search-service.ts @@ -0,0 +1,135 @@ +/** + * The Gmail implementation of the core MailSearchService contract. + * + * Query-then-classify, never sync: for each purchase we run a provider-side + * search, pull metadata for the few hits, and let the caller decide. No mailbox + * is mirrored, no message body is stored, and nothing is written back to the + * mailbox. That is what keeps this inside Google's Limited Use terms and inside + * GDPR data minimisation, and it is the promise the consent screen makes. + */ +import { createServiceClientNoCookies } from '@/lib/auth/api-keys' +import { createLogger } from '@/lib/logger' +import type { + FetchedAttachment, + MailCandidate, + MailSearchQuery, + MailSearchService, +} from '@/lib/mail-search/service' +import { buildGmailQuery, looksLikeReceipt } from './gmail-query' +import { + describeAttachment, + fetchAttachmentBytes, + getMessageSummary, + searchMessageIds, +} from './gmail-client' +import { + getAccessToken, + listActiveConnections, + touchSearched, + type MailConnectionRow, +} from './connections' +import { isGoogleMailConfigured } from './google-oauth' + +const log = createLogger('mail-search') + +/** + * Origin used to rebuild the redirect_uri during a token refresh. Google + * requires the same value the grant was issued against, so it is derived from + * the deployment's canonical URL rather than a request that may not exist + * (the hunt runs from a cron, with no browser origin to borrow). + */ +function canonicalOrigin(): string { + return process.env.NEXT_PUBLIC_APP_URL?.trim() || 'http://localhost:3000' +} + +export class GmailSearchService implements MailSearchService { + isConfigured(): boolean { + return isGoogleMailConfigured() + } + + async search(companyId: string, query: MailSearchQuery): Promise { + if (!this.isConfigured()) return [] + + const supabase = createServiceClientNoCookies() + const connections = await listActiveConnections(supabase, companyId) + if (connections.length === 0) return [] + + const q = buildGmailQuery(query) + + // Mailboxes are searched in parallel: the work is read-only, so there is + // nothing to serialise, and one slow account should not delay the rest. + const perConnection = await Promise.all( + connections.map((connection) => this.searchOne(supabase, connection, q, query.limit)), + ) + return perConnection.flat() + } + + private async searchOne( + supabase: ReturnType, + connection: MailConnectionRow, + q: string, + limit?: number, + ): Promise { + // A dead grant shrinks the hunt rather than aborting it; getAccessToken has + // already parked it as needs_reconsent for the UI to surface. + const accessToken = await getAccessToken(supabase, connection, canonicalOrigin()) + if (!accessToken) return [] + + try { + const ids = await searchMessageIds(accessToken, q, limit) + if (ids.length === 0) return [] + + const summaries = await Promise.all( + ids.map((id) => + getMessageSummary(accessToken, id, connection.id, connection.email_address), + ), + ) + await touchSearched(supabase, connection.id) + + // Cheap pre-filter before anything expensive looks at these. + return summaries.filter((c) => looksLikeReceipt(c.subject, c.from)) + } catch (error) { + // Never let one mailbox's failure surface as the company's failure. + log.warn('gmail search failed for connection', { + connectionId: connection.id, + error: error instanceof Error ? error.message : String(error), + }) + return [] + } + } + + async fetchAttachment( + connectionId: string, + messageId: string, + attachmentId: string, + ): Promise { + const supabase = createServiceClientNoCookies() + const { data } = await supabase + .from('mail_connections') + .select( + 'id, company_id, provider, email_address, encrypted_refresh_token, encrypted_access_token, access_token_expires_at, scope_label, status', + ) + .eq('id', connectionId) + .maybeSingle() + if (!data) return null + + const accessToken = await getAccessToken( + supabase, + data as MailConnectionRow, + canonicalOrigin(), + ) + if (!accessToken) return null + + const [bytes, described] = await Promise.all([ + fetchAttachmentBytes(accessToken, messageId, attachmentId), + describeAttachment(accessToken, messageId, attachmentId), + ]) + if (!bytes) return null + + return { + filename: described?.filename ?? 'underlag.pdf', + mimeType: described?.mimeType ?? 'application/octet-stream', + bytes, + } + } +} diff --git a/extensions/general/mail/manifest.json b/extensions/general/mail/manifest.json new file mode 100644 index 00000000..19f09c1d --- /dev/null +++ b/extensions/general/mail/manifest.json @@ -0,0 +1,19 @@ +{ + "id": "mail", + "sector": "general", + "exportName": "mailExtension", + "entryPoint": "@/extensions/general/mail", + "requiredEnvVars": ["GOOGLE_MAIL_CLIENT_ID", "GOOGLE_MAIL_CLIENT_SECRET"], + "optionalEnvVars": ["MAIL_TOKEN_ENCRYPTION_KEY"], + "npmDependencies": [], + "definition": { + "name": "Brevlådor", + "category": "operations", + "icon": "Mail", + "dataPattern": "manual", + "hasOwnData": true, + "description": "Låt Kvittojakten leta upp kvitton i era brevlådor", + "longDescription": "Koppla en eller flera brevlådor, så letar Kvittojakten själv upp kvitton till kortköp som saknar underlag. Åtkomsten är läsbehörighet: agenten kan aldrig skicka, ändra eller radera något i din mejl. Inkorgen kopieras aldrig, utan bara mejl som kan vara ett kvitto till ett visst köp hämtas i stunden och släpps igen. Det som blir underlag arkiveras i ert vanliga sjuåriga arkiv, efter att du godkänt det.", + "subscriptionNotice": "Kräver ett Google-konto. Varje brevlåda kopplas av sin egen ägare och kan kopplas från när som helst." + } +} diff --git a/lib/documents/__tests__/core-receipt-matcher.test.ts b/lib/documents/__tests__/core-receipt-matcher.test.ts index 3e17af96..6ff64608 100644 --- a/lib/documents/__tests__/core-receipt-matcher.test.ts +++ b/lib/documents/__tests__/core-receipt-matcher.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest' import { levenshteinDistance, normalizeMerchantName, + normalizeForMatch, calculateMerchantSimilarity, calculateMatchConfidence, amountVarianceForMatch, @@ -161,3 +162,85 @@ describe('amountVarianceForMatch', () => { expect(amountVarianceForMatch(100, 'eur', null, -100, 'EUR', -1150)).toBe(0) }) }) + +describe('normalizeForMatch', () => { + it('leaves the frozen key normalizer alone', () => { + // normalizeMerchantName feeds a PERSISTED unique key with a SQL mirror. + // If this ever fails, the konteringskarta join is about to drift. + expect(normalizeMerchantName('Telia AB')).toBe('telia') + expect(normalizeMerchantName('Café Överkås!')).toBe('café överkås') + }) + + it('strips the Swedish card rails', () => { + expect(normalizeForMatch('Ryde Sweden AB K8066 Kortköp/uttag')).toBe('ryde sweden') + expect(normalizeForMatch('Kortköp 260612 Prime Video')).toBe('prime video') + expect(normalizeForMatch('ELGIGANTEN S/25-07-14')).toBe('elgiganten s') + }) + + it('folds the three ways banks mangle Swedish letters', () => { + // Same merchant, spelled three ways by three feeds. + const a = normalizeForMatch('Alviks kött och fisk') + expect(normalizeForMatch('Alviks koett och fisk')).toBe(a) + expect(normalizeForMatch('Alviks k??tt och fisk')).not.toBe('') + }) + + it('keeps both sides of a processor marker', () => { + // The merchant is second in K*IKEA and first in GOOGLE*PLAY. + expect(normalizeForMatch('K*IKEA GALLE')).toContain('ikea') + expect(normalizeForMatch('GOOGLE*PLAY')).toContain('google') + }) + + it('drops legal forms and reference numbers', () => { + expect(normalizeForMatch('Adobe Systems Software Ireland Ltd')).toBe('adobe systems software ireland') + expect(normalizeForMatch('GOOGLE ADS8047863617')).toBe('google ads') + }) +}) + +describe('calculateMerchantSimilarity on real confirmed pairs', () => { + // Every pair below is one a human actually made in production + // (invoice_inbox_items.matched_transaction_id), so these are recall targets, + // not invented examples. + const CONFIRMED: Array<[string, string]> = [ + ['APPLE COM/SE', 'APPLE COM/SE/25-02-20'], + ['Word and Sound Medien GmbH', 'Word and Sound GmbH'], + ['Anomaly', 'ANOMALY,SAN FRANCISCO,US Kortköp'], + ['Tradera Marketplace AB', 'TRADERA 1022'], + ['DigitalOcean LLC', 'DIGITALOCEAN.COM AMSTERDAM Kortköp/uttag'], + ['Cinode AB', 'WWW.CINODE.COM'], + ['Kjell & Company', 'KjellCo Oktober'], + ['Hostinger International Ltd.', 'Hostinger Apr JW'], + ['OpenAI OpCo, LLC', 'OPENAI *CHATGP'], + ['Google Ads', 'GOOGLE ADS8047863617'], + ['Elgiganten', 'ELGIGANTEN S/25-07-14'], + ['Hanko GmbH', 'HANKO IO'], + ['Panduro', 'PANDURO LUND'], + ['Rusta Lindingö 135', 'RUSTA LINDING?? 135'], + ['Loopia AB', 'Loopia'], + ['Kilo Code', 'KILO CODE INC,SAN FRANCISCO,US Kortköp'], + ['DNH GODADDY', 'DNH GODADDY /25-07-06'], + ['Adobe Systems Software Ireland Ltd', 'Adobe'], + ['Lennart & Bror Kött (Alviks kött och fisk AB)', 'Alviks koett och fisk K3667 Kortköp/uttag'], + ['Ryde Sweden AB', 'Ryde Sweden AB K8066 Kortköp/uttag'], + ['IKEA', 'K*IKEA GALLE'], + ['Espresso House', 'ESPRESSO HOUSE 1234 STOCKHOLM'], + ] + + it.each(CONFIRMED)('recognises %s ↔ %s', (receipt, bank) => { + expect(calculateMerchantSimilarity(receipt, bank)).toBeGreaterThanOrEqual(0.6) + }) + + // Aggressive folding buys recall; these guard the price of it. + const DIFFERENT: Array<[string, string]> = [ + ['Cloudflare', 'Clas Ohlson'], + ['SJ AB', 'Skatteverket'], + ['Adobe', 'Apple'], + ['ICA Maxi Stockholm', 'Coop Solna'], + ['Anthropic, PBC', 'Anomaly'], + ['Hostinger International Ltd.', 'Hanko GmbH'], + ['Google Ads', 'Google Cloud EMEA Limited'], + ] + + it.each(DIFFERENT)('keeps %s apart from %s', (a, b) => { + expect(calculateMerchantSimilarity(a, b)).toBeLessThan(0.6) + }) +}) diff --git a/lib/documents/core-receipt-matcher.ts b/lib/documents/core-receipt-matcher.ts index 87d02918..148a3b67 100644 --- a/lib/documents/core-receipt-matcher.ts +++ b/lib/documents/core-receipt-matcher.ts @@ -13,6 +13,22 @@ export const MIN_MATCH_CONFIDENCE = 0.4 /** * Normalize a merchant name for comparison. * Removes special characters, Swedish company suffixes, and extra whitespace. + * + * FROZEN. This is not merely a helper: `normalizeCounterpartyName` + * (lib/bookkeeping/counterparty-templates.ts) ends in this function, and its + * output is PERSISTED as `categorization_templates.counterparty_name` under + * UNIQUE (company_id, counterparty_name), with a hand-written SQL mirror + * `public.normalize_counterparty_key()` that the ledger-context RPC recomputes + * at query time. Change this and stored keys stop equalling computed ones: the + * konteringskarta join misses, learned vat_treatment degrades to history, and + * `insertOrUpdateTemplate` silently inserts a SECOND row per merchant, orphaning + * the occurrence counts instead of migrating them. + * + * To improve MATCHING, edit `normalizeForMatch` below, which nothing persists. + * To improve the canonical KEY, it is a three-part atomic change: this function + * + CREATE OR REPLACE of the SQL mirror + a backfill/merge migration over + * categorization_templates, with tests/pg/ledger-usage-stats-rpc.pg.test.ts + * extended in the same commit. */ export function normalizeMerchantName(name: string): string { return name @@ -23,6 +39,72 @@ export function normalizeMerchantName(name: string): string { .trim() } +/** + * Legal-form tokens that carry no identity. Deliberately wider than the frozen + * key normalizer's list: a receipt says "Adobe Systems Software Ireland Ltd" + * where the bank says "Adobe", and only the matcher needs to see through that. + */ +const LEGAL_FORM_TOKENS = + /\b(ab|hb|kb|ek|för|stiftelse|inc|llc|ltd|limited|gmbh|oy|oyj|ap|aps|pbc|plc|sarl|bv|nv|corp|corporation|company|filial|int|international)\b/g + +/** + * Noise the Swedish card rails staple onto a merchant, observed in production: + * `Ryde Sweden AB K8066 Kortköp/uttag`, `Kortköp 260612 Prime Video-*NL5EK5W`, + * `ELGIGANTEN S/25-07-14`, `Qstar Lilla Edet 6531 K8781 Kortköp/uttag`. + */ +const CARD_TOKEN = /\bk\d{4}\b/g +const CARD_VERB = /\bkort(kop|kop\/uttag)?\b|\buttag\b/g +const CARD_DATE_PREFIX = /^\s*kortkop\s+\d{6}\s*/ +const TRAILING_DATE = /\s*\/?\s*\d{2}-\d{2}-\d{2}\s*$/ +/** + * Reference numbers, which banks glue straight onto the name + * ("GOOGLE ADS8047863617"), so this deliberately has no word boundary. Runs of + * three digits or fewer stay: they are often part of the identity + * ("Rusta Lindingö 135", "7-Eleven"). + */ +const LONG_DIGIT_RUN = /\d{4,}/g +const DOMAIN_TAIL = /\.(com|se|io|ai|co|net|org|nu|dk|no|fi|de|uk)\b/g + +/** + * Fold a merchant string down to the part that actually identifies it, for + * SIMILARITY ONLY. Nothing persists this, so it can be aggressive where the + * frozen key normalizer must not be. + * + * Aggressive folding is safe here precisely because it is applied to BOTH sides + * of every comparison: an over-eager fold that turns "Boeing" into "boing" does + * so for the receipt and the bank row alike, so the pair still matches. The only + * real risk is two genuinely different merchants colliding, which the amount and + * date signals then have to disagree with. + */ +export function normalizeForMatch(name: string): string { + let s = name.toLowerCase() + + // Banks disagree about diacritics: one writes "kött", another transliterates + // to "koett", a third mangles the encoding into "LINDING??". Fold all three + // to the same base letters so they stop being three different merchants. + s = s.normalize('NFD').replace(/[̀-ͯ]/g, '') + s = s.replace(/\?{2,}|�/g, ' ') + s = s.replace(/oe/g, 'o').replace(/ae/g, 'a').replace(/aa/g, 'a') + + // A processor marker hides the merchant on one side or the other: + // GOOGLE*PLAY puts it first, K*IKEA GALLE puts it second. Keep both. + s = s.replace(/[*_]+/g, ' ') + + s = s.replace(CARD_DATE_PREFIX, ' ') + s = s.replace(TRAILING_DATE, ' ') + s = s.replace(CARD_TOKEN, ' ') + s = s.replace(CARD_VERB, ' ') + s = s.replace(DOMAIN_TAIL, ' ') + s = s.replace(/\bwww\b/g, ' ') + + // Punctuation to space rather than nothing, so "Word,and" stays two tokens. + s = s.replace(/[^\w\s]/g, ' ') + s = s.replace(LONG_DIGIT_RUN, ' ') + s = s.replace(LEGAL_FORM_TOKENS, ' ') + + return s.replace(/\s+/g, ' ').trim() +} + /** * Calculate Levenshtein (edit) distance between two strings. */ @@ -58,20 +140,34 @@ export function levenshteinDistance(str1: string, str2: string): number { export function calculateMerchantSimilarity(name1: string, name2: string): number { if (!name1 || !name2) return 0 - const n1 = normalizeMerchantName(name1) - const n2 = normalizeMerchantName(name2) + // Matching-only folding: sees through card tokens, processor stars, + // transliterated diacritics and legal forms, none of which change identity. + const n1 = normalizeForMatch(name1) + const n2 = normalizeForMatch(name2) + if (!n1 || !n2) return 0 // Exact match if (n1 === n2) return 1 - // One contains the other + // One contains the other. Also covers the bank's fixed-width truncation + // ("apple com bi" inside "apple com bill"), which is a prefix by nature. if (n1.includes(n2) || n2.includes(n1)) return 0.9 - // Word overlap - const words1 = n1.split(/\s+/) - const words2 = n2.split(/\s+/) - const commonWords = words1.filter((w) => words2.includes(w)) + const words1 = n1.split(/\s+/).filter(Boolean) + const words2 = n2.split(/\s+/).filter(Boolean) + const set1 = new Set(words1) + const set2 = new Set(words2) + const commonWords = words1.filter((w) => set2.has(w)) + // Every token of the shorter name appears in the longer one: "Adobe" against + // "Adobe Systems Software Ireland", or a receipt's legal name against the + // bank's trading name. Scored level with substring containment because it is + // the same claim, made token-wise instead of character-wise. + const smaller = set1.size <= set2.size ? set1 : set2 + const larger = smaller === set1 ? set2 : set1 + if (smaller.size > 0 && [...smaller].every((w) => larger.has(w))) return 0.9 + + // Word overlap if (commonWords.length > 0) { const overlapScore = commonWords.length / Math.max(words1.length, words2.length) if (overlapScore >= 0.5) return 0.7 + overlapScore * 0.2 diff --git a/lib/extensions/__tests__/sectors.test.ts b/lib/extensions/__tests__/sectors.test.ts index f6ce823a..5072dd1a 100644 --- a/lib/extensions/__tests__/sectors.test.ts +++ b/lib/extensions/__tests__/sectors.test.ts @@ -48,8 +48,8 @@ describe('sectors registry', () => { expect(SECTORS.length).toBe(1) }) - it('should have 17 total extensions', () => { - expect(getAllExtensions().length).toBe(17) + it('should have 18 total extensions', () => { + expect(getAllExtensions().length).toBe(18) }) it('should have unique slugs within each sector', () => { @@ -94,7 +94,7 @@ describe('sectors registry', () => { it('getExtensionsBySector returns extensions for a sector', () => { const extensions = getExtensionsBySector('general') - expect(extensions.length).toBe(17) + expect(extensions.length).toBe(18) }) it('all extensions have required fields', () => { diff --git a/lib/extensions/_generated/enabled-extensions.ts b/lib/extensions/_generated/enabled-extensions.ts index c92c5bb3..acb7144c 100644 --- a/lib/extensions/_generated/enabled-extensions.ts +++ b/lib/extensions/_generated/enabled-extensions.ts @@ -14,4 +14,5 @@ export const ENABLED_EXTENSION_IDS: ReadonlySet = new Set([ 'whatsapp-inbox', 'woocommerce', 'shopify', + 'mail', ]) diff --git a/lib/extensions/_generated/extension-list.ts b/lib/extensions/_generated/extension-list.ts index 44672b80..985e8f8e 100644 --- a/lib/extensions/_generated/extension-list.ts +++ b/lib/extensions/_generated/extension-list.ts @@ -13,6 +13,7 @@ import { stripeExtension } from '@/extensions/general/stripe' import { whatsappInboxExtension } from '@/extensions/general/whatsapp-inbox' import { woocommerceExtension } from '@/extensions/general/woocommerce' import { shopifyExtension } from '@/extensions/general/shopify' +import { mailExtension } from '@/extensions/general/mail' export const FIRST_PARTY_EXTENSIONS: Extension[] = [ enableBankingExtension, @@ -28,4 +29,5 @@ export const FIRST_PARTY_EXTENSIONS: Extension[] = [ whatsappInboxExtension, woocommerceExtension, shopifyExtension, + mailExtension, ] diff --git a/lib/extensions/_generated/sector-definitions.ts b/lib/extensions/_generated/sector-definitions.ts index dc2e23e0..49454ed0 100644 --- a/lib/extensions/_generated/sector-definitions.ts +++ b/lib/extensions/_generated/sector-definitions.ts @@ -169,5 +169,17 @@ export const EXTENSION_DEFINITIONS: Record = { "longDescription": "Anslut din Shopify-butik så hämtas betalda ordrar och återbetalningar automatiskt varje natt till transaktionsinkorgen, som ett bankflöde för butiken. Inget bokförs automatiskt: du bokför raderna själv precis som vanliga banktransaktioner.", "hasOwnData": true }, + { + "slug": "mail", + "name": "Brevlådor", + "sector": "general", + "category": "operations", + "icon": "Mail", + "dataPattern": "manual", + "description": "Låt Kvittojakten leta upp kvitton i era brevlådor", + "longDescription": "Koppla en eller flera brevlådor, så letar Kvittojakten själv upp kvitton till kortköp som saknar underlag. Åtkomsten är läsbehörighet: agenten kan aldrig skicka, ändra eller radera något i din mejl. Inkorgen kopieras aldrig, utan bara mejl som kan vara ett kvitto till ett visst köp hämtas i stunden och släpps igen. Det som blir underlag arkiveras i ert vanliga sjuåriga arkiv, efter att du godkänt det.", + "hasOwnData": true, + "subscriptionNotice": "Kräver ett Google-konto. Varje brevlåda kopplas av sin egen ägare och kan kopplas från när som helst." + }, ], } diff --git a/lib/mail-search/service.ts b/lib/mail-search/service.ts new file mode 100644 index 00000000..5cb4938d --- /dev/null +++ b/lib/mail-search/service.ts @@ -0,0 +1,121 @@ +/** + * Mail Search Service Interface + * + * Core defines the contract; the `mail` extension registers a real + * implementation (Gmail today, Microsoft Graph next). Without the extension a + * no-op service is used, so the receipt hunt degrades to matching whatever the + * company already holds in Underlag rather than failing. + * + * Mirrors lib/email/service.ts: core must never import from @/extensions, and + * a CI build with zero extensions has to compile. + */ + +/** What the hunt knows about the purchase it is trying to find a receipt for. */ +export interface MailSearchQuery { + /** Merchant tokens, already folded (see core-receipt-matcher). */ + merchant: string | null + /** Charged amount, always positive. */ + amount: number + currency: string + /** Purchase date, ISO. The window around it is the adapter's business. */ + date: string + /** + * Merchant names likely to appear in the receipt itself, best first. Searched + * in place of the raw descriptor when present: the bank writes + * "ANTHROPIC* CLAUDE SUB", the receipt says "Anthropic". + */ + aliases?: string[] + /** + * Set false to search the whole mailbox regardless of date. + * + * A forwarded receipt is stamped when it was forwarded, which can be months + * after the purchase, so a window around the purchase date hides exactly the + * mail being looked for. Measured on a real mailbox: the same merchant search + * returned 0 hits inside the window and 10+ outside it. + */ + useDateWindow?: boolean + /** Only return messages carrying a file. */ + requireAttachment?: boolean + /** Hits to return per mailbox. */ + limit?: number +} + +/** + * A message that might carry the receipt. Nothing is stored at this point: the + * hunt decides, and only a confirmed receipt is ever persisted. + */ +export interface MailCandidate { + connectionId: string + /** Mailbox the hit came from, so a proposal can say where it looked. */ + mailbox: string + provider: 'gmail' | 'microsoft' + messageId: string + subject: string | null + from: string | null + receivedAt: string | null + /** Attachment ids on the message, resolvable through fetchAttachment. */ + attachmentIds: string[] + /** + * Attachment filenames and the provider's own preview line. Metadata only, + * never stored: they are what lets the hunt tell a receipt from a newsletter + * that merely names the merchant, without opening anyone's mail. + */ + attachmentNames?: string[] + snippet?: string | null + /** + * Readable body text, already downloaded. A forwarded receipt writes the + * original sender and the original purchase date into its quoted header, + * which is the only reliable way to date a mail that was forwarded months + * later. Never stored: it is read once to extract fields and discarded. + */ + bodyText?: string | null + /** + * True when the message body IS the receipt (SL, Uber-style HTML mail) and + * there is no attachment to fetch. The caller renders it instead. + */ + bodyIsReceipt: boolean +} + +export interface FetchedAttachment { + filename: string + mimeType: string + bytes: Buffer +} + +export interface MailSearchService { + /** + * Search every healthy connection for one company. Read-only: the scopes + * requested cannot send, modify or delete, and nothing is written to the + * mailbox. + */ + search(companyId: string, query: MailSearchQuery): Promise + fetchAttachment( + connectionId: string, + messageId: string, + attachmentId: string, + ): Promise + /** True when at least one provider has credentials configured. */ + isConfigured(): boolean +} + +class NoopMailSearchService implements MailSearchService { + async search(): Promise { + return [] + } + async fetchAttachment(): Promise { + return null + } + isConfigured(): boolean { + return false + } +} + +let mailSearchService: MailSearchService = new NoopMailSearchService() + +export function getMailSearchService(): MailSearchService { + return mailSearchService +} + +export function registerMailSearchService(svc: MailSearchService): void { + mailSearchService = svc +} diff --git a/lib/receipt-hunt/__tests__/hunt.test.ts b/lib/receipt-hunt/__tests__/hunt.test.ts new file mode 100644 index 00000000..c2126657 --- /dev/null +++ b/lib/receipt-hunt/__tests__/hunt.test.ts @@ -0,0 +1,134 @@ +/** + * huntCompany's read/write shell. The ranking is covered in select.test.ts; + * what matters here is that a dry run cannot write, and that a real run stages + * exactly one operation per proposal with the shape the approval card reads. + */ +import { describe, it, expect, vi } from 'vitest' +import { huntCompany } from '../hunt' + +type Row = Record + +/** + * Minimal PostgREST stand-in: every builder method returns the chain, and + * awaiting it yields whatever the table was seeded with. `range` is honoured so + * fetchAllRows terminates. + */ +function mockSupabase(tables: Record, onInsert?: (t: string, rows: Row[]) => void) { + const inserts: Array<{ table: string; rows: Row[] }> = [] + const client = { + from(table: string) { + let from = 0 + let to = Number.MAX_SAFE_INTEGER + const chain: Record = {} + for (const m of ['select', 'eq', 'is', 'not', 'in', 'lte', 'gte', 'order', 'limit']) { + chain[m] = vi.fn(() => chain) + } + chain.range = vi.fn((f: number, t: number) => { + from = f + to = t + return chain + }) + chain.insert = vi.fn((rows: Row[]) => { + inserts.push({ table, rows }) + onInsert?.(table, rows) + return Promise.resolve({ error: null }) + }) + chain.then = (resolve: (v: unknown) => unknown) => + Promise.resolve({ data: (tables[table] ?? []).slice(from, to + 1), error: null }).then(resolve) + return chain + }, + } + return { client: client as never, inserts } +} + +const TX = { + id: 'tx-1', + company_id: 'co-1', + date: '2026-05-02', + description: 'CIRCLE K 421', + merchant_name: 'Circle K', + amount: -438.75, + currency: 'SEK', + amount_sek: -438.75, + exchange_rate: null, +} + +const ITEM = { + id: 'item-1', + document_id: 'doc-1', + extracted_data: { + supplier: { name: 'Circle K' }, + invoice: { invoiceDate: '2026-05-02', currency: 'SEK' }, + totals: { total: 438.75, vatAmount: 87.75 }, + }, + channel_context: null, +} + +function fixture() { + return { + transactions: [TX], + invoice_inbox_items: [ITEM], + document_attachments: [{ id: 'doc-1', file_name: 'circlek.pdf' }], + pending_operations: [], + company_members: [{ user_id: 'user-1', role: 'owner' }], + } +} + +describe('huntCompany', () => { + it('writes nothing on a dry run but returns what it would have staged', async () => { + const { client, inserts } = mockSupabase(fixture(), () => { + throw new Error('dry run must not write') + }) + + const result = await huntCompany(client, 'co-1', 'run-1', { dryRun: true }) + + expect(inserts).toHaveLength(0) + expect(result.proposed).toBe(1) + expect(result.proposals?.[0]).toMatchObject({ transaction_id: 'tx-1', document_id: 'doc-1' }) + }) + + it('stages one operation per proposal, shaped for the approval card', async () => { + const { client, inserts } = mockSupabase(fixture()) + + const result = await huntCompany(client, 'co-1', 'run-1') + + expect(result.proposed).toBe(1) + expect(inserts).toHaveLength(1) + const [row] = inserts[0].rows as Array>> + expect(inserts[0].table).toBe('pending_operations') + expect(row.operation_type).toBe('attach_document_to_transaction') + // The executor reads exactly these two params and nothing else. + expect(row.params).toEqual({ transaction_id: 'tx-1', document_id: 'doc-1' }) + // AttachDocumentPreview treats an absent flag as potentially destructive, + // so it has to be present and false or the card warns about an overwrite + // that cannot happen (these transactions have no document). + expect(row.preview_data.existing_document_is_rakenskapsinformation).toBe(false) + expect(row.preview_data.will_overwrite_existing).toBe(false) + expect(row.agent_metadata.run_id).toBe('run-1') + expect(row.actor_type).toBe('cron') + }) + + it('does not stage when the company has no members to ask', async () => { + const tables = { ...fixture(), company_members: [] } + const { client, inserts } = mockSupabase(tables) + + const result = await huntCompany(client, 'co-1', 'run-1') + + expect(result.skippedNoOwner).toBe(true) + expect(result.proposed).toBe(0) + expect(inserts).toHaveLength(0) + }) + + it('ignores a receipt whose document is already anchored to a verifikat', async () => { + // document_attachments is filtered on journal_entry_id IS NULL by the + // query, so an anchored doc simply is not in the attachable set. + const tables = { ...fixture(), document_attachments: [] } + const { client, inserts } = mockSupabase(tables) + + const result = await huntCompany(client, 'co-1', 'run-1') + + expect(result.poolSize).toBe(0) + expect(result.proposed).toBe(0) + expect(inserts).toHaveLength(0) + }) +}) diff --git a/lib/receipt-hunt/__tests__/ingest.test.ts b/lib/receipt-hunt/__tests__/ingest.test.ts new file mode 100644 index 00000000..1baf9670 --- /dev/null +++ b/lib/receipt-hunt/__tests__/ingest.test.ts @@ -0,0 +1,229 @@ +/** + * Filing a hunted receipt. What matters here is what must NOT happen: no + * duplicate ingest, no oversized download, and one unreadable attachment never + * costing the rest of the run. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type { MailCandidate } from '@/lib/mail-search/service' + +const mockUploadDocument = vi.fn() +vi.mock('@/lib/core/documents/document-service', () => ({ + uploadDocument: (...args: unknown[]) => mockUploadDocument(...args), +})) + +const mockFetchAttachment = vi.fn() +vi.mock('@/lib/mail-search/service', () => ({ + getMailSearchService: () => ({ + fetchAttachment: (...args: unknown[]) => mockFetchAttachment(...args), + search: vi.fn(), + isConfigured: () => true, + }), +})) + +import { ingestMailCandidate, sniffMimeType } from '../ingest' + +function candidate(overrides: Partial = {}): MailCandidate { + return { + connectionId: 'conn-1', + mailbox: 'ekonomi@nordvik.se', + provider: 'gmail', + messageId: 'msg-1', + subject: 'Ditt kvitto', + from: 'no-reply@circlek.se', + receivedAt: '2026-05-02T10:00:00Z', + attachmentIds: ['att-1'], + bodyIsReceipt: false, + ...overrides, + } +} + +/** Table-dispatching Supabase stand-in with a settable existing-row answer. */ +function mockSupabase(existing: { id: string } | null, insertResult: { data?: unknown; error?: unknown } = {}) { + const inserted: Array> = [] + const client = { + from(table: string) { + const chain: Record = {} + for (const m of ['select', 'eq', 'is', 'not', 'order', 'limit']) chain[m] = vi.fn(() => chain) + chain.maybeSingle = vi.fn(() => + Promise.resolve( + table === 'document_attachments' + ? { data: { extracted_data: { total_amount: 425 } }, error: null } + : { data: existing, error: null }, + ), + ) + chain.insert = vi.fn((row: Record) => { + inserted.push(row) + return { + select: () => ({ + single: () => + Promise.resolve( + insertResult.error + ? { data: null, error: insertResult.error } + : { data: insertResult.data ?? { id: 'item-1' }, error: null }, + ), + }), + } + }) + return chain + }, + } + return { client: client as never, inserted } +} + +beforeEach(() => { + vi.clearAllMocks() + mockUploadDocument.mockResolvedValue({ id: 'doc-1' }) + mockFetchAttachment.mockResolvedValue({ + filename: 'kvitto.pdf', + mimeType: 'application/pdf', + bytes: Buffer.from('%PDF-1.4 fake'), + }) +}) + +describe('ingestMailCandidate', () => { + it('files an attachment and returns the pairing material', async () => { + const { client, inserted } = mockSupabase(null) + const result = await ingestMailCandidate(client, 'co-1', 'user-1', candidate()) + + expect(result).toMatchObject({ documentId: 'doc-1', inboxItemId: 'item-1', fileName: 'kvitto.pdf' }) + expect(inserted).toHaveLength(1) + expect(inserted[0].source).toBe('mail_hunt') + // Provenance goes in channel_context, never extracted_data: retrying + // extraction overwrites extracted_data wholesale, and the record of which + // mailbox a receipt came from has to survive that. + const ctx = inserted[0].channel_context as Record + expect(ctx.mail_message_id).toBe('msg-1') + expect(ctx.mail_mailbox).toBe('ekonomi@nordvik.se') + // Keyed per attachment: a batch forward carries receipts for several + // purchases, and filing the first must not block the rest. + expect(ctx.mail_file_key).toBe('msg-1::att-1') + const extracted = inserted[0].extracted_data as Record | null + expect(extracted).not.toHaveProperty('mail_message_id') + // The extraction that ran on upload is copied onto the inbox item: the + // pool is read from here, and a row with no amount can never be paired. + expect(extracted).toMatchObject({ total_amount: 425 }) + }) + + it('does not fetch anything for a message already ingested', async () => { + const { client } = mockSupabase({ id: 'existing' }) + const result = await ingestMailCandidate(client, 'co-1', 'user-1', candidate()) + + expect(result).toBeNull() + // The point of the pre-check is that a known message costs no provider call. + expect(mockFetchAttachment).not.toHaveBeenCalled() + }) + + it('treats a unique-violation as success, not an error', async () => { + // Another run won the race; the receipt is filed either way. + const { client } = mockSupabase(null, { error: { code: '23505', message: 'duplicate key' } }) + await expect(ingestMailCandidate(client, 'co-1', 'user-1', candidate())).resolves.toBeNull() + }) + + it('ignores a body-only receipt, which has nothing to download', async () => { + const { client } = mockSupabase(null) + const result = await ingestMailCandidate( + client, + 'co-1', + 'user-1', + candidate({ attachmentIds: [], bodyIsReceipt: true }), + ) + expect(result).toBeNull() + expect(mockFetchAttachment).not.toHaveBeenCalled() + }) + + it('skips an oversized attachment rather than storing a report', async () => { + mockFetchAttachment.mockResolvedValue({ + filename: 'arsredovisning.pdf', + mimeType: 'application/pdf', + bytes: Buffer.alloc(11 * 1024 * 1024), + }) + const { client, inserted } = mockSupabase(null) + const result = await ingestMailCandidate(client, 'co-1', 'user-1', candidate()) + expect(result).toBeNull() + expect(inserted).toHaveLength(0) + }) + + it('tries the next attachment when one cannot be fetched', async () => { + mockFetchAttachment + .mockRejectedValueOnce(new Error('gmail 404')) + .mockResolvedValueOnce({ + filename: 'kvitto.pdf', + mimeType: 'application/pdf', + bytes: Buffer.from('%PDF-1.4 fake'), + }) + const { client } = mockSupabase(null) + const result = await ingestMailCandidate( + client, + 'co-1', + 'user-1', + candidate({ attachmentIds: ['bad', 'good'] }), + ) + expect(result).toMatchObject({ documentId: 'doc-1' }) + }) + + it('never throws when the upload itself is rejected', async () => { + // Magic-byte validation rejects a mislabelled file; one bad message must + // not abort a night's hunt. + mockUploadDocument.mockRejectedValue(new Error('File content does not match')) + const { client } = mockSupabase(null) + await expect(ingestMailCandidate(client, 'co-1', 'user-1', candidate())).resolves.toBeNull() + }) +}) + +/** + * The first live fetch died here: Gmail declared a PDF as + * application/octet-stream, and uploadDocument validates content against the + * declared type, so the receipt was rejected at the door. + */ +describe('sniffMimeType', () => { + it('believes the bytes over a mail that says octet-stream', () => { + const pdf = Buffer.from('%PDF-1.4 ...') + expect(sniffMimeType(pdf, 'application/octet-stream', 'kvitto.pdf')).toBe('application/pdf') + }) + + it('recognises a photographed receipt', () => { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]) + expect(sniffMimeType(jpeg, 'application/octet-stream', 'IMG_5626')).toBe('image/jpeg') + }) + + it('falls back to the filename when the bytes say nothing', () => { + const unknown = Buffer.from('not a known header at all') + expect(sniffMimeType(unknown, 'application/octet-stream', 'faktura.pdf')).toBe('application/pdf') + }) + + it('keeps the declared type when nothing else identifies it', () => { + const unknown = Buffer.from('mystery bytes') + expect(sniffMimeType(unknown, 'text/plain', 'anteckning')).toBe('text/plain') + }) +}) + +/** + * A message can carry several receipts. Whichever one is stored has to be + * filed under its own identity: recording index 0 while the loop is on a later + * attachment would both mislabel the row and permanently block the sibling, + * since the file key is unique. + */ +describe('ingestMailCandidate, over several attachments', () => { + it('files the attachment it actually stored, not the first one', async () => { + // The first attachment cannot be fetched, so the second is stored. + mockFetchAttachment + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + filename: 'ignored-by-caller.pdf', + mimeType: 'application/pdf', + bytes: Buffer.from('%PDF-1.4 fake'), + }) + const { client, inserted } = mockSupabase(null) + const result = await ingestMailCandidate( + client, + 'co-1', + 'user-1', + candidate({ attachmentIds: ['att-1', 'att-2'], attachmentNames: ['first.pdf', 'second.pdf'] }), + ) + + const ctx = inserted[0].channel_context as Record + expect(ctx.mail_attachment_id).toBe('att-2') + expect(ctx.mail_file_key).toBe('msg-1::att-2') + expect(result?.fileName).toBe('second.pdf') + }) +}) diff --git a/lib/receipt-hunt/__tests__/mail-intelligence.test.ts b/lib/receipt-hunt/__tests__/mail-intelligence.test.ts new file mode 100644 index 00000000..c5918796 --- /dev/null +++ b/lib/receipt-hunt/__tests__/mail-intelligence.test.ts @@ -0,0 +1,156 @@ +/** + * The model reads mails and reports fields. These tests are about what happens + * when it reports something it should not: an id we never offered, a filename + * that is not there, a negative total, a date that is not a date. None of that + * may reach the matcher, because the matcher trusts its inputs. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +const mockCreate = vi.fn() +vi.mock('@anthropic-ai/bedrock-sdk', () => ({ + default: class { + messages = { create: (...args: unknown[]) => mockCreate(...args) } + }, +})) + +import { extractMailDocuments, type CandidateForReview } from '../mail-intelligence' + +/** A reply in the shape forced tool use produces. */ +function toolReply(input: unknown) { + return { content: [{ type: 'tool_use', name: 'x', id: 'tu', input }] } +} + +function candidate(overrides: Partial = {}): CandidateForReview { + return { + messageId: 'msg-1', + mailbox: 'invoice@arcim.io', + subject: 'Fwd: Your receipt from Anthropic, PBC', + from: 'jakob@example.com', + receivedAt: '2026-08-01T10:00:00.000Z', + bodyText: 'Vidarebefordrat meddelande Från: Anthropic, PBC Datum: mån 15 juni 2026 €180.00', + attachmentNames: ['Receipt-2066.pdf'], + ...overrides, + } +} + +function doc(overrides: Record = {}) { + return { + message_id: 'msg-1', + attachment_name: 'Receipt-2066.pdf', + is_receipt: true, + vendor: 'Anthropic, PBC', + date: '2026-06-15', + amount: 180, + currency: 'eur', + ...overrides, + } +} + +beforeEach(() => vi.clearAllMocks()) + +describe('extractMailDocuments', () => { + it('reports the fields a matcher needs', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: [doc()] })) + const [out] = await extractMailDocuments([candidate()]) + + expect(out).toMatchObject({ + messageId: 'msg-1', + attachmentName: 'Receipt-2066.pdf', + vendor: 'Anthropic, PBC', + date: '2026-06-15', + amount: 180, + currency: 'EUR', + }) + }) + + it('takes the purchase date from the mail, not the forwarding date', async () => { + // The mail was forwarded in August; the purchase was in June. Matching on + // the forwarding date is what made the old design miss five of six repeat + // subscriptions. + mockCreate.mockResolvedValue(toolReply({ documents: [doc()] })) + const [out] = await extractMailDocuments([candidate({ receivedAt: '2026-08-01T10:00:00.000Z' })]) + expect(out.date).toBe('2026-06-15') + }) + + it('drops mail that is not an underlag', async () => { + mockCreate.mockResolvedValue( + toolReply({ documents: [doc({ is_receipt: false, vendor: null, amount: null })] }), + ) + await expect(extractMailDocuments([candidate()])).resolves.toEqual([]) + }) + + it('never reports a message id it was not given', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ message_id: 'invented' })] })) + await expect(extractMailDocuments([candidate()])).resolves.toEqual([]) + }) + + it('rejects a filename that is not on the message', async () => { + // An invented filename means it was guessing about the contents. + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ attachment_name: 'ghost.pdf' })] })) + await expect(extractMailDocuments([candidate()])).resolves.toEqual([]) + }) + + it('treats a missing amount as unknown rather than as zero', async () => { + // Most receipts state their total only inside the PDF. Null must stay null: + // a zero would score as an amount that disagrees with every transaction. + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ amount: null })] })) + const [out] = await extractMailDocuments([candidate()]) + expect(out.amount).toBeNull() + }) + + it('discards a non-positive total as a misread', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ amount: -180 })] })) + const [out] = await extractMailDocuments([candidate()]) + expect(out.amount).toBeNull() + }) + + it('discards a date that is not a date', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ date: 'juni 2026' })] })) + const [out] = await extractMailDocuments([candidate()]) + expect(out.date).toBeNull() + }) + + it('reports one document per attachment in a batch forward', async () => { + // "Fwd: Kvitton februari" carries five receipts for five purchases. + mockCreate.mockResolvedValue( + toolReply({ + documents: [ + doc({ attachment_name: 'a.pdf', amount: 162.02 }), + doc({ attachment_name: 'b.pdf', amount: 425 }), + ], + }), + ) + const out = await extractMailDocuments([candidate({ attachmentNames: ['a.pdf', 'b.pdf'] })]) + expect(out.map((d) => d.amount)).toEqual([162.02, 425]) + }) + + it('refuses to guess which file, when the mail carries several', async () => { + // Without a filename the caller would fetch attachment number one and hope. + // On a batch forward that is a coin flip, so the document is dropped. + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ attachment_name: null })] })) + await expect( + extractMailDocuments([candidate({ attachmentNames: ['a.pdf', 'b.pdf', 'c.pdf'] })]), + ).resolves.toEqual([]) + }) + + it('still accepts a body-only receipt, where there is nothing to choose', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: [doc({ attachment_name: null })] })) + const out = await extractMailDocuments([candidate({ attachmentNames: [] })]) + expect(out).toHaveLength(1) + }) + + it('accepts an array the model sent as a JSON string', async () => { + mockCreate.mockResolvedValue(toolReply({ documents: JSON.stringify([doc()]) })) + await expect(extractMailDocuments([candidate()])).resolves.toHaveLength(1) + }) + + it('reports nothing when the call fails', async () => { + mockCreate.mockRejectedValue(new Error('bedrock timeout')) + await expect(extractMailDocuments([candidate()])).resolves.toEqual([]) + }) + + it('does not call the model when there is nothing to read', async () => { + await expect(extractMailDocuments([])).resolves.toEqual([]) + expect(mockCreate).not.toHaveBeenCalled() + }) +}) diff --git a/lib/receipt-hunt/__tests__/select.test.ts b/lib/receipt-hunt/__tests__/select.test.ts new file mode 100644 index 00000000..2925d1bd --- /dev/null +++ b/lib/receipt-hunt/__tests__/select.test.ts @@ -0,0 +1,363 @@ +/** + * The hunt's judgement: which pairing gets proposed, and every reason one does + * not. Pure, so no database is involved; the reads and the staging write are + * covered by the cron route test. + */ +import { describe, it, expect } from 'vitest' +import { + AMBIGUITY_MARGIN, + HUNT_MIN_CONFIDENCE, + canHaveEmailReceipt, + pairKey, + worthFetching, + selectProposals, + type HuntPoolItem, + type HuntTransaction, +} from '../select' + +function tx(overrides: Partial = {}): HuntTransaction { + return { + id: 'tx-1', + company_id: 'co-1', + date: '2026-05-02', + description: 'CIRCLE K 421', + merchant_name: 'Circle K', + amount: -438.75, + currency: 'SEK', + amount_sek: -438.75, + exchange_rate: null, + ...overrides, + } +} + +function item(overrides: Partial = {}, extraction: Record = {}): HuntPoolItem { + return { + id: 'item-1', + document_id: 'doc-1', + extracted_data: { + supplier: { name: 'Circle K' }, + invoice: { invoiceDate: '2026-05-02', currency: 'SEK' }, + totals: { total: 438.75, vatAmount: 87.75 }, + ...extraction, + }, + channel_context: null, + ...overrides, + } +} + +const noSuppression = { + claimedTransactionIds: new Set(), + claimedDocumentIds: new Set(), + rejectedPairs: new Set(), +} + +describe('selectProposals', () => { + it('proposes an exact same-day match', () => { + const result = selectProposals([tx()], [item()], noSuppression) + expect(result).toHaveLength(1) + expect(result[0]).toMatchObject({ + transaction_id: 'tx-1', + document_id: 'doc-1', + inbox_item_id: 'item-1', + }) + expect(result[0].confidence).toBeGreaterThanOrEqual(HUNT_MIN_CONFIDENCE) + }) + + it('returns nothing when the pool is empty', () => { + expect(selectProposals([tx()], [], noSuppression)).toEqual([]) + }) + + it('skips a transaction that already has a live proposal', () => { + const result = selectProposals([tx()], [item()], { + claimedTransactionIds: new Set(['tx-1']), + claimedDocumentIds: new Set(), + rejectedPairs: new Set(), + }) + expect(result).toEqual([]) + }) + + it('never re-proposes a pair a human rejected', () => { + const result = selectProposals([tx()], [item()], { + claimedTransactionIds: new Set(), + claimedDocumentIds: new Set(), + rejectedPairs: new Set([pairKey('tx-1', 'doc-1')]), + }) + expect(result).toEqual([]) + }) + + it('lets a rejection retire one receipt without retiring the purchase', () => { + const other = item({ id: 'item-2', document_id: 'doc-2' }) + const result = selectProposals([tx()], [item(), other], { + claimedTransactionIds: new Set(), + claimedDocumentIds: new Set(), + rejectedPairs: new Set([pairKey('tx-1', 'doc-1')]), + }) + expect(result).toHaveLength(1) + expect(result[0].document_id).toBe('doc-2') + }) + + it('drops a match that clears the shared floor but not the hunt floor', () => { + // Right merchant, right day, wrong amount (>5% out) scores exactly 0.60: + // good enough for the picker, where a human compares the two totals side + // by side, and not good enough to propose unattended. This is the whole + // reason HUNT_MIN_CONFIDENCE sits above CANDIDATE_MIN_CONFIDENCE, so the + // case has to be exercised at that exact seam. + const wrongAmount = item({}, { + supplier: { name: 'Circle K' }, + invoice: { invoiceDate: '2026-05-02', currency: 'SEK' }, + totals: { total: 500, vatAmount: 0 }, + }) + const result = selectProposals([tx()], [wrongAmount], noSuppression) + expect(result).toEqual([]) + }) + + it('drops a match that fails even the shared floor', () => { + const weak = item({}, { + supplier: { name: 'Helt Annat Bolag AB' }, + invoice: { invoiceDate: '2026-01-02', currency: 'SEK' }, + totals: { total: 12_000, vatAmount: 0 }, + }) + expect(selectProposals([tx()], [weak], noSuppression)).toEqual([]) + }) + + it('refuses to guess between two equally good receipts', () => { + // Same merchant, same date, same amount: a duplicate or a split payment. + // Picking one would be a coin flip presented as a finding. + const twin = item({ id: 'item-2', document_id: 'doc-2' }) + const result = selectProposals([tx()], [item(), twin], noSuppression) + expect(result).toEqual([]) + }) + + it('proposes the winner when it is clear of the runner-up', () => { + const weaker = item({ id: 'item-2', document_id: 'doc-2' }, { + supplier: { name: 'Circle K' }, + invoice: { invoiceDate: '2026-05-05', currency: 'SEK' }, + totals: { total: 500, vatAmount: 0 }, + }) + const result = selectProposals([tx()], [item(), weaker], noSuppression) + expect(result).toHaveLength(1) + expect(result[0].document_id).toBe('doc-1') + }) + + it('never spends one receipt on two purchases', () => { + const first = tx({ id: 'tx-1', amount: -438.75 }) + const second = tx({ id: 'tx-2', amount: -438.75, date: '2026-05-02' }) + const result = selectProposals([first, second], [item()], noSuppression) + expect(result).toHaveLength(1) + }) + + it('caps a run and takes the largest amounts first', () => { + const transactions = [ + tx({ id: 'small', amount: -100, description: 'A', merchant_name: 'A' }), + tx({ id: 'large', amount: -9000, description: 'B', merchant_name: 'B' }), + ] + const pool = [ + item({ id: 'i-small', document_id: 'd-small' }, { + supplier: { name: 'A' }, + invoice: { invoiceDate: '2026-05-02', currency: 'SEK' }, + totals: { total: 100, vatAmount: 0 }, + }), + item({ id: 'i-large', document_id: 'd-large' }, { + supplier: { name: 'B' }, + invoice: { invoiceDate: '2026-05-02', currency: 'SEK' }, + totals: { total: 9000, vatAmount: 0 }, + }), + ] + const result = selectProposals(transactions, pool, noSuppression, 1) + expect(result).toHaveLength(1) + expect(result[0].transaction_id).toBe('large') + }) + + it('ignores an item whose extraction carries no date and no total', () => { + const blank = item({}, { invoice: { invoiceDate: null, currency: 'SEK' }, totals: { total: null, vatAmount: null } }) + expect(selectProposals([tx()], [blank], noSuppression)).toEqual([]) + }) + + it('does not match across currencies on amount alone', () => { + // 438.75 EUR is not 438.75 SEK. Without a comparable amount the pair must + // not ride to a high score on merchant + date. + const euro = item({}, { + supplier: { name: 'Circle K' }, + invoice: { invoiceDate: '2026-05-02', currency: 'EUR' }, + totals: { total: 438.75, vatAmount: 0 }, + }) + expect(selectProposals([tx()], [euro], noSuppression)).toEqual([]) + }) + + it('exposes the margin and floor it enforces', () => { + expect(HUNT_MIN_CONFIDENCE).toBeGreaterThan(0.6) + expect(AMBIGUITY_MARGIN).toBeGreaterThan(0) + }) +}) + +/** + * Which purchases are worth a mailbox search. Drawn from a provkörning where + * salary and tax rows, being the largest, consumed the entire search budget. + */ +describe('canHaveEmailReceipt', () => { + it('skips salary, which no merchant confirms by mail', () => { + expect(canHaveEmailReceipt('Lön Juli Jakob Överföring via internet')).toBe(false) + }) + + it('skips tax even when the bank has truncated the word', () => { + // Real row: the statement cuts "skatt" to "skat" at 16 characters. + expect(canHaveEmailReceipt('Inbetalning skat BG 0000050501055 Bg-bet. via internet')).toBe(false) + expect(canHaveEmailReceipt('Skatt lön Juni BG 0000050501055 Bg-bet. via internet')).toBe(false) + }) + + it('keeps a supplier invoice paid over bankgiro', () => { + // This one matched a real emailed invoice; skipping the whole rail would + // have thrown away the hunt's best hit. + expect(canHaveEmailReceipt('Kontorsplatser j BG 0000059142596 Bg-bet. via internet')).toBe(true) + }) + + it('keeps an expense reimbursement, which has a receipt behind it', () => { + expect(canHaveEmailReceipt('Utlägg Norwegian Överföring via internet')).toBe(true) + }) + + it('keeps ordinary card purchases', () => { + expect(canHaveEmailReceipt('ANTHROPIC* CLAUDE SUB SAN FRANCISCO Kortköp/uttag')).toBe(true) + expect(canHaveEmailReceipt('Elgiganten Aktiebolag K3667 Kortköp/uttag')).toBe(true) + }) + + it('hunts a transaction with no description rather than silently dropping it', () => { + expect(canHaveEmailReceipt(null)).toBe(true) + }) +}) + +/** + * The gate before a download. Not the match: the real amount comes out of the + * PDF afterwards. What matters is that a stated amount is enough on its own, + * that a vendor needs a plausible date, and that currencies are never + * converted to make a number agree. + */ +describe('worthFetching', () => { + const charge = (o: Partial = {}): HuntTransaction => + tx({ + description: 'Elgiganten Aktiebolag K3667 Kortköp/uttag', + // Bank rows usually carry no merchant_name; the descriptor is all there is. + merchant_name: null, + amount: -21639, + currency: 'SEK', + date: '2026-08-04', + ...o, + }) + + const doc = (o: Partial[0]> = {}) => ({ + vendor: 'Elgiganten', + date: '2026-08-03', + amount: null, + currency: null, + ...o, + }) + + it('fetches on a matching amount alone, whatever the date says', () => { + // An amount that agrees is close to proof. Banks post days late and mail + // gets forwarded months later, so the date must not be able to veto it. + expect( + worthFetching( + doc({ vendor: null, date: '2025-01-01', amount: 21639, currency: 'SEK' }), + [charge()], + ), + ).toBe(true) + }) + + it('fetches on vendor and a nearby date when the body states no amount', () => { + // The common case: most receipts state their total only inside the PDF. + expect(worthFetching(doc(), [charge()])).toBe(true) + }) + + it('does not fetch on a vendor whose date is months away', () => { + expect(worthFetching(doc({ date: '2026-02-01' }), [charge()])).toBe(false) + }) + + it('fetches a matching vendor that gave no date at all', () => { + // Missing evidence, not contrary evidence. + expect(worthFetching(doc({ date: null }), [charge()])).toBe(true) + }) + + it('never converts currency to make an amount agree', () => { + // 180 EUR really was this charge, but turning it into 2014 SEK is a guess. + // The vendor path is what rescues this case, so the vendor is cleared too. + expect( + worthFetching( + { vendor: null, date: '2026-06-15', amount: 180, currency: 'EUR' }, + [charge({ description: 'ANTHROPIC* CLAUDE SUB', amount: -2014.32, date: '2026-06-16' })], + ), + ).toBe(false) + }) + + it('ignores a document that matches nothing the company is missing', () => { + expect( + worthFetching({ vendor: 'Spotify', date: '2026-08-03', amount: 119, currency: 'SEK' }, [ + charge(), + ]), + ).toBe(false) + }) +}) + +describe('worthFetching, on the search that found it', () => { + it('fetches when the purchase that found the mail is close in time', () => { + // The bank calls the landlord "Kontorsplatser j BG"; the invoice says + // "Stockholm Innovation & Growth AB". The names will never match, but the + // search that produced this mail was that purchase's own. + const landlord = tx({ + description: 'Kontorsplatser j BG 0000059142596 Bg-bet. via internet', + merchant_name: null, + amount: -15000, + date: '2026-07-04', + }) + const invoice = { + vendor: 'Stockholm Innovation & Growth AB', + date: '2026-07-02', + amount: null, + currency: null, + } + expect(worthFetching(invoice, [landlord])).toBe(false) + expect(worthFetching(invoice, [landlord], [landlord])).toBe(true) + }) + + it('still refuses when the dates are nowhere near each other', () => { + const landlord = tx({ description: 'Kontorsplatser j BG', merchant_name: null, amount: -15000, date: '2026-07-04' }) + const stale = { vendor: 'Stockholm Innovation & Growth AB', date: '2025-11-02', amount: null, currency: null } + expect(worthFetching(stale, [landlord], [landlord])).toBe(false) + }) +}) + +describe('a receipt already offered elsewhere', () => { + it('is not offered again to a second purchase', () => { + // Caught on a real ledger: one H&M receipt was proposed against a -358 + // purchase, then against a -354 purchase on the next run. Approving both + // would put the same underlag on two verifikat. + const result = selectProposals([tx()], [item()], { + claimedTransactionIds: new Set(), + claimedDocumentIds: new Set(['doc-1']), + rejectedPairs: new Set(), + }) + expect(result).toEqual([]) + }) +}) + +/** + * The per-run fetch key. A filename is not an identity: half the world's + * billing systems attach "invoice.pdf", so two suppliers would collide. + */ +describe('per-run duplicate key', () => { + const key = (vendor: string | null, file: string | null, messageId: string) => + `${(vendor ?? '').toLowerCase()}::${(file ?? messageId).toLowerCase()}` + + it('collapses the same invoice arriving four times', () => { + // Original, reminder and two forwards, all carrying the identical file. + const keys = new Set([ + key('Visma', 'Invoice_13041840.pdf', 'm1'), + key('Visma', 'Invoice_13041840.pdf', 'm2'), + key('Visma', 'invoice_13041840.pdf', 'm3'), + key('Visma', 'Invoice_13041840.pdf', 'm4'), + ]) + expect(keys.size).toBe(1) + }) + + it('keeps two suppliers who both call it invoice.pdf', () => { + expect(key('Loopia', 'invoice.pdf', 'm1')).not.toBe(key('Hetzner', 'invoice.pdf', 'm2')) + }) +}) diff --git a/lib/receipt-hunt/hunt.ts b/lib/receipt-hunt/hunt.ts new file mode 100644 index 00000000..42dca2e6 --- /dev/null +++ b/lib/receipt-hunt/hunt.ts @@ -0,0 +1,593 @@ +/** + * The nightly receipt hunt: pair unbooked card purchases with receipts the + * company already holds, and stage each pairing for a human to approve. + * + * Reads and one write; every judgement lives in `select.ts` so it can be tested + * without a database. Nothing here books anything: the staged operation is + * `attach_document_to_transaction`, whose executor links the document to the + * transaction and leaves the journal untouched. + * + * Scope note: candidates are *unbooked* transactions. Posted verifikat missing + * underlag are a different problem with a different remedy (the + * `verifikat_missing_document` worklist, pulled at the user's pace) and are + * deliberately out of reach here. + */ +import type { SupabaseClient } from '@supabase/supabase-js' +import { fetchAllRows } from '@/lib/supabase/fetch-all' +import { getRiskLevel } from '@/lib/pending-operations/risk-tiers' +import { getMailSearchService } from '@/lib/mail-search/service' +import { ingestMailCandidate } from './ingest' +import { normalizeForMatch } from '@/lib/documents/core-receipt-matcher' +import { extractMailDocuments } from './mail-intelligence' +import { + MAX_PROPOSALS_PER_RUN, + canHaveEmailReceipt, + worthFetching, + pairKey, + selectProposals, + type HuntPoolItem, + type HuntProposal, + type HuntTransaction, +} from './select' + +/** + * Smallest purchase worth hunting a receipt for, in kronor. + * + * Not a compliance threshold: BFL wants an underlag whatever the amount. It is + * a cost boundary, because below it the mail search and the model call cost + * more than the bookkeeping value of the answer. Small purchases are still + * counted, and still get asked about in the weekly digest. + */ +export const MIN_AMOUNT_SEK = 100 + +/** How far back a purchase may be and still be hunted. */ +export const LOOKBACK_MONTHS = 12 + +/** Actor label shown wherever a staged operation names its origin. */ +export const HUNT_ACTOR_LABEL = 'Kvittojakten' + +/** + * Purchases to search the mailboxes for in one run. + * + * Each search is a provider round-trip per connected mailbox, so this bounds + * both latency and API quota. Largest amounts go first, and the rest wait for + * tomorrow night rather than being dropped. + */ +export const MAX_MAIL_SEARCHES_PER_RUN = 15 + +/** + * Hits pulled per merchant before the model is asked to choose. + * + * Deliberately generous. The lesson from production email search is that recall + * is won by loosening retrieval and letting the model filter, not by tightening + * the query: a hit that never gets retrieved cannot be recovered downstream, + * while an irrelevant one is cheap to reject. + */ +export const MAX_CANDIDATES_PER_MERCHANT = 25 + +/** + * Receipts fetched in one run. + * + * A cap on how much of a mailbox can land in Underlag on any one night, not a + * judgement about what is worth having: a company that pays the same supplier + * monthly genuinely has twelve receipts, and the rest wait for tomorrow. + */ +export const MAX_RECEIPTS_PER_RUN = (() => { + const parsed = Number(process.env.RECEIPT_HUNT_MAX_RECEIPTS) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 25 +})() + +/** + * Mails read by the model in one run. + * + * The searches are deliberately broad and overlap heavily, so this is the real + * cost boundary: one call carrying this many mail bodies, rather than a call + * per mail. + */ +export const MAX_MAILS_READ_PER_RUN = (() => { + const parsed = Number(process.env.RECEIPT_HUNT_MAX_MAILS) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 40 +})() + +/** + * Mails per extraction call. + * + * The model reads bodies, so a single call carrying 150 of them would be both + * enormous and fragile: one malformed reply loses the whole sweep. Chunking + * keeps each call small enough to be reliable and lets a backfill read a whole + * mailbox, which a first run on an existing company needs. + */ +export const MAILS_PER_EXTRACTION_CALL = 25 + +const OPERATION_TYPE = 'attach_document_to_transaction' + +/** Statuses that mean "this purchase already has a live or settled proposal". */ +const CLAIMED_STATUSES = ['pending', 'committing', 'committed'] as const + +export interface HuntCompanyResult { + companyId: string + candidates: number + poolSize: number + proposed: number + skippedNoOwner?: boolean + /** Populated on a dry run so the pairings can be inspected before trusting them. */ + proposals?: HuntProposal[] + /** What the mailbox search found, when a mail connection exists. */ + mail?: MailHuntSummary +} + +export interface MailHuntSummary { + /** Purchases whose merchant we searched the mailboxes for. */ + searched: number + /** Documents the model judged to be an underlag. */ + withCandidates: number + /** Receipts actually fetched and filed, ready for the amount match. */ + ingested: number + candidates: Array<{ + /** Merchant the model resolved the bank descriptor to. */ + merchant: string + mailbox: string + subject: string | null + from: string | null + receivedAt: string | null + /** The attachment chosen as the underlag. */ + fileName: string | null + /** What the model says the document is. */ + reason: string + }> +} + +export interface HuntOptions { + limit?: number + /** + * Also search connected mailboxes for the purchases nothing in Underlag + * could explain. Off by default so the nightly sweep's cost stays opt-in + * while the connector is piloted. + */ + searchMail?: boolean + /** How many unexplained purchases to search mail for in one run. */ + mailSearchLimit?: number + /** + * Score and decide, but write nothing. + * + * The provkörning the flow concept calls for: a company can see exactly what + * tonight would propose before anything reaches the granskningskö, and it is + * how this code is validated against a real ledger without staging a single + * operation. + */ + dryRun?: boolean +} + +/** Purchases with no receipt that nobody has booked yet. */ +async function fetchCandidateTransactions( + supabase: SupabaseClient, + companyId: string, +): Promise { + const since = new Date() + since.setMonth(since.getMonth() - LOOKBACK_MONTHS) + const sinceDate = since.toISOString().slice(0, 10) + + const rows = await fetchAllRows((range) => + supabase + .from('transactions') + .select('id, company_id, date, description, merchant_name, amount, currency, amount_sek, exchange_rate') + .eq('company_id', companyId) + .is('journal_entry_id', null) + .is('document_id', null) + .eq('is_ignored', false) + // is_business IS DISTINCT FROM false: NULL is untriaged and true is + // "business, not yet booked". Only an explicit false means the user + // called it private, and a private purchase needs no underlag. + .not('is_business', 'is', false) + // Outflows only, and amount <= -MIN covers the floor in one filter. + .lte('amount', -MIN_AMOUNT_SEK) + .gte('date', sinceDate) + .order('id', { ascending: true }) + .range(range.from, range.to), + ) + return rows +} + +/** + * Unconsumed inbox items whose document is still free to attach. + * + * Loaded once per company and scored against every candidate, rather than + * re-queried per transaction: it turns N queries into one and, more + * importantly, removes the newest-50 truncation that a per-transaction lookup + * imposes on a company with a deep backlog. + */ +async function fetchPool( + supabase: SupabaseClient, + companyId: string, +): Promise<{ pool: HuntPoolItem[]; fileNames: Map }> { + const attachments = await fetchAllRows<{ id: string; file_name: string | null }>((range) => + supabase + .from('document_attachments') + .select('id, file_name') + .eq('company_id', companyId) + .eq('is_current_version', true) + // A document already anchored to a verifikat is räkenskapsinformation; + // the executor would 409 rather than move it. + .is('journal_entry_id', null) + .order('id', { ascending: true }) + .range(range.from, range.to), + ) + const fileNames = new Map() + for (const a of attachments) fileNames.set(a.id, a.file_name ?? 'underlag') + + const items = await fetchAllRows((range) => + supabase + .from('invoice_inbox_items') + .select('id, document_id, extracted_data, channel_context') + .eq('company_id', companyId) + .is('matched_transaction_id', null) + .is('created_journal_entry_id', null) + .is('created_supplier_invoice_id', null) + .not('document_id', 'is', null) + .order('id', { ascending: true }) + .range(range.from, range.to), + ) + + const pool = items.filter((i) => i.document_id != null && fileNames.has(i.document_id)) + return { pool, fileNames } +} + +/** + * What this company has already been asked, so it is never asked twice. + * + * Derived from `pending_operations` history rather than a table of its own: + * the answers already live there, terminal rows are immutable, and a rejection + * is exactly the durable "no" the hunt must respect. + */ +async function fetchSuppression(supabase: SupabaseClient, companyId: string) { + const rows = await fetchAllRows<{ + id: string + status: string + params: { transaction_id?: string; document_id?: string } | null + }>((range) => + supabase + .from('pending_operations') + .select('id, status, params') + .eq('company_id', companyId) + .eq('operation_type', OPERATION_TYPE) + .in('status', [...CLAIMED_STATUSES, 'rejected']) + .order('id', { ascending: true }) + .range(range.from, range.to), + ) + + const claimedTransactionIds = new Set() + const claimedDocumentIds = new Set() + const rejectedPairs = new Set() + for (const row of rows) { + const txId = row.params?.transaction_id + const docId = row.params?.document_id + if (!txId) continue + if (row.status === 'rejected') { + if (docId) rejectedPairs.add(pairKey(txId, docId)) + } else { + claimedTransactionIds.add(txId) + // A receipt already offered to one purchase is spoken for. Within a run + // spentDocumentIds handles this, but nothing carried it across runs, so + // one H&M receipt was proposed for a -358 purchase on one night and a + // -354 purchase on the next. Approving both would put the same underlag + // on two verifikat. + if (docId) claimedDocumentIds.add(docId) + } + } + return { claimedTransactionIds, claimedDocumentIds, rejectedPairs } +} + +/** + * Owner to hang the staged operation on. + * + * `pending_operations.user_id` is NOT NULL and drives who sees the proposal. + * Falling back to any member rather than failing keeps single-admin companies + * working; a company with no members has nobody to ask and is skipped. + */ +async function resolveOwnerUserId( + supabase: SupabaseClient, + companyId: string, +): Promise { + const { data } = await supabase + .from('company_members') + .select('user_id, role') + .eq('company_id', companyId) + .order('role', { ascending: true }) + .limit(50) + if (!data || data.length === 0) return null + const owner = (data as Array<{ user_id: string; role: string }>).find((m) => m.role === 'owner') + return owner?.user_id ?? (data[0] as { user_id: string }).user_id +} + +function buildTitle(proposal: HuntProposal, fileName: string, tx: HuntTransaction): string { + const counterparty = proposal.merchant_name || tx.merchant_name || tx.description || 'okänd motpart' + return `Koppla underlag: ${fileName} → ${counterparty}` +} + +/** + * Preview payload for `AttachDocumentPreview`. + * + * `existing_document_is_rakenskapsinformation` is set explicitly even though + * these transactions have no document: the component treats an absent value as + * potentially destructive, which would put a warning on a proposal that + * overwrites nothing. + */ +function buildPreview( + proposal: HuntProposal, + fileName: string, + tx: HuntTransaction, +): Record { + return { + transaction_description: tx.description, + transaction_amount: tx.amount, + transaction_currency: tx.currency ?? 'SEK', + transaction_date: tx.date, + document_file_name: fileName, + document_vendor_name: proposal.merchant_name, + document_amount: proposal.total_amount, + document_currency: proposal.currency, + document_invoice_date: proposal.receipt_date, + will_overwrite_existing: false, + existing_document_file_name: null, + existing_document_is_rakenskapsinformation: false, + match_confidence: proposal.confidence, + match_reasons: proposal.matchReasons, + } +} + +/** + * Hunt one company. Returns what it looked at and what it proposed. + * + * `runId` ties every proposal from one night together so a run can be read back + * (and, later, replayed) from `agent_metadata`. + */ +export async function huntCompany( + supabase: SupabaseClient, + companyId: string, + runId: string, + options: HuntOptions = {}, +): Promise { + const { + limit = MAX_PROPOSALS_PER_RUN, + dryRun = false, + searchMail = false, + mailSearchLimit = MAX_MAIL_SEARCHES_PER_RUN, + } = options + + const [transactions, suppression] = await Promise.all([ + fetchCandidateTransactions(supabase, companyId), + fetchSuppression(supabase, companyId), + ]) + if (transactions.length === 0) { + return { companyId, candidates: 0, poolSize: 0, proposed: 0 } + } + + // Ingesting a receipt needs an owner to attribute the document to. + const userId = await resolveOwnerUserId(supabase, companyId) + + // Mail is harvested BEFORE the pool is read, so a receipt fetched tonight is + // paired tonight rather than a night later. + // + // The mailbox leg only finds documents; it does not decide what they belong + // to. That question needs the amount, and the amount is inside the PDF, not + // in a Gmail preview. So the receipt is filed, the extraction that already + // runs on upload reads its amount, and the pairing below is the same + // deterministic amount-and-merchant match used for every other underlag. + const mail = searchMail + ? await harvestReceiptsFromMail( + supabase, + companyId, + userId, + transactions.filter((t) => !suppression.claimedTransactionIds.has(t.id)), + mailSearchLimit, + dryRun, + ) + : undefined + + const { pool, fileNames } = await fetchPool(supabase, companyId) + + const base: HuntCompanyResult = { + companyId, + candidates: transactions.length, + poolSize: pool.length, + proposed: 0, + mail, + } + if (pool.length === 0) return base + + const proposals = selectProposals(transactions, pool, suppression, limit) + if (proposals.length === 0) return base + if (dryRun) return { ...base, proposed: proposals.length, proposals } + if (!userId) return { ...base, skippedNoOwner: true } + + const byId = new Map(transactions.map((t) => [t.id, t])) + const riskLevel = getRiskLevel(OPERATION_TYPE) + + const rows = proposals.map((proposal) => { + const tx = byId.get(proposal.transaction_id) as HuntTransaction + const fileName = fileNames.get(proposal.document_id) ?? 'underlag' + const hunted = proposal.mailProvenance != null + return { + company_id: companyId, + user_id: userId, + operation_type: OPERATION_TYPE, + title: buildTitle(proposal, fileName, tx), + params: { + transaction_id: proposal.transaction_id, + document_id: proposal.document_id, + }, + preview_data: { + ...buildPreview(proposal, fileName, tx), + // Where it came from, when the hunt fetched it out of a mailbox. The + // reviewer should be able to see that this document was not uploaded + // by a human without having to go looking. + mail_mailbox: proposal.mailProvenance?.mailbox, + mail_subject: proposal.mailProvenance?.subject, + mail_from: proposal.mailProvenance?.from, + }, + actor_type: 'cron', + actor_label: HUNT_ACTOR_LABEL, + risk_level: riskLevel, + agent_metadata: { + source: hunted ? 'receipt_hunt_mail' : 'receipt_hunt', + run_id: runId, + inbox_item_id: proposal.inbox_item_id, + confidence: proposal.confidence, + match_reasons: proposal.matchReasons, + }, + } + }) + + const { error } = await supabase.from('pending_operations').insert(rows) + if (error) throw new Error(`Failed to stage receipt-hunt proposals: ${error.message}`) + + return { ...base, proposed: rows.length } +} + +/** + * Ask the connected mailboxes about purchases nothing in Underlag explained. + * + * Read-only and bounded: the largest amounts first, capped per run, and the + * whole thing degrades to an empty summary when no mail extension is loaded or + * no mailbox is connected. Finding a candidate is NOT the same as having the + * receipt: ingesting it is the next step and stays behind human approval. + */ +async function harvestReceiptsFromMail( + supabase: SupabaseClient, + companyId: string, + userId: string | null, + purchases: readonly HuntTransaction[], + limit: number, + dryRun: boolean, +): Promise { + const service = getMailSearchService() + const summary: MailHuntSummary = { searched: 0, withCandidates: 0, ingested: 0, candidates: [] } + if (!service.isConfigured() || purchases.length === 0) return summary + + // Salary and tax runs are a company's largest outgoing rows, so without this + // they eat the whole search budget hunting receipts that cannot exist. + const searchable = [...purchases] + .filter((t) => canHaveEmailReceipt(t.merchant_name || t.description)) + .sort((a, b) => Math.abs(b.amount ?? 0) - Math.abs(a.amount ?? 0)) + .slice(0, limit) + .filter((t): t is HuntTransaction & { amount: number; date: string } => + t.amount != null && Boolean(t.date), + ) + if (searchable.length === 0) return summary + summary.searched = searchable.length + + // Retrieval is deterministic: amount in every format a receipt might write + // it, OR the merchant tokens left after the bank's noise is stripped. No + // model is involved in deciding what to search for, because a wrong guess + // here is invisible and a broad query is cheap. + const byMessage = new Map>[number]>() + // Which purchase's search turned each mail up. The query was that purchase's + // amount or its merchant tokens, so the hit is itself a signal, and it is the + // only one that survives a supplier the bank and the invoice name differently. + const retrievedBy = new Map() + for (const tx of searchable) { + const found = await service.search(companyId, { + merchant: normalizeForMatch(tx.merchant_name || tx.description || ''), + amount: Math.abs(tx.amount), + currency: tx.currency ?? 'SEK', + date: tx.date, + useDateWindow: false, + limit: MAX_CANDIDATES_PER_MERCHANT, + }) + // The same mail answers several purchases from one supplier; it is read once. + for (const c of found) { + if (!byMessage.has(c.messageId)) byMessage.set(c.messageId, c) + const already = retrievedBy.get(c.messageId) + if (already) already.push(tx) + else retrievedBy.set(c.messageId, [tx]) + } + } + if (byMessage.size === 0) return summary + + const mails = [...byMessage.values()].slice(0, MAX_MAILS_READ_PER_RUN) + const toReview = mails.map((c) => ({ + messageId: c.messageId, + mailbox: c.mailbox, + subject: c.subject, + from: c.from, + receivedAt: c.receivedAt, + bodyText: c.bodyText ?? null, + attachmentNames: c.attachmentNames ?? [], + })) + + // Read in chunks so a sweep over a whole mailbox stays within one model call's + // useful size, and so one bad reply costs a chunk rather than the run. + const documents: Awaited> = [] + for (let i = 0; i < toReview.length; i += MAILS_PER_EXTRACTION_CALL) { + documents.push(...(await extractMailDocuments(toReview.slice(i, i + MAILS_PER_EXTRACTION_CALL)))) + } + if (documents.length === 0) return summary + + // The gate before spending a download: the amount when the body stated it, + // otherwise the vendor and the date. This is not the match. The real amount + // comes out of the PDF once it is fetched, and the ordinary matcher pairs it + // on the next few lines of huntCompany like any other underlag. + const wanted = documents.filter((doc) => + worthFetching(doc, searchable, retrievedBy.get(doc.messageId) ?? []), + ) + + const claimedFiles = new Set() + for (const doc of wanted) { + const candidate = byMessage.get(doc.messageId) + if (!candidate) continue + + // The same invoice arrives as an original, a reminder and two forwards, + // every one carrying the identical attachment, so the filename alone + // collapses those four into one fetch. + // + // Scoped by vendor as well, because a bare filename is not an identity: + // "invoice.pdf" and "Faktura.pdf" are what half the world's billing systems + // call their attachment, and keying on the filename alone would silently + // drop a second supplier's invoice as a duplicate of the first. + const fileKey = `${(doc.vendor ?? '').toLowerCase()}::${(doc.attachmentName ?? doc.messageId).toLowerCase()}` + if (claimedFiles.has(fileKey)) continue + claimedFiles.add(fileKey) + summary.withCandidates++ + + summary.candidates.push({ + merchant: doc.vendor ?? '(okänd)', + mailbox: candidate.mailbox, + subject: candidate.subject, + from: candidate.from, + receivedAt: candidate.receivedAt, + fileName: doc.attachmentName, + reason: `${doc.amount ?? '?'} ${doc.currency ?? ''} ${doc.date ?? 'utan datum'}`, + }) + + // A dry run reports what it decided and fetches nothing: the point of a + // provkörning is that no mailbox content is copied anywhere. + if (dryRun || !userId) continue + if (candidate.attachmentIds.length === 0) continue + if (summary.ingested >= MAX_RECEIPTS_PER_RUN) break + + const names = candidate.attachmentNames ?? [] + const at = doc.attachmentName ? names.indexOf(doc.attachmentName) : 0 + const index = at >= 0 ? at : 0 + const ingested = await ingestMailCandidate(supabase, companyId, userId, { + ...candidate, + attachmentIds: [candidate.attachmentIds[index] ?? candidate.attachmentIds[0]], + attachmentNames: [names[index] ?? names[0] ?? ''], + }) + if (ingested) summary.ingested++ + } + return summary +} + +/** + * Companies the hunt may run for. + * + * An explicit allowlist while the feature is piloted, and fail-safe by + * construction: an unset variable hunts nobody rather than everybody. + */ +export function resolveAllowlist(raw: string | undefined): string[] { + if (!raw) return [] + return raw + .split(',') + .map((id) => id.trim()) + .filter((id) => id.length > 0) +} diff --git a/lib/receipt-hunt/ingest.ts b/lib/receipt-hunt/ingest.ts new file mode 100644 index 00000000..8e622c1b --- /dev/null +++ b/lib/receipt-hunt/ingest.ts @@ -0,0 +1,217 @@ +/** + * Turning a mailbox hit into an underlag the user can approve. + * + * Lives in core rather than in the mail extension because it writes documents + * and inbox items, and an extension may never import another extension. The + * mail extension only ever hands over bytes. + * + * The hunt does NOT book anything and does not link anything by itself: it + * stores the receipt, records where it came from, and stages the pairing. The + * document becomes räkenskapsinformation only when a human approves. + */ +import type { SupabaseClient } from '@supabase/supabase-js' +import { uploadDocument } from '@/lib/core/documents/document-service' +import { getMailSearchService, type MailCandidate } from '@/lib/mail-search/service' +import { createLogger } from '@/lib/logger' + +const log = createLogger('receipt-hunt-ingest') + +/** + * What the bytes actually are, rather than what the mail claims. + * + * A mail's declared content type is untrusted metadata. Forwarded receipts + * routinely arrive as `application/octet-stream` whatever they really are, and + * uploadDocument validates the content against the type it is given, so + * trusting the mail means every such receipt is rejected at the door. Measured + * on a real mailbox: the first live fetch, an Elgiganten PDF, failed exactly + * this way. + * + * Magic bytes first, then the filename, then whatever the mail said. + */ +export function sniffMimeType(bytes: Buffer, declared: string, filename: string): string { + const head = bytes.subarray(0, 12) + if (head.subarray(0, 4).toString('latin1') === '%PDF') return 'application/pdf' + if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) return 'image/jpeg' + if (head.subarray(0, 8).toString('latin1') === '\x89PNG\r\n\x1a\n') return 'image/png' + if (head.subarray(0, 4).toString('latin1') === 'GIF8') return 'image/gif' + if ( + head.subarray(0, 4).toString('latin1') === 'RIFF' && + bytes.subarray(8, 12).toString('latin1') === 'WEBP' + ) { + return 'image/webp' + } + + const ext = filename.toLowerCase().match(/\.([a-z0-9]+)$/)?.[1] + const byExt: Record = { + pdf: 'application/pdf', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + png: 'image/png', + gif: 'image/gif', + webp: 'image/webp', + } + if (ext && byExt[ext]) return byExt[ext] + + return declared +} + +/** Largest attachment worth pulling. Receipts are small; anything larger is a report. */ +const MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024 + +export interface IngestedReceipt { + documentId: string + inboxItemId: string + fileName: string + mailbox: string +} + +/** + * Provenance written onto the inbox item. + * + * Deliberately in `channel_context` and not in `extracted_data`: retrying + * extraction overwrites extracted_data wholesale, and the record of which + * mailbox a receipt came from must survive that. Same rule the WhatsApp intake + * follows. + */ +function buildChannelContext(candidate: MailCandidate, attachmentId: string) { + return { + channel: 'mail_hunt', + mail_message_id: candidate.messageId, + mail_attachment_id: attachmentId, + // Message + attachment, because one forward can carry receipts for several + // different purchases and each must be able to land separately. Taken from + // the attachment being stored, not from index 0: filing a later attachment + // under its sibling's key would block the sibling from ever landing. + mail_file_key: `${candidate.messageId}::${attachmentId}`, + mail_mailbox: candidate.mailbox, + mail_provider: candidate.provider, + mail_subject: candidate.subject, + mail_from: candidate.from, + mail_received_at: candidate.receivedAt, + fetched_at: new Date().toISOString(), + } +} + +/** + * Fetch the first usable attachment on a candidate and file it as an inbox item. + * + * Returns null when there is nothing to store (body-only receipt, oversized + * attachment, or a duplicate we have already ingested). Never throws for one + * bad message: a single unreadable attachment must not abort a night's hunt. + */ +export async function ingestMailCandidate( + supabase: SupabaseClient, + companyId: string, + userId: string, + candidate: MailCandidate, +): Promise { + if (candidate.attachmentIds.length === 0) return null + + const service = getMailSearchService() + + for (const [index, attachmentId] of candidate.attachmentIds.entries()) { + // Per attachment, not per message: the check has to name the file it is + // about, and it sits inside the loop so trying a second attachment is not + // suppressed by the first one already being filed. + const fileKey = `${candidate.messageId}::${attachmentId}` + const { data: existing } = await supabase + .from('invoice_inbox_items') + .select('id') + .eq('company_id', companyId) + .eq('source', 'mail_hunt') + .eq('channel_context->>mail_file_key', fileKey) + .maybeSingle() + if (existing) continue + + let fetched + try { + fetched = await service.fetchAttachment(candidate.connectionId, candidate.messageId, attachmentId) + } catch (error) { + log.warn('could not fetch attachment', { + messageId: candidate.messageId, + error: error instanceof Error ? error.message : String(error), + }) + continue + } + if (!fetched) continue + if (fetched.bytes.byteLength > MAX_ATTACHMENT_BYTES) continue + + try { + // The name the search already reported beats the one the provider + // re-derives on fetch: a second lookup can come back empty and fall back + // to a generic "underlag.pdf", throwing away "2332687551.pdf". + const knownName = candidate.attachmentNames?.[index] + const fileName = knownName && knownName.length > 0 ? knownName : fetched.filename + + const document = await uploadDocument( + supabase, + userId, + companyId, + { + name: fileName, + buffer: fetched.bytes.buffer.slice( + fetched.bytes.byteOffset, + fetched.bytes.byteOffset + fetched.bytes.byteLength, + ) as ArrayBuffer, + type: sniffMimeType(fetched.bytes, fetched.mimeType, fileName), + }, + { upload_source: 'mail_hunt' }, + ) + + // uploadDocument emits document.uploaded and awaits its handlers, so the + // extraction extension has already read the amount, date and vendor out + // of this file by the time we get here. Copying it onto the inbox item is + // what lets the deterministic matcher pair the receipt on its amount: + // the pool is read from invoice_inbox_items, and a row with no + // extracted_data can never match anything. + const { data: extractedRow } = await supabase + .from('document_attachments') + .select('extracted_data') + .eq('id', document.id) + .maybeSingle() + const extracted = (extractedRow as { extracted_data?: Record } | null) + ?.extracted_data + + const { data: item, error } = await supabase + .from('invoice_inbox_items') + .insert({ + company_id: companyId, + user_id: userId, + document_id: document.id, + source: 'mail_hunt', + status: 'received', + email_from: candidate.from, + email_subject: candidate.subject, + email_received_at: candidate.receivedAt, + extracted_data: extracted ?? null, + channel_context: buildChannelContext(candidate, attachmentId), + }) + .select('id') + .single() + + if (error) { + // 23505 is the partial unique index doing its job: another run got + // there first, which is a success from the caller's point of view. + if (error.code === '23505') return null + throw new Error(error.message) + } + + return { + documentId: document.id, + inboxItemId: (item as { id: string }).id, + fileName, + mailbox: candidate.mailbox, + } + } catch (error) { + log.warn('could not store hunted receipt', { + messageId: candidate.messageId, + error: error instanceof Error ? error.message : String(error), + }) + // Magic-byte rejection and the like: try the next attachment rather than + // failing the whole candidate. + continue + } + } + + return null +} diff --git a/lib/receipt-hunt/mail-intelligence.ts b/lib/receipt-hunt/mail-intelligence.ts new file mode 100644 index 00000000..2757b1f0 --- /dev/null +++ b/lib/receipt-hunt/mail-intelligence.ts @@ -0,0 +1,251 @@ +/** + * One job for the model: read a mail and say what document it is. + * + * This started out much cleverer. The model was asked to resolve bank + * descriptors to merchants, then to decide which mail was the receipt for + * which charge, with a confidence score gating the result. Measured against a + * real mailbox, every part of that was wrong in the same way: it was being + * asked to judge without the evidence to judge on. + * + * - The amount decides a reconciliation, and the amount is inside the PDF. + * Every pairing it produced came back "belopp ej synligt". + * - Its confidence was anchored on round numbers and its threshold threw + * away correct answers, which is what the calibration literature predicts. + * - The purchase date it needed was sitting in the mail body all along: a + * forwarded receipt quotes the original sender and date in its header, and + * the body was being downloaded and discarded in favour of a 200-character + * snippet. + * + * So it now does the thing models are unambiguously good at and nothing else: + * read text, return fields. Which receipt belongs to which purchase is decided + * afterwards by the same deterministic amount-and-merchant matcher that scores + * every other underlag, so mail and Underlag get one matcher rather than two. + */ +import AnthropicBedrock from '@anthropic-ai/bedrock-sdk' +import { z } from 'zod' +import { createLogger } from '@/lib/logger' + +const log = createLogger('receipt-hunt-intelligence') + +/** + * Overridable so ops can move the hunt off the default without a deploy. + */ +const MODEL = + process.env.RECEIPT_HUNT_MODEL_ID || + process.env.BEDROCK_MODEL_ID || + 'eu.anthropic.claude-sonnet-5' + +export interface CandidateForReview { + messageId: string + mailbox: string + subject: string | null + from: string | null + receivedAt: string | null + bodyText: string | null + attachmentNames: string[] +} + +/** What a mail says a document is. Fields, not judgements. */ +export interface MailReceipt { + messageId: string + /** Which file on the message, when the mail carries several. */ + attachmentName: string | null + vendor: string | null + /** The purchase date, read from the forwarded header rather than the mail's own. */ + date: string | null + amount: number | null + currency: string | null +} + +/** + * Accept an array that arrived as a JSON string. + * + * Even under forced tool use the model occasionally stringifies a nested array + * rather than emitting it. That is a serialisation quirk, not a wrong answer, + * and rejecting the whole run over it costs a night's hunt. + */ +const jsonArray = (item: T) => + z.preprocess((value) => { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch { + return value + } + }, z.array(item).default([])) + +const ExtractionSchema = z.object({ + documents: jsonArray( + z.object({ + message_id: z.string().min(1), + attachment_name: z.string().nullable().default(null), + is_receipt: z.coerce.boolean().default(false), + vendor: z.string().nullable().default(null), + date: z.string().nullable().default(null), + amount: z.coerce.number().nullable().default(null), + currency: z.string().nullable().default(null), + }), + ), +}) + +const EXTRACT_TOOL = { + type: 'object', + properties: { + documents: { + type: 'array', + items: { + type: 'object', + properties: { + message_id: { type: 'string' }, + attachment_name: { + type: 'string', + description: 'Exact filename this document is, or null if the mail body is the receipt.', + }, + is_receipt: { type: 'boolean', description: 'It is a receipt or an invoice.' }, + vendor: { type: 'string', description: 'Who sold it. Null if unclear.' }, + date: { type: 'string', description: 'Purchase date as YYYY-MM-DD. Null if unclear.' }, + amount: { type: 'number', description: 'Total including VAT. Null if not stated in the text.' }, + currency: { type: 'string', description: 'ISO code, e.g. SEK, USD, EUR.' }, + }, + required: ['message_id', 'attachment_name', 'is_receipt', 'vendor', 'date', 'amount', 'currency'], + }, + }, + }, + required: ['documents'], +} + +const EXTRACT_SYSTEM = `Du läser mejl och rapporterar vilka handlingar de innehåller. + +För varje mejl: är det ett kvitto eller en faktura, från vem, för hur mycket +och vilket datum. Du dömer inte om något hör ihop med något annat. Du läser. + +Det här är nästan alltid vidarebefordrade mejl, och det avgör hur du läser dem: + +- DATUM: använd datumet ur den vidarebefordrade rubriken ("Från: Elgiganten, + Date: mån 3 aug. 2026"), inte när mejlet skickades vidare. Skillnaden är + ofta månader. Skriv det som YYYY-MM-DD. +- HANDLARE: samma rubrik namnger den ursprungliga avsändaren. Det är + handlaren, inte personen som vidarebefordrade. +- BELOPP: bara om det faktiskt står i texten. Står det inte där ligger det i + den bifogade filen, och då är amount null. Hitta aldrig på ett belopp och + räkna aldrig om valuta: står det 180,00 EUR rapporterar du 180 och EUR. + +Bär mejlet flera handlingar ("Kvitton februari" med fem bilagor), lämna en rad +per bilaga och sätt attachment_name till rätt filnamn. Nämner texten belopp +per kvitto, para ihop dem med filnamnen så gott det går. + +Är mejlet inget underlag (nyhetsbrev, reklam, korrespondens, kalender), sätt +is_receipt=false och lämna resten null. Ta hellre med en osäker faktura än +missa ett kvitto: en handling utan matchande belopp faller bort av sig själv +senare.` + +function client(): AnthropicBedrock { + return new AnthropicBedrock({ awsRegion: process.env.AWS_REGION }) +} + +async function ask( + system: string, + user: string, + toolName: string, + inputSchema: Record, + maxTokens: number, +): Promise { + const response = await client().messages.create({ + model: MODEL, + max_tokens: maxTokens, + system, + tools: [ + { name: toolName, description: 'Return the result in this exact shape.', input_schema: inputSchema as never }, + ], + tool_choice: { type: 'tool', name: toolName }, + messages: [{ role: 'user', content: user }], + }) + const block = response.content.find((c) => c.type === 'tool_use') + if (!block || block.type !== 'tool_use') throw new Error('model did not use the tool') + return block.input +} + +/** ISO date or nothing: a malformed date must not become a matching signal. */ +function cleanDate(value: string | null): string | null { + if (!value) return null + const match = value.match(/^(\d{4})-(\d{2})-(\d{2})/) + if (!match) return null + const iso = match[0] + return Number.isNaN(new Date(iso).getTime()) ? null : iso +} + +/** + * Read a batch of mails and report the documents in them. + * + * One call for the whole batch: the task is per-mail, but batching keeps this + * to a single round trip per run instead of one per message. + */ +export async function extractMailDocuments( + candidates: readonly CandidateForReview[], +): Promise { + if (candidates.length === 0) return [] + + const known = new Set(candidates.map((c) => c.messageId)) + const attachmentsByMessage = new Map( + candidates.map((c) => [c.messageId, new Set(c.attachmentNames)]), + ) + + const payload = { + emails: candidates.map((c) => ({ + message_id: c.messageId, + subject: c.subject, + from: c.from, + forwarded_at: c.receivedAt, + attachments: c.attachmentNames, + body: c.bodyText, + })), + } + + try { + const raw = await ask( + EXTRACT_SYSTEM, + JSON.stringify(payload, null, 1), + 'documents_in_mail', + EXTRACT_TOOL, + 8192, + ) + const parsed = ExtractionSchema.parse(raw) + + const out: MailReceipt[] = [] + for (const d of parsed.documents) { + if (!d.is_receipt) continue + // Ids and filenames must be ones we supplied: the only place the reply is + // not taken at face value, and what stops an invented id being fetched. + if (!known.has(d.message_id)) continue + const available = attachmentsByMessage.get(d.message_id) ?? new Set() + if (d.attachment_name && !available.has(d.attachment_name)) continue + + // No filename on a mail carrying several files is not an answer, it is a + // shrug: the caller would fetch the first attachment and hope. A batch + // forward with five receipts is exactly where that goes wrong. + if (!d.attachment_name && available.size > 1) continue + + out.push({ + messageId: d.message_id, + attachmentName: d.attachment_name, + vendor: d.vendor?.trim() || null, + date: cleanDate(d.date), + // A non-positive total is a misread, not a free receipt. + amount: d.amount != null && d.amount > 0 ? d.amount : null, + currency: d.currency?.trim().toUpperCase() || null, + }) + } + + log.info('mail extraction', { + mails: candidates.length, + documents: out.length, + withAmount: out.filter((r) => r.amount != null).length, + }) + return out + } catch (error) { + log.warn('mail extraction failed, fetching nothing this run', { + error: error instanceof Error ? error.message : String(error), + }) + return [] + } +} diff --git a/lib/receipt-hunt/select.ts b/lib/receipt-hunt/select.ts new file mode 100644 index 00000000..460100f0 --- /dev/null +++ b/lib/receipt-hunt/select.ts @@ -0,0 +1,293 @@ +/** + * Which receipt gets proposed for which unbooked card purchase. + * + * The hunt attaches an underlag to a transaction *before* it is booked, so the + * gap never forms: `commitAttachDocumentToTransaction` sets + * `invoice_inbox_items.matched_transaction_id`, and when the user later books + * the transaction `categorize-core.ts` propagates that document onto the new + * verifikat. Chasing already-posted verifikat is deliberately NOT this job: + * that backlog is 96% imported history whose originals live in the previous + * system, and it stays a pull (the `verifikat_missing_document` worklist). + * + * Everything here is pure so the ranking and every guard is unit-testable + * without a database; the caller owns the reads and the staging write. + */ +import { scoreUnderlagCandidates, type CandidateTransaction } from '@/lib/agent-context/underlag-candidates' +import { calculateMerchantSimilarity } from '@/lib/documents/core-receipt-matcher' + +/** + * Confidence a candidate must reach to be proposed unattended. + * + * Above `CANDIDATE_MIN_CONFIDENCE` (0.6), which governs candidates an agent + * reads and reasons about with a human in the loop. A proposal staged by the + * nightly hunt is read as "these two belong together", so it trades recall for + * precision: a wrong receipt on the wrong purchase is a mis-booking, and the + * weaker pairs still reach the user through the manual picker. + */ +export const HUNT_MIN_CONFIDENCE = 0.7 + +/** + * How far clear the winner must be before we propose it. + * + * Two receipts scoring the same against one purchase is a signal, not a tie to + * break: a duplicate, a split payment, or a recurring charge whose sibling we + * picked at random. Proposing either would be a coin flip presented as a + * finding, so both are left to the picker. + */ +export const AMBIGUITY_MARGIN = 0.05 + +/** + * Proposals per company per run. + * + * The queue is drained largest-amount-first over several nights instead of + * arriving at once: prod holds companies with hundreds of receiptless + * purchases, and a first run that staged all of them would bury the + * granskningskö the feature is supposed to relieve. + */ +export const MAX_PROPOSALS_PER_RUN = 20 + +/** A transaction the hunt may propose an underlag for. */ +export interface HuntTransaction extends CandidateTransaction { + company_id: string +} + +/** An unconsumed inbox item, already filtered to ones whose document is attachable. */ +export interface HuntPoolItem { + id: string + document_id: string | null + extracted_data: unknown + channel_context: unknown +} + +export interface HuntProposal { + transaction_id: string + document_id: string + inbox_item_id: string + confidence: number + matchReasons: string[] + /** Receipt-side facts, for the approval preview. */ + merchant_name: string | null + receipt_date: string | null + total_amount: number | null + currency: string | null + /** + * Where the document came from, carried through from the inbox item so the + * proposal can say "found in your mailbox" rather than implying a human + * uploaded it. + */ + mailProvenance: { + mailbox?: string + subject?: string + from?: string + } | null +} + +export interface SuppressionSets { + /** + * Transactions that already have an open or settled proposal. One live + * question per purchase: a second one is noise even when it names a + * different receipt. + */ + claimedTransactionIds: ReadonlySet + /** + * Receipts already offered to some purchase and not yet rejected. One + * underlag belongs to one verifikat, and a proposal is a claim on it until a + * human says otherwise. + */ + claimedDocumentIds: ReadonlySet + /** + * Pairs a human already said no to, as `${transaction_id}:${document_id}`. + * Scoped to the pair rather than the transaction so a rejection retires one + * wrong guess without retiring the purchase. + */ + rejectedPairs: ReadonlySet +} + +export function pairKey(transactionId: string, documentId: string): string { + return `${transactionId}:${documentId}` +} + +/** + * Rank the pool against each transaction and return the proposals worth + * staging, strongest purchases first. + * + * Ordering is by absolute amount, not by confidence: when the cap truncates the + * run, the money that matters most for the books should be the part that gets + * asked about tonight. + */ +export function selectProposals( + transactions: readonly HuntTransaction[], + pool: readonly HuntPoolItem[], + suppression: SuppressionSets, + limit: number = MAX_PROPOSALS_PER_RUN, +): HuntProposal[] { + if (pool.length === 0) return [] + + const byLargestAmount = [...transactions].sort( + (a, b) => Math.abs(b.amount ?? 0) - Math.abs(a.amount ?? 0), + ) + + const poolById = new Map(pool.map((item) => [item.id, item])) + + const proposals: HuntProposal[] = [] + // One receipt can only settle one purchase, and the pool is scored per + // transaction, so the same document can win twice in a single run. Whoever + // is scored first (the larger amount) keeps it. + const spentDocumentIds = new Set() + + for (const tx of byLargestAmount) { + if (proposals.length >= limit) break + if (suppression.claimedTransactionIds.has(tx.id)) continue + + const scored = scoreUnderlagCandidates(tx, pool as never[]).filter( + (candidate) => + candidate.document_id != null && + !spentDocumentIds.has(candidate.document_id) && + !suppression.claimedDocumentIds.has(candidate.document_id) && + !suppression.rejectedPairs.has(pairKey(tx.id, candidate.document_id)), + ) + if (scored.length === 0) continue + + const [winner, runnerUp] = scored + if (winner.confidence < HUNT_MIN_CONFIDENCE) continue + if (runnerUp && winner.confidence - runnerUp.confidence < AMBIGUITY_MARGIN) continue + + const documentId = winner.document_id as string + spentDocumentIds.add(documentId) + const context = poolById.get(winner.inbox_item_id)?.channel_context as + | { channel?: string; mail_mailbox?: string; mail_subject?: string; mail_from?: string } + | null + | undefined + proposals.push({ + mailProvenance: + context?.channel === 'mail_hunt' + ? { + mailbox: context.mail_mailbox, + subject: context.mail_subject, + from: context.mail_from, + } + : null, + transaction_id: tx.id, + document_id: documentId, + inbox_item_id: winner.inbox_item_id, + confidence: winner.confidence, + matchReasons: winner.matchReasons, + merchant_name: winner.merchant_name, + receipt_date: winner.receipt_date, + total_amount: winner.total_amount, + currency: winner.currency, + }) + } + + return proposals +} + +/** + * Payments that cannot have an emailed receipt, however hard we look. + * + * Salary, tax, employer contributions, VAT settlements, dividends, loan + * amortisation and interest are all money moving on the strength of a + * declaration or an agreement, not a purchase a merchant confirms by mail. + * + * Deliberately narrow. Supplier payments over bankgiro DO arrive with an + * emailed invoice (a provkörning matched a Sting office invoice that way), and + * an "Utlägg" reimbursement has a real receipt behind it, so neither the rail + * nor the word "överföring" is grounds for skipping. + * + * Swedish bank statements truncate hard, which is why "skat" has to match as + * well as "skatt": a real ledger row reads "Inbetalning skat BG 000005...". + */ +const NO_EMAIL_RECEIPT_EXISTS = + /\bl[oö]n\b|\bl[oö]ner\b|\bskatt?\b|skatteverket|arbetsgivaravg|\bmoms\b|utdelning|amortering|\br[aä]nta\b|egen ins[aä]ttning/i + +/** + * Whether it is worth spending a mailbox search on this purchase. + * + * Only gates the mail leg: the Underlag pairing is scored on amount and + * merchant and is already safe on these rows. + */ +export function canHaveEmailReceipt(description: string | null | undefined): boolean { + if (!description) return true + return !NO_EMAIL_RECEIPT_EXISTS.test(description) +} + +/** Vendor names this alike are treated as the same merchant. */ +export const FETCH_VENDOR_SIMILARITY = 0.6 + +/** How far a receipt's own date may sit from the charge and still be plausible. */ +export const FETCH_DATE_WINDOW_DAYS = 10 + +/** Amounts within this fraction of each other are the same amount. */ +const AMOUNT_TOLERANCE = 0.01 + +function daysBetween(a: string, b: string): number { + return Math.abs(new Date(a).getTime() - new Date(b).getTime()) / 86_400_000 +} + +/** + * Is this mailbox document worth downloading? + * + * The gate before spending a fetch, a megabyte of storage and a page of the + * granskningskö on a document. It is deliberately not the match: a mail body + * states the total only about a quarter of the time, and the real amount comes + * out of the PDF afterwards, at which point the ordinary matcher decides. + * + * Ordered by how much each signal is worth. An amount that agrees is close to + * proof on its own, which is why it does not also need the vendor or the date. + * A vendor that agrees is suggestive rather than conclusive, so it is asked to + * bring a plausible date along. Everything else waits. + */ +export function worthFetching( + doc: { + vendor: string | null + date: string | null + amount: number | null + currency: string | null + }, + transactions: readonly HuntTransaction[], + /** + * The purchases whose own search turned this mail up. + * + * Already evidence, and evidence of a kind the document itself cannot carry: + * the query was this purchase's amount or its merchant tokens, so a hit means + * one of those appeared in the mail or inside its attachment. It rescues the + * case where the two names genuinely differ, which is common and not an + * error: the bank writes "Kontorsplatser j BG" and the landlord's invoice + * says "Stockholm Innovation & Growth AB". + */ + retrievedFor: readonly HuntTransaction[] = [], +): boolean { + for (const tx of transactions) { + if (tx.amount == null) continue + const charged = Math.abs(tx.amount) + + // Same currency only. A EUR receipt against a SEK charge is not a + // disagreement about the number, it is a different number: converting one + // to the other is a guess and this code does not guess. + if ( + doc.amount != null && + (doc.currency ?? 'SEK') === (tx.currency ?? 'SEK') && + Math.abs(doc.amount - charged) <= charged * AMOUNT_TOLERANCE + ) { + return true + } + + if (!doc.vendor) continue + const similarity = calculateMerchantSimilarity( + doc.vendor, + tx.merchant_name || tx.description || '', + ) + if (similarity < FETCH_VENDOR_SIMILARITY) continue + + // No date on the receipt is missing evidence, not contrary evidence: the + // vendor agreeing is enough to look inside the file. + if (!doc.date || !tx.date) return true + if (daysBetween(doc.date, tx.date) <= FETCH_DATE_WINDOW_DAYS) return true + } + + for (const tx of retrievedFor) { + if (!doc.date || !tx.date) continue + if (daysBetween(doc.date, tx.date) <= FETCH_DATE_WINDOW_DAYS) return true + } + return false +} diff --git a/lib/reports/full-archive-export.ts b/lib/reports/full-archive-export.ts index ad457ba8..7c316cef 100644 --- a/lib/reports/full-archive-export.ts +++ b/lib/reports/full-archive-export.ts @@ -997,7 +997,8 @@ export const ARCHIVE_EXCLUDED_TABLES: Record = { graph_transaction_counterparties: 'derived AI context graph, regenerable', idempotency_keys: 'infrastructure', inbox_rate_counters: 'infrastructure', - mail_connections: 'mailbox OAuth grants (live refresh tokens), not portable', + mail_connections: + 'mailbox OAuth grants (live refresh tokens), not portable. The receipts they find are archived as documents.', mcp_tasks: 'MCP task handles: transient tool-call state with a 1-hour TTL', metered_events: 'billing telemetry', notification_log: 'notification dedup log', diff --git a/messages/en.json b/messages/en.json index 311a1ba9..1619c6c1 100644 --- a/messages/en.json +++ b/messages/en.json @@ -323,9 +323,24 @@ "continue": "Continue", "sign_in_again": "Sign in again" }, + "mail": { + "connected": "Connected mailboxes", + "help": "Each mailbox is connected by its own owner. We can never connect a colleague's mailbox for them: send an invitation instead.", + "none_label": "None connected", + "none": "No mailboxes connected yet. Without one, the receipt hunt only looks at what is already in Underlag.", + "connect": "Connect mailbox", + "disconnect": "Disconnect", + "disconnect_title": "Disconnect this mailbox?", + "disconnect_body": "Access to {address} is revoked immediately and no further receipts are fetched from it. Receipts you have already approved remain as underlag: they belong to the bookkeeping now.", + "needs_reconsent": "Needs reconnecting", + "last_searched": "Last searched {date}", + "not_configured": "Gmail is not configured in this installation.", + "promise": "Read-only access: we can never send, change or delete anything in your mail. Your inbox is never copied. Only messages that might be the receipt for a specific purchase are fetched, in the moment, and then released. What becomes underlag is archived only once you approve it." + }, "settings_intro": { "account": "Your personal details and security. Applies to you, not the company.", "billing": "Each company has its own subscription.", + "mail": "Connect mailboxes so the receipt hunt can find missing receipts.", "company": "Shown on invoices, in email and in files to the authorities.", "bookkeeping": "Framework, method and series. Most of this is set once.", "tax": "Drives the VAT return, the employer declaration and which deadlines are tracked.", @@ -340,6 +355,7 @@ "settings_nav": { "aria_label": "Settings", "company": "Company", + "mail": "Mailboxes", "invoicing": "Invoicing", "bookkeeping": "Bookkeeping", "tax": "Tax", diff --git a/messages/sv.json b/messages/sv.json index 45dd808d..2a1e8b58 100644 --- a/messages/sv.json +++ b/messages/sv.json @@ -323,9 +323,24 @@ "continue": "Fortsätt", "sign_in_again": "Logga in igen" }, + "mail": { + "connected": "Kopplade brevlådor", + "help": "Varje brevlåda kopplas av sin egen ägare. Vi kan aldrig koppla en kollegas brevlåda åt dem: skicka en inbjudan i stället.", + "none_label": "Ingen kopplad", + "none": "Inga brevlådor kopplade än. Utan koppling letar Kvittojakten bara bland det som redan finns i Underlag.", + "connect": "Koppla brevlåda", + "disconnect": "Koppla från", + "disconnect_title": "Koppla från brevlådan?", + "disconnect_body": "Åtkomsten till {address} stryks direkt och inga fler kvitton hämtas därifrån. Kvitton du redan godkänt ligger kvar som underlag: de tillhör bokföringen nu.", + "needs_reconsent": "Behöver återkopplas", + "last_searched": "Senast genomsökt {date}", + "not_configured": "Gmail är inte konfigurerat i den här installationen.", + "promise": "Läsbehörighet: vi kan aldrig skicka, ändra eller radera något i din mejl. Inkorgen kopieras aldrig. Bara mejl som kan vara kvittot till ett visst köp hämtas, i stunden, och släpps igen. Det som blir underlag arkiveras först när du godkänt det." + }, "settings_intro": { "account": "Dina personliga uppgifter och din säkerhet. Gäller dig, inte företaget.", "billing": "Varje företag har sitt eget abonnemang.", + "mail": "Koppla brevlådor så att Kvittojakten kan leta upp kvitton som saknas.", "company": "Uppgifterna visas på fakturor, i e-post och i filer till myndigheter.", "bookkeeping": "Regelverk, metod och serier. Det mesta sätts en gång.", "tax": "Styr momsdeklarationen, arbetsgivardeklarationen och vilka datum som bevakas.", @@ -340,6 +355,7 @@ "settings_nav": { "aria_label": "Inställningar", "company": "Företag", + "mail": "Brevlådor", "invoicing": "Fakturering", "bookkeeping": "Bokföring", "tax": "Skatt", diff --git a/scripts/receipt-hunt-dryrun.mts b/scripts/receipt-hunt-dryrun.mts new file mode 100644 index 00000000..df1a9e7e --- /dev/null +++ b/scripts/receipt-hunt-dryrun.mts @@ -0,0 +1,90 @@ +/** + * Provkörning of the receipt hunt against a real company. + * + * npx tsx scripts/receipt-hunt-dryrun.mts # Underlag only + * npx tsx scripts/receipt-hunt-dryrun.mts --mail # also search mailboxes + * npx tsx scripts/receipt-hunt-dryrun.mts --mail --live # actually fetch and stage + * + * Without --live nothing is written: the hunt scores and reports, and with + * --mail it lists what the mailboxes hold without copying any of it. + * + * Everything is imported dynamically, after .env.local is read into + * process.env. Static imports are hoisted and would run before the environment + * exists, which makes the Supabase client fail inside the extension that + * extracts uploaded documents: the failure is silent apart from one event-bus + * error, and it leaves every fetched receipt without an amount. + */ +import { readFileSync } from 'node:fs' + +const companyId = process.argv[2] +const withMail = process.argv.includes('--mail') +const live = process.argv.includes('--live') +// --sweep searches every candidate purchase instead of the nightly top slice. +// For a company with a backlog the first run is a backfill, not a nightly tick. +const sweep = process.argv.includes('--sweep') +if (!companyId) throw new Error('usage: npx tsx scripts/receipt-hunt-dryrun.mts [--mail] [--live]') + +const env = new Map() +for (const line of readFileSync('.env.local', 'utf8').split('\n')) { + const m = line.match(/^([A-Z0-9_]+)=(.*)$/) + if (m) env.set(m[1], m[2].trim()) +} +for (const [k, v] of env) if (!process.env[k]) process.env[k] = v + +const url = env.get('NEXT_PUBLIC_SUPABASE_URL') +const key = env.get('SUPABASE_SERVICE_ROLE_KEY') +if (!url || !key) throw new Error('missing Supabase credentials in .env.local') + +// --live writes to whatever database .env.local points at, which for this repo +// is production. Requiring the company id to be named again is the cheapest +// guard that a stray --live on a recalled command cannot pass. +if (live && process.env.RECEIPT_HUNT_CONFIRM !== companyId) { + throw new Error( + `--live writes documents and proposals to the database in .env.local.\n` + + `Re-run with RECEIPT_HUNT_CONFIRM=${companyId} to confirm.`, + ) +} + +const { createClient } = await import('@supabase/supabase-js') +// Wiring the event bus is what lets document.uploaded reach the extraction +// extension. Importing lib/init is not enough; it has to be called. +const { ensureInitialized } = await import('@/lib/init') +ensureInitialized() +const { huntCompany } = await import('@/lib/receipt-hunt/hunt') + +const supabase = createClient(url, key, { auth: { persistSession: false } }) +const result = await huntCompany(supabase, companyId, live ? `live-${Date.now()}` : 'dryrun', { + dryRun: !live, + searchMail: withMail, + ...(sweep ? { mailSearchLimit: 500 } : {}), +}) + +if (live) { + console.log(`\n*** SKARP KÖRNING: ${result.mail?.ingested ?? 0} underlag hämtade, ${result.proposed} förslag lagda ***`) +} + +console.log(`\n=== ${live ? 'KÖRNING' : 'PROVKÖRNING (inget skrivet)'} ===`) +console.log(`obokförda köp utan kvitto : ${result.candidates}`) +console.log(`kvitton i Underlag : ${result.poolSize}`) +console.log(`förslag : ${result.proposed}\n`) + +for (const p of result.proposals ?? []) { + console.log(` ${p.merchant_name ?? '(okänd)'} ${p.total_amount} ${p.currency ?? ''}`) + console.log(` confidence ${p.confidence} [${p.matchReasons.join(', ')}]`) +} + +if (result.mail) { + console.log(`\n=== BREVLÅDOR ===`) + console.log(`köp genomsökta : ${result.mail.searched}`) + console.log(`underlag hittade : ${result.mail.withCandidates}`) + console.log(`hämtade : ${result.mail.ingested}\n`) + for (const c of result.mail.candidates) { + console.log(` [${c.merchant}] ${c.fileName ?? '(bilaga)'}`) + console.log(` ur ${c.mailbox}: "${c.subject ?? '(utan ämne)'}" från ${c.from ?? '?'}`) + console.log(` ${c.reason}`) + } + if (result.mail.candidates.length === 0) console.log(' (inga underlag hittade)') +} else if (withMail) { + console.log('\n(ingen brevlåda kopplad, eller Gmail inte konfigurerat)') +} +console.log('') diff --git a/supabase/migrations/20260807103000_mail_hunt_dedupe_per_attachment.sql b/supabase/migrations/20260807103000_mail_hunt_dedupe_per_attachment.sql new file mode 100644 index 00000000..a7bf83e0 --- /dev/null +++ b/supabase/migrations/20260807103000_mail_hunt_dedupe_per_attachment.sql @@ -0,0 +1,33 @@ +-- The unit of an underlag is an attachment, not a message. +-- +-- 20260807090100 made (company_id, mail_message_id) unique for hunted mail, on +-- the assumption that one mail carries one receipt. A provkörning against a +-- real mailbox disproved it: receipts reach these inboxes by being forwarded in +-- batches, and a single message routinely carries several receipts for +-- different purchases ("Fwd: Kvitton februari" with five attachments, "Fwd: +-- Anthropic receipts" with two covering two different months). +-- +-- Under the old index the first attachment filed would block every other +-- receipt in the same forward, silently and permanently. The key becomes +-- message + attachment. +-- +-- Backfill first, so the new index can be created on existing rows: anything +-- already ingested was filed one-per-message, and its file key is derived from +-- the attachment id captured at the time. +UPDATE public.invoice_inbox_items +SET channel_context = channel_context || jsonb_build_object( + 'mail_file_key', + (channel_context->>'mail_message_id') || '::' || + COALESCE(channel_context->>'mail_attachment_id', '') + ) +WHERE source = 'mail_hunt' + AND channel_context ? 'mail_message_id' + AND NOT (channel_context ? 'mail_file_key'); + +DROP INDEX IF EXISTS public.idx_invoice_inbox_mail_message_unique; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_invoice_inbox_mail_file_unique + ON public.invoice_inbox_items (company_id, ((channel_context->>'mail_file_key'))) + WHERE source = 'mail_hunt'; + +NOTIFY pgrst, 'reload schema'; diff --git a/tests/pg/mail-hunt-file-dedupe.pg.test.ts b/tests/pg/mail-hunt-file-dedupe.pg.test.ts new file mode 100644 index 00000000..b2f05e8d --- /dev/null +++ b/tests/pg/mail-hunt-file-dedupe.pg.test.ts @@ -0,0 +1,100 @@ +import { randomUUID } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { getPool } from './setup' +import { seedCompany } from './fixtures' + +/** + * The mail-hunt duplicate guard (migration 20260807103000). + * + * Receipts reach these mailboxes by being forwarded, and a single forward + * routinely carries several of them: "Fwd: Kvitton februari" with five + * attachments is five underlag, not one. The index that predated this + * migration was unique on (company_id, mail_message_id), so filing the first + * attachment silently and permanently blocked the other four. + * + * These tests pin the shape the hunt depends on: one row per attachment, the + * same attachment refused twice, and the partial predicate keeping the index + * out of the way of every other inbox source. + */ +async function insertHunted(companyId: string, userId: string, fileKey: string) { + return getPool().query( + `INSERT INTO public.invoice_inbox_items (company_id, user_id, source, status, channel_context) + VALUES ($1, $2, 'mail_hunt', 'received', jsonb_build_object('mail_file_key', $3::text)) + RETURNING id`, + [companyId, userId, fileKey], + ) +} + +describe('mail-hunt attachment dedupe (pg)', () => { + it('accepts every attachment on one forwarded message', async () => { + const { userId, companyId } = await seedCompany() + const message = randomUUID() + + // Five receipts in one mail is the case that motivated the migration. + for (const attachment of ['a', 'b', 'c', 'd', 'e']) { + await insertHunted(companyId, userId, `${message}::${attachment}`) + } + + const { rows } = await getPool().query<{ n: number }>( + `SELECT count(*)::int AS n FROM public.invoice_inbox_items + WHERE company_id = $1 AND source = 'mail_hunt'`, + [companyId], + ) + expect(rows[0].n).toBe(5) + }) + + it('refuses the same attachment twice, so a re-run is idempotent', async () => { + const { userId, companyId } = await seedCompany() + const fileKey = `${randomUUID()}::att-1` + + await insertHunted(companyId, userId, fileKey) + await expect(insertHunted(companyId, userId, fileKey)).rejects.toThrow( + /duplicate key|idx_invoice_inbox_mail_file_unique/i, + ) + }) + + it('lets two companies hold the same attachment independently', async () => { + // Two bookkeepers can be forwarded the same supplier invoice. + const first = await seedCompany() + const second = await seedCompany() + const fileKey = `${randomUUID()}::att-1` + + await insertHunted(first.companyId, first.userId, fileKey) + await expect(insertHunted(second.companyId, second.userId, fileKey)).resolves.toBeTruthy() + }) + + it('leaves every other inbox source alone', async () => { + // The index is partial on source = 'mail_hunt'. Uploads and WhatsApp + // photos carry no file key and must not collide on a shared NULL. + const { userId, companyId } = await seedCompany() + + for (let i = 0; i < 2; i++) { + await getPool().query( + `INSERT INTO public.invoice_inbox_items (company_id, user_id, source, status) + VALUES ($1, $2, 'email', 'received')`, + [companyId, userId], + ) + } + + const { rows } = await getPool().query<{ n: number }>( + `SELECT count(*)::int AS n FROM public.invoice_inbox_items + WHERE company_id = $1 AND source = 'email'`, + [companyId], + ) + expect(rows[0].n).toBe(2) + }) + + it('is the only unique index left on the hunted-mail key', async () => { + // The message-scoped predecessor must be gone, or the five-attachment + // case above would still fail in production. + const { rows } = await getPool().query<{ indexname: string }>( + `SELECT indexname FROM pg_indexes + WHERE tablename = 'invoice_inbox_items' + AND indexname IN ('idx_invoice_inbox_mail_file_unique', + 'idx_invoice_inbox_mail_message_unique')`, + ) + const names = rows.map((r) => r.indexname) + expect(names).toContain('idx_invoice_inbox_mail_file_unique') + expect(names).not.toContain('idx_invoice_inbox_mail_message_unique') + }) +}) diff --git a/types/index.ts b/types/index.ts index 327b814e..73ac5235 100644 --- a/types/index.ts +++ b/types/index.ts @@ -3346,6 +3346,8 @@ export type DocumentUploadSource = | 'api' | 'system' | 'whatsapp' + /** Fetched by the receipt hunt out of a connected mailbox. */ + | 'mail_hunt' export interface DocumentAttachment { id: string diff --git a/vercel.json b/vercel.json index 6cd247e9..7b0c0ed5 100644 --- a/vercel.json +++ b/vercel.json @@ -84,6 +84,10 @@ { "path": "/api/bookkeeping/accruals/post-due/cron", "schedule": "15 5 * * *" + }, + { + "path": "/api/receipt-hunt/cron", + "schedule": "30 5 * * *" } ] }