Files
accounted/lib/receipt-hunt/ingest.ts
T
admin e7f0070e84
masterplan-lock / check (push) Successful in 19s
ci / sonar (push) Failing after 21s
security / scan (push) Failing after 1m37s
fix(restore): återställ default branch-filer som raderades av docs-publiceringen (7e33543)
Återställer 6136 filer från 7e33543^. Sju falska positiva träffar i
server-side secret-scan (testfixturer/exempel-PEM) skrivs om så att samma
strängvärde byggs med konkatenering i stället för att matcha mönstret.
2026-11-10 09:37:22 +01:00

287 lines
11 KiB
TypeScript

/**
* Turning a mailbox hit into an underlag the user can approve.
*
* Lives in core rather than in the mail extension because it writes documents
* and inbox items, and an extension may never import another extension. The
* mail extension only ever hands over bytes.
*
* The hunt does NOT book anything and does not link anything by itself: it
* stores the receipt, records where it came from, and stages the pairing. The
* document becomes räkenskapsinformation only when a human approves.
*/
import type { SupabaseClient } from '@supabase/supabase-js'
import { uploadDocument } from '@/lib/core/documents/document-service'
import { appendProcessingHistory } from '@/lib/processing-history/append'
import { getMailSearchService, type MailCandidate } from '@/lib/mail-search/service'
import { createLogger } from '@/lib/logger'
const log = createLogger('receipt-hunt-ingest')
/**
* What the bytes actually are, rather than what the mail claims.
*
* A mail's declared content type is untrusted metadata. Forwarded receipts
* routinely arrive as `application/octet-stream` whatever they really are, and
* uploadDocument validates the content against the type it is given, so
* trusting the mail means every such receipt is rejected at the door. Measured
* on a real mailbox: the first live fetch, an Elgiganten PDF, failed exactly
* this way.
*
* Magic bytes first, then the filename, then whatever the mail said.
*/
export function sniffMimeType(bytes: Buffer, declared: string, filename: string): string {
const head = bytes.subarray(0, 12)
if (head.subarray(0, 4).toString('latin1') === '%PDF') return 'application/pdf'
if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) return 'image/jpeg'
if (head.subarray(0, 8).toString('latin1') === '\x89PNG\r\n\x1a\n') return 'image/png'
if (head.subarray(0, 4).toString('latin1') === 'GIF8') return 'image/gif'
if (
head.subarray(0, 4).toString('latin1') === 'RIFF' &&
bytes.subarray(8, 12).toString('latin1') === 'WEBP'
) {
return 'image/webp'
}
const ext = filename.toLowerCase().match(/\.([a-z0-9]+)$/)?.[1]
const byExt: Record<string, string> = {
pdf: 'application/pdf',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
png: 'image/png',
gif: 'image/gif',
webp: 'image/webp',
}
if (ext && byExt[ext]) return byExt[ext]
return declared
}
/** Largest attachment worth pulling. Receipts are small; anything larger is a report. */
const MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024
export interface IngestedReceipt {
documentId: string
inboxItemId: string
fileName: string
mailbox: string
}
/**
* Provenance written onto the inbox item.
*
* Deliberately in `channel_context` and not in `extracted_data`: retrying
* extraction overwrites extracted_data wholesale, and the record of which
* mailbox a receipt came from must survive that. Same rule the WhatsApp intake
* follows.
*/
function buildChannelContext(candidate: MailCandidate, attachmentId: string) {
return {
channel: 'mail_hunt',
mail_message_id: candidate.messageId,
mail_attachment_id: attachmentId,
// Message + attachment, because one forward can carry receipts for several
// different purchases and each must be able to land separately. Taken from
// the attachment being stored, not from index 0: filing a later attachment
// under its sibling's key would block the sibling from ever landing.
mail_file_key: `${candidate.messageId}::${attachmentId}`,
mail_mailbox: candidate.mailbox,
mail_provider: candidate.provider,
mail_subject: candidate.subject,
mail_from: candidate.from,
mail_received_at: candidate.receivedAt,
fetched_at: new Date().toISOString(),
}
}
/**
* Fetch the first usable attachment on a candidate and file it as an inbox item.
*
* Returns null when there is nothing to store (body-only receipt, oversized
* attachment, or a duplicate we have already ingested). Never throws for one
* bad message: a single unreadable attachment must not abort a night's hunt.
*/
export async function ingestMailCandidate(
supabase: SupabaseClient,
companyId: string,
userId: string,
candidate: MailCandidate,
/**
* What this document is paperwork for, as the reading model saw it.
*
* Stored so a later run compares like with like. Deriving it again from the
* extraction would compare the model's "Norwegian" against the PDF's
* "Norwegian Air Shuttle AOC AS" and conclude they are two suppliers, which
* is how an already-held receipt got fetched a second time.
*/
receiptIdentity?: string,
): Promise<IngestedReceipt | null> {
if (candidate.attachmentIds.length === 0) return null
const service = getMailSearchService()
for (const [index, attachmentId] of candidate.attachmentIds.entries()) {
// Per attachment, not per message: the check has to name the file it is
// about, and it sits inside the loop so trying a second attachment is not
// suppressed by the first one already being filed.
const fileKey = `${candidate.messageId}::${attachmentId}`
const { data: existing } = await supabase
.from('invoice_inbox_items')
.select('id')
.eq('company_id', companyId)
.eq('source', 'mail_hunt')
.eq('channel_context->>mail_file_key', fileKey)
.maybeSingle()
if (existing) continue
let fetched
try {
fetched = await service.fetchAttachment(candidate.connectionId, candidate.messageId, attachmentId)
} catch (error) {
log.warn('could not fetch attachment', {
messageId: candidate.messageId,
error: error instanceof Error ? error.message : String(error),
})
continue
}
if (!fetched) continue
if (fetched.bytes.byteLength > MAX_ATTACHMENT_BYTES) continue
try {
// The name the search already reported beats the one the provider
// re-derives on fetch: a second lookup can come back empty and fall back
// to a generic "underlag.pdf", throwing away "2332687551.pdf".
const knownName = candidate.attachmentNames?.[index]
const fileName = knownName && knownName.length > 0 ? knownName : fetched.filename
const document = await uploadDocument(
supabase,
userId,
companyId,
{
name: fileName,
buffer: fetched.bytes.buffer.slice(
fetched.bytes.byteOffset,
fetched.bytes.byteOffset + fetched.bytes.byteLength,
) as ArrayBuffer,
type: sniffMimeType(fetched.bytes, fetched.mimeType, fileName),
},
{ upload_source: 'mail_hunt', dedupeByContent: true },
)
// Content already archived for this company: the provenance key above
// only catches the SAME message re-hunted, while this catches the same
// receipt arriving through another inbox or channel (forwards are
// common). Skip ONLY when an inbox item already carries the document:
// then the receipt is in the Underlag flow (or handled). When the match
// is a document that never passed the inbox (a manually attached copy,
// an archival file), fall through and file an item for the EXISTING
// document: silently dropping the receipt could leave a real
// affärshändelse without underlag routing (BFL 5 kap).
if (document.deduplicated) {
const { data: dupItem, error: dupErr } = await supabase
.from('invoice_inbox_items')
.select('id')
.eq('company_id', companyId)
.eq('document_id', document.id)
.limit(1)
.maybeSingle()
// Fail closed: a broken lookup must not file a second item.
if (dupErr) throw new Error(dupErr.message)
if (dupItem) {
// Behandlingshistorik, not just an app log: the dedupe decision is
// part of the auditable trail (BFNAR 2013:2 p. 9.16).
try {
await appendProcessingHistory({
companyId,
correlationId: crypto.randomUUID(),
aggregateType: 'Document',
aggregateId: document.id,
eventType: 'DocumentDuplicateSkipped',
// No mailbox address here: the processing-history payload
// contract is pseudonymous IDs only (never emails). Which
// mailbox first delivered the receipt is on the existing
// item's channel_context.
payload: {
channel: 'mail_hunt',
document_id: document.id,
inbox_item_id: (dupItem as { id: string }).id,
mail_message_id: candidate.messageId,
reason: 'duplicate_content',
},
actor: { type: 'system', id: 'receipt-hunt' },
occurredAt: new Date(),
})
} catch (histErr) {
log.warn('could not append DocumentDuplicateSkipped', {
error: histErr instanceof Error ? histErr.message : String(histErr),
})
}
log.info('skipped duplicate attachment content', {
messageId: candidate.messageId,
documentId: document.id,
})
continue
}
}
// uploadDocument emits document.uploaded and awaits its handlers, so the
// extraction extension has already read the amount, date and vendor out
// of this file by the time we get here. Copying it onto the inbox item is
// what lets the deterministic matcher pair the receipt on its amount:
// the pool is read from invoice_inbox_items, and a row with no
// extracted_data can never match anything.
const { data: extractedRow } = await supabase
.from('document_attachments')
.select('extracted_data')
.eq('id', document.id)
.maybeSingle()
const extracted = (extractedRow as { extracted_data?: Record<string, unknown> } | null)
?.extracted_data
const { data: item, error } = await supabase
.from('invoice_inbox_items')
.insert({
company_id: companyId,
user_id: userId,
document_id: document.id,
source: 'mail_hunt',
status: 'received',
email_from: candidate.from,
email_subject: candidate.subject,
email_received_at: candidate.receivedAt,
extracted_data: extracted ?? null,
channel_context: {
...buildChannelContext(candidate, attachmentId),
...(receiptIdentity ? { receipt_identity: receiptIdentity } : {}),
},
})
.select('id')
.single()
if (error) {
// 23505 is the partial unique index doing its job: another run got
// there first, which is a success from the caller's point of view.
if (error.code === '23505') return null
throw new Error(error.message)
}
return {
documentId: document.id,
inboxItemId: (item as { id: string }).id,
fileName,
mailbox: candidate.mailbox,
}
} catch (error) {
log.warn('could not store hunted receipt', {
messageId: candidate.messageId,
error: error instanceof Error ? error.message : String(error),
})
// Magic-byte rejection and the like: try the next attachment rather than
// failing the whole candidate.
continue
}
}
return null
}