diff --git a/extensions/general/skatteverket/__tests__/skattekonto-sync-invalid-rows.test.ts b/extensions/general/skatteverket/__tests__/skattekonto-sync-invalid-rows.test.ts new file mode 100644 index 00000000..ae9c8c05 --- /dev/null +++ b/extensions/general/skatteverket/__tests__/skattekonto-sync-invalid-rows.test.ts @@ -0,0 +1,378 @@ +/** + * Tests for sync resilience against SKV transaktioner rows that cannot + * satisfy the table's NOT NULL columns (transaktionsdatum, transaktionstext, + * belopp_skatteverket). + * + * SKV has been observed returning a row without beloppSkatteverket; before + * the guard, one such row failed the whole batch upsert (23502) and with it + * every sync for the company: post-connect, manual and the nightly cron. + * The guard skips unusable rows, syncs the rest, and reports the count. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { createQueuedMockSupabase } from '@/tests/helpers' +import { eventBus } from '@/lib/events/bus' + +const { supabase, enqueue, reset, findCalls } = createQueuedMockSupabase() + +const getSaldoMock = vi.fn() +const getTransaktionerMock = vi.fn() +vi.mock('../lib/skattekonto-client', () => ({ + getSaldo: (...args: unknown[]) => getSaldoMock(...args), + getTransaktioner: (...args: unknown[]) => getTransaktionerMock(...args), +})) + +vi.mock('../lib/agi-tax-settlement', () => ({ + settleAgiTaxPayments: vi.fn().mockResolvedValue(undefined), +})) + +const warnMock = vi.fn() +vi.mock('@/lib/logger', () => ({ + createLogger: () => ({ + info: vi.fn(), + warn: (...args: unknown[]) => warnMock(...args), + error: vi.fn(), + debug: vi.fn(), + }), +})) + +import { syncSkattekonto, SKATTEKONTO_SKIPPED_ROWS_KEY } from '../lib/skattekonto-sync' +import type { ExtensionContext } from '@/lib/extensions/types' + +const settingsSetMock = vi.fn().mockResolvedValue(undefined) + +function makeCtx(): ExtensionContext { + return { + supabase, + companyId: 'company-1', + userId: 'user-1', + settings: { + get: vi.fn().mockResolvedValue(null), + set: settingsSetMock, + }, + emit: vi.fn().mockResolvedValue(undefined), + } as unknown as ExtensionContext +} + +function skippedRowsWrites() { + return settingsSetMock.mock.calls.filter( + ([key]) => key === SKATTEKONTO_SKIPPED_ROWS_KEY, + ) +} + +function makeSaldo() { + return { + nastaAvstamningsdatum: '2026-09-05', + senastUppdaterad: '2026-08-17', + informationstext: [], + saldoSkatteverket: 1000, + saldoKronofogden: 0, + rantaSkatteverket: 0, + rantaKronofogden: 0, + ocrNummer: '1234567897', + } +} + +const VALID_BOOKED_A = { + transaktionsidentitet: 9001, + transaktionsdatum: '2026-07-13', + ranteberakningsdatum: '2026-07-13', + transaktionstext: 'Arbetsgivaravgift juni 2026', + beloppSkatteverket: -15710, + beloppKronofogden: 0, +} + +const VALID_BOOKED_B = { + transaktionsidentitet: 9002, + transaktionsdatum: '2026-07-14', + ranteberakningsdatum: '2026-07-14', + transaktionstext: 'Inbetalning bokförd 260714', + beloppSkatteverket: 20000, + beloppKronofogden: 0, +} + +// The observed failure shape: a booked row where SKV omitted the amount. +const BOOKED_NO_BELOPP = { + transaktionsidentitet: 9003, + transaktionsdatum: '2026-07-15', + ranteberakningsdatum: null, + transaktionstext: 'Överföring till Kronofogden', + beloppSkatteverket: undefined as unknown as number, + beloppKronofogden: -500, +} + +const VALID_UPCOMING = { + transaktionsidentitet: null, + transaktionsdatum: '2026-09-12', + forfallodatum: '2026-09-12', + ranteberakningsdatum: null, + transaktionstext: 'Moms augusti 2026', + beloppSkatteverket: -8400, + beloppKronofogden: 0, +} + +const UPCOMING_NULL_BELOPP = { + transaktionsidentitet: null, + transaktionsdatum: '2026-09-12', + forfallodatum: '2026-09-12', + ranteberakningsdatum: null, + transaktionstext: 'Preliminär debitering', + beloppSkatteverket: null as unknown as number, + beloppKronofogden: null, +} + +describe('syncSkattekonto: rows missing NOT NULL fields', () => { + beforeEach(() => { + vi.clearAllMocks() + eventBus.clear() + reset() + getSaldoMock.mockResolvedValue(makeSaldo()) + settingsSetMock.mockResolvedValue(undefined) + }) + + it('skips unusable rows, upserts the rest, and reports the count', async () => { + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: [VALID_BOOKED_A, BOOKED_NO_BELOPP, VALID_BOOKED_B], + kommandeTransaktioner: [VALID_UPCOMING, UPCOMING_NULL_BELOPP], + }) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + enqueue({ data: [] }) // existing dedup_key lookup + enqueue({ data: [] }) // takeover candidate scan (both booked rows are new) + enqueue({ data: null }) // upsert + + const result = await syncSkattekonto(makeCtx()) + + expect(result.booked).toBe(2) + expect(result.upcoming).toBe(1) + expect(result.skipped).toBe(2) + + const upserts = findCalls('skattekonto_transactions', 'upsert') + expect(upserts).toHaveLength(1) + const rows = upserts[0][0] as Array> + expect(rows).toHaveLength(3) + for (const row of rows) { + expect(typeof row.belopp_skatteverket).toBe('number') + } + + // The log is minimized (no transaktionstext, no amounts); the raw rows + // are retained in the company-scoped extension_data trace instead. + expect(warnMock).toHaveBeenCalledWith( + 'skipped transaktioner rows missing required fields', + { + companyId: 'company-1', + skipped: 2, + traceTruncated: false, + rows: [ + { + status: 'booked', + missing: ['beloppSkatteverket'], + transaktionsidentitet: 9003, + transaktionsdatum: '2026-07-15', + }, + { + status: 'upcoming', + missing: ['beloppSkatteverket'], + transaktionsidentitet: null, + transaktionsdatum: '2026-09-12', + }, + ], + }, + ) + const logged = JSON.stringify(warnMock.mock.calls) + expect(logged).not.toContain('Överföring till Kronofogden') + expect(logged).not.toContain('Preliminär debitering') + + const writes = skippedRowsWrites() + expect(writes).toHaveLength(1) + expect(writes[0][1]).toEqual( + expect.objectContaining({ + rows: [ + expect.objectContaining({ + status: 'booked', + missing: ['beloppSkatteverket'], + row: BOOKED_NO_BELOPP, + }), + expect.objectContaining({ + status: 'upcoming', + missing: ['beloppSkatteverket'], + row: UPCOMING_NULL_BELOPP, + }), + ], + }), + ) + }) + + it('keeps aged-out trace entries and drops entries whose id resolved', async () => { + // Previous sync traced two rows; the current payload contains neither as + // skipped: 9003 now arrives complete (resolved: the table has it), while + // the id-less upcoming row has aged out of SKV's window entirely. + const previousTrace = { + updatedAt: '2026-08-01T00:00:00.000Z', + rows: [ + { + status: 'booked', + missing: ['beloppSkatteverket'], + row: BOOKED_NO_BELOPP, + firstSeenAt: '2026-08-01T00:00:00.000Z', + lastSeenAt: '2026-08-01T00:00:00.000Z', + }, + { + status: 'upcoming', + missing: ['beloppSkatteverket'], + row: UPCOMING_NULL_BELOPP, + firstSeenAt: '2026-08-01T00:00:00.000Z', + lastSeenAt: '2026-08-01T00:00:00.000Z', + }, + ], + } + const completed9003 = { + ...BOOKED_NO_BELOPP, + beloppSkatteverket: -500, + } + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: [completed9003], + kommandeTransaktioner: [], + }) + const ctx = makeCtx() + ;(ctx.settings.get as ReturnType).mockImplementation( + (key: string) => + Promise.resolve(key === SKATTEKONTO_SKIPPED_ROWS_KEY ? previousTrace : null), + ) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + enqueue({ data: [] }) // existing dedup_key lookup + enqueue({ data: [] }) // takeover candidate scan + enqueue({ data: null }) // upsert + + const result = await syncSkattekonto(ctx) + + expect(result.skipped).toBe(0) + const writes = skippedRowsWrites() + expect(writes).toHaveLength(1) + const record = writes[0][1] as { rows: Array<{ row: { transaktionstext: string } }> } + // 9003 resolved into the table and left the trace; the aged-out id-less + // row survives with its original firstSeenAt. + expect(record.rows).toHaveLength(1) + expect(record.rows[0]).toEqual( + expect.objectContaining({ + status: 'upcoming', + firstSeenAt: '2026-08-01T00:00:00.000Z', + row: UPCOMING_NULL_BELOPP, + }), + ) + }) + + it('reports the full skipped count and flags truncation past the trace cap', async () => { + const manyInvalid = Array.from({ length: 60 }, (_, i) => ({ + transaktionsidentitet: 20000 + i, + transaktionsdatum: '2026-07-01', + ranteberakningsdatum: null, + transaktionstext: `Rad ${i}`, + beloppSkatteverket: undefined as unknown as number, + beloppKronofogden: 0, + })) + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: manyInvalid, + kommandeTransaktioner: [], + }) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + + const result = await syncSkattekonto(makeCtx()) + + expect(result.skipped).toBe(60) + const record = skippedRowsWrites()[0][1] as { truncated?: boolean; rows: unknown[] } + expect(record.rows).toHaveLength(50) + expect(record.truncated).toBe(true) + expect(warnMock).toHaveBeenCalledWith( + 'skipped transaktioner rows missing required fields', + expect.objectContaining({ skipped: 60, traceTruncated: true }), + ) + }) + + it('skips rows missing transaktionsdatum or transaktionstext', async () => { + const bookedNoDatum = { + ...VALID_BOOKED_A, + transaktionsidentitet: 9010, + transaktionsdatum: undefined as unknown as string, + } + const bookedEmptyText = { + ...VALID_BOOKED_B, + transaktionsidentitet: 9011, + transaktionstext: '' as string, + } + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: [bookedNoDatum, bookedEmptyText, VALID_BOOKED_A], + kommandeTransaktioner: [], + }) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + enqueue({ data: [] }) // existing dedup_key lookup + enqueue({ data: [] }) // takeover candidate scan + enqueue({ data: null }) // upsert + + const result = await syncSkattekonto(makeCtx()) + + expect(result.booked).toBe(1) + expect(result.skipped).toBe(2) + const upserts = findCalls('skattekonto_transactions', 'upsert') + expect((upserts[0][0] as unknown[]).length).toBe(1) + + const writes = skippedRowsWrites() + expect(writes[0][1]).toEqual( + expect.objectContaining({ + rows: [ + expect.objectContaining({ missing: ['transaktionsdatum'] }), + expect.objectContaining({ missing: ['transaktionstext'] }), + ], + }), + ) + }) + + it('completes with zero writes when every row is unusable', async () => { + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: [BOOKED_NO_BELOPP], + kommandeTransaktioner: [UPCOMING_NULL_BELOPP], + }) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + + const result = await syncSkattekonto(makeCtx()) + + expect(result.booked).toBe(0) + expect(result.upcoming).toBe(0) + expect(result.skipped).toBe(2) + expect(findCalls('skattekonto_transactions', 'upsert')).toHaveLength(0) + }) + + it('does not warn or skip on a fully valid payload', async () => { + getTransaktionerMock.mockResolvedValue({ + tidigareTransaktioner: [VALID_BOOKED_A], + kommandeTransaktioner: [VALID_UPCOMING], + }) + + enqueue({ data: { org_number: '556677-8899', entity_type: 'aktiebolag' } }) // company_settings + enqueue({ data: [] }) // fiscal_periods + enqueue({ data: [] }) // existing dedup_key lookup + enqueue({ data: [] }) // takeover candidate scan + enqueue({ data: null }) // upsert + + const result = await syncSkattekonto(makeCtx()) + + expect(result.skipped).toBe(0) + expect(warnMock).not.toHaveBeenCalled() + const upserts = findCalls('skattekonto_transactions', 'upsert') + expect(upserts).toHaveLength(1) + expect((upserts[0][0] as unknown[]).length).toBe(2) + + // The trace self-clears: a clean payload overwrites any previous rows. + const writes = skippedRowsWrites() + expect(writes).toHaveLength(1) + expect(writes[0][1]).toEqual(expect.objectContaining({ rows: [] })) + }) +}) diff --git a/extensions/general/skatteverket/lib/skattekonto-sync.ts b/extensions/general/skatteverket/lib/skattekonto-sync.ts index 53b3cc71..9d645718 100644 --- a/extensions/general/skatteverket/lib/skattekonto-sync.ts +++ b/extensions/general/skatteverket/lib/skattekonto-sync.ts @@ -27,6 +27,14 @@ const log = createLogger('skattekonto-sync') const BALANCE_SNAPSHOT_KEY = 'skattekonto_balance_snapshot' const LAST_SYNCED_AT_KEY = 'skattekonto_last_synced_at' +const SKIPPED_ROWS_KEY = 'skattekonto_skipped_rows' + +/** + * Cap the retained skipped-row trace: it mirrors the CURRENT sync response + * (overwritten every run), so the cap only guards against a pathological + * payload, not against growth over time. + */ +const MAX_SKIPPED_ROWS_RETAINED = 50 /** SKV's default lookback for tidigare transaktioner when no datumFrom is sent. */ const SKV_DEFAULT_WINDOW_DAYS = 555 @@ -57,6 +65,8 @@ export interface SkattekontoSyncResult { booked: number /** Number of new or updated upcoming rows */ upcoming: number + /** Rows dropped because SKV omitted a field the table requires */ + skipped: number /** Saldo at end of sync (mirrors snapshot) */ saldoSkatteverket: number saldoKronofogden: number @@ -64,6 +74,80 @@ export interface SkattekontoSyncResult { syncedAt: string } +/** + * A transaktioner row is usable only when it can satisfy the table's NOT NULL + * columns (transaktionsdatum, transaktionstext, belopp_skatteverket). SKV has + * been observed returning rows without beloppSkatteverket despite the spec + * typing it as required; a single such row used to fail the whole batch + * upsert (23502) and with it every sync for the company, permanently. + * Skipping is deliberate: coalescing to 0 kr would make the row renderable, + * matchable and bookable with an invented amount. A skipped row returns on a + * later sync if SKV completes it. + */ +type IncomingTransaction = + | SkatteverketBookedTransaction + | SkatteverketUpcomingTransaction + +function missingRequiredFields(tx: IncomingTransaction): string[] { + const missing: string[] = [] + if (!(typeof tx.transaktionsdatum === 'string' && tx.transaktionsdatum.length > 0)) { + missing.push('transaktionsdatum') + } + if (!(typeof tx.transaktionstext === 'string' && tx.transaktionstext.length > 0)) { + missing.push('transaktionstext') + } + if (!(typeof tx.beloppSkatteverket === 'number' && Number.isFinite(tx.beloppSkatteverket))) { + missing.push('beloppSkatteverket') + } + return missing +} + +function isUsableTransaction(tx: IncomingTransaction): boolean { + return missingRequiredFields(tx).length === 0 +} + +/** + * Durable, company-scoped trace of the rows the sync could not store, kept in + * extension_data and overwritten on every sync (an empty rows list means the + * latest response had none). The raw payload lives HERE, in the tenant's own + * storage, so the omission stays reconcilable and re-processable (BFL 5 kap + * fullständighet, BFNAR 2013:2 kap 8 behandlingshistorik); the application + * log deliberately carries only minimized descriptors without free text or + * amounts (GDPR data minimization: an enskild firma's skattekonto is the + * owner's personal tax account). + */ +export interface SkattekontoSkippedRowEntry { + status: 'booked' | 'upcoming' + missing: string[] + row: IncomingTransaction + firstSeenAt: string + lastSeenAt: string +} + +export interface SkattekontoSkippedRowsRecord { + updatedAt: string + /** + * True when a pathological payload exceeded MAX_SKIPPED_ROWS_RETAINED and + * entries had to be dropped from this trace (never from the skipped count). + */ + truncated?: boolean + rows: SkattekontoSkippedRowEntry[] +} + +/** + * Identity for a skipped row inside the trace: SKV's stable id when present, + * otherwise the raw material an id-less row can offer. Only used to merge + * trace entries across syncs; never fed to the table's dedup_key. + */ +function skippedRowKey(status: 'booked' | 'upcoming', row: IncomingTransaction): string { + if (row.transaktionsidentitet != null) return `id:${row.transaktionsidentitet}` + return `raw:${status}:${JSON.stringify([ + row.transaktionsdatum ?? null, + row.transaktionstext ?? null, + 'forfallodatum' in row ? (row.forfallodatum ?? null) : null, + ])}` +} + // Dedup key computation moved to core (lib/skatteverket/skattekonto-dedup): // the skattekontoutdrag file importer is a second producer of this table and // must compute byte-identical keys. Re-exported so existing extension-side @@ -211,12 +295,73 @@ export async function syncSkattekonto( ) const previousBalance = previousSnapshot?.saldo.saldoSkatteverket ?? null - const bookedRows = transaktioner.tidigareTransaktioner.map(tx => - bookedToRow(ctx.companyId, tx), + const tidigare = transaktioner.tidigareTransaktioner.filter(isUsableTransaction) + const kommande = transaktioner.kommandeTransaktioner.filter(isUsableTransaction) + const currentSkipped = [ + ...transaktioner.tidigareTransaktioner + .filter(tx => !isUsableTransaction(tx)) + .map(row => ({ status: 'booked' as const, missing: missingRequiredFields(row), row })), + ...transaktioner.kommandeTransaktioner + .filter(tx => !isUsableTransaction(tx)) + .map(row => ({ status: 'upcoming' as const, missing: missingRequiredFields(row), row })), + ] + // The count reflects THIS sync's payload in full, independent of the + // trace cap below. + const skipped = currentSkipped.length + + // Merge with the previously retained trace instead of overwriting it: a + // skipped row that ages out of SKV's ~555-day window would otherwise + // vanish from every later payload and take its only record with it + // (BFNAR 2013:2 kap 8 behandlingshistorik). An entry leaves the trace only + // when its transaktionsidentitet shows up among the valid rows: the data + // then lives in skattekonto_transactions itself, which is the better + // record. + const previousTrace = await ctx.settings.get(SKIPPED_ROWS_KEY) + const now = new Date().toISOString() + const byKey = new Map() + for (const entry of previousTrace?.rows ?? []) { + byKey.set(skippedRowKey(entry.status, entry.row), entry) + } + for (const cur of currentSkipped) { + const key = skippedRowKey(cur.status, cur.row) + const prior = byKey.get(key) + byKey.set(key, { + ...cur, + firstSeenAt: prior?.firstSeenAt ?? now, + lastSeenAt: now, + }) + } + const resolvedIds = new Set( + [...tidigare, ...kommande] + .map(tx => tx.transaktionsidentitet) + .filter((id): id is number => id != null), ) - const upcomingRows = transaktioner.kommandeTransaktioner.map(tx => - upcomingToRow(ctx.companyId, tx), + const mergedEntries = [...byKey.values()].filter( + e => e.row.transaktionsidentitet == null || !resolvedIds.has(e.row.transaktionsidentitet), ) + const skippedRecord: SkattekontoSkippedRowsRecord = { + updatedAt: now, + truncated: mergedEntries.length > MAX_SKIPPED_ROWS_RETAINED || undefined, + rows: mergedEntries.slice(0, MAX_SKIPPED_ROWS_RETAINED), + } + await ctx.settings.set(SKIPPED_ROWS_KEY, skippedRecord) + if (skipped > 0) { + log.warn('skipped transaktioner rows missing required fields', { + companyId: ctx.companyId, + skipped, + traceTruncated: skippedRecord.truncated === true, + rows: skippedRecord.rows.map(r => ({ + status: r.status, + missing: r.missing, + transaktionsidentitet: r.row.transaktionsidentitet ?? null, + transaktionsdatum: + typeof r.row.transaktionsdatum === 'string' ? r.row.transaktionsdatum : null, + })), + }) + } + + const bookedRows = tidigare.map(tx => bookedToRow(ctx.companyId, tx)) + const upcomingRows = kommande.map(tx => upcomingToRow(ctx.companyId, tx)) // Upsert in two steps to keep the conflict target consistent. We rely on // the (company_id, dedup_key) unique constraint defined in the migration. @@ -447,7 +592,7 @@ export async function syncSkattekonto( } // First-appearance upcoming transactions. - for (const tx of transaktioner.kommandeTransaktioner) { + for (const tx of kommande) { const key = computeDedupKey(tx) if (existingMap.has(key)) continue await ctx.emit({ @@ -466,6 +611,7 @@ export async function syncSkattekonto( return { booked: bookedRows.length, upcoming: upcomingRows.length, + skipped, saldoSkatteverket: saldo.saldoSkatteverket, saldoKronofogden: saldo.saldoKronofogden, syncedAt: new Date().toISOString(), @@ -474,3 +620,4 @@ export async function syncSkattekonto( export const SKATTEKONTO_BALANCE_SNAPSHOT_KEY = BALANCE_SNAPSHOT_KEY export const SKATTEKONTO_LAST_SYNCED_AT_KEY = LAST_SYNCED_AT_KEY +export const SKATTEKONTO_SKIPPED_ROWS_KEY = SKIPPED_ROWS_KEY