fix(api): stabilize report pagination + declare real { data, meta } envelope on v1 single/write endpoints (#811)
* fix(reports): stabilize fetchAllRows paging to stop doubled/dropped balances (#790, #791) PostgREST `.range()` paging is only correct when the underlying query has a stable TOTAL order. Several aggregating report queries (general ledger, trial balance, grundbok, supplier/AR ledgers, etc.) paginated without `.order()`, so on datasets larger than one 1000-row page Postgres could return rows in a different order between requests — silently DUPLICATING or SKIPPING rows on a page boundary and doubling or dropping financial totals. - fetch-all.ts: document the ordering invariant and add an optional `dedupeBy` defense-in-depth that drops cross-page duplicates and warns when it fires (surfaces a missing `.order()` in logs instead of corrupting money). - Add a stable `.order()` (line PK or account_number) to every paginated query in lib/reports/ and the account-balances route; pass `dedupeBy` on the money-aggregating line queries. - Add fetch-all unit tests and update report test fixtures to carry row ids. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(api): declare the real { data, meta } envelope on v1 single/write/204 endpoints (#794) The OpenAPI generator derives each endpoint's documented body purely from its registered `response.success` Zod schema, and that schema is never validated at runtime — so a route could advertise a shape its handler never sends. #802 fixed this for list endpoints; the same drift was latent on single-resource and write endpoints, which declared the bare resource schema instead of the `{ data, meta }` envelope the handlers actually return. - registry.ts: extend `ResponseMetaSchema` with the optional `audit` block and `partial_expansions` list that writes/expansions emit; add the `NoBodyResponse` sentinel so 204 DELETE handlers document a bare 204 instead of a phantom 200. - Wrap every single/write endpoint's `response.success` in `dataEnvelope(...)` (or `NoBodyResponse` for 204s) across the v1 routes. - Add a response-envelope contract test that fails CI if any JSON endpoint forgets to wrap its schema, with binary downloads and 204s as the only exemptions. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(reports): extend paging dedupeBy to rc-basis-gaps and opening-balances Address PR review: these two money-aggregating line queries already had the stable `.order('id')` (so paging was correct) but didn't carry `id` in the select, so they couldn't use the `dedupeBy` defense-in-depth that general-ledger and trial-balance got. Select `id` and pass `dedupeBy: r => r.id` so the whole report layer applies the ordering invariant consistently. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
ae17b304d7
commit
fce6faff2c
@@ -0,0 +1,88 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { fetchAllRows } from '../fetch-all'
|
||||
|
||||
const PAGE_SIZE = 1000
|
||||
|
||||
type Row = { id: string; value?: number }
|
||||
|
||||
/**
|
||||
* Build a queryFn that serves predefined pages keyed by the `from` offset.
|
||||
* Mirrors how `fetchAllRows` drives PostgREST `.range(from, to)`.
|
||||
*/
|
||||
function pagedQuery(pages: Record<number, Row[]>) {
|
||||
return ({ from }: { from: number; to: number }) =>
|
||||
Promise.resolve({ data: pages[from] ?? [], error: null })
|
||||
}
|
||||
|
||||
function makeRows(start: number, count: number): Row[] {
|
||||
return Array.from({ length: count }, (_, i) => ({ id: String(start + i), value: 1 }))
|
||||
}
|
||||
|
||||
describe('fetchAllRows', () => {
|
||||
it('returns a single page as-is and stops (page < PAGE_SIZE)', async () => {
|
||||
const rows = makeRows(0, 3)
|
||||
const out = await fetchAllRows<Row>(pagedQuery({ 0: rows }))
|
||||
expect(out).toHaveLength(3)
|
||||
expect(out.map((r) => r.id)).toEqual(['0', '1', '2'])
|
||||
})
|
||||
|
||||
it('paginates across multiple pages and concatenates in order', async () => {
|
||||
const page1 = makeRows(0, PAGE_SIZE) // full page → fetch continues
|
||||
const page2 = makeRows(PAGE_SIZE, 5) // partial page → stop
|
||||
const out = await fetchAllRows<Row>(pagedQuery({ 0: page1, [PAGE_SIZE]: page2 }))
|
||||
expect(out).toHaveLength(PAGE_SIZE + 5)
|
||||
expect(out[0].id).toBe('0')
|
||||
expect(out[out.length - 1].id).toBe(String(PAGE_SIZE + 4))
|
||||
})
|
||||
|
||||
it('throws when the query returns an error', async () => {
|
||||
await expect(
|
||||
fetchAllRows<Row>(() => Promise.resolve({ data: null, error: { message: 'boom' } })),
|
||||
).rejects.toThrow('boom')
|
||||
})
|
||||
|
||||
it('returns [] when the first page is empty', async () => {
|
||||
const out = await fetchAllRows<Row>(pagedQuery({ 0: [] }))
|
||||
expect(out).toEqual([])
|
||||
})
|
||||
|
||||
// ── The regression-critical behaviour: an unstable cross-page order ──
|
||||
// (a query missing a stable .order()) can return the same row on two
|
||||
// pages. This is the mechanism behind the doubled-balance bugs (#790/#791).
|
||||
|
||||
it('dedupeBy drops a row duplicated across page boundaries (keeps first)', async () => {
|
||||
const page1 = makeRows(0, PAGE_SIZE) // ids 0..999
|
||||
// Unstable order: page 2 re-serves id "999" (already on page 1) plus a new id.
|
||||
const page2: Row[] = [
|
||||
{ id: '999', value: 1 },
|
||||
{ id: '1000', value: 1 },
|
||||
]
|
||||
const out = await fetchAllRows<Row>(
|
||||
pagedQuery({ 0: page1, [PAGE_SIZE]: page2 }),
|
||||
{ dedupeBy: (r) => r.id },
|
||||
)
|
||||
// 1001 unique ids (0..1000), the duplicate "999" removed → no doubling.
|
||||
expect(out).toHaveLength(PAGE_SIZE + 1)
|
||||
const ids = out.map((r) => r.id)
|
||||
expect(ids.filter((id) => id === '999')).toHaveLength(1)
|
||||
expect(new Set(ids).size).toBe(out.length)
|
||||
})
|
||||
|
||||
it('without dedupeBy, cross-page duplicates pass through (unsafe default)', async () => {
|
||||
const page1 = makeRows(0, PAGE_SIZE)
|
||||
const page2: Row[] = [{ id: '999', value: 1 }]
|
||||
const out = await fetchAllRows<Row>(pagedQuery({ 0: page1, [PAGE_SIZE]: page2 }))
|
||||
expect(out).toHaveLength(PAGE_SIZE + 1)
|
||||
expect(out.map((r) => r.id).filter((id) => id === '999')).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('dedupeBy is a no-op for a single page (no cross-page duplicates possible)', async () => {
|
||||
const rows: Row[] = [
|
||||
{ id: 'a' },
|
||||
{ id: 'b' },
|
||||
{ id: 'a' }, // an intra-page repeat is left untouched — single page is trusted
|
||||
]
|
||||
const out = await fetchAllRows<Row>(pagedQuery({ 0: rows }), { dedupeBy: (r) => r.id })
|
||||
expect(out).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
@@ -1,11 +1,41 @@
|
||||
import { createLogger } from '@/lib/logger'
|
||||
|
||||
const log = createLogger('fetch-all')
|
||||
|
||||
const PAGE_SIZE = 1000
|
||||
|
||||
export interface FetchAllRowsOptions<T> {
|
||||
/**
|
||||
* Stable de-duplication key. When supplied AND more than one page was
|
||||
* fetched, rows are de-duplicated by this key after all pages are collected,
|
||||
* and a warn is logged if any duplicates were dropped.
|
||||
*
|
||||
* This is a safety net, NOT the fix: PostgREST `.range()` paging is only
|
||||
* correct when the underlying query has a stable TOTAL order (see the
|
||||
* ordering invariant below). If a duplicate is ever observed here it means a
|
||||
* caller's query is missing that `.order()` — the warn surfaces the
|
||||
* regression in logs instead of letting it silently double financial totals.
|
||||
* Note this only catches *duplicates*; *skipped* rows can only be prevented
|
||||
* by ordering on a unique column at the call site.
|
||||
*/
|
||||
dedupeBy?: (row: T) => string | number
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all rows from a Supabase query by paginating through results.
|
||||
* Overcomes PostgREST's default 1000-row limit.
|
||||
*
|
||||
* **Ordering invariant:** any query that can return more than `PAGE_SIZE` rows
|
||||
* MUST `.order()` on a unique column (e.g. the table's `id` PK). Postgres
|
||||
* returns rows in an undefined order that can differ between the two `.range()`
|
||||
* requests, so without a stable total order, rows on a page boundary are
|
||||
* silently DUPLICATED and/or SKIPPED across pages. For aggregating reports
|
||||
* (general ledger, trial balance, grundbok) that means doubled or missing
|
||||
* balances. Order is purely for paging stability — callers that need a
|
||||
* different display order should re-sort after fetching.
|
||||
*
|
||||
* The callback receives `{ from, to }` range values — append `.range(from, to)`
|
||||
* to your query builder:
|
||||
* to your query builder, AFTER a stable `.order()`:
|
||||
*
|
||||
* ```ts
|
||||
* const accounts = await fetchAllRows(({ from, to }) =>
|
||||
@@ -13,27 +43,62 @@ const PAGE_SIZE = 1000
|
||||
* .from('chart_of_accounts')
|
||||
* .select('account_number, account_name')
|
||||
* .eq('company_id', companyId)
|
||||
* .order('account_number', { ascending: true }) // stable total order
|
||||
* .range(from, to)
|
||||
* )
|
||||
* ```
|
||||
*
|
||||
* Pass `{ dedupeBy }` as defense-in-depth for queries where a missing/regressed
|
||||
* order would corrupt money:
|
||||
*
|
||||
* ```ts
|
||||
* const lines = await fetchAllRows(
|
||||
* ({ from, to }) => q.order('id').range(from, to),
|
||||
* { dedupeBy: (r) => r.id },
|
||||
* )
|
||||
* ```
|
||||
*/
|
||||
export async function fetchAllRows<T>(
|
||||
queryFn: (range: { from: number; to: number }) => PromiseLike<{
|
||||
data: T[] | null
|
||||
error: { message: string } | null
|
||||
}>
|
||||
}>,
|
||||
options?: FetchAllRowsOptions<T>
|
||||
): Promise<T[]> {
|
||||
const allRows: T[] = []
|
||||
let from = 0
|
||||
let pages = 0
|
||||
|
||||
while (true) {
|
||||
const { data, error } = await queryFn({ from, to: from + PAGE_SIZE - 1 })
|
||||
if (error) throw new Error(error.message)
|
||||
if (!data || data.length === 0) break
|
||||
allRows.push(...data)
|
||||
pages += 1
|
||||
if (data.length < PAGE_SIZE) break
|
||||
from += PAGE_SIZE
|
||||
}
|
||||
|
||||
// Duplicates are only possible across page boundaries, so single-page results
|
||||
// never need the dedup pass.
|
||||
if (options?.dedupeBy && pages > 1) {
|
||||
const seen = new Set<string | number>()
|
||||
const deduped: T[] = []
|
||||
for (const row of allRows) {
|
||||
const key = options.dedupeBy(row)
|
||||
if (seen.has(key)) continue
|
||||
seen.add(key)
|
||||
deduped.push(row)
|
||||
}
|
||||
const dropped = allRows.length - deduped.length
|
||||
if (dropped > 0) {
|
||||
log.warn(
|
||||
'fetchAllRows dropped duplicate rows across pages — a paginated query is missing a stable .order() on a unique column',
|
||||
{ dropped, total: allRows.length, pages }
|
||||
)
|
||||
return deduped
|
||||
}
|
||||
}
|
||||
|
||||
return allRows
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user