fix(workbench): numerisk decimal → källans exakta sträng #446
@@ -13,7 +13,7 @@ function isCalendarDate(year: number, month: number, day: number): boolean {
|
||||
return day <= days[month - 1]!;
|
||||
}
|
||||
|
||||
const pad = (n: number): string => String(n).padStart(2, "0");
|
||||
const pad = (n: number): string => String(n).padStart(2, '0');
|
||||
|
||||
/** Returns every unambiguous ISO rendering of the span (empty when ambiguous). */
|
||||
export function dateCandidates(text: string): string[] {
|
||||
@@ -43,31 +43,44 @@ export function dateCandidates(text: string): string[] {
|
||||
|
||||
/** Returns every unambiguous decimal rendering of the span. */
|
||||
export function decimalCandidates(text: string): string[] {
|
||||
const t = text.trim().replace(/[\s\u00a0\u202f]/g, "");
|
||||
const t = text.trim().replace(/[\s\u00a0\u202f]/g, '');
|
||||
if (!/^[+-]?[\d.,]+$/.test(t) || !/\d/.test(t)) return [];
|
||||
const sign = t.startsWith("-") ? "-" : "";
|
||||
const body = t.replace(/^[+-]/, "");
|
||||
const hasComma = body.includes(",");
|
||||
const hasDot = body.includes(".");
|
||||
const sign = t.startsWith('-') ? '-' : '';
|
||||
const body = t.replace(/^[+-]/, '');
|
||||
const hasComma = body.includes(',');
|
||||
const hasDot = body.includes('.');
|
||||
const out = new Set<string>();
|
||||
const add = (value: string) => {
|
||||
if (/^(0|[1-9]\d*)(\.\d+)?$/.test(value)) out.add(`${sign}${value}`);
|
||||
};
|
||||
if (hasComma && hasDot) {
|
||||
const lastComma = body.lastIndexOf(",");
|
||||
const lastDot = body.lastIndexOf(".");
|
||||
if (lastComma > lastDot) add(body.replace(/\./g, "").replace(",", "."));
|
||||
else add(body.replace(/,/g, ""));
|
||||
const lastComma = body.lastIndexOf(',');
|
||||
const lastDot = body.lastIndexOf('.');
|
||||
if (lastComma > lastDot) add(body.replace(/\./g, '').replace(',', '.'));
|
||||
else add(body.replace(/,/g, ''));
|
||||
} else if (hasComma) {
|
||||
const parts = body.split(",");
|
||||
const parts = body.split(',');
|
||||
if (parts.length === 2 && parts[1]!.length <= 2) add(`${parts[0]}.${parts[1]}`);
|
||||
add(body.replace(/,/g, ""));
|
||||
add(body.replace(/,/g, ''));
|
||||
} else if (hasDot) {
|
||||
const parts = body.split(".");
|
||||
const parts = body.split('.');
|
||||
if (parts.length === 2 && parts[1]!.length <= 2) add(body);
|
||||
add(body.replace(/\./g, ""));
|
||||
add(body.replace(/\./g, ''));
|
||||
} else {
|
||||
add(body);
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical decimal string: no exponent, no leading '+', no trailing zeros in
|
||||
* the fraction. Used only for equality checks; the delivered value keeps its
|
||||
* original precision.
|
||||
*/
|
||||
export function canonicalDecimal(value: string): string | undefined {
|
||||
const match = /^([+-]?)(0|[1-9]\d*)(?:\.(\d+))?$/.exec(value.trim());
|
||||
if (!match) return undefined;
|
||||
const sign = match[1] === '-' ? '-' : '';
|
||||
const fraction = (match[3] ?? '').replace(/0+$/, '');
|
||||
return `${sign}${match[2]}${fraction ? `.${fraction}` : ''}`;
|
||||
}
|
||||
|
||||
@@ -15,9 +15,10 @@ function collectMatches(
|
||||
text: string,
|
||||
field: OutputFieldContract,
|
||||
raw: unknown,
|
||||
): Array<{ start: number; end: number }> {
|
||||
const matches: Array<{ start: number; end: number }> = [];
|
||||
const push = (start: number, end: number) => matches.push({ start, end });
|
||||
): Array<{ start: number; end: number; normalized?: string }> {
|
||||
const matches: Array<{ start: number; end: number; normalized?: string }> = [];
|
||||
const push = (start: number, end: number, normalized?: string) =>
|
||||
matches.push(normalized === undefined ? { start, end } : { start, end, normalized });
|
||||
if (field.type === 'string') {
|
||||
if (typeof raw !== 'string' || raw.length === 0) return [];
|
||||
let index = text.indexOf(raw);
|
||||
@@ -35,11 +36,17 @@ function collectMatches(
|
||||
return matches;
|
||||
}
|
||||
if (field.type === 'decimal') {
|
||||
if (typeof raw !== 'string') return [];
|
||||
if (typeof raw !== 'string' && !(typeof raw === 'number' && Number.isFinite(raw))) return [];
|
||||
for (const match of text.matchAll(/[+-]?\d+(?:[\s\u00a0\u202f.,]\d+)*/g)) {
|
||||
if (decimalCandidates(match[0]).includes(raw)) {
|
||||
push(match.index!, match.index! + match[0].length);
|
||||
const candidates = decimalCandidates(match[0]);
|
||||
if (typeof raw === 'string') {
|
||||
if (candidates.includes(raw)) push(match.index!, match.index! + match[0].length);
|
||||
continue;
|
||||
}
|
||||
// A JSON number lost the source formatting. Accept a numeric match and
|
||||
// deliver the SOURCE's decimal string, so precision is never rounded.
|
||||
const matched = candidates.find((candidate) => Number(candidate) === raw);
|
||||
if (matched !== undefined) push(match.index!, match.index! + match[0].length, matched);
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
@@ -63,6 +70,30 @@ function collectMatches(
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic type coercion for values a local model typed differently.
|
||||
* The contract is about the VALUE, not the JSON type: a decimal delivered as
|
||||
* a JSON number and an integer delivered as a digit string are the same value.
|
||||
* Anything that does not coerce cleanly is left untouched and fails validation.
|
||||
*/
|
||||
function coerceFieldValues(
|
||||
value: Record<string, unknown>,
|
||||
fields: readonly OutputFieldContract[],
|
||||
): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = { ...value };
|
||||
for (const field of fields) {
|
||||
if (!Object.prototype.hasOwnProperty.call(out, field.name)) continue;
|
||||
const raw = out[field.name];
|
||||
if (field.type === 'integer' && typeof raw === 'string' && /^-?(0|[1-9]\d*)$/.test(raw.trim())) {
|
||||
const parsed = Number(raw.trim());
|
||||
if (Number.isSafeInteger(parsed)) out[field.name] = parsed;
|
||||
} else if (field.type === 'boolean' && (raw === 'true' || raw === 'false')) {
|
||||
out[field.name] = raw === 'true';
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic evidence for values the model returned.
|
||||
*
|
||||
@@ -85,8 +116,10 @@ function evidenceFromValues(
|
||||
}
|
||||
const raw = value[field.name];
|
||||
const matches: FieldEvidence[] = [];
|
||||
let normalized = raw;
|
||||
for (const source of sources) {
|
||||
for (const span of collectMatches(source.text, field, raw)) {
|
||||
if (span.normalized !== undefined) normalized = span.normalized;
|
||||
matches.push({
|
||||
field: field.name,
|
||||
sourceId: source.id,
|
||||
@@ -98,6 +131,8 @@ function evidenceFromValues(
|
||||
}
|
||||
if (matches.length !== 1) return undefined;
|
||||
evidence.push(matches[0]!);
|
||||
const out = value;
|
||||
out[field.name] = normalized;
|
||||
}
|
||||
return evidence;
|
||||
}
|
||||
@@ -218,6 +253,7 @@ export async function structureDocument(input: {
|
||||
asObject = candidate;
|
||||
}
|
||||
if (!asObject) return { status: 'needs_review', issues: [{ field: '', code: 'invalid_output' }], model };
|
||||
asObject = coerceFieldValues(asObject, snapshot.fields);
|
||||
|
||||
// 1. Trust model-supplied evidence only if it validates as-is.
|
||||
if (Array.isArray((candidate as { evidence?: unknown }).evidence)) {
|
||||
|
||||
@@ -12,7 +12,10 @@ export interface OutputFieldContract {
|
||||
type: OutputFieldType;
|
||||
required: boolean;
|
||||
}
|
||||
export interface TextSource { id: string; text: string }
|
||||
export interface TextSource {
|
||||
id: string;
|
||||
text: string;
|
||||
}
|
||||
export interface FieldEvidence {
|
||||
field: string;
|
||||
sourceId: string;
|
||||
@@ -20,10 +23,12 @@ export interface FieldEvidence {
|
||||
end: number;
|
||||
transform: 'exact' | 'trim' | 'integer' | 'boolean' | 'decimal' | 'date';
|
||||
}
|
||||
export interface ValidationIssue { field: string; code: string }
|
||||
export interface ValidationIssue {
|
||||
field: string;
|
||||
code: string;
|
||||
}
|
||||
export type OutputValidation =
|
||||
| { status: 'mechanically_validated'; issues: [] }
|
||||
| { status: 'needs_review'; issues: ValidationIssue[] };
|
||||
{ status: 'mechanically_validated'; issues: [] } | { status: 'needs_review'; issues: ValidationIssue[] };
|
||||
|
||||
/** ISO dates are validated as calendar dates without timezone conversion. */
|
||||
function isIsoDate(value: string): boolean {
|
||||
@@ -47,59 +52,109 @@ export function validateStructuredOutput(input: {
|
||||
}): OutputValidation {
|
||||
const issues: ValidationIssue[] = [];
|
||||
const add = (field: string, code: string) => issues.push({ field, code });
|
||||
if (!Array.isArray(input.fields) || input.fields.length === 0 || !Array.isArray(input.sources) || !Array.isArray(input.evidence)) {
|
||||
if (
|
||||
!Array.isArray(input.fields) ||
|
||||
input.fields.length === 0 ||
|
||||
!Array.isArray(input.sources) ||
|
||||
!Array.isArray(input.evidence)
|
||||
) {
|
||||
return { status: 'needs_review', issues: [{ field: '', code: 'invalid_contract' }] };
|
||||
}
|
||||
const fields = new Map<string, OutputFieldContract>();
|
||||
for (const field of input.fields) {
|
||||
if (!field || typeof field.name !== 'string' || !field.name.trim() || fields.has(field.name)
|
||||
|| !['string', 'integer', 'boolean', 'decimal', 'date'].includes(field.type) || typeof field.required !== 'boolean') {
|
||||
if (
|
||||
!field ||
|
||||
typeof field.name !== 'string' ||
|
||||
!field.name.trim() ||
|
||||
fields.has(field.name) ||
|
||||
!['string', 'integer', 'boolean', 'decimal', 'date'].includes(field.type) ||
|
||||
typeof field.required !== 'boolean'
|
||||
) {
|
||||
return { status: 'needs_review', issues: [{ field: '', code: 'invalid_contract' }] };
|
||||
}
|
||||
fields.set(field.name, field);
|
||||
}
|
||||
const sources = new Map<string, string>();
|
||||
for (const source of input.sources) {
|
||||
if (!source || typeof source.id !== 'string' || !source.id.trim() || typeof source.text !== 'string' || sources.has(source.id)) {
|
||||
if (
|
||||
!source ||
|
||||
typeof source.id !== 'string' ||
|
||||
!source.id.trim() ||
|
||||
typeof source.text !== 'string' ||
|
||||
sources.has(source.id)
|
||||
) {
|
||||
return { status: 'needs_review', issues: [{ field: '', code: 'invalid_source' }] };
|
||||
}
|
||||
sources.set(source.id, source.text);
|
||||
}
|
||||
if (!input.value || typeof input.value !== 'object' || Array.isArray(input.value)
|
||||
|| ![Object.prototype, null].includes(Object.getPrototypeOf(input.value))) {
|
||||
if (
|
||||
!input.value ||
|
||||
typeof input.value !== 'object' ||
|
||||
Array.isArray(input.value) ||
|
||||
![Object.prototype, null].includes(Object.getPrototypeOf(input.value))
|
||||
) {
|
||||
return { status: 'needs_review', issues: [{ field: '', code: 'invalid_output' }] };
|
||||
}
|
||||
const output = input.value as Record<string, unknown>;
|
||||
for (const key of Object.keys(output)) if (!fields.has(key)) add(key, 'unexpected_field');
|
||||
for (const evidence of input.evidence) {
|
||||
if (!evidence || typeof evidence.field !== 'string' || !fields.has(evidence.field)
|
||||
|| !Object.prototype.hasOwnProperty.call(output, evidence.field)) add(evidence?.field ?? '', 'unexpected_evidence');
|
||||
if (
|
||||
!evidence ||
|
||||
typeof evidence.field !== 'string' ||
|
||||
!fields.has(evidence.field) ||
|
||||
!Object.prototype.hasOwnProperty.call(output, evidence.field)
|
||||
)
|
||||
add(evidence?.field ?? '', 'unexpected_evidence');
|
||||
}
|
||||
for (const [name, field] of fields) {
|
||||
const present = Object.prototype.hasOwnProperty.call(output, name);
|
||||
if (!present) { if (field.required) add(name, 'missing_required'); continue; }
|
||||
if (!present) {
|
||||
if (field.required) add(name, 'missing_required');
|
||||
continue;
|
||||
}
|
||||
const value = output[name];
|
||||
if ((['string', 'decimal', 'date'].includes(field.type) && typeof value !== 'string')
|
||||
|| (field.type === 'boolean' && typeof value !== 'boolean')
|
||||
|| (field.type === 'integer' && (typeof value !== 'number' || !Number.isSafeInteger(value)))) {
|
||||
add(name, 'invalid_type'); continue;
|
||||
if (
|
||||
(['string', 'decimal', 'date'].includes(field.type) && typeof value !== 'string') ||
|
||||
(field.type === 'boolean' && typeof value !== 'boolean') ||
|
||||
(field.type === 'integer' && (typeof value !== 'number' || !Number.isSafeInteger(value)))
|
||||
) {
|
||||
add(name, 'invalid_type');
|
||||
continue;
|
||||
}
|
||||
const evidence = input.evidence.filter((entry) => entry?.field === name);
|
||||
if (evidence.length !== 1) { add(name, 'evidence_required'); continue; }
|
||||
if (evidence.length !== 1) {
|
||||
add(name, 'evidence_required');
|
||||
continue;
|
||||
}
|
||||
const ref = evidence[0]!;
|
||||
const text = sources.get(ref.sourceId);
|
||||
if (typeof text !== 'string' || !Number.isSafeInteger(ref.start) || !Number.isSafeInteger(ref.end)
|
||||
|| ref.start < 0 || ref.end <= ref.start || ref.end > text.length) {
|
||||
add(name, 'invalid_source_range'); continue;
|
||||
if (
|
||||
typeof text !== 'string' ||
|
||||
!Number.isSafeInteger(ref.start) ||
|
||||
!Number.isSafeInteger(ref.end) ||
|
||||
ref.start < 0 ||
|
||||
ref.end <= ref.start ||
|
||||
ref.end > text.length
|
||||
) {
|
||||
add(name, 'invalid_source_range');
|
||||
continue;
|
||||
}
|
||||
const excerpt = text.slice(ref.start, ref.end);
|
||||
let matched = false;
|
||||
if (field.type === 'string' && ref.transform === 'exact') matched = excerpt === value;
|
||||
else if (field.type === 'string' && ref.transform === 'trim') matched = excerpt.trim() === value;
|
||||
else if (field.type === 'integer' && ref.transform === 'integer' && /^-?(0|[1-9][0-9]*)$/.test(excerpt.trim())) {
|
||||
else if (
|
||||
field.type === 'integer' &&
|
||||
ref.transform === 'integer' &&
|
||||
/^-?(0|[1-9][0-9]*)$/.test(excerpt.trim())
|
||||
) {
|
||||
const number = Number(excerpt.trim());
|
||||
matched = Number.isSafeInteger(number) && number === value;
|
||||
} else if (field.type === 'boolean' && ref.transform === 'boolean' && ['true', 'false'].includes(excerpt.trim())) {
|
||||
} else if (
|
||||
field.type === 'boolean' &&
|
||||
ref.transform === 'boolean' &&
|
||||
['true', 'false'].includes(excerpt.trim())
|
||||
) {
|
||||
matched = (excerpt.trim() === 'true') === value;
|
||||
} else if (field.type === 'decimal' && ref.transform === 'decimal' && typeof value === 'string') {
|
||||
// Keep the exact decimal string: floating point would lose money precision.
|
||||
@@ -111,5 +166,7 @@ export function validateStructuredOutput(input: {
|
||||
}
|
||||
if (!matched) add(name, 'evidence_mismatch');
|
||||
}
|
||||
return issues.length ? { status: 'needs_review', issues } : { status: 'mechanically_validated', issues: [] };
|
||||
return issues.length
|
||||
? { status: 'needs_review', issues }
|
||||
: { status: 'mechanically_validated', issues: [] };
|
||||
}
|
||||
|
||||
@@ -181,6 +181,21 @@ describe('structure document workflow', () => {
|
||||
expect(out.evidence.map((e) => e.transform)).toEqual(['decimal', 'date']);
|
||||
expect(out.evidence[0]).toMatchObject({ start: 8, end: 17 });
|
||||
});
|
||||
it('rewrites a JSON-number decimal to the exact source string (no rounding)', async () => {
|
||||
const out = await structureDocument({
|
||||
request,
|
||||
countTokens: vi.fn(async () => 50),
|
||||
instructions: 'Extract the amount',
|
||||
fields: [{ name: 'amount', type: 'decimal' as const, required: true }],
|
||||
sources: [{ id: 'p1', text: 'Belopp: 12 500,00 SEK' }],
|
||||
gateway: {
|
||||
complete: vi.fn(async () => ({ ...response, content: JSON.stringify({ amount: 12500 }) })),
|
||||
},
|
||||
});
|
||||
expect(out.status).toBe('mechanically_validated');
|
||||
if (out.status !== 'mechanically_validated') throw new Error('expected validated');
|
||||
expect(out.value.amount).toBe('12500.00');
|
||||
});
|
||||
it('rejects malformed contracts before a model call', async () => {
|
||||
const complete = vi.fn(async () => response);
|
||||
await expect(structureDocument({ ...base, fields: [], gateway: { complete } })).rejects.toThrow(
|
||||
|
||||
Reference in New Issue
Block a user