feat: add 0pt.query_source, 0pt.document_ocr, 0pt.write_external_record capabilities
Three new, tenant-agnostic P0 capabilities: - 0pt.query_source: bounded, timeout/row-capped ad-hoc SQL query execution against registered database sources, via a Fabric/OneLake adapter (Azure AD credential-based mssql connection), with AUD0 audit events and an app/api/0pt/sources/[sourceId]/query endpoint. - 0pt.document_ocr (document capture + extraction): mailbox-based document intake via Microsoft Graph, OCR/field extraction via Azure Document Intelligence, with capture/extraction API endpoints and services that never fabricate extracted values. - 0pt.write_external_record (external writeback): a generic writeback-adapter contract plus a concrete Odoo bills adapter that matches partners via Fabric, backing an external-writeback-service for pushing records to downstream systems of record. Known interim gaps: - src/adapters/local-blob-store.ts is an interim ST0RE substitute pending the real ST0RE artifact-store integration. - mssql is used by the Fabric/OneLake adapter but deliberately not yet added to package.json (pending `pnpm add mssql` once the concurrent session's package.json/dependency changes settle). A separate, concurrently-active session has unrelated in-flight changes in this same working tree (data-platform catalog/lineage/feature-store, dataset quality, lakehouse/Iceberg docs, deploy scaffolding, package.json/ pnpm-lock.yaml updates, etc.) — none of that is touched or included here. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,31 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { documentExtractionService } from "@/src/services/document-extraction-service";
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) {
|
||||
try {
|
||||
const { documentId } = await params;
|
||||
const body = await request.json();
|
||||
const result = await documentExtractionService.extractDocument(documentId, {
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
model_id: body.model_id,
|
||||
});
|
||||
return NextResponse.json(result, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return NextResponse.json({ error: message }, { status: 400 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(_request: NextRequest, { params }: { params: Promise<{ documentId: string }> }) {
|
||||
try {
|
||||
const { documentId } = await params;
|
||||
const extractions = await documentExtractionService.listExtractionsForDocument(documentId);
|
||||
return NextResponse.json(extractions);
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return NextResponse.json({ error: message }, { status: 404 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { documentCaptureService } from "@/src/services/document-capture-service";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const result = await documentCaptureService.captureFromMailbox(
|
||||
{
|
||||
mailbox: {
|
||||
tenantId: body.mailbox?.tenant_id,
|
||||
clientId: body.mailbox?.client_id,
|
||||
clientSecret: body.mailbox?.client_secret,
|
||||
mailboxAddress: body.mailbox?.mailbox_address,
|
||||
folder: body.mailbox?.folder,
|
||||
},
|
||||
since_iso: body.since_iso,
|
||||
top: body.top,
|
||||
max_pages: body.max_pages,
|
||||
},
|
||||
{
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
},
|
||||
);
|
||||
return NextResponse.json(result, { status: 201 });
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return NextResponse.json({ error: message }, { status: 400 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const tenantId = searchParams.get("tenant_id") ?? undefined;
|
||||
const projectId = searchParams.get("project_id") ?? undefined;
|
||||
const documents = await documentCaptureService.listDocuments({ tenant_id: tenantId, project_id: projectId });
|
||||
return NextResponse.json(documents);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
return NextResponse.json({
|
||||
status: "ok",
|
||||
product: "0pt",
|
||||
version: "1.0.0",
|
||||
endpoints: [
|
||||
"/api/0pt/sources",
|
||||
"/api/0pt/datasets",
|
||||
"/api/0pt/data-contracts",
|
||||
"/api/0pt/quality/checks",
|
||||
"/api/0pt/quality/reports",
|
||||
"/api/0pt/features/views",
|
||||
"/api/0pt/documents/capture",
|
||||
"/api/0pt/documents/{documentId}/extraction",
|
||||
],
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* app/api/0pt/sources/[sourceId]/query/route.ts
|
||||
* ================================================
|
||||
* 0pt.query_source — bounded synchronous query execution for a registered
|
||||
* Source. Additive: does not modify sibling ../route.ts or ../../route.ts.
|
||||
*
|
||||
* NOTE — in-memory scope: this module instantiates its own SourceService,
|
||||
* matching the existing sources/route.ts and sources/[sourceId]/route.ts
|
||||
* pattern (each API route module owns a fresh in-memory registry, so state
|
||||
* is not shared across route modules yet — see the "Singleton note" in
|
||||
* src/services/document-capture-service.ts for prior art on this same repo
|
||||
* limitation). A source registered via POST /api/0pt/sources will not be
|
||||
* resolvable here until SourceService is backed by shared persistence
|
||||
* (sql/002_p0_dataset_quality.sql already defines `opt_sources` for that).
|
||||
* Details: docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*/
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { SourceService } from "@/src/services/source-service";
|
||||
import { SourceQueryService, SourceQueryTimeoutError, SourceQueryUnsupportedError } from "@/src/services/source-query-service";
|
||||
|
||||
const sourceService = new SourceService();
|
||||
const queryService = new SourceQueryService({ sourceService });
|
||||
|
||||
export async function POST(request: NextRequest, { params }: { params: Promise<{ sourceId: string }> }) {
|
||||
try {
|
||||
const { sourceId } = await params;
|
||||
const body = await request.json();
|
||||
|
||||
if (typeof body.query !== "string" || body.query.trim().length === 0) {
|
||||
return NextResponse.json({ error: "query (string) is required" }, { status: 400 });
|
||||
}
|
||||
if (typeof body.tenant_id !== "string" || typeof body.project_id !== "string") {
|
||||
return NextResponse.json({ error: "tenant_id and project_id are required" }, { status: 400 });
|
||||
}
|
||||
if (!body.connection_config || typeof body.connection_config !== "object") {
|
||||
return NextResponse.json({ error: "connection_config is required" }, { status: 400 });
|
||||
}
|
||||
|
||||
const result = await queryService.querySource({
|
||||
source_id: sourceId,
|
||||
query: body.query,
|
||||
params: body.params,
|
||||
tenant_id: body.tenant_id,
|
||||
project_id: body.project_id,
|
||||
actor_id: body.actor_id,
|
||||
correlation_id: body.correlation_id,
|
||||
max_rows: body.max_rows,
|
||||
timeout_ms: body.timeout_ms,
|
||||
connection_config: body.connection_config,
|
||||
});
|
||||
|
||||
return NextResponse.json(result, { status: 200 });
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
if (error instanceof SourceQueryTimeoutError) {
|
||||
return NextResponse.json({ error: message }, { status: 504 });
|
||||
}
|
||||
if (error instanceof SourceQueryUnsupportedError) {
|
||||
return NextResponse.json({ error: message }, { status: 422 });
|
||||
}
|
||||
const status = /not found/i.test(message) ? 404 : 400;
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
# 0pt.query_source — Bounded Source Query Flow
|
||||
|
||||
Additive companion to `docs/P0_DATASET_QUALITY_FLOW.md`. Adds the capability
|
||||
to actually **read data** from a registered Source, not just register its
|
||||
metadata.
|
||||
|
||||
## What this is
|
||||
|
||||
`0pt.query_source`: given a Source already registered via `0pt.create_source`
|
||||
(`source_type = "database"`, `adapter_id = "fabric_onelake_sql"`), run a
|
||||
caller-supplied SQL query against the customer's own Azure Fabric / OneLake
|
||||
SQL analytics endpoint and return rows — synchronously, bounded by a max-row
|
||||
and max-time ceiling.
|
||||
|
||||
**Tenant-agnostic by construction.** Any SIAX customer registers their own
|
||||
Fabric/OneLake warehouse (their own Entra tenant, their own service
|
||||
principal, their own SQL endpoint). Nothing in the adapter or service layer
|
||||
is hardcoded to one customer — every credential and endpoint is a parameter
|
||||
supplied per call, never an env var fallback.
|
||||
|
||||
## Why bounded + synchronous, not just "run any query"
|
||||
|
||||
This path is for small, interactive/preview-style reads (validate a Source
|
||||
is reachable, sample a table, preview a query before scheduling it). It is
|
||||
*not* a general query engine:
|
||||
|
||||
- `max_rows`: default 1,000, hard ceiling 10,000 (`SOURCE_QUERY_DEFAULT_MAX_ROWS` /
|
||||
`SOURCE_QUERY_HARD_MAX_ROWS` in `source-query-service.ts`). Excess rows are
|
||||
truncated and the response is flagged `truncated: true`.
|
||||
- `timeout_ms`: default 10s, hard ceiling 30s (`SOURCE_QUERY_DEFAULT_TIMEOUT_MS` /
|
||||
`SOURCE_QUERY_HARD_MAX_TIMEOUT_MS`). A caller-requested value above the
|
||||
ceiling is clamped down, not rejected.
|
||||
- Anything heavier — large exports, long scans, scheduled/recurring pulls —
|
||||
must go through **N0D as a job** instead, the same split
|
||||
`quality-check-service.ts` already uses (`runQualityChecks` → CL0UD policy
|
||||
→ N0D job, not an inline synchronous call).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Adapter: src/adapters/fabric-onelake-adapter.ts
|
||||
- acquireFabricAccessToken(): Entra service-principal OAuth2
|
||||
client-credentials flow (POST login.microsoftonline.com/{tenantId}
|
||||
/oauth2/v2.0/token) — generalized from v0-datahub's
|
||||
scripts/ops/cloud-bootstrap.mjs `aadToken()`.
|
||||
- FabricOneLakeAdapter.query(): opens a TDS pool to the Fabric SQL
|
||||
analytics endpoint using that token (azure-active-directory-access-token
|
||||
auth), same shape as v0-datahub's lib/warehouse/fabric-sql.ts
|
||||
`buildConfig()`/`runFabricQuery()`, generalized to caller-supplied
|
||||
tenant/client/secret/endpoint/database instead of one customer's env vars.
|
||||
|
||||
Service: src/services/source-query-service.ts
|
||||
- SourceQueryService.querySource(): looks up the Source via the existing
|
||||
SourceService.getSource() (read-only — does not modify source-service.ts),
|
||||
validates source_type/adapter_id, clamps max_rows/timeout_ms, races the
|
||||
adapter query against a timeout, truncates rows, emits an AUD0 event,
|
||||
always releases the adapter's connection.
|
||||
|
||||
Route: app/api/0pt/sources/[sourceId]/query/route.ts
|
||||
- POST { query, params?, tenant_id, project_id, actor_id?, correlation_id?,
|
||||
max_rows?, timeout_ms?, connection_config } -> QuerySourceResult
|
||||
- 400 on missing query/tenant_id/project_id/connection_config
|
||||
- 404 when the source does not exist
|
||||
- 422 (SourceQueryUnsupportedError) when source_type != "database" or
|
||||
adapter_id != "fabric_onelake_sql"
|
||||
- 504 (SourceQueryTimeoutError) on timeout
|
||||
|
||||
AUD0: event_type "0pt.source_queried", capability_id "0pt.query_source",
|
||||
resource_type "source", risk_level "L4" (live external data access with a
|
||||
tenant's own credentials — one level above the "L3" used for Source/
|
||||
Dataset/QualityCheck metadata writes, matching the "L4 = restricted
|
||||
sensitivity" precedent in src/data-platform/services/catalog-service.ts).
|
||||
Emitted on both success (ok:true, row_count, truncated, duration_ms) and
|
||||
failure (ok:false, error) via the same `Aud0Client` pattern as
|
||||
source-service.ts / quality-check-service.ts.
|
||||
```
|
||||
|
||||
## Known gaps / deliberately left out
|
||||
|
||||
- **`mssql` is not a declared dependency yet.** package.json and
|
||||
pnpm-lock.yaml already have unrelated uncommitted changes from a separate
|
||||
in-flight session at the time this capability was built, so adding a new
|
||||
dependency there was avoided to prevent a collision. The adapter loads
|
||||
`mssql` lazily via a non-literal dynamic import (works with any bundler/
|
||||
typecheck without the package installed) and throws a clear, actionable
|
||||
error if it's missing. Once package.json is settled, `pnpm add mssql` is
|
||||
the only change needed to make `query()` hit a real Fabric endpoint — no
|
||||
code changes required.
|
||||
- **In-memory SourceService is not shared across API route modules.**
|
||||
Same pre-existing limitation as `sources/route.ts` /
|
||||
`sources/[sourceId]/route.ts` (each Next.js route module instantiates its
|
||||
own `SourceService`, so a Source created via `POST /api/0pt/sources` is not
|
||||
yet visible to `POST /api/0pt/sources/[sourceId]/query` in the same running
|
||||
process). `document-capture-service.ts` hit the same issue and added a
|
||||
shared singleton for its own two routes; doing the same for Source would
|
||||
mean editing `source-service.ts` and/or the two existing `sources/route.ts`
|
||||
files, which was out of scope here to avoid touching that flagged area.
|
||||
The real fix is the same one already pointed at by `sql/002_p0_dataset_quality.sql`
|
||||
(`opt_sources` table) — back `SourceService` with real persistence.
|
||||
- **No CL0UD policy check before executing a query.** `quality-check-service.ts`
|
||||
calls `Cl0udClient.evaluatePolicy()` before creating a N0D job; this
|
||||
synchronous path does not, since it is scoped as a bounded preview read, not
|
||||
a governed job. Worth revisiting once policy-gating on `0pt.query_source`
|
||||
is defined.
|
||||
- **No SQL rewriting/injection of `TOP`/`LIMIT`.** Row bounding is applied to
|
||||
whatever the adapter returns, not pushed into the query itself (arbitrary
|
||||
caller SQL can't be safely rewritten). A query that itself returns far more
|
||||
rows than `max_rows` still executes in full on the warehouse before being
|
||||
truncated client-side — only the *response* is bounded, not warehouse-side
|
||||
compute. Documented, not silently hidden.
|
||||
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* src/adapters/document-intelligence-adapter.ts
|
||||
* ================================================
|
||||
* Tenant-agnostic Azure AI Document Intelligence client.
|
||||
*
|
||||
* Generalized from v0-datahub/lib/invoice-inbox/doc-intelligence.ts, which
|
||||
* hardcoded the `prebuilt-invoice` model and a fixed invoice-shaped return
|
||||
* type (amount_gross, vendor, vat, ...). Here `modelId` is a CALLER-SUPPLIED
|
||||
* parameter — "prebuilt-invoice", "prebuilt-receipt", "prebuilt-layout",
|
||||
* "prebuilt-read", or any custom-trained model id — and the adapter returns
|
||||
* a generic field map instead of assuming any particular document shape.
|
||||
*
|
||||
* Evidence discipline (correctness-critical, not a style choice): a field is
|
||||
* only ever populated with the value + confidence Document Intelligence
|
||||
* actually reported. If DI did not return a value, the field comes back as
|
||||
* { value: null, confidence: <DI's confidence if any, else null>, found: false }.
|
||||
* Nothing here ever defaults, guesses, or backfills a value.
|
||||
*/
|
||||
|
||||
export interface DocumentIntelligenceConfig {
|
||||
endpoint: string;
|
||||
apiKey: string;
|
||||
/** Default: "2024-11-30" — the API version the production invoice pipeline used. */
|
||||
apiVersion?: string;
|
||||
/** Delay between result-polling attempts, ms. Default 2000. (Tests override to run fast.) */
|
||||
pollIntervalMs?: number;
|
||||
/** Delay before retrying after a 429, ms. Default 6000. (Tests override to run fast.) */
|
||||
rateLimitRetryMs?: number;
|
||||
/** Max result-polling attempts before timing out. Default 40. */
|
||||
maxPollAttempts?: number;
|
||||
/** Max analyze-request attempts (each 429 counts as one). Default 4. */
|
||||
maxAnalyzeAttempts?: number;
|
||||
}
|
||||
|
||||
export interface ExtractedField {
|
||||
value: string | number | boolean | null;
|
||||
confidence: number | null;
|
||||
found: boolean;
|
||||
}
|
||||
|
||||
export interface DocumentIntelligenceResult {
|
||||
modelId: string;
|
||||
apiVersion: string;
|
||||
/** Full extracted text, when the API returns one (present across prebuilt models). */
|
||||
content: string | null;
|
||||
/** Top-level structured fields (documents[0].fields), normalized. Empty for pure layout/read. */
|
||||
fields: Record<string, ExtractedField>;
|
||||
/** Array-of-object fields (e.g. invoice/receipt line items), each item pre-normalized. */
|
||||
arrayFields: Record<string, Array<Record<string, ExtractedField>>>;
|
||||
/** documents[0].confidence, when the model reports a whole-document confidence. */
|
||||
documentConfidence: number | null;
|
||||
pageCount: number | null;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Normalize one Document Intelligence field object into an evidence-based ExtractedField. */
|
||||
function normalizeField(raw: unknown): ExtractedField {
|
||||
const f = raw as Record<string, unknown> | null | undefined;
|
||||
if (!f) return { value: null, confidence: null, found: false };
|
||||
let value: string | number | boolean | null = null;
|
||||
switch (f.type) {
|
||||
case "string":
|
||||
value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null;
|
||||
break;
|
||||
case "number":
|
||||
value = (f.valueNumber as number | undefined) ?? null;
|
||||
break;
|
||||
case "integer":
|
||||
value = (f.valueInteger as number | undefined) ?? null;
|
||||
break;
|
||||
case "date":
|
||||
value = (f.valueDate as string | undefined) ?? null;
|
||||
break;
|
||||
case "time":
|
||||
value = (f.valueTime as string | undefined) ?? null;
|
||||
break;
|
||||
case "phoneNumber":
|
||||
value = (f.valuePhoneNumber as string | undefined) ?? null;
|
||||
break;
|
||||
case "countryRegion":
|
||||
value = (f.valueCountryRegion as string | undefined) ?? null;
|
||||
break;
|
||||
case "selectionMark":
|
||||
value = (f.valueSelectionMark as string | undefined) ?? null;
|
||||
break;
|
||||
case "currency": {
|
||||
const cur = f.valueCurrency as Record<string, unknown> | undefined;
|
||||
value = typeof cur?.amount === "number" ? (cur.amount as number) : null;
|
||||
break;
|
||||
}
|
||||
case "boolean":
|
||||
value = typeof f.valueBoolean === "boolean" ? (f.valueBoolean as boolean) : null;
|
||||
break;
|
||||
default:
|
||||
value = (f.valueString as string | undefined) ?? (f.content as string | undefined) ?? null;
|
||||
}
|
||||
const found = value !== null && value !== undefined && value !== "";
|
||||
const confidence = typeof f.confidence === "number" ? (f.confidence as number) : null;
|
||||
return { value: found ? value : null, confidence, found };
|
||||
}
|
||||
|
||||
function normalizeObjectFields(obj: Record<string, unknown> | null | undefined): Record<string, ExtractedField> {
|
||||
const out: Record<string, ExtractedField> = {};
|
||||
for (const [key, raw] of Object.entries(obj ?? {})) {
|
||||
out[key] = normalizeField(raw);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class DocumentIntelligenceAdapter {
|
||||
constructor(private config: DocumentIntelligenceConfig) {}
|
||||
|
||||
static isConfigured(config: Partial<DocumentIntelligenceConfig>): boolean {
|
||||
return !!(config.endpoint && config.apiKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a document buffer with the given model. Throws on any failure —
|
||||
* callers must never interpret a caught error as "no fields found". A
|
||||
* genuinely empty field is represented by { found: false } *inside a
|
||||
* successful result*; a failed call must propagate as an error, not a
|
||||
* fabricated empty success.
|
||||
*/
|
||||
async analyzeDocument(buf: Buffer, modelId: string): Promise<DocumentIntelligenceResult> {
|
||||
const apiVersion = this.config.apiVersion || "2024-11-30";
|
||||
const base = this.config.endpoint.replace(/\/$/, "");
|
||||
const analyzeUrl = `${base}/documentintelligence/documentModels/${encodeURIComponent(modelId)}:analyze?api-version=${apiVersion}`;
|
||||
|
||||
const maxAnalyzeAttempts = this.config.maxAnalyzeAttempts ?? 4;
|
||||
const rateLimitRetryMs = this.config.rateLimitRetryMs ?? 6000;
|
||||
const pollIntervalMs = this.config.pollIntervalMs ?? 2000;
|
||||
const maxPollAttempts = this.config.maxPollAttempts ?? 40;
|
||||
|
||||
let operationLocation = "";
|
||||
for (let attempt = 0; attempt < maxAnalyzeAttempts; attempt++) {
|
||||
const res = await fetch(analyzeUrl, {
|
||||
method: "POST",
|
||||
headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ base64Source: buf.toString("base64") }),
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await sleep(rateLimitRetryMs);
|
||||
continue;
|
||||
}
|
||||
if (res.status !== 202) {
|
||||
const text = await res.text().catch(() => "");
|
||||
throw new Error(`Document Intelligence analyze request failed: ${res.status} ${text.slice(0, 300)}`);
|
||||
}
|
||||
operationLocation = res.headers.get("operation-location") || "";
|
||||
break;
|
||||
}
|
||||
if (!operationLocation) {
|
||||
throw new Error(
|
||||
"Document Intelligence analyze did not return an operation-location (rate-limited after retries, or misconfigured)",
|
||||
);
|
||||
}
|
||||
|
||||
let payload: Record<string, unknown> | null = null;
|
||||
for (let i = 0; i < maxPollAttempts; i++) {
|
||||
await sleep(pollIntervalMs);
|
||||
const pollRes = await fetch(operationLocation, { headers: { "Ocp-Apim-Subscription-Key": this.config.apiKey } });
|
||||
const json = (await pollRes.json()) as Record<string, unknown>;
|
||||
if (json.status === "succeeded") {
|
||||
payload = json;
|
||||
break;
|
||||
}
|
||||
if (json.status === "failed") {
|
||||
throw new Error(`Document Intelligence analyze failed: ${JSON.stringify(json.error ?? json).slice(0, 500)}`);
|
||||
}
|
||||
}
|
||||
if (!payload) {
|
||||
throw new Error("Document Intelligence analyze timed out waiting for a result");
|
||||
}
|
||||
|
||||
const analyzeResult = (payload.analyzeResult as Record<string, unknown>) ?? {};
|
||||
const documents = analyzeResult.documents as Array<Record<string, unknown>> | undefined;
|
||||
const doc = documents?.[0];
|
||||
const rawFields = (doc?.fields as Record<string, unknown>) ?? {};
|
||||
|
||||
const fields: Record<string, ExtractedField> = {};
|
||||
const arrayFields: Record<string, Array<Record<string, ExtractedField>>> = {};
|
||||
for (const [key, raw] of Object.entries(rawFields)) {
|
||||
const f = raw as Record<string, unknown>;
|
||||
if (f?.type === "array" && Array.isArray(f.valueArray)) {
|
||||
arrayFields[key] = (f.valueArray as Array<Record<string, unknown>>).map((item) =>
|
||||
normalizeObjectFields(item?.valueObject as Record<string, unknown> | undefined),
|
||||
);
|
||||
} else {
|
||||
fields[key] = normalizeField(f);
|
||||
}
|
||||
}
|
||||
|
||||
const pages = analyzeResult.pages as unknown[] | undefined;
|
||||
|
||||
return {
|
||||
modelId,
|
||||
apiVersion,
|
||||
content: typeof analyzeResult.content === "string" ? (analyzeResult.content as string) : null,
|
||||
fields,
|
||||
arrayFields,
|
||||
documentConfidence: typeof doc?.confidence === "number" ? (doc.confidence as number) : null,
|
||||
pageCount: Array.isArray(pages) ? pages.length : null,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* src/adapters/fabric-onelake-adapter.ts
|
||||
* ========================================
|
||||
* Tenant-agnostic Azure Fabric / OneLake SQL analytics endpoint adapter
|
||||
* (adapter_id = "fabric_onelake_sql" on a 0pt Source).
|
||||
*
|
||||
* Generalized from two proven patterns in v0-datahub:
|
||||
* - scripts/ops/cloud-bootstrap.mjs `aadToken()` — Entra service-principal
|
||||
* OAuth2 client-credentials flow (POST login.microsoftonline.com/{tenant}
|
||||
* /oauth2/v2.0/token, grant_type=client_credentials).
|
||||
* - lib/warehouse/fabric-sql.ts `buildConfig()` / `runFabricQuery()` — using
|
||||
* that token as an `azure-active-directory-access-token` credential on a
|
||||
* `mssql` TDS connection to the Fabric SQL analytics endpoint, and
|
||||
* returning `result.recordset` as plain rows.
|
||||
*
|
||||
* Both of those originals hardcode ONE customer's tenant/client/secret/
|
||||
* endpoint as module-level constants or env vars (AZ_SP_*, FABRIC_SQL_ENDPOINT,
|
||||
* kv-pts-datahub, ...). Nothing here is hardcoded: tenantId, clientId,
|
||||
* clientSecret, sqlEndpoint and database are always caller-supplied, because
|
||||
* this is a SIAX platform capability — any SIAX customer registers their own
|
||||
* Fabric/OneLake warehouse as a Source and queries it through this adapter.
|
||||
*
|
||||
* Dependency note: `mssql` (the TDS driver used by fabric-sql.ts) is NOT a
|
||||
* declared dependency of this repo yet, and package.json/pnpm-lock.yaml
|
||||
* already have unrelated uncommitted changes from a separate in-flight
|
||||
* session — so it is deliberately not added here to avoid colliding with
|
||||
* that work. The driver is loaded lazily via a non-literal dynamic import
|
||||
* (see `loadDriver`) so:
|
||||
* - this file has zero hard dependency at import/bundle time (Next.js/
|
||||
* webpack cannot statically resolve a non-literal specifier, so it does
|
||||
* not fail the build even though "mssql" isn't installed), and
|
||||
* - `tsc --noEmit` does not attempt to resolve module types for it either
|
||||
* (a widened `string`-typed specifier makes `import(...)` type as
|
||||
* `Promise<any>` instead of triggering module resolution).
|
||||
* Once `mssql` is added to package.json (`pnpm add mssql`), query() works
|
||||
* against a real endpoint with no code change here. Until then, `query()`
|
||||
* throws a clear, actionable error — see docs/P0_QUERY_SOURCE_FLOW.md.
|
||||
*/
|
||||
|
||||
const DEFAULT_SQL_SCOPE = "https://database.windows.net/.default";
|
||||
const TOKEN_REFRESH_SKEW_MS = 60_000;
|
||||
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface FabricOneLakeCredentials {
|
||||
/** Entra (Azure AD) tenant id the service-principal app registration lives in. */
|
||||
tenantId: string;
|
||||
/** Entra app (client) id — app-only, client-credentials grant. */
|
||||
clientId: string;
|
||||
/** Entra app client secret. Never logged, never persisted by this adapter. */
|
||||
clientSecret: string;
|
||||
}
|
||||
|
||||
export interface FabricOneLakeConnectionConfig {
|
||||
credentials: FabricOneLakeCredentials;
|
||||
/** Fabric SQL analytics endpoint host, e.g. "xxxx.datawarehouse.fabric.microsoft.com" (scheme optional). */
|
||||
sqlEndpoint: string;
|
||||
/** Target database / lakehouse SQL endpoint name, e.g. "lh_customer_datahub". */
|
||||
database: string;
|
||||
/** AAD scope used for the SQL token. Defaults to the Azure SQL / Fabric warehouse resource scope. */
|
||||
scope?: string;
|
||||
/** Per-request timeout passed to the mssql pool config. Default 30s. */
|
||||
requestTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface FabricAccessToken {
|
||||
accessToken: string;
|
||||
expiresAtEpochMs: number;
|
||||
}
|
||||
|
||||
export interface AdapterQueryResult<T = Record<string, unknown>> {
|
||||
rows: T[];
|
||||
rowCount: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* OAuth2 client-credentials token acquisition for an Azure AD service
|
||||
* principal — generalized 1:1 from v0-datahub's `aadToken()`. Exported
|
||||
* standalone (not just as a private method) so it can be unit-tested and
|
||||
* reused by callers that only need a token (e.g. to probe connectivity)
|
||||
* without opening a SQL connection.
|
||||
*/
|
||||
export async function acquireFabricAccessToken(
|
||||
credentials: FabricOneLakeCredentials,
|
||||
scope: string = DEFAULT_SQL_SCOPE,
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
): Promise<FabricAccessToken> {
|
||||
const { tenantId, clientId, clientSecret } = credentials;
|
||||
if (!tenantId || !clientId || !clientSecret) {
|
||||
throw new Error("fabric_onelake_adapter: tenantId, clientId and clientSecret are all required");
|
||||
}
|
||||
const body = new URLSearchParams({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
grant_type: "client_credentials",
|
||||
scope,
|
||||
});
|
||||
const res = await fetchImpl(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-www-form-urlencoded" },
|
||||
body,
|
||||
});
|
||||
const json: { access_token?: string; expires_in?: number; error?: string; error_description?: string } = await res.json();
|
||||
if (!res.ok || !json.access_token) {
|
||||
throw new Error(
|
||||
`fabric_onelake_adapter: AAD token request failed (scope=${scope}): ${json.error ?? res.status} ${json.error_description ?? ""}`.trim(),
|
||||
);
|
||||
}
|
||||
const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000;
|
||||
return { accessToken: json.access_token, expiresAtEpochMs: Date.now() + expiresInMs };
|
||||
}
|
||||
|
||||
function stripScheme(endpoint: string): string {
|
||||
return endpoint.replace(/^https?:\/\//, "").replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** Minimal shape of the pieces of the `mssql` module this adapter actually uses. Kept as a structural type, never imported from the real package (see file header). */
|
||||
interface MssqlLike {
|
||||
ConnectionPool: new (config: Record<string, unknown>) => {
|
||||
connect(): Promise<unknown>;
|
||||
request(): {
|
||||
input(name: string, value: unknown): unknown;
|
||||
query(text: string): Promise<{ recordset?: unknown[] }>;
|
||||
};
|
||||
close(): Promise<void>;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapter for Source.adapter_id === "fabric_onelake_sql". One instance holds
|
||||
* one registered Source's connection config; nothing is shared across
|
||||
* tenants/customers, and no config here ever falls back to an env var.
|
||||
*/
|
||||
export class FabricOneLakeAdapter {
|
||||
private token: FabricAccessToken | null = null;
|
||||
private pool: InstanceType<MssqlLike["ConnectionPool"]> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly config: FabricOneLakeConnectionConfig,
|
||||
private readonly fetchImpl: typeof fetch = fetch,
|
||||
) {}
|
||||
|
||||
private async getToken(): Promise<string> {
|
||||
if (this.token && this.token.expiresAtEpochMs - TOKEN_REFRESH_SKEW_MS > Date.now()) {
|
||||
return this.token.accessToken;
|
||||
}
|
||||
this.token = await acquireFabricAccessToken(this.config.credentials, this.config.scope, this.fetchImpl);
|
||||
return this.token.accessToken;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the `mssql` TDS driver at call time via a *non-literal* specifier.
|
||||
* `moduleName` is deliberately widened to `string` (not the literal
|
||||
* `"mssql"`) so TypeScript types the result as `Promise<any>` instead of
|
||||
* resolving module types, and so bundlers cannot statically inline the
|
||||
* specifier into a build-time-resolved chunk. Overridable for tests.
|
||||
*/
|
||||
private async loadDriver(): Promise<MssqlLike> {
|
||||
const moduleName: string = "mssql";
|
||||
try {
|
||||
const mod = await import(moduleName);
|
||||
return (mod.default ?? mod) as MssqlLike;
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`fabric_onelake_adapter: "mssql" package is not installed/resolvable. Add it to package.json ` +
|
||||
`(dependencies.mssql) to enable live Fabric SQL query execution. Original error: ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async getPool(): Promise<InstanceType<MssqlLike["ConnectionPool"]>> {
|
||||
if (this.pool) return this.pool;
|
||||
const sql = await this.loadDriver();
|
||||
const token = await this.getToken();
|
||||
const poolConfig = {
|
||||
server: stripScheme(this.config.sqlEndpoint),
|
||||
database: this.config.database,
|
||||
options: { encrypt: true, trustServerCertificate: false },
|
||||
authentication: {
|
||||
type: "azure-active-directory-access-token",
|
||||
options: { token },
|
||||
},
|
||||
pool: { max: 1, min: 0, idleTimeoutMillis: 30_000 },
|
||||
requestTimeout: this.config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
|
||||
};
|
||||
const pool = new sql.ConnectionPool(poolConfig);
|
||||
await pool.connect();
|
||||
this.pool = pool;
|
||||
return pool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query and return rows + timing. This adapter does NOT enforce
|
||||
* row/time bounds itself — that is the job of the caller (see
|
||||
* src/services/source-query-service.ts), which wraps this in a timeout and
|
||||
* truncates the result for the synchronous path.
|
||||
*/
|
||||
async query<T = Record<string, unknown>>(queryText: string, params?: Record<string, unknown>): Promise<AdapterQueryResult<T>> {
|
||||
const start = Date.now();
|
||||
const pool = await this.getPool();
|
||||
const request = pool.request();
|
||||
if (params) {
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
request.input(key, value);
|
||||
}
|
||||
}
|
||||
const result = await request.query(queryText);
|
||||
const rows = (result.recordset ?? []) as T[];
|
||||
return { rows, rowCount: rows.length, durationMs: Date.now() - start };
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.pool) {
|
||||
await this.pool.close();
|
||||
this.pool = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createFabricOneLakeAdapter(config: FabricOneLakeConnectionConfig, fetchImpl?: typeof fetch): FabricOneLakeAdapter {
|
||||
return new FabricOneLakeAdapter(config, fetchImpl);
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* src/adapters/graph-mailbox-adapter.ts
|
||||
* ======================================
|
||||
* Tenant-agnostic Microsoft Graph mailbox poller (app-only client-credentials).
|
||||
*
|
||||
* Generalized from v0-datahub/lib/invoice-inbox/poll.ts: there the mailbox
|
||||
* (invoice@protosell.se), the Entra app, and the folder were all hardcoded
|
||||
* module constants. Here every one of those is a CALLER-SUPPLIED parameter —
|
||||
* any SIAX customer can register any mailbox (invoices, HR forms, contracts,
|
||||
* receipts, ...) and poll it through this adapter. Nothing about invoices,
|
||||
* vendors, or Protosell is baked in here — that logic (vendor-kb matching,
|
||||
* dup-hash, DQ scoring, invoice classification) was intentionally left
|
||||
* behind; it does not belong in a generic capture adapter.
|
||||
*/
|
||||
|
||||
const GRAPH_BASE = "https://graph.microsoft.com/v1.0";
|
||||
|
||||
export interface GraphMailboxConfig {
|
||||
/** Entra (Azure AD) tenant id the app registration lives in. */
|
||||
tenantId: string;
|
||||
/** Entra app (client) id — app-only, client-credentials grant. */
|
||||
clientId: string;
|
||||
/** Entra app client secret. Never logged, never persisted by this adapter. */
|
||||
clientSecret: string;
|
||||
/** Mailbox address to poll, e.g. "invoices@customer.com". */
|
||||
mailboxAddress: string;
|
||||
/** Mail folder to poll. Default: "inbox". */
|
||||
folder?: string;
|
||||
}
|
||||
|
||||
export interface GraphMessage {
|
||||
id: string;
|
||||
internetMessageId: string | null;
|
||||
conversationId: string | null;
|
||||
receivedAt: string | null;
|
||||
senderName: string | null;
|
||||
senderEmail: string | null;
|
||||
toRecipients: string[];
|
||||
subject: string | null;
|
||||
bodyPreview: string | null;
|
||||
/** HTML body stripped to plain text. Falls back to bodyPreview when body is absent. */
|
||||
bodyText: string | null;
|
||||
hasAttachments: boolean;
|
||||
webLink: string | null;
|
||||
}
|
||||
|
||||
export interface GraphAttachment {
|
||||
id: string;
|
||||
name: string | null;
|
||||
contentType: string | null;
|
||||
sizeBytes: number | null;
|
||||
isPdf: boolean;
|
||||
contentBytes: Buffer;
|
||||
}
|
||||
|
||||
export interface PollMessagesOptions {
|
||||
/** Only return messages received at/after this ISO timestamp (incremental polling). */
|
||||
sinceISO?: string;
|
||||
/** Page size per Graph request. Default 50. */
|
||||
top?: number;
|
||||
/** Max number of pages to walk. Default 2 (raise for backfills). */
|
||||
maxPages?: number;
|
||||
}
|
||||
|
||||
/** Strip HTML to plain text. Generic — no document-type-specific parsing. */
|
||||
function stripHtml(html: string): string {
|
||||
return (html || "")
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
||||
.replace(/<[^>]+>/g, " ")
|
||||
.replace(/ /gi, " ")
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export class GraphMailboxAdapter {
|
||||
private cachedToken: { token: string; expiresAt: number } | null = null;
|
||||
|
||||
constructor(private config: GraphMailboxConfig) {}
|
||||
|
||||
static isConfigured(config: Partial<GraphMailboxConfig>): boolean {
|
||||
return !!(config.tenantId && config.clientId && config.clientSecret && config.mailboxAddress);
|
||||
}
|
||||
|
||||
private async getToken(): Promise<string> {
|
||||
if (this.cachedToken && this.cachedToken.expiresAt > Date.now() + 30_000) {
|
||||
return this.cachedToken.token;
|
||||
}
|
||||
const { tenantId, clientId, clientSecret } = this.config;
|
||||
if (!tenantId || !clientId || !clientSecret) {
|
||||
throw new Error("GraphMailboxAdapter: missing tenantId/clientId/clientSecret");
|
||||
}
|
||||
const body = new URLSearchParams({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
grant_type: "client_credentials",
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
});
|
||||
const res = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-www-form-urlencoded" },
|
||||
body,
|
||||
});
|
||||
const json: Record<string, unknown> = await res.json();
|
||||
if (!res.ok) {
|
||||
throw new Error(`Graph token request failed: ${json.error || res.status} ${json.error_description || ""}`);
|
||||
}
|
||||
const expiresInMs = (typeof json.expires_in === "number" ? json.expires_in : 3600) * 1000;
|
||||
this.cachedToken = { token: json.access_token as string, expiresAt: Date.now() + expiresInMs };
|
||||
return this.cachedToken.token;
|
||||
}
|
||||
|
||||
private async get(path: string): Promise<any> {
|
||||
const token = await this.getToken();
|
||||
const url = path.startsWith("http") ? path : `${GRAPH_BASE}${path}`;
|
||||
const res = await fetch(url, { headers: { authorization: `Bearer ${token}` } });
|
||||
const json: any = await res.json();
|
||||
if (!res.ok) {
|
||||
throw new Error(`Graph GET ${path} -> ${json.error?.code || res.status}: ${json.error?.message || ""}`);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
/** List messages in the configured mailbox/folder, newest first, optionally incremental. */
|
||||
async pollMessages(opts: PollMessagesOptions = {}): Promise<GraphMessage[]> {
|
||||
const folder = this.config.folder || "inbox";
|
||||
const top = opts.top ?? 50;
|
||||
const maxPages = opts.maxPages ?? 2;
|
||||
const select =
|
||||
"$select=id,internetMessageId,conversationId,receivedDateTime,from,toRecipients,subject,bodyPreview,hasAttachments,webLink,body";
|
||||
let sinceFilter = "";
|
||||
if (opts.sinceISO) {
|
||||
sinceFilter = `&$filter=receivedDateTime ge ${new Date(opts.sinceISO).toISOString()}`;
|
||||
}
|
||||
let url =
|
||||
`${GRAPH_BASE}/users/${encodeURIComponent(this.config.mailboxAddress)}/mailFolders/${encodeURIComponent(folder)}` +
|
||||
`/messages?$top=${top}&$orderby=receivedDateTime desc&${select}${sinceFilter}`;
|
||||
|
||||
const out: GraphMessage[] = [];
|
||||
for (let page = 0; page < maxPages && url; page++) {
|
||||
const data = await this.get(url);
|
||||
for (const m of data.value || []) {
|
||||
out.push({
|
||||
id: m.id,
|
||||
internetMessageId: m.internetMessageId || null,
|
||||
conversationId: m.conversationId || null,
|
||||
receivedAt: m.receivedDateTime || null,
|
||||
senderName: m.from?.emailAddress?.name || null,
|
||||
senderEmail: m.from?.emailAddress?.address || null,
|
||||
toRecipients: (m.toRecipients || []).map((r: any) => r.emailAddress?.address).filter(Boolean),
|
||||
subject: m.subject || null,
|
||||
bodyPreview: m.bodyPreview || null,
|
||||
bodyText: m.body?.content ? stripHtml(m.body.content) : m.bodyPreview || null,
|
||||
hasAttachments: !!m.hasAttachments,
|
||||
webLink: m.webLink || null,
|
||||
});
|
||||
}
|
||||
url = data["@odata.nextLink"] || "";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** List and download every file attachment on a message. */
|
||||
async listAttachments(messageId: string): Promise<GraphAttachment[]> {
|
||||
const data = await this.get(
|
||||
`/users/${encodeURIComponent(this.config.mailboxAddress)}/messages/${messageId}/attachments`,
|
||||
);
|
||||
const out: GraphAttachment[] = [];
|
||||
for (const a of data.value || []) {
|
||||
if (a["@odata.type"] !== "#microsoft.graph.fileAttachment" || !a.contentBytes) continue;
|
||||
const contentType: string | null = a.contentType || null;
|
||||
const name: string | null = a.name || null;
|
||||
const isPdf = (contentType || "").includes("pdf") || /\.pdf$/i.test(name || "");
|
||||
out.push({
|
||||
id: a.id,
|
||||
name,
|
||||
contentType,
|
||||
sizeBytes: typeof a.size === "number" ? a.size : null,
|
||||
isPdf,
|
||||
contentBytes: Buffer.from(a.contentBytes, "base64"),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* src/adapters/local-blob-store.ts
|
||||
* ==================================
|
||||
* INTERIM local-disk blob store — NOT the real ST0RE bytes backend.
|
||||
*
|
||||
* KNOWN GAP: src/clients/st0re-client.ts only registers artifact METADATA
|
||||
* (a pointer: provider/bucket/key/region). It has no endpoint that accepts
|
||||
* and persists raw bytes — ST0RE does not yet expose a real object-storage
|
||||
* write path for 0pt to call. Until it does, document-capture-service.ts
|
||||
* and document-extraction-service.ts write attachment/extraction bytes here
|
||||
* so the capability is functionally real end-to-end (download → persist →
|
||||
* read back for extraction) instead of silently discarding the PDF bytes
|
||||
* and only faking a metadata pointer.
|
||||
*
|
||||
* This is single-node, unencrypted-at-rest beyond the host filesystem, and
|
||||
* has no retention/lifecycle management — do not treat it as production
|
||||
* object storage. Swap it for a real ST0RE (or direct MinIO/Azure Blob)
|
||||
* client the day one exists; every call site that constructs this class is
|
||||
* marked "INTERIM" so the seam is easy to find.
|
||||
*/
|
||||
import { promises as fs } from "fs";
|
||||
import path from "path";
|
||||
|
||||
export interface LocalBlobWriteResult {
|
||||
provider: "local_interim";
|
||||
bucket: string;
|
||||
key: string;
|
||||
absolutePath: string;
|
||||
sizeBytes: number;
|
||||
}
|
||||
|
||||
export class LocalBlobStore {
|
||||
constructor(private rootDir: string = process.env.OPT_LOCAL_BLOB_DIR || path.join(process.cwd(), ".data", "artifacts")) {}
|
||||
|
||||
async write(bucket: string, key: string, bytes: Buffer): Promise<LocalBlobWriteResult> {
|
||||
const safeKey = key.replace(/\.\./g, "_");
|
||||
const absolutePath = path.join(this.rootDir, bucket, safeKey);
|
||||
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
|
||||
await fs.writeFile(absolutePath, bytes);
|
||||
return { provider: "local_interim", bucket, key: safeKey, absolutePath, sizeBytes: bytes.length };
|
||||
}
|
||||
|
||||
async read(bucket: string, key: string): Promise<Buffer> {
|
||||
const absolutePath = path.join(this.rootDir, bucket, key);
|
||||
return fs.readFile(absolutePath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* src/adapters/odoo-bills-adapter.ts
|
||||
* =====================================
|
||||
* `WritebackAdapter` (../adapters/writeback-adapter.ts) implementation for
|
||||
* Odoo Vendor Bills (`account.move`, `move_type: 'in_invoice'`) --
|
||||
* registered under `adapterId: "odoo_bills"` onto an
|
||||
* `ExternalWritebackService` (../services/external-writeback-service.ts).
|
||||
*
|
||||
* PORTED FROM v0-datahub's `lib/odoo/invoice-bridge.ts` (read in full before
|
||||
* writing this file), generalized as one WritebackAdapter implementation
|
||||
* rather than a standalone module with its own Supabase-coupled pipeline
|
||||
* function (`processInboxToOdoo`). What is a DIRECT PORT of proven logic,
|
||||
* vs. what is NEW/GENERALIZED, stated precisely:
|
||||
*
|
||||
* DIRECT PORT (same behavior, same guard order, same reasoning):
|
||||
* - `isReminder`/`duplicateFlag` guard -- invoice-bridge.ts's
|
||||
* `processInboxToOdoo`: "if (invoice.is_reminder || invoice.duplicate_flag)
|
||||
* return { ok:false, error: ... }" (a reminder or a detected duplicate is
|
||||
* NEVER posted to Odoo). Ported verbatim as this adapter's first guard in
|
||||
* `write()`.
|
||||
* - "already posted, skip" guard -- invoice-bridge.ts: "if
|
||||
* (invoice.odoo_move_id) { ...hoppar över...; return { ok:true, odoo_id:
|
||||
* invoice.odoo_move_id } }". Ported as the `existingExternalId` check --
|
||||
* this adapter's own idempotency defense-in-depth layer, complementing
|
||||
* (not replacing) `ExternalWritebackService`'s own in-memory claim -- see
|
||||
* that service's own module doc comment and writeback-adapter.ts's
|
||||
* `WritebackAdapter.write` doc comment for why BOTH layers exist.
|
||||
* - Partner matching by org-nr -- invoice-bridge.ts's
|
||||
* `matchInvoiceToOdooPartner`: digits-only comparison against
|
||||
* `odoo_export__res_partner.vat` so "556xxx-xxxx", "SE556xxxxxxxx01", and
|
||||
* "5561234567" all match the same partner. Ported as
|
||||
* `matchPartnerViaFabric` below, with ONE necessary substitution: the
|
||||
* original queried via v0-datahub's own `@/lib/warehouse/client`
|
||||
* (Next.js-app-local, not importable from this repo) -- this port uses
|
||||
* 0pt's own already-built, equally generic
|
||||
* `../adapters/fabric-onelake-adapter.ts` (built this same pass, for
|
||||
* 0pt.document_ocr's own needs) against the SAME table name
|
||||
* (`odoo_export__res_partner`) and the SAME digits-only `LIKE` query
|
||||
* shape, parameterized instead of string-interpolated.
|
||||
* - Vendor Bill request shape -- invoice-bridge.ts's `pushInvoiceToOdoo`:
|
||||
* POSTs `{ model: "account.move", invoice: { partner_id, ref,
|
||||
* invoice_date, invoice_date_due, invoice_line_ids: [...] } }` to
|
||||
* `/api/odoo/invoices/write`. Ported verbatim as this adapter's own POST
|
||||
* body -- same endpoint path convention, same field names, so an existing
|
||||
* `/api/odoo/invoices/write` deployment needs no change to accept calls
|
||||
* from this adapter.
|
||||
* - Line synthesis fallback -- invoice-bridge.ts's `processInboxToOdoo`
|
||||
* builds a single summary `invoice_line_ids` entry
|
||||
* (`description: subject||vendor||"Faktura {number}"`, `quantity: 1,
|
||||
* unit_price: amount_excl_vat`) when no itemized lines exist. Ported as
|
||||
* `synthesizeSummaryLine` below, used only when `payload.lines` is absent.
|
||||
*
|
||||
* GENERALIZED / NEW (did not exist in the original, or could not be ported
|
||||
* 1:1):
|
||||
* - The Supabase read/claim/update-status statements in
|
||||
* `processInboxToOdoo` (the `invoice_inbox` table CRUD, and its own
|
||||
* atomic `UPDATE ... WHERE odoo_move_id IS NULL` claim) are NOT ported
|
||||
* here -- that is `ExternalWritebackService`'s job now (its own
|
||||
* adapter-agnostic in-memory claim, see that service's module doc
|
||||
* comment), not this adapter's. This adapter is stateless per call.
|
||||
* - `baseUrl`/`apiKey`/the Fabric credentials are all constructor
|
||||
* parameters -- the original hardcoded `process.env.ODOO_BASE_URL`/
|
||||
* `process.env.NEXT_PUBLIC_APP_URL`/etc as module-level env reads (v0-
|
||||
* datahub is a single-tenant app; this repo is a SIAX platform capability
|
||||
* any customer's tenant can register their own Odoo instance against, the
|
||||
* same "nothing hardcoded, everything a caller-supplied credential"
|
||||
* doctrine ../adapters/fabric-onelake-adapter.ts's own header already
|
||||
* documents for the identical reason).
|
||||
* - `fetchImpl`/`matchPartner` are injectable (default to the real `fetch`
|
||||
* / `matchPartnerViaFabric`) purely so tests never make a real network
|
||||
* call -- the original had no test coverage for this path at all
|
||||
* (v0-datahub's invoice-bridge.ts has no accompanying test file); this
|
||||
* port is genuinely tested (see tests/p0/odoo-bills-adapter.test.ts).
|
||||
*/
|
||||
import type { WritebackAdapter, WritebackAdapterContext, WritebackRequest, WritebackResult } from "./writeback-adapter";
|
||||
import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig } from "./fabric-onelake-adapter";
|
||||
|
||||
/** One itemized Vendor Bill line -- same shape as invoice-bridge.ts's `InvoiceForOdoo.lines`. */
|
||||
export interface OdooBillLine {
|
||||
description: string;
|
||||
quantity: number;
|
||||
unit_price: number;
|
||||
vat_rate?: number;
|
||||
account_bas?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The structured record this adapter expects as `WritebackRequest.payload`
|
||||
* -- same field set as invoice-bridge.ts's `InvoiceForOdoo`, plus the guard
|
||||
* fields (`is_reminder`/`duplicate_flag`/`existing_external_id`) that
|
||||
* `processInboxToOdoo` read directly off its own `invoice_inbox` row (this
|
||||
* adapter has no row of its own to read them from, so the caller supplies
|
||||
* them on the payload instead).
|
||||
*/
|
||||
export interface OdooBillPayload {
|
||||
vendor_name: string;
|
||||
vendor_org_nr?: string;
|
||||
/** Pre-resolved Odoo res.partner id -- when absent and `vendor_org_nr` is set, this adapter
|
||||
* attempts to resolve one via partner matching (see this file's module doc comment). */
|
||||
partner_id?: number;
|
||||
invoice_number: string;
|
||||
ocr?: string;
|
||||
amount_gross: number;
|
||||
amount_excl_vat?: number;
|
||||
vat?: number;
|
||||
currency: string;
|
||||
invoice_date: string;
|
||||
due_date?: string;
|
||||
description?: string;
|
||||
lines?: readonly OdooBillLine[];
|
||||
/** Mirrors invoice_inbox.is_reminder -- a payment reminder must never be posted as a new Bill. */
|
||||
is_reminder?: boolean;
|
||||
/** Mirrors invoice_inbox.duplicate_flag -- a detected duplicate must never be posted. */
|
||||
duplicate_flag?: boolean;
|
||||
/** When the caller already knows this record was posted (e.g. re-delivering a webhook), pass
|
||||
* the known Odoo id here -- this adapter treats it as already-done and returns it unchanged,
|
||||
* exactly mirroring invoice-bridge.ts's own `invoice.odoo_move_id` early-return. */
|
||||
existing_external_id?: string;
|
||||
}
|
||||
|
||||
export type OdooPartnerMatcher = (
|
||||
orgNr: string,
|
||||
ctx: WritebackAdapterContext,
|
||||
) => Promise<{ partner_id?: number; partner_name?: string }>;
|
||||
|
||||
export interface OdooBillsAdapterConfig {
|
||||
/** Base URL of the deployment exposing `/api/odoo/invoices/write` (same route shape as
|
||||
* v0-datahub's own `/api/odoo/invoices/write` this was ported from). */
|
||||
baseUrl: string;
|
||||
/** Bearer credential for that route. Optional ONLY when `allowUnauthenticated` is explicitly
|
||||
* set (e.g. a route that authenticates by network policy instead) -- see `isConfigured`. */
|
||||
apiKey?: string;
|
||||
allowUnauthenticated?: boolean;
|
||||
/** Fabric/OneLake connection used for org-nr -> Odoo partner matching (this file's own
|
||||
* `matchPartnerViaFabric`), when `matchPartner` is not explicitly supplied. Optional --
|
||||
* without it (and without an explicit `matchPartner`), partner matching is skipped entirely
|
||||
* and the Bill is posted without a partner link, mirroring invoice-bridge.ts's own
|
||||
* `matchInvoiceToOdooPartner` catch-all ("Ingen partner match — skickar utan
|
||||
* partner-koppling"). */
|
||||
fabric?: FabricOneLakeConnectionConfig;
|
||||
/** Table name for the partner lookup query -- defaults to `odoo_export__res_partner`, the
|
||||
* exact table invoice-bridge.ts's own `matchInvoiceToOdooPartner` queried. */
|
||||
partnerTable?: string;
|
||||
/** Injectable partner matcher, for tests or a caller with its own partner-resolution source.
|
||||
* When supplied, this takes priority over `fabric`. */
|
||||
matchPartner?: OdooPartnerMatcher;
|
||||
/** Injectable `fetch`, so tests never make a real network call. */
|
||||
fetchImpl?: typeof fetch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Digits-only org-nr -> Odoo partner match, ported 1:1 from invoice-bridge.ts's
|
||||
* `matchInvoiceToOdooPartner` (see this file's module doc comment). The only
|
||||
* substitution is the query transport: `FabricOneLakeAdapter.query` instead
|
||||
* of v0-datahub's `@/lib/warehouse/client` -- same table, same digits-only
|
||||
* `LIKE` comparison so "556xxx-xxxx"/"SE556xxxxxxxx01"/"5561234567" all
|
||||
* resolve to the same partner, now parameterized instead of string-interpolated.
|
||||
*/
|
||||
export async function matchPartnerViaFabric(
|
||||
fabric: FabricOneLakeAdapter,
|
||||
orgNr: string,
|
||||
partnerTable: string = "odoo_export__res_partner",
|
||||
): Promise<{ partner_id?: number; partner_name?: string }> {
|
||||
const digits = orgNr.replace(/[^0-9]/g, "");
|
||||
if (!digits) return {};
|
||||
try {
|
||||
const result = await fabric.query<{ id: number; name: string; vat: string }>(
|
||||
`SELECT TOP 1 id, name, vat FROM ${partnerTable} WHERE REPLACE(REPLACE(vat,'-',''),' ','') LIKE '%' + @digits + '%'`,
|
||||
{ digits },
|
||||
);
|
||||
if (result.rows.length > 0) {
|
||||
return { partner_id: result.rows[0]!.id, partner_name: result.rows[0]!.name };
|
||||
}
|
||||
return {};
|
||||
} catch {
|
||||
// Same catch-all as invoice-bridge.ts's own matchInvoiceToOdooPartner: an unreachable/
|
||||
// misconfigured warehouse must never fail the whole write -- it degrades to "post without
|
||||
// a partner link", not a thrown error.
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds the single synthesized `invoice_line_ids` entry used when `payload.lines` is absent
|
||||
* -- ported verbatim from invoice-bridge.ts's `processInboxToOdoo` line-synthesis block. */
|
||||
function synthesizeSummaryLine(payload: OdooBillPayload): OdooBillLine {
|
||||
const exclVat = payload.amount_excl_vat ?? payload.amount_gross;
|
||||
const description = (payload.description || payload.vendor_name || `Faktura ${payload.invoice_number}`).trim();
|
||||
return { description, quantity: 1, unit_price: exclVat };
|
||||
}
|
||||
|
||||
function isOdooBillPayload(payload: Readonly<Record<string, unknown>>): payload is OdooBillPayload & Record<string, unknown> {
|
||||
return (
|
||||
typeof payload.vendor_name === "string" &&
|
||||
typeof payload.invoice_number === "string" &&
|
||||
typeof payload.amount_gross === "number" &&
|
||||
typeof payload.currency === "string" &&
|
||||
typeof payload.invoice_date === "string"
|
||||
);
|
||||
}
|
||||
|
||||
export class OdooBillsAdapter implements WritebackAdapter {
|
||||
readonly adapterId = "odoo_bills";
|
||||
private fabricAdapter: FabricOneLakeAdapter | null;
|
||||
|
||||
constructor(private config: OdooBillsAdapterConfig) {
|
||||
this.fabricAdapter = config.fabric ? new FabricOneLakeAdapter(config.fabric, config.fetchImpl) : null;
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return !!this.config.baseUrl && (!!this.config.apiKey || this.config.allowUnauthenticated === true);
|
||||
}
|
||||
|
||||
async write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise<WritebackResult> {
|
||||
if (!this.isConfigured()) {
|
||||
return { ok: false, error: "odoo_not_configured" };
|
||||
}
|
||||
if (!isOdooBillPayload(request.payload)) {
|
||||
return { ok: false, error: "invalid_payload_missing_required_fields" };
|
||||
}
|
||||
const payload = request.payload as unknown as OdooBillPayload;
|
||||
|
||||
// Guard 1: reminders/duplicates are never posted -- ported verbatim from
|
||||
// invoice-bridge.ts's processInboxToOdoo (see this file's module doc comment).
|
||||
if (payload.is_reminder) return { ok: false, error: "is_reminder" };
|
||||
if (payload.duplicate_flag) return { ok: false, error: "duplicate" };
|
||||
|
||||
// Guard 2: already posted -- idempotent no-op, ported verbatim.
|
||||
if (payload.existing_external_id) {
|
||||
return { ok: true, externalId: payload.existing_external_id };
|
||||
}
|
||||
|
||||
// Partner match (best-effort, never fails the write) -- ported verbatim in spirit.
|
||||
let partnerId = payload.partner_id;
|
||||
if (!partnerId && payload.vendor_org_nr) {
|
||||
if (this.config.matchPartner) {
|
||||
const matched = await this.config.matchPartner(payload.vendor_org_nr, ctx);
|
||||
partnerId = matched.partner_id;
|
||||
} else if (this.fabricAdapter) {
|
||||
const matched = await matchPartnerViaFabric(this.fabricAdapter, payload.vendor_org_nr, this.config.partnerTable);
|
||||
partnerId = matched.partner_id;
|
||||
}
|
||||
}
|
||||
|
||||
const lines = payload.lines && payload.lines.length > 0 ? payload.lines : [synthesizeSummaryLine(payload)];
|
||||
|
||||
const fetchImpl = this.config.fetchImpl ?? fetch;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetchImpl(`${this.config.baseUrl}/api/odoo/invoices/write`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(this.config.apiKey ? { Authorization: `Bearer ${this.config.apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: "account.move",
|
||||
invoice: {
|
||||
partner_id: partnerId,
|
||||
ref: payload.invoice_number,
|
||||
invoice_date: payload.invoice_date,
|
||||
invoice_date_due: payload.due_date,
|
||||
invoice_line_ids: lines.map((l) => ({ name: l.description, quantity: l.quantity, price_unit: l.unit_price })),
|
||||
},
|
||||
}),
|
||||
});
|
||||
} catch (e) {
|
||||
return { ok: false, error: `odoo_network_error: ${e instanceof Error ? e.message : String(e)}` };
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
return { ok: false, error: `odoo_http_${res.status}` };
|
||||
}
|
||||
const result: { ok?: boolean; odoo_id?: number; error?: string } = await res.json();
|
||||
if (!result.ok || !result.odoo_id) {
|
||||
return { ok: false, error: result.error ?? "odoo_write_failed" };
|
||||
}
|
||||
return { ok: true, externalId: String(result.odoo_id) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* src/adapters/writeback-adapter.ts
|
||||
* ===================================
|
||||
* The adapter interface `ExternalWritebackService` (../services/
|
||||
* external-writeback-service.ts) writes through. One implementation per
|
||||
* external system (e.g. odoo-bills-adapter.ts's OdooBillsAdapter for Odoo
|
||||
* Vendor Bills) -- the service itself has zero knowledge of Odoo, Salesforce,
|
||||
* or any other concrete external system; it only knows "an adapter with this
|
||||
* adapterId exists, is configured, and can write a record."
|
||||
*/
|
||||
|
||||
export interface WritebackAdapterContext {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
}
|
||||
|
||||
export interface WritebackRequest {
|
||||
recordType: string;
|
||||
payload: Readonly<Record<string, unknown>>;
|
||||
/** Same idempotency key the caller gave ExternalWritebackService -- forwarded so an adapter
|
||||
* with its own persistent storage (unlike the generic service's in-memory claim) can ALSO
|
||||
* claim defensively at its own layer. See WritebackAdapter.write's own doc comment. */
|
||||
idempotencyKey: string;
|
||||
}
|
||||
|
||||
export interface WritebackResult {
|
||||
ok: boolean;
|
||||
/** The external system's own id for the record that was created (or already existed --
|
||||
* see WritebackAdapter.write's own doc comment on idempotent re-calls), when ok is true. */
|
||||
externalId?: string;
|
||||
/** A short, stable, machine-readable reason code when ok is false (e.g. "is_reminder",
|
||||
* "duplicate", "odoo_http_500") -- NOT a free-form human sentence, so callers/tests can
|
||||
* assert on it reliably. */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One external-system write adapter. Registered onto an
|
||||
* `ExternalWritebackService` via `registerAdapter` under a stable `adapterId`
|
||||
* (e.g. "odoo_bills") that `ExternalWriteRequest.adapter_id` references.
|
||||
*/
|
||||
export interface WritebackAdapter {
|
||||
readonly adapterId: string;
|
||||
|
||||
/** Whether this adapter has everything it needs (base URL, credentials, etc) to actually
|
||||
* write -- mirrors DocumentIntelligenceAdapter.isConfigured's "refuse to fabricate a result"
|
||||
* convention (../adapters/document-intelligence-adapter.ts): an unconfigured adapter must
|
||||
* never be silently treated as a no-op success. */
|
||||
isConfigured(): boolean;
|
||||
|
||||
/**
|
||||
* Writes one record to the external system.
|
||||
*
|
||||
* IDEMPOTENCY, TWO LAYERS (per this task's own instruction not to regress the proven
|
||||
* idempotent-claim logic ported from v0-datahub's invoice-bridge.ts):
|
||||
* 1. `ExternalWritebackService` (the caller) already claims `idempotencyKey` in its own
|
||||
* in-memory map BEFORE ever calling this method, so under normal operation `write` is
|
||||
* only ever called once per logical record, from one service instance.
|
||||
* 2. That in-memory claim is process-local and does not survive a restart, nor protect a
|
||||
* multi-instance deployment. An adapter backed by its own persistent storage (like
|
||||
* OdooBillsAdapter, when a caller supplies an already-posted record's
|
||||
* `existing_external_id` in the payload) MUST ALSO refuse to re-post a record that
|
||||
* already carries an external id -- exactly the "Redan bokförd i Odoo... hoppar över"
|
||||
* early-return invoice-bridge.ts's own `processInboxToOdoo` performs. This is the
|
||||
* genuine defense-in-depth the original proven code had; a `WritebackAdapter`
|
||||
* implementation with real persistent storage should follow the same doctrine.
|
||||
*/
|
||||
write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise<WritebackResult>;
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* src/services/document-capture-service.ts
|
||||
* =========================================
|
||||
* Orchestrates the capture half of the 0pt.document_ocr capability:
|
||||
* poll a customer-registered mailbox (Microsoft Graph, app-only) →
|
||||
* download PDF attachments → persist the bytes as an artifact →
|
||||
* enqueue extraction (an N0D job placeholder; a worker or the
|
||||
* /extraction API route performs the real Document Intelligence call).
|
||||
*
|
||||
* Tenant-agnostic: mailbox address + Graph credentials + folder are all
|
||||
* caller-supplied per invocation — nothing here is stored keyed only by a
|
||||
* hardcoded mailbox. Nothing here assumes invoices, vendors, or Protosell;
|
||||
* that logic (vendor-kb matching, dup-hash, DQ scoring, invoice
|
||||
* classification) was intentionally left behind in
|
||||
* v0-datahub/lib/invoice-inbox — it does not belong in a generic capture
|
||||
* pipeline that also has to serve HR forms, contracts, receipts, etc.
|
||||
*
|
||||
* KNOWN GAP — ST0RE bytes storage: st0re-client.ts registers artifact
|
||||
* METADATA only (a pointer to provider/bucket/key); it has no real
|
||||
* bytes-upload endpoint yet. Until one exists, this service writes
|
||||
* attachment bytes to LocalBlobStore (src/adapters/local-blob-store.ts), a
|
||||
* clearly-marked interim local-disk store, then registers that location
|
||||
* with ST0RE so the artifact/lineage metadata is still correct. Search this
|
||||
* file for "INTERIM" to find the seam to replace.
|
||||
*
|
||||
* Singleton note: unlike source-service.ts / dataset-service.ts (which are
|
||||
* instantiated fresh per API route file because each request is
|
||||
* self-contained), DocumentCaptureService holds the only in-memory record
|
||||
* of *which bytes belong to which document_id* — document-extraction-service
|
||||
* must read the exact same instance to find them. So this module also
|
||||
* exports a shared `documentCaptureService` singleton for both API routes
|
||||
* to import, instead of each route constructing its own instance.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { GraphMailboxAdapter, type GraphMailboxConfig, type GraphMessage, type GraphAttachment } from "../adapters/graph-mailbox-adapter";
|
||||
import { LocalBlobStore } from "../adapters/local-blob-store";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import type { DocumentRecord, DocumentStatus } from "../types/document";
|
||||
|
||||
export interface CaptureMailboxContext {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
}
|
||||
|
||||
export interface CaptureMailboxInput {
|
||||
mailbox: GraphMailboxConfig;
|
||||
since_iso?: string;
|
||||
top?: number;
|
||||
max_pages?: number;
|
||||
}
|
||||
|
||||
export interface CaptureMailboxResult {
|
||||
scanned_messages: number;
|
||||
captured_documents: DocumentRecord[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
interface StoredBytesLocation {
|
||||
bucket: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export class DocumentCaptureService {
|
||||
private documents: Map<string, DocumentRecord> = new Map();
|
||||
private bytesByDocumentId: Map<string, StoredBytesLocation> = new Map();
|
||||
private st0re: St0reClient;
|
||||
private n0d: N0dClient;
|
||||
private aud0: Aud0Client;
|
||||
private blobStore: LocalBlobStore;
|
||||
|
||||
constructor(opts?: { st0reBaseUrl?: string; n0dBaseUrl?: string; aud0BaseUrl?: string; blobStore?: LocalBlobStore }) {
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.n0d = createN0dClient(opts?.n0dBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
|
||||
}
|
||||
|
||||
async captureFromMailbox(input: CaptureMailboxInput, ctx: CaptureMailboxContext): Promise<CaptureMailboxResult> {
|
||||
if (!ctx.tenant_id || !ctx.project_id || !ctx.actor_id) {
|
||||
throw new Error("captureFromMailbox: missing tenant_id/project_id/actor_id");
|
||||
}
|
||||
if (!GraphMailboxAdapter.isConfigured(input.mailbox)) {
|
||||
throw new Error(
|
||||
"captureFromMailbox: missing mailbox.tenantId/clientId/clientSecret/mailboxAddress",
|
||||
);
|
||||
}
|
||||
const graph = new GraphMailboxAdapter(input.mailbox);
|
||||
const result: CaptureMailboxResult = { scanned_messages: 0, captured_documents: [], errors: [] };
|
||||
|
||||
const messages = await graph.pollMessages({
|
||||
sinceISO: input.since_iso,
|
||||
top: input.top,
|
||||
maxPages: input.max_pages,
|
||||
});
|
||||
|
||||
for (const message of messages) {
|
||||
result.scanned_messages++;
|
||||
if (!message.hasAttachments) continue;
|
||||
try {
|
||||
const attachments = await graph.listAttachments(message.id);
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment.isPdf) continue;
|
||||
const document = await this.captureAttachment(input.mailbox, message, attachment, ctx);
|
||||
result.captured_documents.push(document);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
result.errors.push(`${message.id}: ${msg.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async captureAttachment(
|
||||
mailbox: GraphMailboxConfig,
|
||||
message: GraphMessage,
|
||||
attachment: GraphAttachment,
|
||||
ctx: CaptureMailboxContext,
|
||||
): Promise<DocumentRecord> {
|
||||
const documentId = `doc_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
const bucket = "0pt-documents";
|
||||
const safeName = (attachment.name || "attachment.pdf").replace(/[^A-Za-z0-9._-]/g, "_");
|
||||
const key = `${ctx.tenant_id}/${ctx.project_id}/${message.id}/${attachment.id}/${safeName}`;
|
||||
|
||||
// INTERIM — see file header. Real bytes persistence until ST0RE exposes an upload endpoint.
|
||||
const written = await this.blobStore.write(bucket, key, attachment.contentBytes);
|
||||
const checksum = crypto.createHash("sha256").update(attachment.contentBytes).digest("hex");
|
||||
|
||||
let record: DocumentRecord = {
|
||||
document_id: documentId,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
mailbox_address: mailbox.mailboxAddress,
|
||||
graph_message_id: message.id,
|
||||
graph_attachment_id: attachment.id,
|
||||
filename: attachment.name ?? undefined,
|
||||
content_type: attachment.contentType ?? undefined,
|
||||
size_bytes: attachment.sizeBytes ?? attachment.contentBytes.length,
|
||||
received_at: message.receivedAt ?? undefined,
|
||||
sender_email: message.senderEmail ?? undefined,
|
||||
sender_name: message.senderName ?? undefined,
|
||||
subject: message.subject ?? undefined,
|
||||
status: "captured",
|
||||
correlation_id: ctx.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.documents.set(documentId, record);
|
||||
this.bytesByDocumentId.set(documentId, { bucket: written.bucket, key: written.key });
|
||||
|
||||
// Register the artifact pointer with ST0RE (metadata only — see file header gap note).
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
artifact_type: "document",
|
||||
app_id: "0pt",
|
||||
produced_by: "0pt.capture_document",
|
||||
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
|
||||
content_type: attachment.contentType || "application/pdf",
|
||||
size_bytes: written.sizeBytes,
|
||||
checksum_sha256: checksum,
|
||||
pii_level: "medium",
|
||||
retention_policy: "document_capture_standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
// Enqueue extraction. A worker (out of scope here) or a caller hitting the
|
||||
// /documents/[documentId]/extraction route performs the real DI call.
|
||||
const job = await this.n0d.createJob({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
job_type: "0pt.document_extraction_job",
|
||||
capability_id: "0pt.extract_document",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: ctx.actor_id,
|
||||
input_artifact_ids: [artifact.artifact_id],
|
||||
output_artifact_type: "document_extraction",
|
||||
requires_backbone: true,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "cpu_io_medium",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
record = {
|
||||
...record,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
n0d_job_id: job.job_id,
|
||||
status: "extraction_requested",
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
this.documents.set(documentId, record);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_document_captured_${documentId}`,
|
||||
event_type: "0pt.document_captured",
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.capture_document",
|
||||
resource_type: "document",
|
||||
resource_id: documentId,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
mailbox_address: mailbox.mailboxAddress,
|
||||
graph_message_id: message.id,
|
||||
filename: attachment.name,
|
||||
size_bytes: written.sizeBytes,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
n0d_job_id: job.job_id,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_document_captured_${documentId}`,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.capture_document",
|
||||
source_type: "mailbox",
|
||||
source_id: mailbox.mailboxAddress,
|
||||
target_type: "document",
|
||||
target_id: documentId,
|
||||
lineage_type: "produced",
|
||||
metadata: { graph_message_id: message.id, st0re_artifact_id: artifact.artifact_id },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
async getDocument(documentId: string): Promise<DocumentRecord> {
|
||||
const record = this.documents.get(documentId);
|
||||
if (!record) throw new Error(`Document ${documentId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async listDocuments(filter?: { tenant_id?: string; project_id?: string; status?: DocumentStatus }): Promise<DocumentRecord[]> {
|
||||
let results = Array.from(this.documents.values());
|
||||
if (filter?.tenant_id) results = results.filter((r) => r.tenant_id === filter.tenant_id);
|
||||
if (filter?.project_id) results = results.filter((r) => r.project_id === filter.project_id);
|
||||
if (filter?.status) results = results.filter((r) => r.status === filter.status);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Read back the raw bytes captured for a document. INTERIM local store — see file header. */
|
||||
async getDocumentBytes(documentId: string): Promise<Buffer> {
|
||||
const loc = this.bytesByDocumentId.get(documentId);
|
||||
if (!loc) throw new Error(`Document ${documentId} has no stored bytes`);
|
||||
return this.blobStore.read(loc.bucket, loc.key);
|
||||
}
|
||||
|
||||
setStatus(documentId: string, status: DocumentStatus): void {
|
||||
const record = this.documents.get(documentId);
|
||||
if (!record) throw new Error(`Document ${documentId} not found`);
|
||||
record.status = status;
|
||||
record.updated_at = new Date().toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared singleton — see the "Singleton note" in the file header for why. */
|
||||
export const documentCaptureService = new DocumentCaptureService();
|
||||
@@ -0,0 +1,267 @@
|
||||
/**
|
||||
* src/services/document-extraction-service.ts
|
||||
* ==============================================
|
||||
* Evidence-based field extraction for a captured document.
|
||||
*
|
||||
* Calls DocumentIntelligenceAdapter and NEVER fabricates a value: a field
|
||||
* key is only populated with a non-null value when Document Intelligence
|
||||
* actually returned one, and it always carries DI's own confidence score
|
||||
* (or null when DI reported none). Fields DI did not find are left blank —
|
||||
* { value: null, confidence: <DI's confidence if any>, found: false } —
|
||||
* never defaulted, guessed, or backfilled from anything else. This mirrors
|
||||
* the evidence-based discipline in
|
||||
* v0-datahub/lib/invoice-inbox/doc-intelligence.ts exactly (it never
|
||||
* invented an amount/vendor/date that DI didn't return); it is a
|
||||
* correctness property, not a style preference, so it is enforced twice —
|
||||
* once inside the adapter's field normalizer, and again defensively here.
|
||||
*
|
||||
* modelId is caller-supplied ("prebuilt-invoice", "prebuilt-receipt",
|
||||
* "prebuilt-layout", or a custom model id) — this service does not assume
|
||||
* any particular document type. Invoice-specific reasoning (vendor
|
||||
* matching, dup-hash, DQ scoring, invoice classification) was intentionally
|
||||
* left in v0-datahub/lib/invoice-inbox and is NOT ported here.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import {
|
||||
DocumentIntelligenceAdapter,
|
||||
type DocumentIntelligenceConfig,
|
||||
type ExtractedField as AdapterField,
|
||||
} from "../adapters/document-intelligence-adapter";
|
||||
import { LocalBlobStore } from "../adapters/local-blob-store";
|
||||
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
|
||||
import { createSt0reClient, type St0reClient } from "../clients/st0re-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { DocumentCaptureService, documentCaptureService } from "./document-capture-service";
|
||||
import type { DocumentExtractionRecord, ExtractedField } from "../types/document-extraction";
|
||||
|
||||
export interface ExtractDocumentContext {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
/** e.g. "prebuilt-invoice" | "prebuilt-receipt" | "prebuilt-layout" | a custom model id. */
|
||||
model_id: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Defensive re-application of the "never fabricate" rule at the service
|
||||
* boundary. Even though the adapter's normalizeField() already enforces
|
||||
* this, a field is only ever considered found here if it is BOTH marked
|
||||
* found by the adapter AND carries a genuinely non-empty value — so a
|
||||
* malformed/partial adapter response can never leak a fabricated value
|
||||
* through this service.
|
||||
*/
|
||||
function toEvidenceFields(fields: Record<string, AdapterField>): Record<string, ExtractedField> {
|
||||
const out: Record<string, ExtractedField> = {};
|
||||
for (const [key, f] of Object.entries(fields)) {
|
||||
const found = !!f?.found && f.value !== null && f.value !== undefined && f.value !== "";
|
||||
out[key] = { value: found ? f.value : null, confidence: f?.confidence ?? null, found };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class DocumentExtractionService {
|
||||
private extractions: Map<string, DocumentExtractionRecord> = new Map();
|
||||
private cl0ud: Cl0udClient;
|
||||
private st0re: St0reClient;
|
||||
private aud0: Aud0Client;
|
||||
private blobStore: LocalBlobStore;
|
||||
private di: DocumentIntelligenceAdapter | null;
|
||||
|
||||
constructor(
|
||||
private captureService: DocumentCaptureService,
|
||||
opts?: {
|
||||
cl0udBaseUrl?: string;
|
||||
st0reBaseUrl?: string;
|
||||
aud0BaseUrl?: string;
|
||||
documentIntelligence?: DocumentIntelligenceConfig;
|
||||
blobStore?: LocalBlobStore;
|
||||
},
|
||||
) {
|
||||
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
|
||||
this.st0re = createSt0reClient(opts?.st0reBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.blobStore = opts?.blobStore ?? new LocalBlobStore();
|
||||
this.di =
|
||||
opts?.documentIntelligence && DocumentIntelligenceAdapter.isConfigured(opts.documentIntelligence)
|
||||
? new DocumentIntelligenceAdapter(opts.documentIntelligence)
|
||||
: null;
|
||||
}
|
||||
|
||||
async extractDocument(documentId: string, ctx: ExtractDocumentContext): Promise<DocumentExtractionRecord> {
|
||||
if (!this.di) {
|
||||
throw new Error(
|
||||
"DocumentExtractionService: Document Intelligence is not configured (missing endpoint/apiKey) — refusing to fabricate a result",
|
||||
);
|
||||
}
|
||||
if (!ctx.model_id) {
|
||||
throw new Error("extractDocument: missing model_id");
|
||||
}
|
||||
const document = await this.captureService.getDocument(documentId);
|
||||
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
capability_id: "0pt.extract_document",
|
||||
resource_type: "document",
|
||||
resource_id: documentId,
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") throw new Error(`Policy denied: ${policy.reason}`);
|
||||
if (policy.decision === "approval_required") throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
|
||||
const extractionId = `dex_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
let record: DocumentExtractionRecord = {
|
||||
extraction_id: extractionId,
|
||||
document_id: documentId,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
model_id: ctx.model_id,
|
||||
status: "requested",
|
||||
fields: {},
|
||||
line_items: [],
|
||||
fields_found_count: 0,
|
||||
fields_total_count: 0,
|
||||
average_confidence: null,
|
||||
document_confidence: null,
|
||||
correlation_id: ctx.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.extractions.set(extractionId, record);
|
||||
|
||||
try {
|
||||
const bytes = await this.captureService.getDocumentBytes(documentId);
|
||||
const diResult = await this.di.analyzeDocument(bytes, ctx.model_id);
|
||||
|
||||
const fields = toEvidenceFields(diResult.fields);
|
||||
const lineItems = Object.values(diResult.arrayFields).flatMap((items) => items.map((item) => toEvidenceFields(item)));
|
||||
|
||||
const foundCount = Object.values(fields).filter((f) => f.found).length;
|
||||
// Average only over FOUND fields — a not-found field's confidence (e.g. "31% sure this
|
||||
// region isn't a date after all") describes DI's certainty about absence, not a value,
|
||||
// and must not drag down the confidence of the values that were actually extracted.
|
||||
const confidences = Object.values(fields)
|
||||
.filter((f) => f.found)
|
||||
.map((f) => f.confidence)
|
||||
.filter((c): c is number => c != null);
|
||||
const averageConfidence = confidences.length ? confidences.reduce((a, b) => a + b, 0) / confidences.length : null;
|
||||
|
||||
// Persist the extraction result as an artifact too — same interim-storage gap as
|
||||
// document-capture-service.ts (see that file's header for the full explanation).
|
||||
const payloadJson = Buffer.from(
|
||||
JSON.stringify({ model_id: ctx.model_id, fields, line_items: lineItems, content: diResult.content }),
|
||||
"utf8",
|
||||
);
|
||||
const bucket = "0pt-document-extractions";
|
||||
const key = `${ctx.tenant_id}/${ctx.project_id}/${documentId}/${extractionId}.json`;
|
||||
const written = await this.blobStore.write(bucket, key, payloadJson);
|
||||
const artifact = await this.st0re.createArtifact({
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
artifact_type: "document_extraction",
|
||||
app_id: "0pt",
|
||||
produced_by: "0pt.extract_document",
|
||||
object_location: { provider: written.provider, bucket: written.bucket, key: written.key, region: "local" },
|
||||
content_type: "application/json",
|
||||
size_bytes: written.sizeBytes,
|
||||
checksum_sha256: crypto.createHash("sha256").update(payloadJson).digest("hex"),
|
||||
pii_level: "medium",
|
||||
retention_policy: "document_extraction_standard",
|
||||
data_residency: "eu",
|
||||
risk_level: "L3",
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
record = {
|
||||
...record,
|
||||
status: "completed",
|
||||
fields,
|
||||
line_items: lineItems,
|
||||
fields_found_count: foundCount,
|
||||
fields_total_count: Object.keys(fields).length,
|
||||
average_confidence: averageConfidence,
|
||||
document_confidence: diResult.documentConfidence,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
this.extractions.set(extractionId, record);
|
||||
this.captureService.setStatus(documentId, "extracted");
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_document_extracted_${extractionId}`,
|
||||
event_type: "0pt.document_extracted",
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
actor_id: ctx.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.extract_document",
|
||||
resource_type: "document_extraction",
|
||||
resource_id: extractionId,
|
||||
risk_level: "L3",
|
||||
payload: {
|
||||
document_id: documentId,
|
||||
model_id: ctx.model_id,
|
||||
fields_found_count: foundCount,
|
||||
fields_total_count: Object.keys(fields).length,
|
||||
average_confidence: averageConfidence,
|
||||
st0re_artifact_id: artifact.artifact_id,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_document_extracted_${extractionId}`,
|
||||
tenant_id: ctx.tenant_id,
|
||||
project_id: ctx.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.extract_document",
|
||||
source_type: "document",
|
||||
source_id: documentId,
|
||||
target_type: "document_extraction",
|
||||
target_id: extractionId,
|
||||
lineage_type: "derived_from",
|
||||
metadata: { model_id: ctx.model_id, st0re_artifact_id: artifact.artifact_id },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: ctx.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
} catch (e: unknown) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
|
||||
this.extractions.set(extractionId, record);
|
||||
try {
|
||||
this.captureService.setStatus(documentId, "extraction_failed");
|
||||
} catch {
|
||||
/* document may already be gone in edge-case tests — extraction failure record still stands */
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async getExtraction(extractionId: string): Promise<DocumentExtractionRecord> {
|
||||
const record = this.extractions.get(extractionId);
|
||||
if (!record) throw new Error(`Document extraction ${extractionId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async listExtractionsForDocument(documentId: string): Promise<DocumentExtractionRecord[]> {
|
||||
return Array.from(this.extractions.values()).filter((r) => r.document_id === documentId);
|
||||
}
|
||||
}
|
||||
|
||||
function buildDocumentIntelligenceConfigFromEnv(): DocumentIntelligenceConfig | undefined {
|
||||
const endpoint = process.env.DOC_INTELLIGENCE_ENDPOINT;
|
||||
const apiKey = process.env.DOC_INTELLIGENCE_KEY;
|
||||
if (!endpoint || !apiKey) return undefined;
|
||||
return { endpoint, apiKey, apiVersion: process.env.DOC_INTELLIGENCE_API_VERSION };
|
||||
}
|
||||
|
||||
/** Shared singleton (see document-capture-service.ts "Singleton note") wired to the shared capture service. */
|
||||
export const documentExtractionService = new DocumentExtractionService(documentCaptureService, {
|
||||
documentIntelligence: buildDocumentIntelligenceConfigFromEnv(),
|
||||
});
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* src/services/external-writeback-service.ts
|
||||
* =============================================
|
||||
* `0pt.write_external_record` -- "POST a validated, already-approved
|
||||
* structured record to a registered external-system adapter." Generic in its
|
||||
* core: this service has no Odoo/invoice/Protosell-specific logic anywhere
|
||||
* in it -- it resolves an `adapter_id` to a registered `WritebackAdapter`
|
||||
* (../adapters/writeback-adapter.ts) and delegates the actual write to it.
|
||||
* `../adapters/odoo-bills-adapter.ts`'s OdooBillsAdapter is ONE such
|
||||
* adapter, registered by a caller the same way any other adapter would be --
|
||||
* this service never imports it directly.
|
||||
*
|
||||
* WHY THIS EXISTS / WHERE IT SITS IN THE CHAIN: `act0/src/
|
||||
* approval-writeback-templates.ts`'s generic `0pt_document_approval_then_
|
||||
* external_write` Template chains a captured+extracted document
|
||||
* (0pt.document_ocr's real terminal event, `0pt.document_extracted`) through
|
||||
* an Approval Engine gate, THEN calls this capability. By the time
|
||||
* `writeExternalRecord` runs, a human has already approved the write -- this
|
||||
* service still re-checks CL0UD policy (per docs/PRODUCT_PLAN.md section
|
||||
* 4.1's hard rule that CL0UD is always re-consulted, the same rule
|
||||
* quality-check-service.ts/document-extraction-service.ts already follow),
|
||||
* but does not itself re-verify the upstream Approval record (ACT0 owns
|
||||
* approvals; 0pt's own capability boundary is "did CL0UD say yes for THIS
|
||||
* call", same separation of concerns every other 0pt service in this repo
|
||||
* already draws).
|
||||
*
|
||||
* IDEMPOTENCY (per this task's own instruction not to regress the proven
|
||||
* pattern ported from v0-datahub's `lib/odoo/invoice-bridge.ts`
|
||||
* `processInboxToOdoo`): that original code CLAIMS a row (an atomic
|
||||
* Postgres `UPDATE ... WHERE odoo_move_id IS NULL AND odoo_state != 'posting'`)
|
||||
* BEFORE ever calling the external system, so two concurrent triggers (a
|
||||
* manual approve racing a cron, or a retried ACT0 step racing itself) cannot
|
||||
* both post the same record. This service ports that SAME "claim before you
|
||||
* call out" shape forward as a generic, adapter-agnostic in-memory claim,
|
||||
* keyed on `(adapter_id, idempotency_key)`:
|
||||
* - the claim check-and-set happens SYNCHRONOUSLY, before this function's
|
||||
* first `await` -- JavaScript's single-threaded run-to-first-await
|
||||
* semantics make this exactly as race-free, within one process, as the
|
||||
* original's atomic SQL UPDATE was across concurrent DB clients (see
|
||||
* `writeExternalRecord`'s own inline comment for exactly where the line
|
||||
* is drawn, and tests/p0/external-writeback-service.test.ts's own
|
||||
* concurrency test, which calls `writeExternalRecord` twice with the
|
||||
* same idempotency key via `Promise.all` and asserts the underlying
|
||||
* adapter's `write` is invoked exactly once).
|
||||
* - HONEST SCOPE: this in-memory claim is process-local -- it does not
|
||||
* survive a restart and does not protect a multi-instance deployment,
|
||||
* unlike the original's real Postgres row lock. An adapter with its own
|
||||
* persistent storage (OdooBillsAdapter) additionally refuses to
|
||||
* re-post a record whose payload already carries an `existing_external_id`
|
||||
* -- see WritebackAdapter.write's own doc comment for why this
|
||||
* second layer matters and odoo-bills-adapter.ts for exactly how it is
|
||||
* ported. A production deployment backing this service with real
|
||||
* persistent storage (same "pure core, thin adapter" doctrine every
|
||||
* other 0pt service in this repo already documents) should replace this
|
||||
* in-memory Map with a real claim row, at which point the SAME
|
||||
* synchronous-claim-before-any-await shape still applies, now backed by
|
||||
* a real transaction instead of a JS Map.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { createCl0udClient, type Cl0udClient } from "../clients/cl0ud-client";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import type { WritebackAdapter } from "../adapters/writeback-adapter";
|
||||
import type { ExternalWriteRecord, ExternalWriteRequest } from "../types/external-writeback";
|
||||
|
||||
export interface ExternalWritebackServiceOptions {
|
||||
cl0udBaseUrl?: string;
|
||||
aud0BaseUrl?: string;
|
||||
adapters?: readonly WritebackAdapter[];
|
||||
}
|
||||
|
||||
export class ExternalWritebackService {
|
||||
private writes: Map<string, ExternalWriteRecord> = new Map();
|
||||
/** `${adapter_id}::${idempotency_key}` -> write_id. See this module's own doc comment's
|
||||
* "IDEMPOTENCY" section for exactly what this does and does not guarantee. */
|
||||
private claimedByIdempotencyKey: Map<string, string> = new Map();
|
||||
private adapters: Map<string, WritebackAdapter> = new Map();
|
||||
private cl0ud: Cl0udClient;
|
||||
private aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: ExternalWritebackServiceOptions) {
|
||||
this.cl0ud = createCl0udClient(opts?.cl0udBaseUrl);
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
for (const adapter of opts?.adapters ?? []) {
|
||||
this.registerAdapter(adapter);
|
||||
}
|
||||
}
|
||||
|
||||
registerAdapter(adapter: WritebackAdapter): void {
|
||||
this.adapters.set(adapter.adapterId, adapter);
|
||||
}
|
||||
|
||||
getAdapter(adapterId: string): WritebackAdapter {
|
||||
const adapter = this.adapters.get(adapterId);
|
||||
if (!adapter) throw new Error(`ExternalWritebackService: no adapter registered for '${adapterId}'`);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
async writeExternalRecord(request: ExternalWriteRequest): Promise<ExternalWriteRecord> {
|
||||
const adapter = this.adapters.get(request.adapter_id);
|
||||
if (!adapter) {
|
||||
throw new Error(`ExternalWritebackService: no adapter registered for '${request.adapter_id}'`);
|
||||
}
|
||||
if (!adapter.isConfigured()) {
|
||||
throw new Error(
|
||||
`ExternalWritebackService: adapter '${request.adapter_id}' is not configured — refusing to fabricate a write`,
|
||||
);
|
||||
}
|
||||
|
||||
// --- SYNCHRONOUS claim section: no `await` above this line, and none until the claim is
|
||||
// set below. This is what makes the claim race-free under this module's own doc comment's
|
||||
// "IDEMPOTENCY" doctrine -- see that comment for the full explanation and its own scope
|
||||
// caveat (process-local only).
|
||||
const claimKey = `${request.adapter_id}::${request.idempotency_key}`;
|
||||
const existingWriteId = this.claimedByIdempotencyKey.get(claimKey);
|
||||
if (existingWriteId) {
|
||||
const existing = this.writes.get(existingWriteId);
|
||||
if (existing && existing.status === "written") {
|
||||
return existing; // already posted -- idempotent no-op, matches invoice-bridge.ts's own
|
||||
// "Redan bokförd i Odoo... hoppar över" early return.
|
||||
}
|
||||
if (existing && existing.status === "claimed") {
|
||||
// A write for this exact idempotency key is already in flight (or, in this in-memory
|
||||
// reference implementation, was left 'claimed' by a caller that never awaited the
|
||||
// result) -- do not race a second POST to the external system. Matches
|
||||
// invoice-bridge.ts's own "Redan under postning eller bokförd — hoppar över" claim-
|
||||
// conflict response.
|
||||
return { ...existing, status: "skipped_duplicate" };
|
||||
}
|
||||
// existing.status === 'failed' (or the record is missing for any other reason): fall
|
||||
// through and re-claim -- a failed write is safe and expected to retry, matching
|
||||
// invoice-bridge.ts's own "Release the 'posting' claim... can be retried later" behavior.
|
||||
}
|
||||
|
||||
const writeId = `ewr_${crypto.randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
let record: ExternalWriteRecord = {
|
||||
write_id: writeId,
|
||||
adapter_id: request.adapter_id,
|
||||
idempotency_key: request.idempotency_key,
|
||||
record_type: request.record_type,
|
||||
status: "claimed",
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
correlation_id: request.correlation_id,
|
||||
created_at: now,
|
||||
};
|
||||
this.writes.set(writeId, record);
|
||||
this.claimedByIdempotencyKey.set(claimKey, writeId);
|
||||
// --- End synchronous claim section. Every branch below is async and safe to interleave. ---
|
||||
|
||||
try {
|
||||
const policy = await this.cl0ud.evaluatePolicy({
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
capability_id: "0pt.write_external_record",
|
||||
resource_type: "external_write",
|
||||
resource_id: writeId,
|
||||
risk_level: "L4",
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
if (policy.decision === "deny") {
|
||||
throw new Error(`Policy denied: ${policy.reason}`);
|
||||
}
|
||||
if (policy.decision === "approval_required") {
|
||||
// Per this service's own module doc comment: the ACT0 Approval Engine gate that
|
||||
// precedes this capability in the approval-writeback-templates.ts chain is a SEPARATE
|
||||
// mechanism from CL0UD's own policy decision here -- if CL0UD independently still says
|
||||
// approval_required for this specific call, this service does not silently bypass it
|
||||
// (same "never lie about what happened" doctrine every other 0pt service in this repo
|
||||
// follows for a denied/approval_required decision).
|
||||
throw new Error(`Policy requires approval: ${policy.reason}`);
|
||||
}
|
||||
|
||||
const result = await adapter.write(
|
||||
{ recordType: request.record_type, payload: request.payload, idempotencyKey: request.idempotency_key },
|
||||
{
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
correlation_id: request.correlation_id,
|
||||
},
|
||||
);
|
||||
if (!result.ok) {
|
||||
throw new Error(result.error ?? "adapter_write_failed");
|
||||
}
|
||||
|
||||
record = { ...record, status: "written", external_id: result.externalId, updated_at: new Date().toISOString() };
|
||||
this.writes.set(writeId, record);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_external_record_written_${writeId}`,
|
||||
event_type: "0pt.external_record_written",
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
actor_id: request.actor_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_external_record",
|
||||
resource_type: "external_write",
|
||||
resource_id: writeId,
|
||||
risk_level: "L4",
|
||||
payload: {
|
||||
adapter_id: request.adapter_id,
|
||||
record_type: request.record_type,
|
||||
idempotency_key: request.idempotency_key,
|
||||
external_id: result.externalId,
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
|
||||
await this.aud0.ingestLineageEvent({
|
||||
lineage_event_id: `lineage_0pt_external_write_${writeId}`,
|
||||
tenant_id: request.tenant_id,
|
||||
project_id: request.project_id,
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.write_external_record",
|
||||
// `idempotency_key` is, by convention (not enforced by this generic service), the
|
||||
// upstream record's stable id -- e.g. a document_extraction_id when this write was
|
||||
// triggered by act0's document_approval_then_external_write template.
|
||||
source_type: "document_extraction",
|
||||
source_id: request.idempotency_key,
|
||||
target_type: "external_record",
|
||||
target_id: result.externalId ?? writeId,
|
||||
lineage_type: "produced",
|
||||
metadata: { adapter_id: request.adapter_id, record_type: request.record_type },
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: request.correlation_id,
|
||||
});
|
||||
|
||||
return record;
|
||||
} catch (e: unknown) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
record = { ...record, status: "failed", error: message, updated_at: new Date().toISOString() };
|
||||
this.writes.set(writeId, record);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async getWrite(writeId: string): Promise<ExternalWriteRecord> {
|
||||
const record = this.writes.get(writeId);
|
||||
if (!record) throw new Error(`External write ${writeId} not found`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async findByIdempotencyKey(adapterId: string, idempotencyKey: string): Promise<ExternalWriteRecord | undefined> {
|
||||
const writeId = this.claimedByIdempotencyKey.get(`${adapterId}::${idempotencyKey}`);
|
||||
return writeId ? this.writes.get(writeId) : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared singleton, same "cross-route in-memory state" convention document-capture-service.ts's
|
||||
* own header documents (this service's writes/claims must be visible across whatever API routes
|
||||
* call writeExternalRecord vs getWrite) -- no adapters pre-registered; a caller (an API route,
|
||||
* a test, or a future 0pt/api/documents/writeback route) registers whichever adapters it needs
|
||||
* via registerAdapter before calling writeExternalRecord. */
|
||||
export const externalWritebackService = new ExternalWritebackService();
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* src/services/source-query-service.ts
|
||||
* =======================================
|
||||
* 0pt.query_source — bounded, timeout-guarded synchronous query execution
|
||||
* against a registered Source (source_type = "database",
|
||||
* adapter_id = "fabric_onelake_sql" today; extensible via `ADAPTER_REGISTRY`).
|
||||
*
|
||||
* This is deliberately NOT a general-purpose query engine: it enforces a
|
||||
* max-rows and max-time ceiling on every call and is meant for small,
|
||||
* interactive/preview-style reads. Anything heavier (large exports, long
|
||||
* scans, scheduled pulls) must go through N0D as a job — see
|
||||
* docs/P0_QUERY_SOURCE_FLOW.md — the same "synchronous vs. N0D job" split
|
||||
* `quality-check-service.ts` uses for quality checks.
|
||||
*
|
||||
* Reuses the existing SourceService (src/services/source-service.ts) purely
|
||||
* through its public `getSource()` — this file does not modify that class or
|
||||
* the Source/CreateSourceInput types, which already carry the `adapter_id`
|
||||
* field this capability depends on.
|
||||
*/
|
||||
|
||||
import type { Source } from "../types/source";
|
||||
import { SourceService } from "./source-service";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { FabricOneLakeAdapter, type FabricOneLakeConnectionConfig, type AdapterQueryResult } from "../adapters/fabric-onelake-adapter";
|
||||
|
||||
/** Default bound applied when the caller does not specify one. */
|
||||
export const SOURCE_QUERY_DEFAULT_MAX_ROWS = 1000;
|
||||
/** Absolute ceiling — a caller cannot raise max_rows above this on the synchronous path. */
|
||||
export const SOURCE_QUERY_HARD_MAX_ROWS = 10_000;
|
||||
/** Default timeout applied when the caller does not specify one. */
|
||||
export const SOURCE_QUERY_DEFAULT_TIMEOUT_MS = 10_000;
|
||||
/** Absolute ceiling — a caller cannot raise timeout_ms above this on the synchronous path. */
|
||||
export const SOURCE_QUERY_HARD_MAX_TIMEOUT_MS = 30_000;
|
||||
|
||||
/** Adapter ids supported by the synchronous query path today. */
|
||||
export type SupportedQueryAdapterId = "fabric_onelake_sql";
|
||||
|
||||
export class SourceQueryTimeoutError extends Error {}
|
||||
export class SourceQueryUnsupportedError extends Error {}
|
||||
|
||||
export interface QuerySourceInput {
|
||||
source_id: string;
|
||||
query: string;
|
||||
params?: Record<string, unknown>;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id?: string;
|
||||
correlation_id?: string;
|
||||
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_ROWS]. Defaults to SOURCE_QUERY_DEFAULT_MAX_ROWS. */
|
||||
max_rows?: number;
|
||||
/** Clamped to [1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS]. Defaults to SOURCE_QUERY_DEFAULT_TIMEOUT_MS. */
|
||||
timeout_ms?: number;
|
||||
/**
|
||||
* Connection config for the source's adapter. Never persisted by this
|
||||
* service and never defaulted from an env var — tenant-agnostic by
|
||||
* construction, same as the adapter itself. Today only the
|
||||
* "fabric_onelake_sql" shape is supported.
|
||||
*/
|
||||
connection_config: FabricOneLakeConnectionConfig;
|
||||
}
|
||||
|
||||
export interface QuerySourceResult {
|
||||
source_id: string;
|
||||
rows: Record<string, unknown>[];
|
||||
row_count: number;
|
||||
truncated: boolean;
|
||||
duration_ms: number;
|
||||
bounds: { max_rows: number; timeout_ms: number };
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
if (!Number.isFinite(value)) return min;
|
||||
return Math.min(Math.max(value, min), max);
|
||||
}
|
||||
|
||||
export class SourceQueryService {
|
||||
private sourceService: SourceService;
|
||||
private aud0: Aud0Client;
|
||||
|
||||
constructor(opts?: { sourceService?: SourceService; aud0BaseUrl?: string }) {
|
||||
// Accepts an existing SourceService so callers (e.g. the API route) share
|
||||
// one in-memory registry instead of each constructing an empty one.
|
||||
this.sourceService = opts?.sourceService ?? new SourceService();
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
}
|
||||
|
||||
async querySource(input: QuerySourceInput): Promise<QuerySourceResult> {
|
||||
const startedAt = Date.now();
|
||||
const source = await this.getQueryableSource(input.source_id);
|
||||
|
||||
const maxRows = clamp(input.max_rows ?? SOURCE_QUERY_DEFAULT_MAX_ROWS, 1, SOURCE_QUERY_HARD_MAX_ROWS);
|
||||
const timeoutMs = clamp(input.timeout_ms ?? SOURCE_QUERY_DEFAULT_TIMEOUT_MS, 1, SOURCE_QUERY_HARD_MAX_TIMEOUT_MS);
|
||||
const correlationId = input.correlation_id ?? `corr_0pt_query_source_${input.source_id}_${startedAt}`;
|
||||
|
||||
const adapter = this.buildAdapter(source, input.connection_config);
|
||||
try {
|
||||
const { rows, durationMs } = await this.runWithTimeout(adapter.query(input.query, input.params), timeoutMs);
|
||||
|
||||
const truncated = rows.length > maxRows;
|
||||
const boundedRows = truncated ? rows.slice(0, maxRows) : rows;
|
||||
|
||||
await this.emitQueriedEvent({
|
||||
source,
|
||||
input,
|
||||
correlationId,
|
||||
rowCount: boundedRows.length,
|
||||
truncated,
|
||||
durationMs,
|
||||
ok: true,
|
||||
});
|
||||
|
||||
return {
|
||||
source_id: source.source_id,
|
||||
rows: boundedRows,
|
||||
row_count: boundedRows.length,
|
||||
truncated,
|
||||
duration_ms: durationMs,
|
||||
bounds: { max_rows: maxRows, timeout_ms: timeoutMs },
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
await this.emitQueriedEvent({
|
||||
source,
|
||||
input,
|
||||
correlationId,
|
||||
rowCount: 0,
|
||||
truncated: false,
|
||||
durationMs: Date.now() - startedAt,
|
||||
ok: false,
|
||||
error: message,
|
||||
});
|
||||
throw err instanceof Error ? err : new Error(message);
|
||||
} finally {
|
||||
// Each call opens its own short-lived adapter/pool (see buildAdapter) —
|
||||
// always release it, whether the query succeeded, failed, or timed out.
|
||||
await adapter.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private async getQueryableSource(sourceId: string): Promise<Source> {
|
||||
const source = await this.sourceService.getSource(sourceId);
|
||||
if (source.source_type !== "database") {
|
||||
throw new SourceQueryUnsupportedError(
|
||||
`Source ${sourceId} has source_type "${source.source_type}", not "database" — the synchronous 0pt.query_source path only ` +
|
||||
`supports database sources. Read non-database sources through a N0D job instead.`,
|
||||
);
|
||||
}
|
||||
if (source.adapter_id !== "fabric_onelake_sql") {
|
||||
throw new SourceQueryUnsupportedError(
|
||||
`Source ${sourceId} has adapter_id "${source.adapter_id ?? "(none)"}" — the synchronous 0pt.query_source path currently only ` +
|
||||
`supports adapter_id "fabric_onelake_sql". Register the source with that adapter_id, or run the read through N0D as a job instead.`,
|
||||
);
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
private buildAdapter(source: Source, connectionConfig: FabricOneLakeConnectionConfig): FabricOneLakeAdapter {
|
||||
if (!connectionConfig) {
|
||||
throw new Error(`Source ${source.source_id}: connection_config is required to query a fabric_onelake_sql source`);
|
||||
}
|
||||
return new FabricOneLakeAdapter(connectionConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Races the adapter query against a timeout. Note this bounds *waiting*
|
||||
* time on the caller's side — it does not cancel an in-flight TDS request
|
||||
* on timeout (the `mssql` driver has no cooperative cancellation hook used
|
||||
* here), so a timed-out query may still complete against the warehouse
|
||||
* after this call has already thrown. That is an accepted trade-off for a
|
||||
* synchronous, best-effort preview path; anything where that matters
|
||||
* belongs on the N0D job path instead.
|
||||
*/
|
||||
private async runWithTimeout<T>(promise: Promise<AdapterQueryResult<T>>, timeoutMs: number): Promise<AdapterQueryResult<T>> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(
|
||||
new SourceQueryTimeoutError(
|
||||
`source query exceeded the bounded synchronous timeout of ${timeoutMs}ms — run this query through N0D as a job instead of the synchronous 0pt.query_source path`,
|
||||
),
|
||||
);
|
||||
}, timeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async emitQueriedEvent(args: {
|
||||
source: Source;
|
||||
input: QuerySourceInput;
|
||||
correlationId: string;
|
||||
rowCount: number;
|
||||
truncated: boolean;
|
||||
durationMs: number;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
}): Promise<void> {
|
||||
const { source, input, correlationId, rowCount, truncated, durationMs, ok, error } = args;
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_source_queried_${source.source_id}_${Date.now()}`,
|
||||
event_type: "0pt.source_queried",
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
actor_id: input.actor_id ?? "system",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.query_source",
|
||||
resource_type: "source",
|
||||
resource_id: source.source_id,
|
||||
risk_level: "L4",
|
||||
payload: {
|
||||
adapter_id: source.adapter_id,
|
||||
ok,
|
||||
row_count: rowCount,
|
||||
truncated,
|
||||
duration_ms: durationMs,
|
||||
query_preview: input.query.slice(0, 200),
|
||||
...(error ? { error } : {}),
|
||||
},
|
||||
created_at: new Date().toISOString(),
|
||||
correlation_id: correlationId,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { ExtractedField } from "../adapters/document-intelligence-adapter";
|
||||
|
||||
export type { ExtractedField };
|
||||
|
||||
export type DocumentExtractionStatus = "requested" | "completed" | "failed";
|
||||
|
||||
export interface DocumentExtractionRecord {
|
||||
extraction_id: string;
|
||||
document_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
model_id: string;
|
||||
status: DocumentExtractionStatus;
|
||||
/** Every key DI reported, evidence-based: never fabricated, always carries a confidence. */
|
||||
fields: Record<string, ExtractedField>;
|
||||
line_items: Array<Record<string, ExtractedField>>;
|
||||
fields_found_count: number;
|
||||
fields_total_count: number;
|
||||
average_confidence: number | null;
|
||||
document_confidence: number | null;
|
||||
st0re_artifact_id?: string;
|
||||
n0d_job_id?: string;
|
||||
error?: string;
|
||||
correlation_id?: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
export type DocumentStatus =
|
||||
| "captured"
|
||||
| "extraction_requested"
|
||||
| "extracted"
|
||||
| "extraction_failed";
|
||||
|
||||
export interface DocumentRecord {
|
||||
document_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
mailbox_address: string;
|
||||
graph_message_id: string;
|
||||
graph_attachment_id: string;
|
||||
filename?: string;
|
||||
content_type?: string;
|
||||
size_bytes?: number;
|
||||
received_at?: string;
|
||||
sender_email?: string;
|
||||
sender_name?: string;
|
||||
subject?: string;
|
||||
st0re_artifact_id?: string;
|
||||
n0d_job_id?: string;
|
||||
status: DocumentStatus;
|
||||
correlation_id?: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* src/types/external-writeback.ts
|
||||
* =================================
|
||||
* Types for `0pt.write_external_record` -- "POST a validated, already-
|
||||
* approved structured record to a registered external-system adapter."
|
||||
* See ../services/external-writeback-service.ts for the service that
|
||||
* implements this capability and ../adapters/writeback-adapter.ts for the
|
||||
* adapter interface any external system plugs into.
|
||||
*/
|
||||
|
||||
export type ExternalWriteStatus = "claimed" | "written" | "failed" | "skipped_duplicate";
|
||||
|
||||
/**
|
||||
* One request to write a record to an external system. Deliberately generic
|
||||
* -- nothing here is Odoo/invoice/Protosell-shaped. `payload` is whatever
|
||||
* structured record the caller (e.g. an ACT0 workflow's 'write_record' step,
|
||||
* or a direct API call) has already validated/approved; the registered
|
||||
* adapter for `adapter_id` is solely responsible for interpreting it.
|
||||
*/
|
||||
export interface ExternalWriteRequest {
|
||||
/** Which registered WritebackAdapter (../adapters/writeback-adapter.ts) handles this write,
|
||||
* e.g. "odoo_bills". Looked up via ExternalWritebackService.registerAdapter. */
|
||||
adapter_id: string;
|
||||
/**
|
||||
* Caller-supplied stable key that is the SAME across retries of "the same logical write" --
|
||||
* e.g. a document_extraction_id, or an upstream system's own row id. This is what makes
|
||||
* `writeExternalRecord` idempotent: calling it twice with the same (adapter_id,
|
||||
* idempotency_key) pair never produces two external writes. See
|
||||
* ExternalWritebackService's own module doc comment for the exact claim mechanism this
|
||||
* ports forward from v0-datahub's proven invoice-bridge.ts Postgres-row CLAIM pattern.
|
||||
*/
|
||||
idempotency_key: string;
|
||||
/** Free-form record-type tag the adapter interprets, e.g. "vendor_bill". Not validated by
|
||||
* the generic service -- purely forwarded to the adapter. */
|
||||
record_type: string;
|
||||
/** The structured record itself -- adapter-specific shape, opaque to the generic service. */
|
||||
payload: Readonly<Record<string, unknown>>;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
}
|
||||
|
||||
/** Persisted record of one write attempt (or claim) -- the generic service's own bookkeeping,
|
||||
* independent of whatever the adapter itself may also persist. */
|
||||
export interface ExternalWriteRecord {
|
||||
write_id: string;
|
||||
adapter_id: string;
|
||||
idempotency_key: string;
|
||||
record_type: string;
|
||||
status: ExternalWriteStatus;
|
||||
/** The external system's own id for the created record (e.g. an Odoo account.move id),
|
||||
* populated once status is "written". */
|
||||
external_id?: string;
|
||||
error?: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
actor_id: string;
|
||||
correlation_id: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import { DocumentCaptureService } from "@/src/services/document-capture-service";
|
||||
import { LocalBlobStore } from "@/src/adapters/local-blob-store";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number; headers?: Record<string, string> } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null },
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe("DocumentCaptureService.captureFromMailbox", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "0pt-blob-"));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("downloads PDF attachments, persists bytes, registers a ST0RE artifact, enqueues an N0D extraction job, and emits AUD0 events", async () => {
|
||||
const pdfBytes = Buffer.from("%PDF-1.4 real bytes for test");
|
||||
const pdfBase64 = pdfBytes.toString("base64");
|
||||
const aud0Calls: { url: string; body: any }[] = [];
|
||||
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 });
|
||||
if (url.includes("/messages?")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
id: "msg-1",
|
||||
receivedDateTime: "2026-07-01T09:00:00Z",
|
||||
from: { emailAddress: { name: "Sender", address: "sender@example.com" } },
|
||||
toRecipients: [],
|
||||
subject: "A document",
|
||||
bodyPreview: "preview",
|
||||
hasAttachments: true,
|
||||
webLink: "https://outlook.office.com/mail/1",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (url.includes("/attachments")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "att-1",
|
||||
name: "doc.pdf",
|
||||
contentType: "application/pdf",
|
||||
size: pdfBytes.length,
|
||||
contentBytes: pdfBase64,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) {
|
||||
aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null });
|
||||
return jsonResponse({});
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const service = new DocumentCaptureService({
|
||||
blobStore: new LocalBlobStore(tmpDir),
|
||||
aud0BaseUrl: "https://aud0.example.com",
|
||||
});
|
||||
const result = await service.captureFromMailbox(
|
||||
{ mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } },
|
||||
{ tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" },
|
||||
);
|
||||
|
||||
expect(result.scanned_messages).toBe(1);
|
||||
expect(result.captured_documents).toHaveLength(1);
|
||||
const doc = result.captured_documents[0];
|
||||
expect(doc.status).toBe("extraction_requested");
|
||||
expect(doc.filename).toBe("doc.pdf");
|
||||
expect(doc.mailbox_address).toBe("docs@customer.com");
|
||||
expect(doc.st0re_artifact_id).toContain("mock_artifact_");
|
||||
expect(doc.n0d_job_id).toContain("mock_job_");
|
||||
|
||||
// Bytes were actually persisted (INTERIM local store) and are readable back byte-for-byte.
|
||||
const storedBytes = await service.getDocumentBytes(doc.document_id);
|
||||
expect(storedBytes.equals(pdfBytes)).toBe(true);
|
||||
|
||||
const captured = aud0Calls.find((c) => c.body?.event_type === "0pt.document_captured");
|
||||
expect(captured).toBeTruthy();
|
||||
expect(captured!.body.payload.filename).toBe("doc.pdf");
|
||||
expect(captured!.body.capability_id).toBe("0pt.capture_document");
|
||||
const lineage = aud0Calls.find((c) => c.url.includes("lineage"));
|
||||
expect(lineage).toBeTruthy();
|
||||
expect(lineage!.body.target_id).toBe(doc.document_id);
|
||||
});
|
||||
|
||||
it("skips non-PDF attachments and records per-message errors without aborting the whole run", async () => {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 });
|
||||
if (url.includes("/messages?")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
id: "msg-ok",
|
||||
receivedDateTime: "2026-07-01T09:00:00Z",
|
||||
hasAttachments: true,
|
||||
from: { emailAddress: {} },
|
||||
toRecipients: [],
|
||||
},
|
||||
{
|
||||
id: "msg-bad",
|
||||
receivedDateTime: "2026-07-01T09:05:00Z",
|
||||
hasAttachments: true,
|
||||
from: { emailAddress: {} },
|
||||
toRecipients: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (url.includes("msg-ok/attachments")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "a1",
|
||||
name: "x.png",
|
||||
contentType: "image/png",
|
||||
contentBytes: Buffer.from("x").toString("base64"),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (url.includes("msg-bad/attachments")) {
|
||||
return { ok: false, status: 500, headers: { get: () => null }, json: async () => ({ error: { code: "Fail", message: "boom" } }) } as unknown as Response;
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const service = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) });
|
||||
const result = await service.captureFromMailbox(
|
||||
{ mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } },
|
||||
{ tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" },
|
||||
);
|
||||
expect(result.scanned_messages).toBe(2);
|
||||
expect(result.captured_documents).toHaveLength(0); // png skipped, msg-bad errored
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0]).toContain("msg-bad");
|
||||
});
|
||||
|
||||
it("rejects when mailbox credentials are incomplete", async () => {
|
||||
const service = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) });
|
||||
await expect(
|
||||
service.captureFromMailbox(
|
||||
{ mailbox: { tenantId: "t", clientId: "", clientSecret: "s", mailboxAddress: "docs@customer.com" } },
|
||||
{ tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr1" },
|
||||
),
|
||||
).rejects.toThrow(/missing mailbox/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
import { DocumentCaptureService } from "@/src/services/document-capture-service";
|
||||
import { DocumentExtractionService } from "@/src/services/document-extraction-service";
|
||||
import { LocalBlobStore } from "@/src/adapters/local-blob-store";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number; headers?: Record<string, string> } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null },
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe("DocumentExtractionService — evidence-based extraction", () => {
|
||||
let tmpDir: string;
|
||||
let aud0Calls: { url: string; body: any }[];
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "0pt-blob-"));
|
||||
aud0Calls = [];
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function seedCapturedDocument(captureService: DocumentCaptureService) {
|
||||
const pdfBytes = Buffer.from("%PDF-1.4 fake invoice pdf");
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes("login.microsoftonline.com")) return jsonResponse({ access_token: "tok", expires_in: 3600 });
|
||||
if (url.includes("/messages?")) {
|
||||
return jsonResponse({
|
||||
value: [{ id: "msg-1", receivedDateTime: "2026-07-01T09:00:00Z", hasAttachments: true, from: { emailAddress: {} }, toRecipients: [] }],
|
||||
});
|
||||
}
|
||||
if (url.includes("/attachments")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "att-1",
|
||||
name: "doc.pdf",
|
||||
contentType: "application/pdf",
|
||||
contentBytes: pdfBytes.toString("base64"),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected fetch during seed: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const result = await captureService.captureFromMailbox(
|
||||
{ mailbox: { tenantId: "t", clientId: "c", clientSecret: "s", mailboxAddress: "docs@customer.com" } },
|
||||
{ tenant_id: "tenant1", project_id: "proj1", actor_id: "actor1", correlation_id: "corr-seed" },
|
||||
);
|
||||
return result.captured_documents[0];
|
||||
}
|
||||
|
||||
it("never fabricates a value for a field Document Intelligence did not return, and preserves DI's confidence either way", async () => {
|
||||
const captureService = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) });
|
||||
const document = await seedCapturedDocument(captureService);
|
||||
|
||||
const extractionService = new DocumentExtractionService(captureService, {
|
||||
blobStore: new LocalBlobStore(tmpDir),
|
||||
aud0BaseUrl: "https://aud0.example.com",
|
||||
documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 },
|
||||
});
|
||||
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (url.includes(":analyze")) {
|
||||
return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/abc" } });
|
||||
}
|
||||
if (url.includes("/op/abc")) {
|
||||
return jsonResponse({
|
||||
status: "succeeded",
|
||||
analyzeResult: {
|
||||
content: "Acme AB\nTotal: 1234.50 SEK",
|
||||
pages: [{}],
|
||||
documents: [
|
||||
{
|
||||
confidence: 0.9,
|
||||
fields: {
|
||||
VendorName: { type: "string", valueString: "Acme AB", confidence: 0.97 },
|
||||
InvoiceTotal: { type: "currency", valueCurrency: { amount: 1234.5 }, confidence: 0.92 },
|
||||
// DueDate was detected as a field region but DI could not read a value —
|
||||
// this is exactly the case that must NOT be fabricated into a fake date.
|
||||
DueDate: { type: "date", confidence: 0.31 },
|
||||
// CustomerAddress is not present in the DI response at all.
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) {
|
||||
aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null });
|
||||
return jsonResponse({});
|
||||
}
|
||||
throw new Error(`unexpected fetch during extraction: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const extraction = await extractionService.extractDocument(document.document_id, {
|
||||
tenant_id: "tenant1",
|
||||
project_id: "proj1",
|
||||
actor_id: "actor1",
|
||||
correlation_id: "corr-extract",
|
||||
model_id: "prebuilt-invoice",
|
||||
});
|
||||
|
||||
expect(extraction.status).toBe("completed");
|
||||
// Found fields carry the real value + DI's own confidence.
|
||||
expect(extraction.fields.VendorName).toEqual({ value: "Acme AB", confidence: 0.97, found: true });
|
||||
expect(extraction.fields.InvoiceTotal).toEqual({ value: 1234.5, confidence: 0.92, found: true });
|
||||
// Not-found field: value is null (never fabricated); confidence is whatever DI reported.
|
||||
expect(extraction.fields.DueDate).toEqual({ value: null, confidence: 0.31, found: false });
|
||||
// A field DI never mentioned at all is simply absent — also never fabricated into existence.
|
||||
expect(extraction.fields.CustomerAddress).toBeUndefined();
|
||||
|
||||
expect(extraction.fields_total_count).toBe(3);
|
||||
expect(extraction.fields_found_count).toBe(2);
|
||||
// Average confidence is computed only over the found fields — DueDate's 0.31 excluded.
|
||||
expect(extraction.average_confidence).toBeCloseTo((0.97 + 0.92) / 2, 5);
|
||||
expect(extraction.document_confidence).toBe(0.9);
|
||||
expect(extraction.st0re_artifact_id).toContain("mock_artifact_");
|
||||
expect(extraction.model_id).toBe("prebuilt-invoice");
|
||||
|
||||
const updatedDoc = await captureService.getDocument(document.document_id);
|
||||
expect(updatedDoc.status).toBe("extracted");
|
||||
|
||||
const extractedEvent = aud0Calls.find((c) => c.body?.event_type === "0pt.document_extracted");
|
||||
expect(extractedEvent).toBeTruthy();
|
||||
expect(extractedEvent!.body.capability_id).toBe("0pt.extract_document");
|
||||
expect(extractedEvent!.body.payload.fields_found_count).toBe(2);
|
||||
expect(extractedEvent!.body.payload.fields_total_count).toBe(3);
|
||||
});
|
||||
|
||||
it("marks the extraction failed — never returns a fabricated success — when Document Intelligence errors", async () => {
|
||||
const captureService = new DocumentCaptureService({ blobStore: new LocalBlobStore(tmpDir) });
|
||||
const document = await seedCapturedDocument(captureService);
|
||||
|
||||
const extractionService = new DocumentExtractionService(captureService, {
|
||||
blobStore: new LocalBlobStore(tmpDir),
|
||||
documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 },
|
||||
});
|
||||
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes(":analyze")) return jsonResponse({}, { status: 500 });
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await expect(
|
||||
extractionService.extractDocument(document.document_id, {
|
||||
tenant_id: "tenant1",
|
||||
project_id: "proj1",
|
||||
actor_id: "actor1",
|
||||
correlation_id: "corr-fail",
|
||||
model_id: "prebuilt-invoice",
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
const updatedDoc = await captureService.getDocument(document.document_id);
|
||||
expect(updatedDoc.status).toBe("extraction_failed");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { DocumentCaptureService } from "@/src/services/document-capture-service";
|
||||
import { DocumentExtractionService } from "@/src/services/document-extraction-service";
|
||||
|
||||
describe("DocumentExtractionService — refuses to fabricate when unconfigured", () => {
|
||||
it("throws instead of returning an empty/fake success when Document Intelligence has no endpoint/apiKey", async () => {
|
||||
const captureService = new DocumentCaptureService();
|
||||
const extractionService = new DocumentExtractionService(captureService); // no documentIntelligence opt passed
|
||||
|
||||
await expect(
|
||||
extractionService.extractDocument("doc_does_not_matter", {
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
actor_id: "a1",
|
||||
correlation_id: "c1",
|
||||
model_id: "prebuilt-invoice",
|
||||
}),
|
||||
).rejects.toThrow(/not configured/i);
|
||||
});
|
||||
|
||||
it("rejects a missing model_id rather than silently picking one", async () => {
|
||||
const captureService = new DocumentCaptureService();
|
||||
const extractionService = new DocumentExtractionService(captureService, {
|
||||
documentIntelligence: { endpoint: "https://di.example.com", apiKey: "key123" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
extractionService.extractDocument("doc_does_not_matter", {
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
actor_id: "a1",
|
||||
correlation_id: "c1",
|
||||
model_id: "",
|
||||
}),
|
||||
).rejects.toThrow(/model_id/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { DocumentIntelligenceAdapter } from "@/src/adapters/document-intelligence-adapter";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number; headers?: Record<string, string> } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null },
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe("DocumentIntelligenceAdapter", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("isConfigured requires endpoint + apiKey", () => {
|
||||
expect(DocumentIntelligenceAdapter.isConfigured({ endpoint: "https://x", apiKey: "k" })).toBe(true);
|
||||
expect(DocumentIntelligenceAdapter.isConfigured({ endpoint: "https://x" })).toBe(false);
|
||||
expect(DocumentIntelligenceAdapter.isConfigured({})).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes structured + array fields and never fabricates a missing value", async () => {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes(":analyze")) {
|
||||
return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/123" } });
|
||||
}
|
||||
if (url.includes("/op/123")) {
|
||||
return jsonResponse({
|
||||
status: "succeeded",
|
||||
analyzeResult: {
|
||||
content: "raw extracted text",
|
||||
pages: [{}, {}],
|
||||
documents: [
|
||||
{
|
||||
confidence: 0.93,
|
||||
fields: {
|
||||
VendorName: { type: "string", valueString: "Acme AB", confidence: 0.98 },
|
||||
InvoiceTotal: { type: "currency", valueCurrency: { amount: 1234.5, currencyCode: "SEK" }, confidence: 0.91 },
|
||||
// Field region detected but no value could be read — must stay blank, not fabricated.
|
||||
DueDate: { type: "date", confidence: 0.4 },
|
||||
Items: {
|
||||
type: "array",
|
||||
valueArray: [
|
||||
{
|
||||
type: "object",
|
||||
valueObject: {
|
||||
Description: { type: "string", valueString: "Widget", confidence: 0.9 },
|
||||
Amount: { type: "currency", valueCurrency: { amount: 100 }, confidence: 0.85 },
|
||||
Unit: { type: "string", confidence: 0.2 }, // present but empty -> found:false
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 });
|
||||
const result = await adapter.analyzeDocument(Buffer.from("%PDF-1.4"), "prebuilt-invoice");
|
||||
|
||||
expect(result.modelId).toBe("prebuilt-invoice");
|
||||
expect(result.content).toBe("raw extracted text");
|
||||
expect(result.documentConfidence).toBe(0.93);
|
||||
expect(result.pageCount).toBe(2);
|
||||
|
||||
expect(result.fields.VendorName).toEqual({ value: "Acme AB", confidence: 0.98, found: true });
|
||||
expect(result.fields.InvoiceTotal).toEqual({ value: 1234.5, confidence: 0.91, found: true });
|
||||
// Not fabricated: value stays null, but DI's own (low) confidence is still recorded.
|
||||
expect(result.fields.DueDate).toEqual({ value: null, confidence: 0.4, found: false });
|
||||
|
||||
expect(result.arrayFields.Items).toHaveLength(1);
|
||||
expect(result.arrayFields.Items[0].Description).toEqual({ value: "Widget", confidence: 0.9, found: true });
|
||||
expect(result.arrayFields.Items[0].Amount).toEqual({ value: 100, confidence: 0.85, found: true });
|
||||
expect(result.arrayFields.Items[0].Unit).toEqual({ value: null, confidence: 0.2, found: false });
|
||||
});
|
||||
|
||||
it("retries once on 429 then proceeds", async () => {
|
||||
let analyzeAttempts = 0;
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes(":analyze")) {
|
||||
analyzeAttempts++;
|
||||
if (analyzeAttempts === 1) return jsonResponse({}, { status: 429 });
|
||||
return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/retry" } });
|
||||
}
|
||||
if (url.includes("/op/retry")) {
|
||||
return jsonResponse({ status: "succeeded", analyzeResult: { documents: [{ fields: {} }] } });
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const adapter = new DocumentIntelligenceAdapter({
|
||||
endpoint: "https://di.example.com",
|
||||
apiKey: "key123",
|
||||
pollIntervalMs: 1,
|
||||
rateLimitRetryMs: 1,
|
||||
});
|
||||
const result = await adapter.analyzeDocument(Buffer.from("x"), "prebuilt-receipt");
|
||||
expect(analyzeAttempts).toBe(2);
|
||||
expect(result.fields).toEqual({});
|
||||
});
|
||||
|
||||
it("throws (never returns a fabricated result) when analyze status is failed", async () => {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes(":analyze")) {
|
||||
return jsonResponse({}, { status: 202, headers: { "operation-location": "https://di.example.com/op/999" } });
|
||||
}
|
||||
if (url.includes("/op/999")) {
|
||||
return jsonResponse({ status: "failed", error: { code: "InvalidDocument" } });
|
||||
}
|
||||
throw new Error("unexpected");
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123", pollIntervalMs: 1 });
|
||||
await expect(adapter.analyzeDocument(Buffer.from("x"), "prebuilt-invoice")).rejects.toThrow(/failed/i);
|
||||
});
|
||||
|
||||
it("throws when the analyze request itself errors (5xx)", async () => {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes(":analyze")) return jsonResponse({}, { status: 500 });
|
||||
throw new Error("unexpected");
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const adapter = new DocumentIntelligenceAdapter({ endpoint: "https://di.example.com", apiKey: "key123" });
|
||||
await expect(adapter.analyzeDocument(Buffer.from("x"), "prebuilt-invoice")).rejects.toThrow(/500/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import { describe, it, expect, vi, afterEach } from "vitest";
|
||||
import { ExternalWritebackService } from "@/src/services/external-writeback-service";
|
||||
import type { WritebackAdapter, WritebackAdapterContext, WritebackRequest, WritebackResult } from "@/src/adapters/writeback-adapter";
|
||||
import type { ExternalWriteRequest } from "@/src/types/external-writeback";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
/** A configurable in-memory WritebackAdapter double -- records every `write` call so tests can
|
||||
* assert on call count (critical for the idempotency/concurrency tests below), and can be told
|
||||
* to fail N times before succeeding, or to always fail/always succeed. */
|
||||
class TestWritebackAdapter implements WritebackAdapter {
|
||||
readonly adapterId = "test_adapter";
|
||||
calls: Array<{ request: WritebackRequest; ctx: WritebackAdapterContext }> = [];
|
||||
configured = true;
|
||||
private failuresRemaining = 0;
|
||||
private failureError = "test_adapter_failure";
|
||||
externalIdPrefix = "ext";
|
||||
|
||||
isConfigured(): boolean {
|
||||
return this.configured;
|
||||
}
|
||||
|
||||
failNextCalls(n: number, error = "test_adapter_failure"): void {
|
||||
this.failuresRemaining = n;
|
||||
this.failureError = error;
|
||||
}
|
||||
|
||||
async write(request: WritebackRequest, ctx: WritebackAdapterContext): Promise<WritebackResult> {
|
||||
this.calls.push({ request, ctx });
|
||||
if (this.failuresRemaining > 0) {
|
||||
this.failuresRemaining -= 1;
|
||||
return { ok: false, error: this.failureError };
|
||||
}
|
||||
return { ok: true, externalId: `${this.externalIdPrefix}_${this.calls.length}` };
|
||||
}
|
||||
}
|
||||
|
||||
function baseRequest(overrides: Partial<ExternalWriteRequest> = {}): ExternalWriteRequest {
|
||||
return {
|
||||
adapter_id: "test_adapter",
|
||||
idempotency_key: "idem-1",
|
||||
record_type: "test_record",
|
||||
payload: { foo: "bar" },
|
||||
tenant_id: "tenant1",
|
||||
project_id: "proj1",
|
||||
actor_id: "actor1",
|
||||
correlation_id: "corr-1",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/** Stubs global fetch to answer CL0UD policy evaluation (always "allow" unless `denyOrApproval`
|
||||
* is set) and swallow AUD0 event/lineage POSTs, recording them for assertions. */
|
||||
function stubCl0udAllowAndAud0(opts: { decision?: "allow" | "deny" | "approval_required" } = {}) {
|
||||
const aud0Calls: { url: string; body: any }[] = [];
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (url.includes("/v1/cl0ud/policies/evaluate")) {
|
||||
return jsonResponse({
|
||||
decision_id: "pdec_test",
|
||||
decision: opts.decision ?? "allow",
|
||||
reason: opts.decision === "deny" ? "test_deny" : opts.decision === "approval_required" ? "test_approval_required" : undefined,
|
||||
});
|
||||
}
|
||||
if (url.includes("/v1/aud0/events") || url.includes("/v1/aud0/lineage/events")) {
|
||||
aud0Calls.push({ url, body: init?.body ? JSON.parse(init.body as string) : null });
|
||||
return jsonResponse({});
|
||||
}
|
||||
throw new Error(`unexpected fetch in test: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
return { aud0Calls, fetchMock };
|
||||
}
|
||||
|
||||
describe("ExternalWritebackService — adapter resolution", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("throws when no adapter is registered for the requested adapter_id", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com" });
|
||||
await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/no adapter registered/);
|
||||
});
|
||||
|
||||
it("throws when the registered adapter reports isConfigured() === false, without calling write()", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
adapter.configured = false;
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/not configured/);
|
||||
expect(adapter.calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ExternalWritebackService — CL0UD policy gate", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("denies the write and marks the record failed when CL0UD denies", async () => {
|
||||
stubCl0udAllowAndAud0({ decision: "deny" });
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/Policy denied/);
|
||||
expect(adapter.calls).toHaveLength(0);
|
||||
const record = await service.findByIdempotencyKey("test_adapter", "idem-1");
|
||||
expect(record?.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("does not silently bypass a CL0UD approval_required decision even though ACT0's own approval step already ran upstream", async () => {
|
||||
stubCl0udAllowAndAud0({ decision: "approval_required" });
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/Policy requires approval/);
|
||||
expect(adapter.calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ExternalWritebackService — successful write", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("writes via the adapter, returns status 'written' with the adapter's external id, and emits AUD0 event + lineage", async () => {
|
||||
const { aud0Calls } = stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({
|
||||
cl0udBaseUrl: "https://cl0ud.example.com",
|
||||
aud0BaseUrl: "https://aud0.example.com",
|
||||
adapters: [adapter],
|
||||
});
|
||||
|
||||
const record = await service.writeExternalRecord(baseRequest());
|
||||
|
||||
expect(record.status).toBe("written");
|
||||
expect(record.external_id).toBe("ext_1");
|
||||
expect(adapter.calls).toHaveLength(1);
|
||||
expect(adapter.calls[0]?.request.recordType).toBe("test_record");
|
||||
expect(adapter.calls[0]?.request.payload).toEqual({ foo: "bar" });
|
||||
|
||||
const writtenEvent = aud0Calls.find((c) => c.body?.event_type === "0pt.external_record_written");
|
||||
expect(writtenEvent).toBeTruthy();
|
||||
expect(writtenEvent!.body.capability_id).toBe("0pt.write_external_record");
|
||||
expect(writtenEvent!.body.risk_level).toBe("L4");
|
||||
expect(writtenEvent!.body.payload.external_id).toBe("ext_1");
|
||||
|
||||
const lineageEvent = aud0Calls.find((c) => c.url.includes("/lineage/"));
|
||||
expect(lineageEvent).toBeTruthy();
|
||||
expect(lineageEvent!.body.source_id).toBe("idem-1");
|
||||
expect(lineageEvent!.body.target_id).toBe("ext_1");
|
||||
expect(lineageEvent!.body.lineage_type).toBe("produced");
|
||||
});
|
||||
});
|
||||
|
||||
describe("ExternalWritebackService — idempotency (sequential re-calls)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("a second sequential call with the same idempotency_key after a successful write is an idempotent no-op — the adapter is never called again", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
|
||||
const first = await service.writeExternalRecord(baseRequest());
|
||||
const second = await service.writeExternalRecord(baseRequest());
|
||||
|
||||
expect(adapter.calls).toHaveLength(1);
|
||||
expect(second.write_id).toBe(first.write_id);
|
||||
expect(second.status).toBe("written");
|
||||
expect(second.external_id).toBe(first.external_id);
|
||||
});
|
||||
|
||||
it("a failed write CAN be retried with the same idempotency_key — this is a deliberate 'claim survives failure for retry' behavior, not a bug", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
adapter.failNextCalls(1, "transient_error");
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
|
||||
await expect(service.writeExternalRecord(baseRequest())).rejects.toThrow(/transient_error/);
|
||||
const failed = await service.findByIdempotencyKey("test_adapter", "idem-1");
|
||||
expect(failed?.status).toBe("failed");
|
||||
|
||||
const retried = await service.writeExternalRecord(baseRequest());
|
||||
expect(retried.status).toBe("written");
|
||||
expect(adapter.calls).toHaveLength(2); // one failed attempt, one successful retry
|
||||
});
|
||||
|
||||
it("different idempotency_key values against the same adapter never collide", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
|
||||
const a = await service.writeExternalRecord(baseRequest({ idempotency_key: "idem-a" }));
|
||||
const b = await service.writeExternalRecord(baseRequest({ idempotency_key: "idem-b" }));
|
||||
|
||||
expect(adapter.calls).toHaveLength(2);
|
||||
expect(a.write_id).not.toBe(b.write_id);
|
||||
expect(a.external_id).not.toBe(b.external_id);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ExternalWritebackService — idempotency under concurrency (the regression-critical case)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("two concurrent writeExternalRecord calls with the SAME idempotency_key result in exactly ONE adapter.write call — the claim is synchronous and race-free within one process", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
|
||||
const [resultA, resultB] = await Promise.all([
|
||||
service.writeExternalRecord(baseRequest()),
|
||||
service.writeExternalRecord(baseRequest()),
|
||||
]);
|
||||
|
||||
// The critical assertion: no matter how the two promises interleave afterwards, the
|
||||
// external system is only ever actually written to once for this idempotency_key. This is
|
||||
// the exact property v0-datahub's invoice-bridge.ts's own atomic Postgres UPDATE claim
|
||||
// protected in production (two concurrent triggers can't both create a Bill) -- see
|
||||
// external-writeback-service.ts's own module doc comment for how this port achieves the
|
||||
// same guarantee via a synchronous, pre-first-await, in-memory claim.
|
||||
expect(adapter.calls).toHaveLength(1);
|
||||
|
||||
// One call wins the claim and completes the real write; the other observes the claim
|
||||
// in-flight and returns 'skipped_duplicate' without ever touching the adapter.
|
||||
expect([resultA.status, resultB.status].sort()).toEqual(["skipped_duplicate", "written"]);
|
||||
});
|
||||
|
||||
it("three concurrent calls with the same idempotency_key still only ever produce one adapter.write call", async () => {
|
||||
stubCl0udAllowAndAud0();
|
||||
const adapter = new TestWritebackAdapter();
|
||||
const service = new ExternalWritebackService({ cl0udBaseUrl: "https://cl0ud.example.com", adapters: [adapter] });
|
||||
|
||||
const results = await Promise.all([
|
||||
service.writeExternalRecord(baseRequest()),
|
||||
service.writeExternalRecord(baseRequest()),
|
||||
service.writeExternalRecord(baseRequest()),
|
||||
]);
|
||||
|
||||
expect(adapter.calls).toHaveLength(1);
|
||||
const writtenCount = results.filter((r) => r.status === "written").length;
|
||||
const skippedCount = results.filter((r) => r.status === "skipped_duplicate").length;
|
||||
expect(writtenCount).toBe(1);
|
||||
expect(skippedCount).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { acquireFabricAccessToken, type FabricOneLakeCredentials } from "@/src/adapters/fabric-onelake-adapter";
|
||||
|
||||
const CREDS: FabricOneLakeCredentials = { tenantId: "tenant-a", clientId: "client-a", clientSecret: "secret-a" };
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
|
||||
}
|
||||
|
||||
describe("acquireFabricAccessToken — OAuth client-credentials flow", () => {
|
||||
it("posts client_credentials grant to the caller's tenant AAD endpoint with the default Fabric SQL scope", async () => {
|
||||
let capturedUrl = "";
|
||||
let capturedBody = "";
|
||||
const fetchMock = vi.fn(async (url: unknown, init: unknown) => {
|
||||
capturedUrl = String(url);
|
||||
capturedBody = String((init as RequestInit).body);
|
||||
return jsonResponse({ access_token: "tok_123", expires_in: 3600 });
|
||||
});
|
||||
|
||||
const token = await acquireFabricAccessToken(CREDS, undefined, fetchMock as unknown as typeof fetch);
|
||||
|
||||
expect(capturedUrl).toBe("https://login.microsoftonline.com/tenant-a/oauth2/v2.0/token");
|
||||
const params = new URLSearchParams(capturedBody);
|
||||
expect(params.get("client_id")).toBe("client-a");
|
||||
expect(params.get("client_secret")).toBe("secret-a");
|
||||
expect(params.get("grant_type")).toBe("client_credentials");
|
||||
expect(params.get("scope")).toBe("https://database.windows.net/.default");
|
||||
expect(token.accessToken).toBe("tok_123");
|
||||
expect(token.expiresAtEpochMs).toBeGreaterThan(Date.now());
|
||||
});
|
||||
|
||||
it("uses a caller-supplied scope instead of the default (e.g. OneLake storage)", async () => {
|
||||
let capturedBody = "";
|
||||
const fetchMock = vi.fn(async (_url: unknown, init: unknown) => {
|
||||
capturedBody = String((init as RequestInit).body);
|
||||
return jsonResponse({ access_token: "tok_storage", expires_in: 3600 });
|
||||
});
|
||||
|
||||
await acquireFabricAccessToken(CREDS, "https://storage.azure.com/.default", fetchMock as unknown as typeof fetch);
|
||||
|
||||
expect(new URLSearchParams(capturedBody).get("scope")).toBe("https://storage.azure.com/.default");
|
||||
});
|
||||
|
||||
it("never hardcodes a tenant — different credentials hit different tenant token URLs", async () => {
|
||||
const seenUrls: string[] = [];
|
||||
const fetchMock = vi.fn(async (url: unknown) => {
|
||||
seenUrls.push(String(url));
|
||||
return jsonResponse({ access_token: "t", expires_in: 3600 });
|
||||
});
|
||||
|
||||
await acquireFabricAccessToken({ tenantId: "tenant-x", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch);
|
||||
await acquireFabricAccessToken({ tenantId: "tenant-y", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch);
|
||||
|
||||
expect(seenUrls).toEqual([
|
||||
"https://login.microsoftonline.com/tenant-x/oauth2/v2.0/token",
|
||||
"https://login.microsoftonline.com/tenant-y/oauth2/v2.0/token",
|
||||
]);
|
||||
});
|
||||
|
||||
it("throws a descriptive error when AAD rejects the credentials", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ error: "invalid_client", error_description: "bad secret" }, 401));
|
||||
await expect(acquireFabricAccessToken(CREDS, undefined, fetchMock as unknown as typeof fetch)).rejects.toThrow(/invalid_client/);
|
||||
});
|
||||
|
||||
it("rejects locally (no network call) when tenantId/clientId/clientSecret are missing", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
await expect(
|
||||
acquireFabricAccessToken({ tenantId: "", clientId: "c", clientSecret: "s" }, undefined, fetchMock as unknown as typeof fetch),
|
||||
).rejects.toThrow(/required/);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("FabricOneLakeAdapter.query — TDS execution via a mocked mssql driver", () => {
|
||||
const CONFIG = {
|
||||
credentials: CREDS,
|
||||
sqlEndpoint: "https://my-endpoint.datawarehouse.fabric.microsoft.com/",
|
||||
database: "lh_test",
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("mssql");
|
||||
});
|
||||
|
||||
it("acquires a token, opens a pooled TDS connection with azure-active-directory-access-token auth, runs the query, and returns rows", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_query", expires_in: 3600 }));
|
||||
|
||||
const queryMock = vi.fn(async () => ({ recordset: [{ id: 1, name: "Alice" }, { id: 2, name: "Bob" }] }));
|
||||
const inputMock = vi.fn();
|
||||
const connectMock = vi.fn(async () => {});
|
||||
const closeMock = vi.fn(async () => {});
|
||||
let capturedConfig: Record<string, unknown> | null = null;
|
||||
|
||||
class FakeConnectionPool {
|
||||
constructor(config: Record<string, unknown>) {
|
||||
capturedConfig = config;
|
||||
}
|
||||
connect = connectMock;
|
||||
request() {
|
||||
return { input: inputMock, query: queryMock };
|
||||
}
|
||||
close = closeMock;
|
||||
}
|
||||
|
||||
vi.doMock("mssql", () => ({ default: { ConnectionPool: FakeConnectionPool } }));
|
||||
|
||||
const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter");
|
||||
const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch);
|
||||
const result = await adapter.query("SELECT * FROM dbo.customers WHERE id > @minId", { minId: 1 });
|
||||
|
||||
expect(result.rows).toEqual([{ id: 1, name: "Alice" }, { id: 2, name: "Bob" }]);
|
||||
expect(result.rowCount).toBe(2);
|
||||
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
||||
expect(inputMock).toHaveBeenCalledWith("minId", 1);
|
||||
expect(connectMock).toHaveBeenCalledTimes(1);
|
||||
expect(capturedConfig).toMatchObject({
|
||||
server: "my-endpoint.datawarehouse.fabric.microsoft.com",
|
||||
database: "lh_test",
|
||||
authentication: { type: "azure-active-directory-access-token", options: { token: "tok_query" } },
|
||||
});
|
||||
|
||||
await adapter.close();
|
||||
expect(closeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reuses the cached token across calls and only re-authenticates once", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_cached", expires_in: 3600 }));
|
||||
const queryMock = vi.fn(async () => ({ recordset: [] }));
|
||||
class FakeConnectionPool {
|
||||
connect = vi.fn(async () => {});
|
||||
request() {
|
||||
return { input: vi.fn(), query: queryMock };
|
||||
}
|
||||
close = vi.fn(async () => {});
|
||||
}
|
||||
vi.doMock("mssql", () => ({ default: { ConnectionPool: FakeConnectionPool } }));
|
||||
|
||||
const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter");
|
||||
const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch);
|
||||
await adapter.query("SELECT 1");
|
||||
await adapter.query("SELECT 2");
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(queryMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("throws a clear, actionable error when the mssql driver is not installed/resolvable", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ access_token: "tok_x", expires_in: 3600 }));
|
||||
// No vi.doMock("mssql", ...) here — this repo does not declare "mssql" as a
|
||||
// dependency (see fabric-onelake-adapter.ts header), so the real dynamic
|
||||
// import genuinely fails, exercising the adapter's real error path.
|
||||
const { FabricOneLakeAdapter } = await import("@/src/adapters/fabric-onelake-adapter");
|
||||
const adapter = new FabricOneLakeAdapter(CONFIG, fetchMock as unknown as typeof fetch);
|
||||
|
||||
await expect(adapter.query("SELECT 1")).rejects.toThrow(/mssql.*not installed/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { GraphMailboxAdapter } from "@/src/adapters/graph-mailbox-adapter";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number; headers?: Record<string, string> } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
headers: { get: (k: string) => (opts.headers ?? {})[k] ?? null },
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe("GraphMailboxAdapter", () => {
|
||||
const config = {
|
||||
tenantId: "tenant-1",
|
||||
clientId: "client-1",
|
||||
clientSecret: "secret-1",
|
||||
mailboxAddress: "docs@customer.com",
|
||||
folder: "inbox",
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("isConfigured requires tenantId/clientId/clientSecret/mailboxAddress", () => {
|
||||
expect(GraphMailboxAdapter.isConfigured(config)).toBe(true);
|
||||
expect(GraphMailboxAdapter.isConfigured({ ...config, clientSecret: "" })).toBe(false);
|
||||
expect(GraphMailboxAdapter.isConfigured({ ...config, mailboxAddress: "" })).toBe(false);
|
||||
});
|
||||
|
||||
it("polls messages across pages, following @odata.nextLink up to maxPages", async () => {
|
||||
const calls: string[] = [];
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
calls.push(url);
|
||||
if (url.includes("login.microsoftonline.com")) {
|
||||
return jsonResponse({ access_token: "tok123", expires_in: 3600 });
|
||||
}
|
||||
if (url.includes("/messages?")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
id: "msg-1",
|
||||
internetMessageId: "<msg-1@customer.com>",
|
||||
conversationId: "conv-1",
|
||||
receivedDateTime: "2026-07-01T10:00:00Z",
|
||||
from: { emailAddress: { name: "Vendor AB", address: "vendor@example.com" } },
|
||||
toRecipients: [{ emailAddress: { address: "docs@customer.com" } }],
|
||||
subject: "Document 1",
|
||||
bodyPreview: "preview",
|
||||
body: { content: "<p>Hello <b>World</b></p>" },
|
||||
hasAttachments: true,
|
||||
webLink: "https://outlook.office.com/mail/1",
|
||||
},
|
||||
],
|
||||
"@odata.nextLink": "https://graph.microsoft.com/v1.0/next-page",
|
||||
});
|
||||
}
|
||||
if (url.includes("next-page")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
id: "msg-2",
|
||||
receivedDateTime: "2026-07-01T11:00:00Z",
|
||||
from: { emailAddress: { name: "Vendor 2", address: "vendor2@example.com" } },
|
||||
toRecipients: [],
|
||||
subject: "Document 2",
|
||||
hasAttachments: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const adapter = new GraphMailboxAdapter(config);
|
||||
const messages = await adapter.pollMessages({ top: 50, maxPages: 2 });
|
||||
|
||||
expect(messages).toHaveLength(2);
|
||||
expect(messages[0].id).toBe("msg-1");
|
||||
expect(messages[0].senderEmail).toBe("vendor@example.com");
|
||||
expect(messages[0].bodyText).toBe("Hello World");
|
||||
expect(messages[1].id).toBe("msg-2");
|
||||
expect(messages[1].hasAttachments).toBe(false);
|
||||
expect(calls.some((u) => u.includes("mailFolders/inbox/messages"))).toBe(true);
|
||||
});
|
||||
|
||||
it("lists only file attachments and flags PDFs by content-type or filename", async () => {
|
||||
const pdfBase64 = Buffer.from("%PDF-1.4 fake pdf bytes").toString("base64");
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.includes("login.microsoftonline.com")) {
|
||||
return jsonResponse({ access_token: "tok123", expires_in: 3600 });
|
||||
}
|
||||
if (url.includes("/attachments")) {
|
||||
return jsonResponse({
|
||||
value: [
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "att-1",
|
||||
name: "invoice.pdf",
|
||||
contentType: "application/pdf",
|
||||
size: 123,
|
||||
contentBytes: pdfBase64,
|
||||
},
|
||||
{
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "att-2",
|
||||
name: "logo.png",
|
||||
contentType: "image/png",
|
||||
size: 50,
|
||||
contentBytes: Buffer.from("not-a-pdf").toString("base64"),
|
||||
},
|
||||
{
|
||||
// No mime type but a .pdf filename should still be detected.
|
||||
"@odata.type": "#microsoft.graph.fileAttachment",
|
||||
id: "att-3",
|
||||
name: "scan.PDF",
|
||||
contentType: "application/octet-stream",
|
||||
size: 999,
|
||||
contentBytes: pdfBase64,
|
||||
},
|
||||
{
|
||||
// Non-file attachment (e.g. embedded email) must be skipped entirely.
|
||||
"@odata.type": "#microsoft.graph.itemAttachment",
|
||||
id: "att-4",
|
||||
name: "embedded-email",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected fetch: ${url}`);
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const adapter = new GraphMailboxAdapter(config);
|
||||
const attachments = await adapter.listAttachments("msg-1");
|
||||
|
||||
expect(attachments).toHaveLength(3); // itemAttachment skipped
|
||||
const pdf = attachments.find((a) => a.name === "invoice.pdf")!;
|
||||
expect(pdf.isPdf).toBe(true);
|
||||
expect(pdf.contentBytes.toString()).toContain("%PDF-1.4");
|
||||
const png = attachments.find((a) => a.name === "logo.png")!;
|
||||
expect(png.isPdf).toBe(false);
|
||||
const scanByExt = attachments.find((a) => a.name === "scan.PDF")!;
|
||||
expect(scanByExt.isPdf).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,298 @@
|
||||
import { describe, it, expect, vi, afterEach } from "vitest";
|
||||
import { OdooBillsAdapter, matchPartnerViaFabric, type OdooBillPayload } from "@/src/adapters/odoo-bills-adapter";
|
||||
import { FabricOneLakeAdapter } from "@/src/adapters/fabric-onelake-adapter";
|
||||
import type { WritebackAdapterContext } from "@/src/adapters/writeback-adapter";
|
||||
|
||||
function jsonResponse(body: unknown, opts: { status?: number } = {}) {
|
||||
return {
|
||||
ok: (opts.status ?? 200) < 400,
|
||||
status: opts.status ?? 200,
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
const CTX: WritebackAdapterContext = {
|
||||
tenant_id: "t_protosell",
|
||||
project_id: "proj_ap",
|
||||
actor_id: "actor1",
|
||||
correlation_id: "corr-1",
|
||||
};
|
||||
|
||||
function basePayload(overrides: Partial<OdooBillPayload> = {}): OdooBillPayload {
|
||||
return {
|
||||
vendor_name: "Acme AB",
|
||||
vendor_org_nr: "556123-4567",
|
||||
invoice_number: "INV-2026-001",
|
||||
amount_gross: 1234.5,
|
||||
amount_excl_vat: 987.6,
|
||||
vat: 246.9,
|
||||
currency: "SEK",
|
||||
invoice_date: "2026-07-01",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("OdooBillsAdapter — isConfigured", () => {
|
||||
it("is not configured without a baseUrl", () => {
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "", apiKey: "key" });
|
||||
expect(adapter.isConfigured()).toBe(false);
|
||||
});
|
||||
|
||||
it("is not configured with a baseUrl but no apiKey and no allowUnauthenticated flag", () => {
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com" });
|
||||
expect(adapter.isConfigured()).toBe(false);
|
||||
});
|
||||
|
||||
it("is configured with baseUrl + apiKey", () => {
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key" });
|
||||
expect(adapter.isConfigured()).toBe(true);
|
||||
});
|
||||
|
||||
it("is configured with baseUrl + explicit allowUnauthenticated, without an apiKey", () => {
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", allowUnauthenticated: true });
|
||||
expect(adapter.isConfigured()).toBe(true);
|
||||
});
|
||||
|
||||
it("write() returns ok:false when not configured, without attempting a network call", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toBe("odoo_not_configured");
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("OdooBillsAdapter — guards ported verbatim from invoice-bridge.ts", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("refuses to post a payment reminder (is_reminder: true) — mirrors 'Påminnelse postas inte till Odoo'", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload({ is_reminder: true }) as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(result).toEqual({ ok: false, error: "is_reminder" });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses to post a detected duplicate (duplicate_flag: true) — mirrors 'Dubblett postas inte till Odoo'", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload({ duplicate_flag: true }) as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(result).toEqual({ ok: false, error: "duplicate" });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("treats a payload with existing_external_id as already posted — idempotent no-op, no network call, mirrors 'Redan bokförd i Odoo... hoppar över'", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload({ existing_external_id: "999" }) as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(result).toEqual({ ok: true, externalId: "999" });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a payload missing required fields without attempting a network call", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: { vendor_name: "Acme AB" }, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toBe("invalid_payload_missing_required_fields");
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("OdooBillsAdapter — real POST shape (ported from invoice-bridge.ts's pushInvoiceToOdoo)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("POSTs account.move with itemized lines when payload.lines is present", async () => {
|
||||
let capturedUrl = "";
|
||||
let capturedInit: RequestInit | undefined;
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
capturedUrl = url;
|
||||
capturedInit = init;
|
||||
return jsonResponse({ ok: true, odoo_id: 4242 });
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "sekret", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
|
||||
const payload = basePayload({
|
||||
partner_id: 77,
|
||||
lines: [{ description: "Konsulttjänst", quantity: 2, unit_price: 500 }],
|
||||
});
|
||||
const result = await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
|
||||
expect(result).toEqual({ ok: true, externalId: "4242" });
|
||||
expect(capturedUrl).toBe("https://odoo.example.com/api/odoo/invoices/write");
|
||||
expect((capturedInit?.headers as Record<string, string>).Authorization).toBe("Bearer sekret");
|
||||
const body = JSON.parse(capturedInit!.body as string);
|
||||
expect(body.model).toBe("account.move");
|
||||
expect(body.invoice.partner_id).toBe(77);
|
||||
expect(body.invoice.ref).toBe("INV-2026-001");
|
||||
expect(body.invoice.invoice_line_ids).toEqual([{ name: "Konsulttjänst", quantity: 2, price_unit: 500 }]);
|
||||
});
|
||||
|
||||
it("synthesizes a single summary line from description/vendor_name when payload.lines is absent — ported verbatim from processInboxToOdoo's line-synthesis fallback", async () => {
|
||||
let capturedInit: RequestInit | undefined;
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
capturedInit = init;
|
||||
return jsonResponse({ ok: true, odoo_id: 100 });
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
|
||||
const payload = basePayload({ description: "Städtjänster juni", amount_excl_vat: 800 });
|
||||
await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
|
||||
const body = JSON.parse(capturedInit!.body as string);
|
||||
expect(body.invoice.invoice_line_ids).toEqual([{ name: "Städtjänster juni", quantity: 1, price_unit: 800 }]);
|
||||
});
|
||||
|
||||
it("falls back to vendor_name, then a synthesized 'Faktura <number>' string, when description is absent", async () => {
|
||||
let capturedInit: RequestInit | undefined;
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
capturedInit = init;
|
||||
return jsonResponse({ ok: true, odoo_id: 101 });
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
|
||||
const payload = basePayload({ vendor_name: "", invoice_number: "INV-777", amount_excl_vat: 50 });
|
||||
await adapter.write({ recordType: "vendor_bill", payload: payload as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
|
||||
const body = JSON.parse(capturedInit!.body as string);
|
||||
expect(body.invoice.invoice_line_ids[0].name).toBe("Faktura INV-777");
|
||||
});
|
||||
|
||||
it("returns ok:false with an odoo_http_<status> error on a non-2xx response", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({}, { status: 500 }));
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
expect(result).toEqual({ ok: false, error: "odoo_http_500" });
|
||||
});
|
||||
|
||||
it("returns ok:false when Odoo responds 200 but ok:false/no odoo_id", async () => {
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ ok: false, error: "odoo_validation_error" }));
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
expect(result).toEqual({ ok: false, error: "odoo_validation_error" });
|
||||
});
|
||||
|
||||
it("returns ok:false on a network error, without throwing", async () => {
|
||||
const fetchMock = vi.fn(async () => {
|
||||
throw new Error("ECONNREFUSED");
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({ baseUrl: "https://odoo.example.com", apiKey: "key", fetchImpl: fetchMock as unknown as typeof fetch });
|
||||
const result = await adapter.write({ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" }, CTX);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toContain("odoo_network_error");
|
||||
});
|
||||
});
|
||||
|
||||
describe("matchPartnerViaFabric — ported from invoice-bridge.ts's matchInvoiceToOdooPartner", () => {
|
||||
it("matches by digits-only comparison so '556123-4567' resolves the same as a plain digit string", async () => {
|
||||
const queryMock = vi.fn(async () => ({
|
||||
rows: [{ id: 42, name: "Acme AB", vat: "SE5561234567 01" }],
|
||||
rowCount: 1,
|
||||
durationMs: 1,
|
||||
}));
|
||||
const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter;
|
||||
|
||||
const result = await matchPartnerViaFabric(fabric, "556123-4567");
|
||||
expect(result).toEqual({ partner_id: 42, partner_name: "Acme AB" });
|
||||
expect(queryMock).toHaveBeenCalledWith(expect.stringContaining("odoo_export__res_partner"), { digits: "5561234567" });
|
||||
});
|
||||
|
||||
it("returns {} (no match) when the org-nr has no digits at all", async () => {
|
||||
const queryMock = vi.fn();
|
||||
const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter;
|
||||
const result = await matchPartnerViaFabric(fabric, "N/A");
|
||||
expect(result).toEqual({});
|
||||
expect(queryMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("degrades to {} (no match) rather than throwing when the warehouse query fails — mirrors invoice-bridge.ts's own catch-all", async () => {
|
||||
const queryMock = vi.fn(async () => {
|
||||
throw new Error("warehouse unreachable");
|
||||
});
|
||||
const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter;
|
||||
const result = await matchPartnerViaFabric(fabric, "556123-4567");
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it("returns {} when no partner row matches", async () => {
|
||||
const queryMock = vi.fn(async () => ({ rows: [], rowCount: 0, durationMs: 1 }));
|
||||
const fabric = { query: queryMock } as unknown as FabricOneLakeAdapter;
|
||||
const result = await matchPartnerViaFabric(fabric, "556123-4567");
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe("OdooBillsAdapter — partner resolution wired into write()", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("uses payload.partner_id directly when already set, without calling matchPartner", async () => {
|
||||
const matchPartner = vi.fn();
|
||||
const fetchMock = vi.fn(async () => jsonResponse({ ok: true, odoo_id: 1 }));
|
||||
const adapter = new OdooBillsAdapter({
|
||||
baseUrl: "https://odoo.example.com",
|
||||
apiKey: "key",
|
||||
matchPartner,
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload({ partner_id: 55 }) as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(matchPartner).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls the injected matchPartner when partner_id is absent but vendor_org_nr is present, and forwards the resolved partner_id to Odoo", async () => {
|
||||
const matchPartner = vi.fn(async () => ({ partner_id: 99, partner_name: "Acme AB" }));
|
||||
let capturedBody: any;
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
capturedBody = JSON.parse(init!.body as string);
|
||||
return jsonResponse({ ok: true, odoo_id: 1 });
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({
|
||||
baseUrl: "https://odoo.example.com",
|
||||
apiKey: "key",
|
||||
matchPartner,
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(matchPartner).toHaveBeenCalledWith("556123-4567", CTX);
|
||||
expect(capturedBody.invoice.partner_id).toBe(99);
|
||||
});
|
||||
|
||||
it("posts without a partner_id when no match is found — 'skickar utan partner-koppling'", async () => {
|
||||
const matchPartner = vi.fn(async () => ({}));
|
||||
let capturedBody: any;
|
||||
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => {
|
||||
capturedBody = JSON.parse(init!.body as string);
|
||||
return jsonResponse({ ok: true, odoo_id: 1 });
|
||||
});
|
||||
const adapter = new OdooBillsAdapter({
|
||||
baseUrl: "https://odoo.example.com",
|
||||
apiKey: "key",
|
||||
matchPartner,
|
||||
fetchImpl: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
await adapter.write(
|
||||
{ recordType: "vendor_bill", payload: basePayload() as unknown as Record<string, unknown>, idempotencyKey: "k1" },
|
||||
CTX,
|
||||
);
|
||||
expect(capturedBody.invoice.partner_id).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,314 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { SourceService } from "@/src/services/source-service";
|
||||
import type { SourceType } from "@/src/types/source";
|
||||
|
||||
const queryMock = vi.fn();
|
||||
const closeMock = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
// Hoisted by Vitest above the static imports below, so
|
||||
// source-query-service.ts (which statically imports FabricOneLakeAdapter)
|
||||
// sees this mock too — no real mssql/AAD network calls happen in this file.
|
||||
vi.mock("@/src/adapters/fabric-onelake-adapter", () => {
|
||||
return {
|
||||
// Must be a real `function` (not an arrow fn) so `new FabricOneLakeAdapter(...)`
|
||||
// in source-query-service.ts works: a constructor that explicitly returns an
|
||||
// object substitutes that object for `this`, per JS semantics.
|
||||
FabricOneLakeAdapter: vi.fn().mockImplementation(function FakeFabricOneLakeAdapter() {
|
||||
return { query: queryMock, close: closeMock };
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
SourceQueryService,
|
||||
SourceQueryTimeoutError,
|
||||
SourceQueryUnsupportedError,
|
||||
SOURCE_QUERY_HARD_MAX_ROWS,
|
||||
SOURCE_QUERY_HARD_MAX_TIMEOUT_MS,
|
||||
} from "@/src/services/source-query-service";
|
||||
|
||||
const BASE_CONNECTION_CONFIG = {
|
||||
credentials: { tenantId: "t", clientId: "c", clientSecret: "s" },
|
||||
sqlEndpoint: "ep.datawarehouse.fabric.microsoft.com",
|
||||
database: "lh_test",
|
||||
};
|
||||
|
||||
async function registerSource(
|
||||
sourceService: SourceService,
|
||||
opts: { source_id: string; adapter_id?: string; source_type?: SourceType },
|
||||
): Promise<string> {
|
||||
await sourceService.createSource({
|
||||
source_id: opts.source_id,
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
source_type: opts.source_type ?? "database",
|
||||
name: "Fabric Warehouse",
|
||||
adapter_id: opts.adapter_id ?? "fabric_onelake_sql",
|
||||
});
|
||||
return opts.source_id;
|
||||
}
|
||||
|
||||
describe("SourceQueryService.querySource — bounded execution", () => {
|
||||
beforeEach(() => {
|
||||
queryMock.mockReset();
|
||||
closeMock.mockClear();
|
||||
});
|
||||
|
||||
it("executes a query against a registered fabric_onelake_sql database source and returns rows", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_query_ok" });
|
||||
queryMock.mockResolvedValue({ rows: [{ id: 1 }, { id: 2 }], rowCount: 2, durationMs: 5 });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
const result = await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.customers",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(result.rows).toEqual([{ id: 1 }, { id: 2 }]);
|
||||
expect(result.row_count).toBe(2);
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.bounds.max_rows).toBeGreaterThan(0);
|
||||
expect(closeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects a non-database source with SourceQueryUnsupportedError", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_file_1", source_type: "file", adapter_id: undefined });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }),
|
||||
).rejects.toBeInstanceOf(SourceQueryUnsupportedError);
|
||||
});
|
||||
|
||||
it("rejects a database source with an unsupported adapter_id", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_bad_adapter", adapter_id: "some_other_adapter" });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }),
|
||||
).rejects.toBeInstanceOf(SourceQueryUnsupportedError);
|
||||
});
|
||||
|
||||
it("propagates a 'not found' error for an unregistered source, without ever building an adapter", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({ source_id: "does_not_exist", query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }),
|
||||
).rejects.toThrow(/not found/i);
|
||||
expect(queryMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("truncates rows to max_rows and flags truncated:true", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_truncate" });
|
||||
const rows = Array.from({ length: 50 }, (_, i) => ({ id: i }));
|
||||
queryMock.mockResolvedValue({ rows, rowCount: rows.length, durationMs: 3 });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
const result = await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.big_table",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
max_rows: 10,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(result.rows).toHaveLength(10);
|
||||
expect(result.row_count).toBe(10);
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.bounds.max_rows).toBe(10);
|
||||
});
|
||||
|
||||
it("does not flag truncated when row count is exactly max_rows", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_exact" });
|
||||
const rows = Array.from({ length: 10 }, (_, i) => ({ id: i }));
|
||||
queryMock.mockResolvedValue({ rows, rowCount: rows.length, durationMs: 1 });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
const result = await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.t",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
max_rows: 10,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.rows).toHaveLength(10);
|
||||
});
|
||||
|
||||
it("clamps a caller-requested max_rows above the hard ceiling instead of allowing an unbounded read", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_clamp_rows" });
|
||||
queryMock.mockResolvedValue({ rows: [{ id: 1 }], rowCount: 1, durationMs: 1 });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
const result = await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT 1",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
max_rows: SOURCE_QUERY_HARD_MAX_ROWS * 10,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(result.bounds.max_rows).toBe(SOURCE_QUERY_HARD_MAX_ROWS);
|
||||
});
|
||||
|
||||
it("clamps a caller-requested timeout_ms above the hard ceiling", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_clamp_timeout" });
|
||||
queryMock.mockResolvedValue({ rows: [], rowCount: 0, durationMs: 1 });
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
const result = await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT 1",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
timeout_ms: SOURCE_QUERY_HARD_MAX_TIMEOUT_MS * 100,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(result.bounds.timeout_ms).toBe(SOURCE_QUERY_HARD_MAX_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it("times out a query that runs longer than the bounded timeout, with SourceQueryTimeoutError", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_timeout" });
|
||||
queryMock.mockImplementation(
|
||||
() => new Promise((resolve) => setTimeout(() => resolve({ rows: [], rowCount: 0, durationMs: 1 }), 300)),
|
||||
);
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.slow_view",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
timeout_ms: 20,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(SourceQueryTimeoutError);
|
||||
});
|
||||
|
||||
it("still releases the adapter (calls close) after a timeout", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_timeout_cleanup" });
|
||||
queryMock.mockImplementation(
|
||||
() => new Promise((resolve) => setTimeout(() => resolve({ rows: [], rowCount: 0, durationMs: 1 }), 300)),
|
||||
);
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT 1",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
timeout_ms: 20,
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(closeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("propagates adapter query errors as-is", async () => {
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_adapter_error" });
|
||||
queryMock.mockRejectedValue(new Error("Invalid object name 'dbo.does_not_exist'"));
|
||||
|
||||
const service = new SourceQueryService({ sourceService });
|
||||
await expect(
|
||||
service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.does_not_exist",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
}),
|
||||
).rejects.toThrow(/Invalid object name/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SourceQueryService — AUD0 0pt.source_queried event emission", () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("emits a 0pt.source_queried AUD0 event with ok:true on success", async () => {
|
||||
queryMock.mockReset();
|
||||
closeMock.mockClear();
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_aud0_ok" });
|
||||
queryMock.mockResolvedValue({ rows: [{ id: 1 }], rowCount: 1, durationMs: 2 });
|
||||
|
||||
const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
|
||||
global.fetch = vi.fn(async (url: unknown, init: unknown) => {
|
||||
calls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) });
|
||||
return new Response("{}", { status: 200 });
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const service = new SourceQueryService({ sourceService, aud0BaseUrl: "http://aud0.test" });
|
||||
await service.querySource({
|
||||
source_id: sourceId,
|
||||
query: "SELECT * FROM dbo.customers",
|
||||
tenant_id: "t1",
|
||||
project_id: "p1",
|
||||
actor_id: "user_42",
|
||||
correlation_id: "corr_abc",
|
||||
connection_config: BASE_CONNECTION_CONFIG,
|
||||
});
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0].url).toBe("http://aud0.test/v1/aud0/events");
|
||||
expect(calls[0].body.event_type).toBe("0pt.source_queried");
|
||||
expect(calls[0].body.capability_id).toBe("0pt.query_source");
|
||||
expect(calls[0].body.app_id).toBe("0pt");
|
||||
expect(calls[0].body.resource_type).toBe("source");
|
||||
expect(calls[0].body.resource_id).toBe(sourceId);
|
||||
expect(calls[0].body.actor_id).toBe("user_42");
|
||||
expect(calls[0].body.correlation_id).toBe("corr_abc");
|
||||
const payload = calls[0].body.payload as Record<string, unknown>;
|
||||
expect(payload.ok).toBe(true);
|
||||
expect(payload.row_count).toBe(1);
|
||||
expect(payload.adapter_id).toBe("fabric_onelake_sql");
|
||||
});
|
||||
|
||||
it("emits a 0pt.source_queried AUD0 event with ok:false and the error message on failure", async () => {
|
||||
queryMock.mockReset();
|
||||
closeMock.mockClear();
|
||||
const sourceService = new SourceService();
|
||||
const sourceId = await registerSource(sourceService, { source_id: "src_aud0_fail" });
|
||||
queryMock.mockRejectedValue(new Error("boom"));
|
||||
|
||||
const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
|
||||
global.fetch = vi.fn(async (url: unknown, init: unknown) => {
|
||||
calls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) });
|
||||
return new Response("{}", { status: 200 });
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const service = new SourceQueryService({ sourceService, aud0BaseUrl: "http://aud0.test" });
|
||||
await expect(
|
||||
service.querySource({ source_id: sourceId, query: "SELECT 1", tenant_id: "t1", project_id: "p1", connection_config: BASE_CONNECTION_CONFIG }),
|
||||
).rejects.toThrow("boom");
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
const payload = calls[0].body.payload as Record<string, unknown>;
|
||||
expect(payload.ok).toBe(false);
|
||||
expect(payload.error).toBe("boom");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user