2026-09-21 11:14:42 +00:00
2 changed files with 363 additions and 0 deletions
@@ -0,0 +1,206 @@
+/**
+ * `0pt/synthetic/factory.ts` — X-737: Synthetic Data Factory.
+ *
+ * Ägande enligt kontraktet: 0pt äger dataset-truth + transformationer;
+ * INF0 äger generering (injectas via port). Pipeline:
+ *
+ *   source DatasetVersion
+ *     → sampling
+ *     → synthesis (INF0 GenerationPort)
+ *     → PII/privacy-check        (fail-closed FÖRE publish)
+ *     → quality-filter
+ *     → dedupe
+ *     → contamination-detection  (rader = identiska med val/test-kandidater)
+ *     → train/val/test-split
+ *     → ny DatasetVersion (0pt-repot)
+ *
+ * PROVENANS PER RAD (acceptans): varje syntetisk rad bär generation-policy-ref
+ * + model-version + source-ref. Privacy-check blockerar publish.
+ */
+
+import { createHash } from 'node:crypto';
+
+export type PiiClass = 'none' | 'low' | 'medium' | 'high';
+
+export interface SourceRow {
+  id: string;
+  fields: Record<string, unknown>;
+}
+
+export interface GenerationConfig {
+  /** Policy-ref för genereringen (CL0UD/0pt-policy — spärras per rad). */
+  generationPolicyRef: string;
+  /** INF0-modellversion som genererade (provener per rad). */
+  modelVersion: string;
+  /** Antal rader som ska syntetiseras. */
+  count: number;
+  /** Högst tillåten PII-klass i publicerad data (över → NEKAS). */
+  maxAllowedPiiClass: PiiClass;
+  /** Quality-tröskel per rad (0–1). */
+  minQualityScore: number;
+  /** Train/val/test-fördelning (default 70/15/15). */
+  split?: { train: number; val: number; test: number };
+}
+
+export interface GeneratedRow {
+  fields: Record<string, unknown>;
+  /** Kvalitetspoäng från genereringen (0–1). */
+  qualityScore: number;
+  provenance: SyntheticRowProvenance;
+}
+
+export interface SyntheticRowProvenance {
+  rowId: string;
+  generationPolicyRef: string;
+  modelVersion: string;
+  sourceDatasetRef: string;
+  sourceRowIds: string[];
+  sha256: string;
+}
+
+/** INF0:s genereringsport (0pt anropar — INF0 äger modellen). */
+export interface GenerationPort {
+  generate(input: {
+    sourceRows: SourceRow[];
+    count: number;
+    policyRef: string;
+  }): Promise<{ rows: GeneratedRow[] }>;
+}
+
+/** Privacy-port (X-728-presidio i INF0/integrations): klassificera rader. */
+export interface PrivacyCheckPort {
+  classifyRow(fields: Record<string, unknown>): Promise<PiiClass>;
+}
+
+const PII_ORDER: Record<PiiClass, number> = { none: 0, low: 1, medium: 2, high: 3 };
+
+export interface PipelineStageReport {
+  stage: 'sampled' | 'synthesized' | 'privacy' | 'quality' | 'dedupe' | 'contamination' | 'split';
+  inputCount: number;
+  outputCount: number;
+}
+
+export interface SyntheticDatasetVersionDraft {
+  sourceDatasetId: string;
+  sourceVersion: number;
+  tenantId: string;
+  projectId: string;
+  schemaVersion: string;
+  description: string;
+  rows: Array<{ split: 'train' | 'val' | 'test'; provenance: SyntheticRowProvenance; fields: Record<string, unknown> }>;
+  stages: PipelineStageReport[];
+  /** Privacy-checks före publish — bevis för acceptansen. */
+  privacyChecks: { checked: number; blocked: number };
+  recordCount: number;
+  datasetDraft: {
+    dataset_id: string;
+    tenant_id: string;
+    project_id: string;
+    schema_version: string;
+    artifact_refs: string[];
+    description: string;
+    record_count: number;
+    created_by: string;
+  };
+}
+
+export class SyntheticDataFactory {
+  constructor(
+    private readonly generation: GenerationPort,
+    private readonly privacy: PrivacyCheckPort,
+  ) {}
+
+  /** Kör hela kedjan; nekar publish om privacy/quality står sig ej. */
+  async run(input: {
+    sourceDatasetId: string;
+    sourceVersion: number;
+    tenantId: string;
+    projectId: string;
+    sourceRows: SourceRow[];
+    evalRowSignatures?: readonly string[];
+    config: GenerationConfig;
+    createdBy: string;
+  }): Promise<SyntheticDatasetVersionDraft> {
+    const stages: PipelineStageReport[] = [];
+    const cfg = input.config;
+
+    // 1. sampling
+    const sampled = input.sourceRows.slice(0, Math.max(1, Math.ceil(input.sourceRows.length / 2)));
+    stages.push({ stage: 'sampled', inputCount: input.sourceRows.length, outputCount: sampled.length });
+
+    // 2. syntes (INF0)
+    const { rows } = await this.generation.generate({
+      sourceRows: sampled,
+      count: cfg.count,
+      policyRef: cfg.generationPolicyRef,
+    });
+    stages.push({ stage: 'synthesized', inputCount: sampled.length, outputCount: rows.length });
+
+    // 3. PII/privacy — fail-closed FÖRE publish
+    const privacyChecks = { checked: 0, blocked: 0 };
+    const privacyOk: GeneratedRow[] = [];
+    for (const row of rows) {
+      privacyChecks.checked += 1;
+      const cls = await this.privacy.classifyRow(row.fields);
+      if (PII_ORDER[cls] > PII_ORDER[cfg.maxAllowedPiiClass]) {
+        privacyChecks.blocked += 1;
+        continue;
+      }
+      privacyOk.push(row);
+    }
+    stages.push({ stage: 'privacy', inputCount: rows.length, outputCount: privacyOk.length });
+
+    // 4. quality-filter
+    const qualityOk = privacyOk.filter((r) => r.qualityScore >= cfg.minQualityScore);
+    stages.push({ stage: 'quality', inputCount: privacyOk.length, outputCount: qualityOk.length });
+
+    // 5. dedupe (sha256 av fält)
+    const seen = new Set<string>();
+    const deduped: GeneratedRow[] = [];
+    for (const r of qualityOk) {
+      const h = createHash('sha256').update(JSON.stringify(r.fields)).digest('hex');
+      if (seen.has(h)) continue;
+      seen.add(h);
+      deduped.push({ ...r, provenance: { ...r.provenance, sha256: h } });
+    }
+    stages.push({ stage: 'dedupe', inputCount: qualityOk.length, outputCount: deduped.length });
+
+    // 6. contamination — rader identiska med val/test-eval-signaturer raderas
+    const evalSigs = new Set(input.evalRowSignatures ?? []);
+    const clean = deduped.filter((r) => !evalSigs.has(r.provenance.sha256));
+    stages.push({ stage: 'contamination', inputCount: deduped.length, outputCount: clean.length });
+
+    // 7. train/val/test-split (deterministiskt via hash)
+    const split = cfg.split ?? { train: 0.7, val: 0.15, test: 0.15 };
+    const outRows: SyntheticDatasetVersionDraft['rows'] = clean.map((r) => {
+      const frac = parseInt(r.provenance.sha256.slice(0, 8), 16) / 0xffffffff;
+      const s: 'train' | 'val' | 'test' = frac < split.train ? 'train' : frac < split.train + split.val ? 'val' : 'test';
+      return { split: s, provenance: r.provenance, fields: r.fields };
+    });
+    stages.push({ stage: 'split', inputCount: clean.length, outputCount: outRows.length });
+
+    const datasetId = `ds-synthetic-${input.sourceDatasetId}`;
+    return {
+      sourceDatasetId: input.sourceDatasetId,
+      sourceVersion: input.sourceVersion,
+      tenantId: input.tenantId,
+      projectId: input.projectId,
+      schemaVersion: 'synthetic/v1',
+      description: `Synthetic Data Factory: ${input.sourceDatasetId}@${input.sourceVersion} (policy ${cfg.generationPolicyRef}, modell ${cfg.modelVersion})`,
+      rows: outRows,
+      stages,
+      privacyChecks,
+      recordCount: outRows.length,
+      datasetDraft: {
+        dataset_id: datasetId,
+        tenant_id: input.tenantId,
+        project_id: input.projectId,
+        schema_version: 'synthetic/v1',
+        artifact_refs: [`st0re://synthetic/${input.sourceDatasetId}/v${input.sourceVersion}`],
+        description: `Synthetic (källa ${input.sourceDatasetId}@${input.sourceVersion}, policy ${cfg.generationPolicyRef})`,
+        record_count: outRows.length,
+        created_by: input.createdBy,
+      },
+    };
+  }
+}
@@ -0,0 +1,157 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { SyntheticDataFactory, type GenerationConfig, type GenerationPort, type PrivacyCheckPort, type SourceRow } from '../src/synthetic/factory';
+
+function sourceRows(n: number): SourceRow[] {
+  return Array.from({ length: n }, (_, i) => ({ id: `src-${i}`, fields: { fönster: i, kundtyp: i % 2 ? 'B2B' : 'B2C' } }));
+}
+
+function makeFactory(opts: { piiPerRow?: Record<number, Parameters<PrivacyCheckPort['classifyRow']>[0] extends never ? never : string> } = {}): {
+  factory: SyntheticDataFactory;
+  generation: GenerationPort;
+  privacy: PrivacyCheckPort;
+} {
+  const generated = Array.from({ length: 12 }, (_, i) => ({
+    fields: { rad: i, frukt: 'apelsin' },
+    qualityScore: i < 9 ? 0.9 : 0.3, // 3 rader under quality-tröskeln
+    provenance: {
+      rowId: `syn-${i}`,
+      generationPolicyRef: 'cl0ud-policy-syn-1',
+      modelVersion: 'inf0-synthesizer@2.1',
+      sourceDatasetRef: 'ds-kunder@3',
+      sourceRowIds: ['src-1', 'src-2'],
+      sha256: '',
+    },
+  }));
+  const generation: GenerationPort = {
+    generate: vi.fn(async (input) => ({ rows: generated.map((r) => ({ ...r, provenance: { ...r.provenance, sourceDatasetRef: `ds-kunder@3` } })) })),
+  };
+  const privacy: PrivacyCheckPort = {
+    classifyRow: vi.fn(async (fields: Record<string, unknown>) => {
+      const rad = Number((fields as { rad: number }).rad);
+      // rad 0 = PII high (blockas vid maxAllowed=medium), rad 1 = medium
+      if (rad === 0) return 'high';
+      if (rad === 1) return 'medium';
+      return 'none';
+    }),
+  };
+  return { factory: new SyntheticDataFactory(generation, privacy), generation, privacy };
+}
+
+const CFG: GenerationConfig = {
+  generationPolicyRef: 'cl0ud-policy-syn-1',
+  modelVersion: 'inf0-synthesizer@2.1',
+  count: 12,
+  maxAllowedPiiClass: 'medium',
+  minQualityScore: 0.8,
+};
+
+describe('X-737: hela pipelinen', () => {
+  it('sampling→synthesis→privacy→quality→dedupe→contamination→split med steg-rapport', async () => {
+    const { factory: synFactory } = makeFactory();
+    const draft = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    expect(draft.stages.map((s: { stage: string }) => s.stage)).toEqual(['sampled', 'synthesized', 'privacy', 'quality', 'dedupe', 'contamination', 'split']);
+    // 12 syntetiska: 1 PII-blockad + 3 quality-bort → 8 kvar
+    expect(draft.recordCount).toBe(8);
+    expect(draft.privacyChecks).toEqual({ checked: 12, blocked: 1 });
+  });
+
+  it('contamination: rader identiska med eval-signaturer raderas', async () => {
+    const { factory: synFactory } = makeFactory();
+    // räkna ut sha för rad 2:s fält (apelsin-rad) och stoppa i eval-signaturer
+    const { createHash } = await import('node:crypto');
+    const sig = createHash('sha256').update(JSON.stringify({ rad: 2, frukt: 'apelsin' })).digest('hex');
+    const draft = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      evalRowSignatures: [sig],
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    expect(draft.recordCount).toBe(7); // 8 − 1 kontaminerad
+    expect(draft.stages.find((s) => s.stage === 'contamination')?.outputCount).toBe(7);
+  });
+});
+
+describe('X-737 acceptans: varje rad har provenance', () => {
+  it('generation-policy + model-version + source-ref på VARJE rad', async () => {
+    const { factory: synFactory } = makeFactory();
+    const draft = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    expect(draft.rows.length).toBeGreaterThan(0);
+    for (const row of draft.rows) {
+      expect(row.provenance.generationPolicyRef).toBe('cl0ud-policy-syn-1');
+      expect(row.provenance.modelVersion).toBe('inf0-synthesizer@2.1');
+      expect(row.provenance.sourceDatasetRef).toBe('ds-kunder@3');
+      expect(row.provenance.sha256).toMatch(/^[0-9a-f]{64}$/);
+    }
+  });
+
+  it('privacy-check FÖRE publish: blockade rader är ALDRIG i draften', async () => {
+    const { factory: synFactory, generation } = makeFactory();
+    const draft = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    expect(generation.generate).toHaveBeenCalled();
+    // rad 0 (PII high) är borta trots att genereringen producerade den
+    expect(draft.rows.find((r: { provenance: { rowId: string } }) => r.provenance.rowId === 'syn-0')).toBeUndefined();
+    // dataset-draften är CreateDatasetVersionInput-kompatibel
+    expect(draft.datasetDraft.dataset_id).toContain('ds-kunder');
+    expect(draft.datasetDraft.record_count).toBe(draft.recordCount);
+  });
+
+  it('train/val/test-split är deterministiskt och täcker alla rader', async () => {
+    const { factory: synFactory } = makeFactory();
+    const draft = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    const splits = draft.rows.map((r) => r.split);
+    for (const s of splits) {
+      expect(['train', 'val', 'test']).toContain(s);
+    }
+    // determinism: samma körning igen → samma split per provenance-rad
+    const draft2 = await synFactory.run({
+      sourceDatasetId: 'ds-kunder',
+      sourceVersion: 3,
+      tenantId: 't_0pt',
+      projectId: 'p-1',
+      sourceRows: sourceRows(10),
+      config: CFG,
+      createdBy: '0pt-synthetic',
+    });
+    const byId = new Map(draft2.rows.map((r) => [r.provenance.rowId, r.split]));
+    for (const r of draft.rows) {
+      expect(byId.get(r.provenance.rowId)).toBe(r.split);
+    }
+  });
+});