diff --git a/evidence/bi-p029/semantic-parity.json b/evidence/bi-p029/semantic-parity.json new file mode 100644 index 0000000..7fd1db1 --- /dev/null +++ b/evidence/bi-p029/semantic-parity.json @@ -0,0 +1,62 @@ +{ + "prompt": "BI-P029", + "title": "Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer", + "recorded_at": "2026-09-07T06:23:27.839Z", + "clickhouse_version": "26.3.10.60", + "golden_metric": { + "id": "m_revenue", + "name": "net_revenue", + "definition_filters": [ + { + "dimension": "channel", + "operator": "ne", + "values": [ + "internal" + ] + } + ], + "semantic_model": { + "model_id": "sm_sales", + "version": 1, + "hash": "c5b9cac1b5fdcfbfb89a1e960b946a9b33b8c29dc14fe9a304c134f7a0fb73a7" + } + }, + "expected_from_typescript": { + "SE": 1250, + "NO": 400, + "DK": 125.5 + }, + "regression_guarded": { + "description": "the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change", + "result": { + "DK": 3458.5, + "NO": 8177, + "SE": 11249 + } + }, + "path_a": { + "compiler": "0pt compileRequestToClickHouse", + "aggregate": "sumIf", + "result": { + "DK": 125.5, + "NO": 400, + "SE": 1250 + } + }, + "path_b": { + "compiler": "MetricFlowProvider.compile + independent renderer", + "aggregate": "sum(if(...))", + "result": { + "DK": 125.5, + "NO": 400, + "SE": 1250 + } + }, + "tolerance": "1e-9 relative", + "verdict": "PASS", + "still_open": [ + "MetricFlow itself was not executed (no Python runtime provisioned for this repo)", + "the Superset integration path is not implemented", + "the AI Analyst path (inf0#29) has not been compared against this provider" + ] +} diff --git a/evidence/bi-p103/baseline.json b/evidence/bi-p103/baseline.json new file mode 100644 index 0000000..019dadc --- /dev/null +++ b/evidence/bi-p103/baseline.json @@ -0,0 +1,173 @@ +{ + "prompt": "BI-P103", + "title": "First measured BI SLO baselines", + "recorded_at": "2026-09-07T06:26:43.457Z", + "host_note": "One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the shape of the distribution on this fixture — not a production SLA.", + "clickhouse_version": "26.3.10.60", + "fixture": { + "rows": 400000, + "table": "slo_fact", + "partitions": "MergeTree ORDER BY (tenant_id, bucket, ts)" + }, + "workload": { + "cold_dashboard_queries": 120, + "cached_dashboard_queries": 120, + "exploration_queries": 220, + "report_queries": 0 + }, + "window": { + "from": "2026-09-07T06:26:39.422Z", + "to": "2026-09-07T06:26:43.447Z" + }, + "catalogue_size": 11, + "fully_measured": false, + "unresolved_queries": 0, + "summary": { + "met": 7, + "breached": 0, + "insufficient_data": 1, + "not_measurable_here": 3, + "measured_fraction": 0.6363636363636364 + }, + "results": [ + { + "objective_id": "dash.availability", + "workload": "interactive_dashboard", + "kind": "availability", + "status": "met", + "target": 0.995, + "unit": "ratio", + "observed": 1, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.p95_first_query", + "workload": "interactive_dashboard", + "kind": "latency_p95_first", + "status": "met", + "target": 3000, + "unit": "ms", + "observed": 8.893520999999964, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.p95_cached", + "workload": "interactive_dashboard", + "kind": "latency_p95_cached", + "status": "met", + "target": 400, + "unit": "ms", + "observed": 0.024498999999877924, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.error_rate", + "workload": "interactive_dashboard", + "kind": "error_rate", + "status": "met", + "target": 0.005, + "unit": "ratio", + "observed": 0, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.freshness", + "workload": "interactive_dashboard", + "kind": "freshness", + "status": "not_measurable_here", + "target": 900, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from analytics_freshness_watermark; BI usage events cannot answer it — Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage event. Usage events cannot measure it and this evaluator does not pretend to." + }, + { + "objective_id": "explore.p95", + "workload": "exploration", + "kind": "latency_p95", + "status": "met", + "target": 10000, + "unit": "ms", + "observed": 17.072708000000148, + "sample_size": 220, + "low_confidence": false, + "detail": "p95 over 220 samples" + }, + { + "objective_id": "explore.p99", + "workload": "exploration", + "kind": "latency_p99", + "status": "met", + "target": 60000, + "unit": "ms", + "observed": 21.640736000000288, + "sample_size": 220, + "low_confidence": false, + "detail": "p99 over 220 samples" + }, + { + "objective_id": "explore.max_result", + "workload": "exploration", + "kind": "max_result_rows", + "status": "met", + "target": 1000000, + "unit": "rows", + "observed": 96, + "sample_size": 220, + "low_confidence": false, + "detail": "largest result seen in the window" + }, + { + "objective_id": "explore.timeout", + "workload": "exploration", + "kind": "timeout", + "status": "not_measurable_here", + "target": 300, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from engine_configuration; BI usage events cannot answer it — The configured ceiling (BI-P012 settings profile), not an observation." + }, + { + "objective_id": "report.completion_sla", + "workload": "report", + "kind": "completion_sla", + "status": "insufficient_data", + "target": 900000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "no report_generated events" + }, + { + "objective_id": "report.delivery_sla", + "workload": "report", + "kind": "delivery_sla", + "status": "not_measurable_here", + "target": 300000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from n0tify_delivery_receipt; BI usage events cannot answer it — Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this objective is declared and reported unmeasured rather than proxied by completion." + } + ], + "measured_but_narrower_than_production": [ + "dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey deployment adds a network round trip, so 0.03 ms here is a floor and not the number a user will see.", + "Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no serialization to the caller. The end-to-end p95 is larger by whatever those cost.", + "One tenant, one client, no concurrency. Contention is not represented." + ], + "not_measured_and_why": [ + "dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event", + "explore.timeout — the configured BI-P012 ceiling, not an observation", + "report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0", + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event" + ] +} diff --git a/src/analytics/contract/compile-clickhouse.ts b/src/analytics/contract/compile-clickhouse.ts index 5ff7071..d31b373 100644 --- a/src/analytics/contract/compile-clickhouse.ts +++ b/src/analytics/contract/compile-clickhouse.ts @@ -38,14 +38,19 @@ const GRAIN_FN: Record = { /** * Aggregation -> conditional-aggregate template. `count_distinct` uses * `uniqExactIf` so a multi-metric request stays one scan instead of N. + * + * The argument is the WHOLE per-metric predicate, not just the metric-key + * comparison, because a metric definition can carry its own filters (BI-P028 + * `MetricSpec.filters`) and those belong INSIDE this metric's `If` — one + * request may carry two metrics that exclude different rows. */ -const AGG_IF: Record string> = { - sum: (p) => `sumIf(value, metric = {${p}:String})`, - avg: (p) => `avgIf(value, metric = {${p}:String})`, - min: (p) => `minIf(value, metric = {${p}:String})`, - max: (p) => `maxIf(value, metric = {${p}:String})`, - count: (p) => `countIf(metric = {${p}:String})`, - count_distinct: (p) => `uniqExactIf(event_id, metric = {${p}:String})`, +const AGG_IF: Record string> = { + sum: (p) => `sumIf(value, ${p})`, + avg: (p) => `avgIf(value, ${p})`, + min: (p) => `minIf(value, ${p})`, + max: (p) => `maxIf(value, ${p})`, + count: (p) => `countIf(${p})`, + count_distinct: (p) => `uniqExactIf(event_id, ${p})`, }; export interface ClickHouseCompileOptions { @@ -124,7 +129,19 @@ export function compileRequestToClickHouse( if (!template) { throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported aggregation "${m.aggregation}"`); } - selectParts.push(`${template(key)} AS ${alias}`); + // The metric's own definition filters (BI-P028) join the metric-key + // comparison inside THIS metric's `If`. Bound, never spliced, and scoped + // to this metric — putting them in the shared WHERE would apply one + // metric's meaning to every other metric in the same request. + const metricConditions = [`metric = {${key}:String}`]; + (m.definition_filters ?? []).forEach((f, j) => { + assertSafeIdentifier(f.field_name, 'metric definition filter dimension'); + const nameKey = `mf${i}_${j}`; + params[`param_${nameKey}`] = f.field_name; + const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`; + metricConditions.push(renderFilter(lhs, nameKey, f.operator, [...f.values], `${i}_${j}m`, params)); + }); + selectParts.push(`${template(metricConditions.join(' AND '))} AS ${alias}`); params[`param_${key}`] = m.event_metric_key; metricKeyParams.push(`{${key}:String}`); metricAliases[alias] = m.name; @@ -156,7 +173,9 @@ export function compileRequestToClickHouse( const nameKey = `fn${i}`; params[`param_${nameKey}`] = entry.dimension.field_name; const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`; - conditions.push(renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], i, params)); + conditions.push( + renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], String(i), params), + ); }); // ── order by: resolved back onto compiler-generated aliases only ────── @@ -202,7 +221,13 @@ function renderFilter( nameKey: string, operator: FilterOperator, values: string[], - index: number, + /** + * Uniquifies this filter's bound-value parameter names. A STRING, not a + * number, because request filters and per-metric definition filters share + * one parameter map: `0` and `0_0m` must not collide, and two collided + * bindings would silently make one filter win over the other. + */ + index: string, params: Record, ): string { const bind = (v: string, suffix: string): string => { diff --git a/src/analytics/contract/compile-semantic.ts b/src/analytics/contract/compile-semantic.ts index 8cb1ac8..bb9c93e 100644 --- a/src/analytics/contract/compile-semantic.ts +++ b/src/analytics/contract/compile-semantic.ts @@ -24,6 +24,14 @@ export interface SemanticMetricRef { /** Canonical SIAX metric id — stable across semantic-engine replacement. */ metric_id: string; name: string; + /** + * The metric DEFINITION's own filters, carried through so the semantic + * target computes the same number as the ClickHouse target. Dropping them + * on one side and not the other is how one canonical metric id comes to + * mean two different things — the divergence BI-P029's DoD is written to + * catch. Structured, like every other filter here: never predicate text. + */ + definition_filters?: readonly SemanticWhere[]; } export interface SemanticGroupBy { @@ -81,7 +89,23 @@ export function compileRequestToSemantic(resolved: ResolvedQuery): SemanticQuery return { dataset_id: resolved.dataset_id, semantic_model_version: resolved.semantic_model_version, - metrics: resolved.metrics.map((m) => ({ metric_id: m.metric_id, name: m.name })), + metrics: resolved.metrics.map((m) => ({ + metric_id: m.metric_id, + name: m.name, + ...(m.definition_filters?.length + ? { + definition_filters: m.definition_filters.map((f) => ({ + // Already resolved onto the physical key; the dimension id is + // the same value here because the definition names a field, not + // a caller-supplied alias. + dimension_id: f.field_name, + name: f.field_name, + operator: f.operator, + values: [...f.values], + })), + } + : {}), + })), groupBy, where: resolved.filters.map((f) => ({ dimension_id: f.dimension.dimension_id, diff --git a/src/analytics/contract/query-request.ts b/src/analytics/contract/query-request.ts index 4f0e9f4..dc7fbaa 100644 --- a/src/analytics/contract/query-request.ts +++ b/src/analytics/contract/query-request.ts @@ -351,6 +351,27 @@ function parseOrderBy(raw: unknown, index: number): AnalyticsOrderBy { * What a metric NAME resolves to. Supplied from `MetricDefinition` rows * (BI-P002/BI-P028); the compilers never invent one. */ +/** + * A filter that belongs to the METRIC DEFINITION rather than to the request + * (BI-P028 `MetricSpec.filters`) — "net revenue excludes internal transfers" + * is part of what the metric MEANS, not something a caller asks for. + * + * It is carried in ALREADY-RESOLVED form: `field_name` is the physical key + * inside `dim_names`, mapped from the definition's dimension name by whoever + * builds the resolution. The compilers stay dumb — they bind and render, they + * never look a dimension up — and `assertSafeIdentifier` still applies. + * + * These filters are NOT optional decoration. A compile path that drops them + * answers a different question under the same canonical metric id, which is + * exactly the divergence BI-P029's DoD exists to catch; see + * tests/integration/semantic-parity.integration.test.ts. + */ +export interface ResolvedMetricFilter { + field_name: string; + operator: FilterOperator; + values: readonly string[]; +} + export interface ResolvedMetric { name: string; metric_id: string; @@ -360,6 +381,14 @@ export interface ResolvedMetric { /** Dimension names this metric may legitimately be sliced by. */ allowed_dimensions: readonly string[]; unit?: string; + /** + * The metric definition's own filters. Applied PER METRIC — folded into that + * metric's conditional aggregate — never into the shared WHERE clause: a + * request may ask for two metrics whose definitions exclude different rows, + * and a shared predicate would silently apply one metric's meaning to the + * other. + */ + definition_filters?: readonly ResolvedMetricFilter[]; } /** What a dimension NAME resolves to. Supplied from `DimensionDefinition` rows. */ diff --git a/src/analytics/metrics/semantic-provider.ts b/src/analytics/metrics/semantic-provider.ts index 30d4ea0..435f1d0 100644 --- a/src/analytics/metrics/semantic-provider.ts +++ b/src/analytics/metrics/semantic-provider.ts @@ -16,8 +16,11 @@ * happen first — `METRICFLOW_PINNED_VERSION` below records the intended pin * and is explicitly marked unverified). * - * So what IS proven, in tests/integration/semantic-parity.integration.test.ts, - * is the part that does not need MetricFlow: the SAME golden metric, taken + * So what IS proven, in tests/integration/semantic-parity.integration.test.ts + * — a path this header cited from the day it was written, while the file + * itself did not exist until BI-P029's gap review; the citation was the + * evidence, which is no evidence at all — is the part that does not need + * MetricFlow: the SAME golden metric, taken * from the SAME registry version, executed against the SAME live ClickHouse * data by TWO INDEPENDENT compilers — 0pt's metric compiler, and a renderer * driven only by the engine-neutral `SemanticQuerySpec` — returns the same @@ -169,10 +172,28 @@ export class MetricFlowProvider implements SiaxSemanticProvider { } } const timeBucket = spec.groupBy.find((g) => g.grain); + // Per-metric definition filters (BI-P028 `MetricSpec.filters`), kept + // SEPARATE from the request's `where`. Folding them together would apply + // one metric's meaning to every metric in the query; omitting them would + // make this provider compute a different number from the ClickHouse + // target for the same canonical metric id. + const metric_filters: Record< + string, + Array<{ dimension: string; operator: string; values: string[] }> + > = {}; + for (const m of spec.metrics) { + if (!m.definition_filters?.length) continue; + metric_filters[m.metric_id] = m.definition_filters.map((f) => ({ + dimension: f.name, + operator: f.operator, + values: [...f.values], + })); + } return { provider: this.provider, query: { metrics: spec.metrics.map((m) => m.metric_id), + ...(Object.keys(metric_filters).length ? { metric_filters } : {}), group_by: spec.groupBy.filter((g) => !g.grain).map((g) => g.name), ...(timeBucket ? { time_granularity: timeBucket.grain } : {}), // STRUCTURED, not a `--where` SQL string. MetricFlow accepts predicate diff --git a/src/analytics/slo/bi-slo.ts b/src/analytics/slo/bi-slo.ts index b4bad4b..c172c94 100644 --- a/src/analytics/slo/bi-slo.ts +++ b/src/analytics/slo/bi-slo.ts @@ -78,8 +78,18 @@ export interface SloObjective { * The catalogue, field-for-field with what BI-P103 lists per workload class. * Targets are STARTING POINTS chosen to be honest about what has been * measured: they are the thresholds the evaluator compares against, and - * evidence/bi-p103 records the first baselines rather than pretending the - * targets were derived from a year of production traffic. + * evidence/bi-p103/baseline.json records the first baselines rather than + * pretending the targets were derived from a year of production traffic. + * + * Those baselines are produced by + * tests/integration/bi-p103-slo-baseline.integration.test.ts, which runs 460 + * real queries against a live ClickHouse and feeds them through this + * evaluator. Read the file's own caveats before quoting a number from it: the + * cached path measured there is the in-process reference cache rather than + * Valkey, every latency is the engine round trip with no HTTP or + * authorization layer around it, and there is no concurrency. Seven of the + * eleven objectives produced a number; the other four say why they could not, + * and `fully_measured` is false. */ export const BI_SLO_CATALOGUE: readonly SloObjective[] = [ // ── interactive dashboard ──────────────────────────────────────────────── diff --git a/tests/integration/bi-p103-slo-baseline.integration.test.ts b/tests/integration/bi-p103-slo-baseline.integration.test.ts new file mode 100644 index 0000000..6fecacd --- /dev/null +++ b/tests/integration/bi-p103-slo-baseline.integration.test.ts @@ -0,0 +1,341 @@ +/** + * tests/integration/bi-p103-slo-baseline.integration.test.ts + * =========================================================== + * BI-P103 — "Measured baselines exist." + * + * `src/analytics/slo/bi-slo.ts` said the first baselines were recorded in + * `evidence/bi-p103`. There was no `evidence/bi-p103`. The catalogue and the + * evaluator were both real and both well tested — against SYNTHETIC events. + * A synthetic event carries whatever duration the test author typed, so a + * suite built only on those proves the arithmetic and nothing about the + * system: it cannot tell you a dashboard query takes 8 ms or 8 s. + * + * So this harness produces the baselines from queries that actually ran: + * + * * ~120 COLD dashboard queries, each a different filter so the BI-P020 + * cache genuinely misses, executed against a live ClickHouse; + * * the same ~120 again through the SAME shipped cache, now hitting, so + * `dash.p95_cached` is a measured cache path and not an assumption; + * * 220 exploration queries, enough for `explore.p99`'s min_samples of 200 + * to be met rather than reported low-confidence. + * + * Every duration is wall-clock around the real call. The telemetry goes + * through the SHIPPED producer (`biUsageEventsForExecution`) and the SHIPPED + * evaluator (`evaluateBiSlos`) — not a copy — so the evidence is produced by + * the code under test. + * + * WHAT THE BASELINE DELIBERATELY DOES NOT CONTAIN. `dash.freshness`, + * `explore.timeout` and `report.delivery_sla` are not measurable from usage + * events and the evaluator says so; `report.completion_sla` has no producer in + * 0pt at all (nothing here emits `aud0.report_generated`). They appear in the + * evidence with their real status rather than being quietly dropped, because a + * baseline table that lists only the rows that worked is how an estate comes + * to believe it measures more than it does. + * + * These numbers describe THIS fixture on THIS machine against a local + * ClickHouse with warm page cache. They are a floor for the shape of the + * distribution, not a production SLA — and the evidence file says which parts + * are narrower than production: the cached path is the in-process reference + * cache rather than Valkey, every latency is the engine round trip with no + * HTTP or authorization layer around it, and there is no concurrency. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * pnpm vitest run tests/integration/bi-p103-slo-baseline.integration.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { InMemoryResultCache, type CacheKeyComponents } from '@/src/analytics/cache/query-cache'; +import { compiledQueryHash } from '@/src/analytics/execution/query-execution-record'; +import { + biUsageEventsForExecution, + type Aud0BiEvent, + type ExecutionTelemetry, +} from '@/src/analytics/slo/usage-events'; +import { BI_SLO_CATALOGUE, evaluateBiSlos, summariseSloReport } from '@/src/analytics/slo/bi-slo'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_p103_${Date.now().toString(36)}`; +const TENANT = 't_alpha'; +const ROWS = 400_000; +const COLD_DASHBOARD_QUERIES = 120; +const EXPLORE_QUERIES = 220; + +const headers: Record = CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } + : {}; + +async function ch(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + url.searchParams.set('query', sql); + const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`); + return text; +} + +interface Measured { + rows: number; + ms: number; +} + +/** One real round trip, wall-clock measured. */ +async function timed(sql: string): Promise { + const started = performance.now(); + const text = await ch(`${sql} FORMAT JSON`); + const ms = performance.now() - started; + const parsed = JSON.parse(text) as { data: unknown[] }; + return { rows: parsed.data.length, ms }; +} + +const events: Aud0BiEvent[] = []; + +function record(t: ExecutionTelemetry): void { + events.push(...biUsageEventsForExecution(t)); +} + +function keyFor(i: number, cachedShape: string): CacheKeyComponents { + return { + tenant_id: TENANT, + dataset_id: 'ds_slo', + semantic_model_version: 1, + metric_versions: { m_amount: 1 }, + compiled_query_hash: compiledQueryHash(cachedShape, { i: String(i) }), + data_snapshot: 'v1', + format: 'json', + }; +} + +let windowFrom: Date; +let windowTo: Date; + +run('BI-P103 — measured baselines, from queries that actually ran', () => { + beforeAll(async () => { + await new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { [TENANT]: { region: 'region' } }, + }).ensureSchema(); + + // A fact table large enough that a scan is not free. Generated inside + // ClickHouse so the fixture cost is not measured as query latency. + await ch( + `CREATE TABLE ${DB}.slo_fact (tenant_id LowCardinality(String), region LowCardinality(String), ` + + `bucket UInt16, ts DateTime, amount Float64) ENGINE = MergeTree ORDER BY (tenant_id, bucket, ts)`, + ); + await ch( + `INSERT INTO ${DB}.slo_fact SELECT 't_alpha', ['SE','NO','DK','FI'][(number % 4) + 1], ` + + `toUInt16(number % 500), toDateTime('2026-09-01 00:00:00') + (number % 86400), ` + + `toFloat64(number % 997) FROM numbers(${ROWS})`, + ); + await ch(`OPTIMIZE TABLE ${DB}.slo_fact FINAL`); + }, 300_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await ch(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it('measures a cold and a cached dashboard path, and an exploration path', async () => { + windowFrom = new Date(); + const cache = new InMemoryResultCache(); + const shape = `SELECT region, sum(amount) FROM ${DB}.slo_fact WHERE tenant_id = ? AND bucket = ? GROUP BY region`; + + // ── cold: every query a different bucket, so the cache genuinely misses ── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const components = keyFor(i, shape); + const started = new Date().toISOString(); + const before = await cache.get(components); + expect(before).toBeUndefined(); // a "cold" sample served from cache is not cold + const m = await timed( + `SELECT region, sum(amount) AS total FROM ${DB}.slo_fact ` + + `WHERE tenant_id = 't_alpha' AND bucket = ${i} GROUP BY region ORDER BY region`, + ); + await cache.set(components, { rows: m.rows }, { ttlSeconds: 300, rowCount: m.rows }); + record({ + query_id: `q_cold_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + + // ── cached: the same questions, through the same shipped cache ────────── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const started = new Date().toISOString(); + const t0 = performance.now(); + const hit = await cache.get(keyFor(i, shape)); + const ms = performance.now() - t0; + expect(hit).toBeDefined(); // a miss recorded as a hit would flatter the baseline + record({ + query_id: `q_hot_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: ms, + row_count: hit?.row_count ?? 0, + cache_hit: true, + }); + } + + // ── exploration: wider, ad-hoc shapes, enough samples for a real p99 ──── + for (let i = 0; i < EXPLORE_QUERIES; i++) { + const started = new Date().toISOString(); + const m = await timed( + `SELECT region, toStartOfHour(ts) AS h, sum(amount) AS total, count() AS n ` + + `FROM ${DB}.slo_fact WHERE tenant_id = 't_alpha' AND bucket >= ${i % 400} ` + + `GROUP BY region, h ORDER BY region, h LIMIT 5000`, + ); + record({ + query_id: `q_explore_${i}`, + surface: 'explore', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + windowTo = new Date(); + + expect(events.length).toBeGreaterThan(2 * (COLD_DASHBOARD_QUERIES * 2 + EXPLORE_QUERIES) - 1); + }, 600_000); + + it('a REAL engine error becomes a failed event and moves the error rate', async () => { + // Proves the availability and error-rate paths are not measuring silence. + // Kept OUT of the baseline window so the recorded baseline describes the + // system's normal behaviour rather than a deliberately broken query. + const started = new Date().toISOString(); + let code: string | undefined; + const t0 = performance.now(); + try { + await ch(`SELECT no_such_column FROM ${DB}.slo_fact LIMIT 1 FORMAT JSON`); + } catch { + code = 'ENGINE_ERROR'; + } + expect(code).toBe('ENGINE_ERROR'); + + const failed = biUsageEventsForExecution({ + query_id: 'q_fail_1', + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'failed', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: performance.now() - t0, + error_code: code, + error_message: 'Unknown expression identifier no_such_column', + }); + expect(failed.map((e) => e.event_type)).toContain('aud0.analytics_query_failed'); + // The engine's message never becomes telemetry. + expect(JSON.stringify(failed)).not.toContain('no_such_column'); + + const withFailure = evaluateBiSlos({ + events: [...events, ...failed], + from: windowFrom, + to: new Date(), + }); + const availability = withFailure.results.find((r) => r.objective_id === 'dash.availability'); + expect(availability?.sample_size).toBeGreaterThanOrEqual(100); + expect(availability?.observed).toBeLessThan(1); + }); + + it('writes the baseline, including every objective it could NOT measure', async () => { + const report = evaluateBiSlos({ events, from: windowFrom, to: windowTo }); + const summary = summariseSloReport(report); + + // The three latency objectives must be real numbers over real samples — + // an evidence file full of insufficient_data is not a baseline. + for (const id of ['dash.p95_first_query', 'dash.p95_cached', 'explore.p95', 'explore.p99']) { + const r = report.results.find((x) => x.objective_id === id); + expect(r?.observed, `${id} produced no number`).toBeTypeOf('number'); + expect(r?.low_confidence, `${id} was computed from too few samples`).not.toBe(true); + } + // ...and the cached path must actually be faster than the cold one, or the + // two were not measuring different things. + const cold = report.results.find((r) => r.objective_id === 'dash.p95_first_query')?.observed ?? 0; + const cached = report.results.find((r) => r.objective_id === 'dash.p95_cached')?.observed ?? 0; + expect(cached).toBeLessThan(cold); + + const dir = path.join(process.cwd(), 'evidence', 'bi-p103'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'baseline.json'), + `${JSON.stringify( + { + prompt: 'BI-P103', + title: 'First measured BI SLO baselines', + recorded_at: new Date().toISOString(), + host_note: + 'One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the ' + + 'shape of the distribution on this fixture — not a production SLA.', + clickhouse_version: JSON.parse(await ch('SELECT version() AS v FORMAT JSON')).data[0].v, + fixture: { + rows: ROWS, + table: 'slo_fact', + partitions: 'MergeTree ORDER BY (tenant_id, bucket, ts)', + }, + workload: { + cold_dashboard_queries: COLD_DASHBOARD_QUERIES, + cached_dashboard_queries: COLD_DASHBOARD_QUERIES, + exploration_queries: EXPLORE_QUERIES, + report_queries: 0, + }, + window: { from: windowFrom.toISOString(), to: windowTo.toISOString() }, + catalogue_size: BI_SLO_CATALOGUE.length, + fully_measured: report.fully_measured, + unresolved_queries: report.unresolved_queries, + summary, + results: report.results.map((r) => ({ + objective_id: r.objective_id, + workload: r.workload, + kind: r.kind, + status: r.status, + target: r.target, + unit: r.unit, + observed: r.observed, + sample_size: r.sample_size, + low_confidence: r.low_confidence ?? false, + detail: r.detail, + })), + measured_but_narrower_than_production: [ + 'dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey ' + + 'deployment adds a network round trip, so 0.03 ms here is a floor and not the number a ' + + 'user will see.', + 'Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no ' + + 'serialization to the caller. The end-to-end p95 is larger by whatever those cost.', + 'One tenant, one client, no concurrency. Contention is not represented.', + ], + not_measured_and_why: [ + 'dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event', + 'explore.timeout — the configured BI-P012 ceiling, not an observation', + 'report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0', + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event", + ], + }, + null, + 2, + )}\n`, + ); + + // The file must be honest about the unmeasured half. + expect(report.fully_measured).toBe(false); + }); +}); diff --git a/tests/integration/semantic-parity.integration.test.ts b/tests/integration/semantic-parity.integration.test.ts new file mode 100644 index 0000000..067ee60 --- /dev/null +++ b/tests/integration/semantic-parity.integration.test.ts @@ -0,0 +1,478 @@ +/** + * tests/integration/semantic-parity.integration.test.ts + * ====================================================== + * BI-P029 — the parity test `src/analytics/metrics/semantic-provider.ts` has + * been citing. + * + * IT DID NOT EXIST. The provider's header named this path as the proof of the + * one DoD surface it claimed was closed, and `providerReadiness().proven` + * asserted that "0pt's metric compiler and an independent spec-driven renderer + * return the same number for the golden metric against live ClickHouse data". + * Nothing in the repository executed that comparison. A citation to a file + * that is not there is the same failure as a green check that gates nothing: + * it survives review because the sentence reads like evidence. + * + * So this file is the comparison, run for real: + * + * PATH A parseAnalyticsQueryRequest -> resolveAnalyticsQuery -> + * compileRequestToClickHouse -> live ClickHouse. + * PATH B the same ResolvedQuery -> compileRequestToSemantic -> + * MetricFlowProvider.compile -> `renderIndependently` below, which + * reads ONLY the provider's structured query object and the registry + * model, writes its own SQL with its own quoting, and shares no line + * of code with compile-clickhouse.ts -> the same live ClickHouse. + * + * The two paths must return the same numbers. `renderIndependently` inlines + * literals where compile-clickhouse binds parameters, groups by the physical + * column expression rather than by a compiler alias, and orders in SQL rather + * than by alias — so the SQL strings genuinely differ, and the test asserts + * that they do. Two identical strings would make the parity claim vacuous. + * + * WHAT THIS STILL DOES NOT PROVE, and `providerReadiness()` still says so: + * MetricFlow itself is not executed here (no Python runtime is provisioned for + * this repo), and neither the Superset path nor the AI Analyst path is + * compared. Three of BI-P029's four DoD surfaces remain open. This closes the + * first one honestly rather than closing all four by assertion. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * pnpm vitest run tests/integration/semantic-parity.integration.test.ts + * + * It CREATEs and DROPs its own database. Never point it at the b0ts + * ClickHouse or at any quarantined dataset. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { + parseAnalyticsQueryRequest, + queryResolutionFrom, + resolveAnalyticsQuery, + type ResolvedQuery, +} from '@/src/analytics/contract/query-request'; +import { compileRequestToClickHouse } from '@/src/analytics/contract/compile-clickhouse'; +import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic'; +import { MetricFlowProvider } from '@/src/analytics/metrics/semantic-provider'; +import { + InMemorySemanticModelStore, + SemanticModelRegistry, + type SemanticModelVersion, +} from '@/src/analytics/metrics/semantic-registry'; +import type { MetricSpec } from '@/src/analytics/metrics/metric-model'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +// Collection-time, never inside a hook: `describe.skipIf(flagSetInBeforeAll)` +// evaluates its flag before any hook has run and therefore skips ALWAYS. +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_parity_${Date.now().toString(36)}`; +const TENANT = 't_alpha'; +const PROJECT = 'p_main'; + +/** + * The golden metric. It carries a DEFINITION FILTER (`channel != 'internal'`) + * on purpose: a metric whose meaning is only "sum this column" cannot tell a + * parity test anything, because every compiler agrees about a bare sum. The + * business meaning of net revenue is that internal transfers are not revenue, + * and that meaning has to survive both compile paths or the two paths are + * answering different questions with the same metric id. + */ +const REVENUE: MetricSpec = { + id: 'm_revenue', + name: 'net_revenue', + description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.', + type: 'simple', + measure: 'amount_sek', + aggregation: 'sum', + dimensions: ['country', 'channel'], + filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }], + timeDimension: 'invoiced_at', + unit: 'SEK', + currency: 'SEK', + format: 'currency', + owner: 'finance@siax.io', + version: 3, + status: 'ACTIVE', +}; + +/** dimension name -> the key stored in `dim_names`. One mapping, both paths. */ +const FIELD_OF: Record = { country: 'country_code', channel: 'sales_channel' }; + +function semanticModel(version = 1): Omit { + return { + model_id: 'sm_sales', + version, + tenant_id: TENANT, + datasets: ['ds_sales'], + relationships: [], + metrics: [REVENUE], + dimensions: [ + { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' }, + ], + valid_from: '2026-09-01T00:00:00Z', + status: 'draft', + }; +} + +/** + * The fixture. Deliberately arranged so that dropping the metric's definition + * filter changes the answer for EVERY country — a fixture where internal rows + * happen to sum to zero would let a broken compiler pass. + */ +const ROWS: Array<{ country: string; channel: string; amount: number }> = [ + { country: 'SE', channel: 'direct', amount: 1000 }, + { country: 'SE', channel: 'partner', amount: 250 }, + { country: 'SE', channel: 'internal', amount: 9999 }, + { country: 'NO', channel: 'direct', amount: 400 }, + { country: 'NO', channel: 'internal', amount: 7777 }, + { country: 'DK', channel: 'partner', amount: 125.5 }, + { country: 'DK', channel: 'internal', amount: 3333 }, +]; + +/** What the metric MEANS, computed in TypeScript, independent of any SQL. */ +const EXPECTED: Record = ROWS.filter((r) => r.channel !== 'internal').reduce( + (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), + {} as Record, +); + +interface ChRow { + [column: string]: string | number; +} + +async function chQuery(sql: string, params: Record = {}): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + const res = await fetch(url.toString(), { method: 'POST', headers, body: sql }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 400)}`); + if (!text.trim()) return []; + return (JSON.parse(text) as { data: ChRow[] }).data; +} + +async function chExec(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + url.searchParams.set('query', sql); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${(await res.text()).slice(0, 400)}`); +} + +// ── PATH B: the independent renderer ─────────────────────────────────────── + +/** Single-quote escaping, written here because this renderer shares nothing. */ +function lit(value: string): string { + return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; +} + +function predicate(field: string, operator: string, values: string[]): string { + const lhs = `dim_values[indexOf(dim_names, ${lit(field)})]`; + switch (operator) { + case 'eq': + return `${lhs} = ${lit(values[0])}`; + case 'ne': + return `${lhs} <> ${lit(values[0])}`; + case 'in': + return `${lhs} IN (${values.map(lit).join(',')})`; + case 'not_in': + return `${lhs} NOT IN (${values.map(lit).join(',')})`; + default: + throw new Error(`independent renderer does not implement operator ${operator}`); + } +} + +/** + * Render the provider's structured query into SQL, reading ONLY that object + * and the registry model. It never sees the ResolvedQuery, the ClickHouse + * compiler's output, or its parameter map. + * + * The aggregate is deliberately expressed differently from compile-clickhouse: + * a `sum(if(...))` rather than a `sumIf(...)`, grouped by the raw expression + * rather than by an alias. Same arithmetic, different SQL — which is the only + * way the comparison means anything. + */ +function renderIndependently( + compiled: ReturnType, + model: SemanticModelVersion, +): string { + const query = compiled.query as { + metrics: string[]; + group_by: string[]; + where: Array<{ dimension: string; operator: string; values: string[] }>; + limit: number; + metric_filters?: Record>; + }; + const byId = new Map(model.metrics.map((m) => [m.id, m])); + + const selects: string[] = []; + const groups: string[] = []; + for (const dimension of query.group_by) { + const expr = `dim_values[indexOf(dim_names, ${lit(FIELD_OF[dimension])})]`; + selects.push(`${expr} AS ${dimension}`); + groups.push(expr); + } + for (const metricId of query.metrics) { + const spec = byId.get(metricId); + if (!spec) throw new Error(`metric ${metricId} is not in the model version`); + // The metric's own definition filters. The provider query carries them + // already resolved onto physical keys; the registry model states them in + // dimension names. Either way they are the metric's MEANING, and a + // renderer that ignored them would compute a different number. + const own = query.metric_filters?.[metricId] + ? query.metric_filters[metricId].map((f) => ({ + field: f.dimension, + operator: f.operator, + values: f.values, + })) + : spec.filters.map((f) => ({ + field: FIELD_OF[f.dimension], + operator: f.operator, + values: f.values ?? [], + })); + const conditions = [ + `metric = ${lit(spec.measure)}`, + ...own.map((f) => predicate(f.field, f.operator, f.values)), + ]; + selects.push(`sum(if(${conditions.join(' AND ')}, value, 0)) AS ${spec.id}`); + } + + const where = [ + `tenant_id = ${lit(TENANT)}`, + `project_id = ${lit(PROJECT)}`, + `dataset_id = ${lit('ds_sales')}`, + ...query.where.map((w) => predicate(FIELD_OF[w.dimension], w.operator, w.values ?? [])), + ]; + + return ( + `SELECT ${selects.join(', ')} FROM ${DB}.opt_analytics_event ` + + `WHERE ${where.join(' AND ')} ` + + (groups.length ? `GROUP BY ${groups.join(', ')} ORDER BY ${groups.join(', ')} ` : '') + + `LIMIT ${query.limit} FORMAT JSON` + ); +} + +// ── the suite ────────────────────────────────────────────────────────────── + +let model: SemanticModelVersion; +let resolved: ResolvedQuery; +/** Filled by the regression test below; recorded in the evidence file. */ +let preFix: Record = {}; + +run('BI-P029 — one golden metric, two independent compilers, one live ClickHouse', () => { + beforeAll(async () => { + const engine = new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { [TENANT]: FIELD_OF }, + }); + await engine.ensureSchema(); + + const body = ROWS.map((r, i) => + JSON.stringify({ + tenant_id: TENANT, + project_id: PROJECT, + dataset_id: 'ds_sales', + metric: 'amount_sek', + event_id: `e${i}`, + event_time: '2026-09-03 12:00:00.000', + value: r.amount, + dim_names: ['country_code', 'sales_channel'], + dim_values: [r.country, r.channel], + }), + ).join('\n'); + await chExec(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, body); + + model = await new SemanticModelRegistry(new InMemorySemanticModelStore()).publish(semanticModel(1)); + + resolved = resolveAnalyticsQuery( + parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['net_revenue'], + dimensions: ['country'], + orderBy: [{ ref: 'country', direction: 'asc' }], + }), + queryResolutionFrom({ + dataset_id: 'ds_sales', + semantic_model_version: 1, + metrics: [ + { + name: 'net_revenue', + metric_id: 'm_revenue', + event_metric_key: 'amount_sek', + aggregation: 'sum', + allowed_dimensions: ['country', 'channel'], + // The metric's OWN filters, resolved onto physical field names. + definition_filters: REVENUE.filters.map((f) => ({ + field_name: FIELD_OF[f.dimension], + operator: f.operator, + values: f.values ?? [], + })), + }, + ], + dimensions: [ + { name: 'country', dimension_id: 'd_country', field_name: 'country_code' }, + { name: 'channel', dimension_id: 'd_channel', field_name: 'sales_channel' }, + ], + }), + ); + }, 120_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await chExec(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it('the fixture is arranged so a dropped definition filter CHANGES every answer', () => { + const withInternal = ROWS.reduce( + (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), + {} as Record, + ); + for (const country of Object.keys(EXPECTED)) { + expect(withInternal[country]).not.toBe(EXPECTED[country]); + } + }); + + it('PATH A — 0pt’s ClickHouse compiler returns the metric’s meaning', async () => { + const compiled = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + const rows = await chQuery(compiled.sql, compiled.params); + const got = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); + expect(got).toEqual(EXPECTED); + }); + + it('PATH B — an independent renderer driven only by the provider spec agrees', async () => { + const spec = compileRequestToSemantic(resolved); + const compiled = new MetricFlowProvider().compile(spec, model); + expect(compiled.canonical_metric_ids).toEqual(['m_revenue']); + + const rows = await chQuery(renderIndependently(compiled, model)); + const got = Object.fromEntries(rows.map((r) => [String(r.country), Number(r.m_revenue)])); + expect(got).toEqual(EXPECTED); + }); + + /** + * The regression this suite was written for. Until BI-P029's gap review, + * `compileRequestToClickHouse` had no notion of a metric's own filters: + * `ResolvedMetric` did not carry them and the compiler never asked. The + * metric definition said "excluding internal transfers" and the executed + * SQL did not, so `net_revenue` returned a number that looked plausible and + * was ~9x too large for this fixture. + * + * Re-run here by stripping the definition filters from the resolution, so + * the number the bug produced is measured by the committed test rather than + * remembered from a console. + */ + it('a resolution WITHOUT the definition filters produces the pre-fix (wrong) number', async () => { + const stripped: ResolvedQuery = { + ...resolved, + metrics: resolved.metrics.map((m) => ({ ...m, definition_filters: undefined })), + }; + const compiled = compileRequestToClickHouse(stripped, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + const rows = await chQuery(compiled.sql, compiled.params); + preFix = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); + + // Not merely different — OVERSTATED, on every country, by the internal + // transfers the definition excludes. + for (const country of Object.keys(EXPECTED)) { + expect(preFix[country]).toBeGreaterThan(EXPECTED[country]); + } + expect(compiled.sql).not.toContain('mf0_0'); + }); + + it('the golden metric returns the SAME number on both paths, within tolerance', async () => { + const a = await chQuery( + ...(() => { + const c = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + return [c.sql, c.params] as const; + })(), + ); + const b = await chQuery( + renderIndependently(new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), model), + ); + + const left = Object.fromEntries(a.map((r) => [String(r.dim_0), Number(r.metric_0)])); + const right = Object.fromEntries(b.map((r) => [String(r.country), Number(r.m_revenue)])); + + expect(Object.keys(left).sort()).toEqual(Object.keys(right).sort()); + for (const key of Object.keys(left)) { + // Relative tolerance, because two different aggregate expressions over + // Float64 may associate the additions differently. + const denominator = Math.max(Math.abs(left[key]), 1); + expect(Math.abs(left[key] - right[key]) / denominator).toBeLessThan(1e-9); + } + + // Parity between two identical strings proves nothing. + const sqlA = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }).sql; + const sqlB = renderIndependently( + new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), + model, + ); + expect(sqlA).not.toBe(sqlB); + expect(sqlA).toContain('sumIf('); + expect(sqlB).toContain('sum(if('); + + fs.mkdirSync(path.join(process.cwd(), 'evidence', 'bi-p029'), { recursive: true }); + fs.writeFileSync( + path.join(process.cwd(), 'evidence', 'bi-p029', 'semantic-parity.json'), + `${JSON.stringify( + { + prompt: 'BI-P029', + title: 'Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer', + recorded_at: new Date().toISOString(), + clickhouse_version: (await chQuery('SELECT version() AS v FORMAT JSON'))[0]?.v, + golden_metric: { + id: REVENUE.id, + name: REVENUE.name, + definition_filters: REVENUE.filters, + semantic_model: { model_id: model.model_id, version: model.version, hash: model.hash }, + }, + expected_from_typescript: EXPECTED, + regression_guarded: { + description: + 'the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change', + result: preFix, + }, + path_a: { compiler: '0pt compileRequestToClickHouse', aggregate: 'sumIf', result: left }, + path_b: { + compiler: 'MetricFlowProvider.compile + independent renderer', + aggregate: 'sum(if(...))', + result: right, + }, + tolerance: '1e-9 relative', + verdict: 'PASS', + still_open: [ + 'MetricFlow itself was not executed (no Python runtime provisioned for this repo)', + 'the Superset integration path is not implemented', + 'the AI Analyst path (inf0#29) has not been compared against this provider', + ], + }, + null, + 2, + )}\n`, + ); + }); +});