From 468a87c00c9a146374ac8b6fe1246342441e3ed5 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Mon, 7 Sep 2026 08:23:43 +0200 Subject: [PATCH 1/2] BI-P029: the cited parity test did not exist, and it caught a real bug `semantic-provider.ts` cited tests/integration/semantic-parity.integration.test.ts as the proof of the one BI-P029 DoD surface it claimed was closed, and `providerReadiness().proven` asserted the comparison had been made against live ClickHouse. The file did not exist. The citation was the evidence. Writing it exposed a divergence that had been shipping: `compileRequestToClickHouse` had no notion of a metric's OWN filters (BI-P028 `MetricSpec.filters`). `ResolvedMetric` did not carry them and the compiler never asked, so a metric defined as "invoiced revenue EXCLUDING internal transfers" executed as "invoiced revenue". On the committed fixture that is 11 249 instead of 1 250 for one country -- a plausible-looking number, 9x too large, under the correct canonical metric id. The registry, the governance checks and the MetricFlow manifest all carried the filter; only the thing that actually ran did not. * `ResolvedMetric` gains `definition_filters`, already resolved onto the physical dim_names key so the compilers stay dumb. * The ClickHouse compiler folds them into THAT metric's conditional aggregate, never into the shared WHERE -- two metrics in one request may exclude different rows, and a shared predicate would apply one metric's meaning to the other. `renderFilter`'s uniquifier becomes a string so request filter `0` and metric filter `0_0m` cannot collide in the parameter map. * `compileRequestToSemantic` carries them through, and `MetricFlowProvider.compile` emits them per metric. The test runs both compile paths against one live ClickHouse: 0pt's `sumIf` compiler, and a renderer driven only by the provider's structured query that shares no SQL-generation code and emits `sum(if(...))` with inlined literals. It asserts the two SQL strings differ, so parity is not vacuous. Backing the fix out reproduces the pre-fix numbers, and the suite records them itself rather than quoting a console: evidence/bi-p029/semantic-parity.json. Three of BI-P029's four DoD surfaces stay OPEN and still say so: MetricFlow itself is not executed (no Python runtime is provisioned here), Superset is not implemented, and the AI Analyst path has not been compared. Co-Authored-By: Claude Opus 5 --- evidence/bi-p029/semantic-parity.json | 62 +++ src/analytics/contract/compile-clickhouse.ts | 45 +- src/analytics/contract/compile-semantic.ts | 26 +- src/analytics/contract/query-request.ts | 29 ++ src/analytics/metrics/semantic-provider.ts | 25 +- .../semantic-parity.integration.test.ts | 478 ++++++++++++++++++ 6 files changed, 652 insertions(+), 13 deletions(-) create mode 100644 evidence/bi-p029/semantic-parity.json create mode 100644 tests/integration/semantic-parity.integration.test.ts diff --git a/evidence/bi-p029/semantic-parity.json b/evidence/bi-p029/semantic-parity.json new file mode 100644 index 0000000..7fd1db1 --- /dev/null +++ b/evidence/bi-p029/semantic-parity.json @@ -0,0 +1,62 @@ +{ + "prompt": "BI-P029", + "title": "Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer", + "recorded_at": "2026-09-07T06:23:27.839Z", + "clickhouse_version": "26.3.10.60", + "golden_metric": { + "id": "m_revenue", + "name": "net_revenue", + "definition_filters": [ + { + "dimension": "channel", + "operator": "ne", + "values": [ + "internal" + ] + } + ], + "semantic_model": { + "model_id": "sm_sales", + "version": 1, + "hash": "c5b9cac1b5fdcfbfb89a1e960b946a9b33b8c29dc14fe9a304c134f7a0fb73a7" + } + }, + "expected_from_typescript": { + "SE": 1250, + "NO": 400, + "DK": 125.5 + }, + "regression_guarded": { + "description": "the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change", + "result": { + "DK": 3458.5, + "NO": 8177, + "SE": 11249 + } + }, + "path_a": { + "compiler": "0pt compileRequestToClickHouse", + "aggregate": "sumIf", + "result": { + "DK": 125.5, + "NO": 400, + "SE": 1250 + } + }, + "path_b": { + "compiler": "MetricFlowProvider.compile + independent renderer", + "aggregate": "sum(if(...))", + "result": { + "DK": 125.5, + "NO": 400, + "SE": 1250 + } + }, + "tolerance": "1e-9 relative", + "verdict": "PASS", + "still_open": [ + "MetricFlow itself was not executed (no Python runtime provisioned for this repo)", + "the Superset integration path is not implemented", + "the AI Analyst path (inf0#29) has not been compared against this provider" + ] +} diff --git a/src/analytics/contract/compile-clickhouse.ts b/src/analytics/contract/compile-clickhouse.ts index 5ff7071..d31b373 100644 --- a/src/analytics/contract/compile-clickhouse.ts +++ b/src/analytics/contract/compile-clickhouse.ts @@ -38,14 +38,19 @@ const GRAIN_FN: Record = { /** * Aggregation -> conditional-aggregate template. `count_distinct` uses * `uniqExactIf` so a multi-metric request stays one scan instead of N. + * + * The argument is the WHOLE per-metric predicate, not just the metric-key + * comparison, because a metric definition can carry its own filters (BI-P028 + * `MetricSpec.filters`) and those belong INSIDE this metric's `If` — one + * request may carry two metrics that exclude different rows. */ -const AGG_IF: Record string> = { - sum: (p) => `sumIf(value, metric = {${p}:String})`, - avg: (p) => `avgIf(value, metric = {${p}:String})`, - min: (p) => `minIf(value, metric = {${p}:String})`, - max: (p) => `maxIf(value, metric = {${p}:String})`, - count: (p) => `countIf(metric = {${p}:String})`, - count_distinct: (p) => `uniqExactIf(event_id, metric = {${p}:String})`, +const AGG_IF: Record string> = { + sum: (p) => `sumIf(value, ${p})`, + avg: (p) => `avgIf(value, ${p})`, + min: (p) => `minIf(value, ${p})`, + max: (p) => `maxIf(value, ${p})`, + count: (p) => `countIf(${p})`, + count_distinct: (p) => `uniqExactIf(event_id, ${p})`, }; export interface ClickHouseCompileOptions { @@ -124,7 +129,19 @@ export function compileRequestToClickHouse( if (!template) { throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported aggregation "${m.aggregation}"`); } - selectParts.push(`${template(key)} AS ${alias}`); + // The metric's own definition filters (BI-P028) join the metric-key + // comparison inside THIS metric's `If`. Bound, never spliced, and scoped + // to this metric — putting them in the shared WHERE would apply one + // metric's meaning to every other metric in the same request. + const metricConditions = [`metric = {${key}:String}`]; + (m.definition_filters ?? []).forEach((f, j) => { + assertSafeIdentifier(f.field_name, 'metric definition filter dimension'); + const nameKey = `mf${i}_${j}`; + params[`param_${nameKey}`] = f.field_name; + const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`; + metricConditions.push(renderFilter(lhs, nameKey, f.operator, [...f.values], `${i}_${j}m`, params)); + }); + selectParts.push(`${template(metricConditions.join(' AND '))} AS ${alias}`); params[`param_${key}`] = m.event_metric_key; metricKeyParams.push(`{${key}:String}`); metricAliases[alias] = m.name; @@ -156,7 +173,9 @@ export function compileRequestToClickHouse( const nameKey = `fn${i}`; params[`param_${nameKey}`] = entry.dimension.field_name; const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`; - conditions.push(renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], i, params)); + conditions.push( + renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], String(i), params), + ); }); // ── order by: resolved back onto compiler-generated aliases only ────── @@ -202,7 +221,13 @@ function renderFilter( nameKey: string, operator: FilterOperator, values: string[], - index: number, + /** + * Uniquifies this filter's bound-value parameter names. A STRING, not a + * number, because request filters and per-metric definition filters share + * one parameter map: `0` and `0_0m` must not collide, and two collided + * bindings would silently make one filter win over the other. + */ + index: string, params: Record, ): string { const bind = (v: string, suffix: string): string => { diff --git a/src/analytics/contract/compile-semantic.ts b/src/analytics/contract/compile-semantic.ts index 8cb1ac8..bb9c93e 100644 --- a/src/analytics/contract/compile-semantic.ts +++ b/src/analytics/contract/compile-semantic.ts @@ -24,6 +24,14 @@ export interface SemanticMetricRef { /** Canonical SIAX metric id — stable across semantic-engine replacement. */ metric_id: string; name: string; + /** + * The metric DEFINITION's own filters, carried through so the semantic + * target computes the same number as the ClickHouse target. Dropping them + * on one side and not the other is how one canonical metric id comes to + * mean two different things — the divergence BI-P029's DoD is written to + * catch. Structured, like every other filter here: never predicate text. + */ + definition_filters?: readonly SemanticWhere[]; } export interface SemanticGroupBy { @@ -81,7 +89,23 @@ export function compileRequestToSemantic(resolved: ResolvedQuery): SemanticQuery return { dataset_id: resolved.dataset_id, semantic_model_version: resolved.semantic_model_version, - metrics: resolved.metrics.map((m) => ({ metric_id: m.metric_id, name: m.name })), + metrics: resolved.metrics.map((m) => ({ + metric_id: m.metric_id, + name: m.name, + ...(m.definition_filters?.length + ? { + definition_filters: m.definition_filters.map((f) => ({ + // Already resolved onto the physical key; the dimension id is + // the same value here because the definition names a field, not + // a caller-supplied alias. + dimension_id: f.field_name, + name: f.field_name, + operator: f.operator, + values: [...f.values], + })), + } + : {}), + })), groupBy, where: resolved.filters.map((f) => ({ dimension_id: f.dimension.dimension_id, diff --git a/src/analytics/contract/query-request.ts b/src/analytics/contract/query-request.ts index 4f0e9f4..dc7fbaa 100644 --- a/src/analytics/contract/query-request.ts +++ b/src/analytics/contract/query-request.ts @@ -351,6 +351,27 @@ function parseOrderBy(raw: unknown, index: number): AnalyticsOrderBy { * What a metric NAME resolves to. Supplied from `MetricDefinition` rows * (BI-P002/BI-P028); the compilers never invent one. */ +/** + * A filter that belongs to the METRIC DEFINITION rather than to the request + * (BI-P028 `MetricSpec.filters`) — "net revenue excludes internal transfers" + * is part of what the metric MEANS, not something a caller asks for. + * + * It is carried in ALREADY-RESOLVED form: `field_name` is the physical key + * inside `dim_names`, mapped from the definition's dimension name by whoever + * builds the resolution. The compilers stay dumb — they bind and render, they + * never look a dimension up — and `assertSafeIdentifier` still applies. + * + * These filters are NOT optional decoration. A compile path that drops them + * answers a different question under the same canonical metric id, which is + * exactly the divergence BI-P029's DoD exists to catch; see + * tests/integration/semantic-parity.integration.test.ts. + */ +export interface ResolvedMetricFilter { + field_name: string; + operator: FilterOperator; + values: readonly string[]; +} + export interface ResolvedMetric { name: string; metric_id: string; @@ -360,6 +381,14 @@ export interface ResolvedMetric { /** Dimension names this metric may legitimately be sliced by. */ allowed_dimensions: readonly string[]; unit?: string; + /** + * The metric definition's own filters. Applied PER METRIC — folded into that + * metric's conditional aggregate — never into the shared WHERE clause: a + * request may ask for two metrics whose definitions exclude different rows, + * and a shared predicate would silently apply one metric's meaning to the + * other. + */ + definition_filters?: readonly ResolvedMetricFilter[]; } /** What a dimension NAME resolves to. Supplied from `DimensionDefinition` rows. */ diff --git a/src/analytics/metrics/semantic-provider.ts b/src/analytics/metrics/semantic-provider.ts index 30d4ea0..435f1d0 100644 --- a/src/analytics/metrics/semantic-provider.ts +++ b/src/analytics/metrics/semantic-provider.ts @@ -16,8 +16,11 @@ * happen first — `METRICFLOW_PINNED_VERSION` below records the intended pin * and is explicitly marked unverified). * - * So what IS proven, in tests/integration/semantic-parity.integration.test.ts, - * is the part that does not need MetricFlow: the SAME golden metric, taken + * So what IS proven, in tests/integration/semantic-parity.integration.test.ts + * — a path this header cited from the day it was written, while the file + * itself did not exist until BI-P029's gap review; the citation was the + * evidence, which is no evidence at all — is the part that does not need + * MetricFlow: the SAME golden metric, taken * from the SAME registry version, executed against the SAME live ClickHouse * data by TWO INDEPENDENT compilers — 0pt's metric compiler, and a renderer * driven only by the engine-neutral `SemanticQuerySpec` — returns the same @@ -169,10 +172,28 @@ export class MetricFlowProvider implements SiaxSemanticProvider { } } const timeBucket = spec.groupBy.find((g) => g.grain); + // Per-metric definition filters (BI-P028 `MetricSpec.filters`), kept + // SEPARATE from the request's `where`. Folding them together would apply + // one metric's meaning to every metric in the query; omitting them would + // make this provider compute a different number from the ClickHouse + // target for the same canonical metric id. + const metric_filters: Record< + string, + Array<{ dimension: string; operator: string; values: string[] }> + > = {}; + for (const m of spec.metrics) { + if (!m.definition_filters?.length) continue; + metric_filters[m.metric_id] = m.definition_filters.map((f) => ({ + dimension: f.name, + operator: f.operator, + values: [...f.values], + })); + } return { provider: this.provider, query: { metrics: spec.metrics.map((m) => m.metric_id), + ...(Object.keys(metric_filters).length ? { metric_filters } : {}), group_by: spec.groupBy.filter((g) => !g.grain).map((g) => g.name), ...(timeBucket ? { time_granularity: timeBucket.grain } : {}), // STRUCTURED, not a `--where` SQL string. MetricFlow accepts predicate diff --git a/tests/integration/semantic-parity.integration.test.ts b/tests/integration/semantic-parity.integration.test.ts new file mode 100644 index 0000000..067ee60 --- /dev/null +++ b/tests/integration/semantic-parity.integration.test.ts @@ -0,0 +1,478 @@ +/** + * tests/integration/semantic-parity.integration.test.ts + * ====================================================== + * BI-P029 — the parity test `src/analytics/metrics/semantic-provider.ts` has + * been citing. + * + * IT DID NOT EXIST. The provider's header named this path as the proof of the + * one DoD surface it claimed was closed, and `providerReadiness().proven` + * asserted that "0pt's metric compiler and an independent spec-driven renderer + * return the same number for the golden metric against live ClickHouse data". + * Nothing in the repository executed that comparison. A citation to a file + * that is not there is the same failure as a green check that gates nothing: + * it survives review because the sentence reads like evidence. + * + * So this file is the comparison, run for real: + * + * PATH A parseAnalyticsQueryRequest -> resolveAnalyticsQuery -> + * compileRequestToClickHouse -> live ClickHouse. + * PATH B the same ResolvedQuery -> compileRequestToSemantic -> + * MetricFlowProvider.compile -> `renderIndependently` below, which + * reads ONLY the provider's structured query object and the registry + * model, writes its own SQL with its own quoting, and shares no line + * of code with compile-clickhouse.ts -> the same live ClickHouse. + * + * The two paths must return the same numbers. `renderIndependently` inlines + * literals where compile-clickhouse binds parameters, groups by the physical + * column expression rather than by a compiler alias, and orders in SQL rather + * than by alias — so the SQL strings genuinely differ, and the test asserts + * that they do. Two identical strings would make the parity claim vacuous. + * + * WHAT THIS STILL DOES NOT PROVE, and `providerReadiness()` still says so: + * MetricFlow itself is not executed here (no Python runtime is provisioned for + * this repo), and neither the Superset path nor the AI Analyst path is + * compared. Three of BI-P029's four DoD surfaces remain open. This closes the + * first one honestly rather than closing all four by assertion. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * pnpm vitest run tests/integration/semantic-parity.integration.test.ts + * + * It CREATEs and DROPs its own database. Never point it at the b0ts + * ClickHouse or at any quarantined dataset. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { + parseAnalyticsQueryRequest, + queryResolutionFrom, + resolveAnalyticsQuery, + type ResolvedQuery, +} from '@/src/analytics/contract/query-request'; +import { compileRequestToClickHouse } from '@/src/analytics/contract/compile-clickhouse'; +import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic'; +import { MetricFlowProvider } from '@/src/analytics/metrics/semantic-provider'; +import { + InMemorySemanticModelStore, + SemanticModelRegistry, + type SemanticModelVersion, +} from '@/src/analytics/metrics/semantic-registry'; +import type { MetricSpec } from '@/src/analytics/metrics/metric-model'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +// Collection-time, never inside a hook: `describe.skipIf(flagSetInBeforeAll)` +// evaluates its flag before any hook has run and therefore skips ALWAYS. +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_parity_${Date.now().toString(36)}`; +const TENANT = 't_alpha'; +const PROJECT = 'p_main'; + +/** + * The golden metric. It carries a DEFINITION FILTER (`channel != 'internal'`) + * on purpose: a metric whose meaning is only "sum this column" cannot tell a + * parity test anything, because every compiler agrees about a bare sum. The + * business meaning of net revenue is that internal transfers are not revenue, + * and that meaning has to survive both compile paths or the two paths are + * answering different questions with the same metric id. + */ +const REVENUE: MetricSpec = { + id: 'm_revenue', + name: 'net_revenue', + description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.', + type: 'simple', + measure: 'amount_sek', + aggregation: 'sum', + dimensions: ['country', 'channel'], + filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }], + timeDimension: 'invoiced_at', + unit: 'SEK', + currency: 'SEK', + format: 'currency', + owner: 'finance@siax.io', + version: 3, + status: 'ACTIVE', +}; + +/** dimension name -> the key stored in `dim_names`. One mapping, both paths. */ +const FIELD_OF: Record = { country: 'country_code', channel: 'sales_channel' }; + +function semanticModel(version = 1): Omit { + return { + model_id: 'sm_sales', + version, + tenant_id: TENANT, + datasets: ['ds_sales'], + relationships: [], + metrics: [REVENUE], + dimensions: [ + { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' }, + ], + valid_from: '2026-09-01T00:00:00Z', + status: 'draft', + }; +} + +/** + * The fixture. Deliberately arranged so that dropping the metric's definition + * filter changes the answer for EVERY country — a fixture where internal rows + * happen to sum to zero would let a broken compiler pass. + */ +const ROWS: Array<{ country: string; channel: string; amount: number }> = [ + { country: 'SE', channel: 'direct', amount: 1000 }, + { country: 'SE', channel: 'partner', amount: 250 }, + { country: 'SE', channel: 'internal', amount: 9999 }, + { country: 'NO', channel: 'direct', amount: 400 }, + { country: 'NO', channel: 'internal', amount: 7777 }, + { country: 'DK', channel: 'partner', amount: 125.5 }, + { country: 'DK', channel: 'internal', amount: 3333 }, +]; + +/** What the metric MEANS, computed in TypeScript, independent of any SQL. */ +const EXPECTED: Record = ROWS.filter((r) => r.channel !== 'internal').reduce( + (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), + {} as Record, +); + +interface ChRow { + [column: string]: string | number; +} + +async function chQuery(sql: string, params: Record = {}): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + const res = await fetch(url.toString(), { method: 'POST', headers, body: sql }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 400)}`); + if (!text.trim()) return []; + return (JSON.parse(text) as { data: ChRow[] }).data; +} + +async function chExec(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + url.searchParams.set('query', sql); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers.Authorization = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${(await res.text()).slice(0, 400)}`); +} + +// ── PATH B: the independent renderer ─────────────────────────────────────── + +/** Single-quote escaping, written here because this renderer shares nothing. */ +function lit(value: string): string { + return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; +} + +function predicate(field: string, operator: string, values: string[]): string { + const lhs = `dim_values[indexOf(dim_names, ${lit(field)})]`; + switch (operator) { + case 'eq': + return `${lhs} = ${lit(values[0])}`; + case 'ne': + return `${lhs} <> ${lit(values[0])}`; + case 'in': + return `${lhs} IN (${values.map(lit).join(',')})`; + case 'not_in': + return `${lhs} NOT IN (${values.map(lit).join(',')})`; + default: + throw new Error(`independent renderer does not implement operator ${operator}`); + } +} + +/** + * Render the provider's structured query into SQL, reading ONLY that object + * and the registry model. It never sees the ResolvedQuery, the ClickHouse + * compiler's output, or its parameter map. + * + * The aggregate is deliberately expressed differently from compile-clickhouse: + * a `sum(if(...))` rather than a `sumIf(...)`, grouped by the raw expression + * rather than by an alias. Same arithmetic, different SQL — which is the only + * way the comparison means anything. + */ +function renderIndependently( + compiled: ReturnType, + model: SemanticModelVersion, +): string { + const query = compiled.query as { + metrics: string[]; + group_by: string[]; + where: Array<{ dimension: string; operator: string; values: string[] }>; + limit: number; + metric_filters?: Record>; + }; + const byId = new Map(model.metrics.map((m) => [m.id, m])); + + const selects: string[] = []; + const groups: string[] = []; + for (const dimension of query.group_by) { + const expr = `dim_values[indexOf(dim_names, ${lit(FIELD_OF[dimension])})]`; + selects.push(`${expr} AS ${dimension}`); + groups.push(expr); + } + for (const metricId of query.metrics) { + const spec = byId.get(metricId); + if (!spec) throw new Error(`metric ${metricId} is not in the model version`); + // The metric's own definition filters. The provider query carries them + // already resolved onto physical keys; the registry model states them in + // dimension names. Either way they are the metric's MEANING, and a + // renderer that ignored them would compute a different number. + const own = query.metric_filters?.[metricId] + ? query.metric_filters[metricId].map((f) => ({ + field: f.dimension, + operator: f.operator, + values: f.values, + })) + : spec.filters.map((f) => ({ + field: FIELD_OF[f.dimension], + operator: f.operator, + values: f.values ?? [], + })); + const conditions = [ + `metric = ${lit(spec.measure)}`, + ...own.map((f) => predicate(f.field, f.operator, f.values)), + ]; + selects.push(`sum(if(${conditions.join(' AND ')}, value, 0)) AS ${spec.id}`); + } + + const where = [ + `tenant_id = ${lit(TENANT)}`, + `project_id = ${lit(PROJECT)}`, + `dataset_id = ${lit('ds_sales')}`, + ...query.where.map((w) => predicate(FIELD_OF[w.dimension], w.operator, w.values ?? [])), + ]; + + return ( + `SELECT ${selects.join(', ')} FROM ${DB}.opt_analytics_event ` + + `WHERE ${where.join(' AND ')} ` + + (groups.length ? `GROUP BY ${groups.join(', ')} ORDER BY ${groups.join(', ')} ` : '') + + `LIMIT ${query.limit} FORMAT JSON` + ); +} + +// ── the suite ────────────────────────────────────────────────────────────── + +let model: SemanticModelVersion; +let resolved: ResolvedQuery; +/** Filled by the regression test below; recorded in the evidence file. */ +let preFix: Record = {}; + +run('BI-P029 — one golden metric, two independent compilers, one live ClickHouse', () => { + beforeAll(async () => { + const engine = new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { [TENANT]: FIELD_OF }, + }); + await engine.ensureSchema(); + + const body = ROWS.map((r, i) => + JSON.stringify({ + tenant_id: TENANT, + project_id: PROJECT, + dataset_id: 'ds_sales', + metric: 'amount_sek', + event_id: `e${i}`, + event_time: '2026-09-03 12:00:00.000', + value: r.amount, + dim_names: ['country_code', 'sales_channel'], + dim_values: [r.country, r.channel], + }), + ).join('\n'); + await chExec(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, body); + + model = await new SemanticModelRegistry(new InMemorySemanticModelStore()).publish(semanticModel(1)); + + resolved = resolveAnalyticsQuery( + parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['net_revenue'], + dimensions: ['country'], + orderBy: [{ ref: 'country', direction: 'asc' }], + }), + queryResolutionFrom({ + dataset_id: 'ds_sales', + semantic_model_version: 1, + metrics: [ + { + name: 'net_revenue', + metric_id: 'm_revenue', + event_metric_key: 'amount_sek', + aggregation: 'sum', + allowed_dimensions: ['country', 'channel'], + // The metric's OWN filters, resolved onto physical field names. + definition_filters: REVENUE.filters.map((f) => ({ + field_name: FIELD_OF[f.dimension], + operator: f.operator, + values: f.values ?? [], + })), + }, + ], + dimensions: [ + { name: 'country', dimension_id: 'd_country', field_name: 'country_code' }, + { name: 'channel', dimension_id: 'd_channel', field_name: 'sales_channel' }, + ], + }), + ); + }, 120_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await chExec(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it('the fixture is arranged so a dropped definition filter CHANGES every answer', () => { + const withInternal = ROWS.reduce( + (acc, r) => ({ ...acc, [r.country]: (acc[r.country] ?? 0) + r.amount }), + {} as Record, + ); + for (const country of Object.keys(EXPECTED)) { + expect(withInternal[country]).not.toBe(EXPECTED[country]); + } + }); + + it('PATH A — 0pt’s ClickHouse compiler returns the metric’s meaning', async () => { + const compiled = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + const rows = await chQuery(compiled.sql, compiled.params); + const got = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); + expect(got).toEqual(EXPECTED); + }); + + it('PATH B — an independent renderer driven only by the provider spec agrees', async () => { + const spec = compileRequestToSemantic(resolved); + const compiled = new MetricFlowProvider().compile(spec, model); + expect(compiled.canonical_metric_ids).toEqual(['m_revenue']); + + const rows = await chQuery(renderIndependently(compiled, model)); + const got = Object.fromEntries(rows.map((r) => [String(r.country), Number(r.m_revenue)])); + expect(got).toEqual(EXPECTED); + }); + + /** + * The regression this suite was written for. Until BI-P029's gap review, + * `compileRequestToClickHouse` had no notion of a metric's own filters: + * `ResolvedMetric` did not carry them and the compiler never asked. The + * metric definition said "excluding internal transfers" and the executed + * SQL did not, so `net_revenue` returned a number that looked plausible and + * was ~9x too large for this fixture. + * + * Re-run here by stripping the definition filters from the resolution, so + * the number the bug produced is measured by the committed test rather than + * remembered from a console. + */ + it('a resolution WITHOUT the definition filters produces the pre-fix (wrong) number', async () => { + const stripped: ResolvedQuery = { + ...resolved, + metrics: resolved.metrics.map((m) => ({ ...m, definition_filters: undefined })), + }; + const compiled = compileRequestToClickHouse(stripped, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + const rows = await chQuery(compiled.sql, compiled.params); + preFix = Object.fromEntries(rows.map((r) => [String(r.dim_0), Number(r.metric_0)])); + + // Not merely different — OVERSTATED, on every country, by the internal + // transfers the definition excludes. + for (const country of Object.keys(EXPECTED)) { + expect(preFix[country]).toBeGreaterThan(EXPECTED[country]); + } + expect(compiled.sql).not.toContain('mf0_0'); + }); + + it('the golden metric returns the SAME number on both paths, within tolerance', async () => { + const a = await chQuery( + ...(() => { + const c = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }); + return [c.sql, c.params] as const; + })(), + ); + const b = await chQuery( + renderIndependently(new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), model), + ); + + const left = Object.fromEntries(a.map((r) => [String(r.dim_0), Number(r.metric_0)])); + const right = Object.fromEntries(b.map((r) => [String(r.country), Number(r.m_revenue)])); + + expect(Object.keys(left).sort()).toEqual(Object.keys(right).sort()); + for (const key of Object.keys(left)) { + // Relative tolerance, because two different aggregate expressions over + // Float64 may associate the additions differently. + const denominator = Math.max(Math.abs(left[key]), 1); + expect(Math.abs(left[key] - right[key]) / denominator).toBeLessThan(1e-9); + } + + // Parity between two identical strings proves nothing. + const sqlA = compileRequestToClickHouse(resolved, { + tenant_id: TENANT, + project_id: PROJECT, + database: DB, + maxResultRows: 1000, + }).sql; + const sqlB = renderIndependently( + new MetricFlowProvider().compile(compileRequestToSemantic(resolved), model), + model, + ); + expect(sqlA).not.toBe(sqlB); + expect(sqlA).toContain('sumIf('); + expect(sqlB).toContain('sum(if('); + + fs.mkdirSync(path.join(process.cwd(), 'evidence', 'bi-p029'), { recursive: true }); + fs.writeFileSync( + path.join(process.cwd(), 'evidence', 'bi-p029', 'semantic-parity.json'), + `${JSON.stringify( + { + prompt: 'BI-P029', + title: 'Golden metric parity: 0pt metric compiler vs an independent spec-driven renderer', + recorded_at: new Date().toISOString(), + clickhouse_version: (await chQuery('SELECT version() AS v FORMAT JSON'))[0]?.v, + golden_metric: { + id: REVENUE.id, + name: REVENUE.name, + definition_filters: REVENUE.filters, + semantic_model: { model_id: model.model_id, version: model.version, hash: model.hash }, + }, + expected_from_typescript: EXPECTED, + regression_guarded: { + description: + 'the same request compiled WITHOUT the metric definition filters — what 0pt returned before this change', + result: preFix, + }, + path_a: { compiler: '0pt compileRequestToClickHouse', aggregate: 'sumIf', result: left }, + path_b: { + compiler: 'MetricFlowProvider.compile + independent renderer', + aggregate: 'sum(if(...))', + result: right, + }, + tolerance: '1e-9 relative', + verdict: 'PASS', + still_open: [ + 'MetricFlow itself was not executed (no Python runtime provisioned for this repo)', + 'the Superset integration path is not implemented', + 'the AI Analyst path (inf0#29) has not been compared against this provider', + ], + }, + null, + 2, + )}\n`, + ); + }); +}); -- 2.54.0 From 7441de542915f7c65be4b1dd4eff69801ce619b6 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Mon, 7 Sep 2026 08:27:37 +0200 Subject: [PATCH 2/2] =?UTF-8?q?BI-P103:=20the=20cited=20evidence/bi-p103?= =?UTF-8?q?=20did=20not=20exist=20=E2=80=94=20measure=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same shape as the BI-P029 finding in the previous commit. `bi-slo.ts` said "evidence/bi-p103 records the first baselines". There was no evidence/bi-p103. The catalogue and the evaluator were both real and both well covered — against SYNTHETIC events. A synthetic event carries whatever duration the test author typed, so that suite proves the arithmetic and nothing about the system: it cannot tell you whether a dashboard query takes 8 ms or 8 s. BI-P103's DoD is one sentence, "measured baselines exist", and it was the part not done. tests/integration/bi-p103-slo-baseline.integration.test.ts runs 460 real queries against a live ClickHouse over a 400 000-row fact table -- 120 cold dashboard queries (each a different bucket so the BI-P020 cache genuinely misses), the same 120 again through the same shipped cache now hitting, and 220 exploration queries, that last number chosen so explore.p99's min_samples of 200 is MET rather than reported low-confidence. Every duration is wall-clock around the real call, and the telemetry goes through the shipped producer and the shipped evaluator, so the evidence is written by the code under test. Measured: 7 of 11 objectives produced a number, `fully_measured` false, measured_fraction 0.64. The four that did not each say why -- dashboard freshness is a property of the dataset, the exploration timeout is a configured ceiling, report completion has no producer in 0pt at all, and report delivery is N0TIFY's receipt. A baseline table listing only the rows that worked is how an estate comes to believe it measures more than it does. The evidence also records what is NARROWER than production, because these numbers will be quoted: the cached path is the in-process reference cache and not Valkey, every latency is the engine round trip with no HTTP or authorization layer, and there is no concurrency. The suite is adversarial about its own inputs: a "cold" sample that came from the cache fails, a cache "hit" that missed fails, a cached p95 that is not faster than the cold p95 fails, and dropping the exploration sample to 60 fails on explore.p99 being low-confidence rather than quietly recording a p99 over 60 requests. Co-Authored-By: Claude Opus 5 --- evidence/bi-p103/baseline.json | 173 +++++++++ src/analytics/slo/bi-slo.ts | 14 +- .../bi-p103-slo-baseline.integration.test.ts | 341 ++++++++++++++++++ 3 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 evidence/bi-p103/baseline.json create mode 100644 tests/integration/bi-p103-slo-baseline.integration.test.ts diff --git a/evidence/bi-p103/baseline.json b/evidence/bi-p103/baseline.json new file mode 100644 index 0000000..019dadc --- /dev/null +++ b/evidence/bi-p103/baseline.json @@ -0,0 +1,173 @@ +{ + "prompt": "BI-P103", + "title": "First measured BI SLO baselines", + "recorded_at": "2026-09-07T06:26:43.457Z", + "host_note": "One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the shape of the distribution on this fixture — not a production SLA.", + "clickhouse_version": "26.3.10.60", + "fixture": { + "rows": 400000, + "table": "slo_fact", + "partitions": "MergeTree ORDER BY (tenant_id, bucket, ts)" + }, + "workload": { + "cold_dashboard_queries": 120, + "cached_dashboard_queries": 120, + "exploration_queries": 220, + "report_queries": 0 + }, + "window": { + "from": "2026-09-07T06:26:39.422Z", + "to": "2026-09-07T06:26:43.447Z" + }, + "catalogue_size": 11, + "fully_measured": false, + "unresolved_queries": 0, + "summary": { + "met": 7, + "breached": 0, + "insufficient_data": 1, + "not_measurable_here": 3, + "measured_fraction": 0.6363636363636364 + }, + "results": [ + { + "objective_id": "dash.availability", + "workload": "interactive_dashboard", + "kind": "availability", + "status": "met", + "target": 0.995, + "unit": "ratio", + "observed": 1, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.p95_first_query", + "workload": "interactive_dashboard", + "kind": "latency_p95_first", + "status": "met", + "target": 3000, + "unit": "ms", + "observed": 8.893520999999964, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.p95_cached", + "workload": "interactive_dashboard", + "kind": "latency_p95_cached", + "status": "met", + "target": 400, + "unit": "ms", + "observed": 0.024498999999877924, + "sample_size": 120, + "low_confidence": false, + "detail": "p95 over 120 samples" + }, + { + "objective_id": "dash.error_rate", + "workload": "interactive_dashboard", + "kind": "error_rate", + "status": "met", + "target": 0.005, + "unit": "ratio", + "observed": 0, + "sample_size": 240, + "low_confidence": false, + "detail": "0 failed of 240 terminal queries" + }, + { + "objective_id": "dash.freshness", + "workload": "interactive_dashboard", + "kind": "freshness", + "status": "not_measurable_here", + "target": 900, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from analytics_freshness_watermark; BI usage events cannot answer it — Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage event. Usage events cannot measure it and this evaluator does not pretend to." + }, + { + "objective_id": "explore.p95", + "workload": "exploration", + "kind": "latency_p95", + "status": "met", + "target": 10000, + "unit": "ms", + "observed": 17.072708000000148, + "sample_size": 220, + "low_confidence": false, + "detail": "p95 over 220 samples" + }, + { + "objective_id": "explore.p99", + "workload": "exploration", + "kind": "latency_p99", + "status": "met", + "target": 60000, + "unit": "ms", + "observed": 21.640736000000288, + "sample_size": 220, + "low_confidence": false, + "detail": "p99 over 220 samples" + }, + { + "objective_id": "explore.max_result", + "workload": "exploration", + "kind": "max_result_rows", + "status": "met", + "target": 1000000, + "unit": "rows", + "observed": 96, + "sample_size": 220, + "low_confidence": false, + "detail": "largest result seen in the window" + }, + { + "objective_id": "explore.timeout", + "workload": "exploration", + "kind": "timeout", + "status": "not_measurable_here", + "target": 300, + "unit": "seconds", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from engine_configuration; BI usage events cannot answer it — The configured ceiling (BI-P012 settings profile), not an observation." + }, + { + "objective_id": "report.completion_sla", + "workload": "report", + "kind": "completion_sla", + "status": "insufficient_data", + "target": 900000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "no report_generated events" + }, + { + "objective_id": "report.delivery_sla", + "workload": "report", + "kind": "delivery_sla", + "status": "not_measurable_here", + "target": 300000, + "unit": "ms", + "sample_size": 0, + "low_confidence": false, + "detail": "measured from n0tify_delivery_receipt; BI usage events cannot answer it — Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this objective is declared and reported unmeasured rather than proxied by completion." + } + ], + "measured_but_narrower_than_production": [ + "dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey deployment adds a network round trip, so 0.03 ms here is a floor and not the number a user will see.", + "Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no serialization to the caller. The end-to-end p95 is larger by whatever those cost.", + "One tenant, one client, no concurrency. Contention is not represented." + ], + "not_measured_and_why": [ + "dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event", + "explore.timeout — the configured BI-P012 ceiling, not an observation", + "report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0", + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event" + ] +} diff --git a/src/analytics/slo/bi-slo.ts b/src/analytics/slo/bi-slo.ts index b4bad4b..c172c94 100644 --- a/src/analytics/slo/bi-slo.ts +++ b/src/analytics/slo/bi-slo.ts @@ -78,8 +78,18 @@ export interface SloObjective { * The catalogue, field-for-field with what BI-P103 lists per workload class. * Targets are STARTING POINTS chosen to be honest about what has been * measured: they are the thresholds the evaluator compares against, and - * evidence/bi-p103 records the first baselines rather than pretending the - * targets were derived from a year of production traffic. + * evidence/bi-p103/baseline.json records the first baselines rather than + * pretending the targets were derived from a year of production traffic. + * + * Those baselines are produced by + * tests/integration/bi-p103-slo-baseline.integration.test.ts, which runs 460 + * real queries against a live ClickHouse and feeds them through this + * evaluator. Read the file's own caveats before quoting a number from it: the + * cached path measured there is the in-process reference cache rather than + * Valkey, every latency is the engine round trip with no HTTP or + * authorization layer around it, and there is no concurrency. Seven of the + * eleven objectives produced a number; the other four say why they could not, + * and `fully_measured` is false. */ export const BI_SLO_CATALOGUE: readonly SloObjective[] = [ // ── interactive dashboard ──────────────────────────────────────────────── diff --git a/tests/integration/bi-p103-slo-baseline.integration.test.ts b/tests/integration/bi-p103-slo-baseline.integration.test.ts new file mode 100644 index 0000000..6fecacd --- /dev/null +++ b/tests/integration/bi-p103-slo-baseline.integration.test.ts @@ -0,0 +1,341 @@ +/** + * tests/integration/bi-p103-slo-baseline.integration.test.ts + * =========================================================== + * BI-P103 — "Measured baselines exist." + * + * `src/analytics/slo/bi-slo.ts` said the first baselines were recorded in + * `evidence/bi-p103`. There was no `evidence/bi-p103`. The catalogue and the + * evaluator were both real and both well tested — against SYNTHETIC events. + * A synthetic event carries whatever duration the test author typed, so a + * suite built only on those proves the arithmetic and nothing about the + * system: it cannot tell you a dashboard query takes 8 ms or 8 s. + * + * So this harness produces the baselines from queries that actually ran: + * + * * ~120 COLD dashboard queries, each a different filter so the BI-P020 + * cache genuinely misses, executed against a live ClickHouse; + * * the same ~120 again through the SAME shipped cache, now hitting, so + * `dash.p95_cached` is a measured cache path and not an assumption; + * * 220 exploration queries, enough for `explore.p99`'s min_samples of 200 + * to be met rather than reported low-confidence. + * + * Every duration is wall-clock around the real call. The telemetry goes + * through the SHIPPED producer (`biUsageEventsForExecution`) and the SHIPPED + * evaluator (`evaluateBiSlos`) — not a copy — so the evidence is produced by + * the code under test. + * + * WHAT THE BASELINE DELIBERATELY DOES NOT CONTAIN. `dash.freshness`, + * `explore.timeout` and `report.delivery_sla` are not measurable from usage + * events and the evaluator says so; `report.completion_sla` has no producer in + * 0pt at all (nothing here emits `aud0.report_generated`). They appear in the + * evidence with their real status rather than being quietly dropped, because a + * baseline table that lists only the rows that worked is how an estate comes + * to believe it measures more than it does. + * + * These numbers describe THIS fixture on THIS machine against a local + * ClickHouse with warm page cache. They are a floor for the shape of the + * distribution, not a production SLA — and the evidence file says which parts + * are narrower than production: the cached path is the in-process reference + * cache rather than Valkey, every latency is the engine round trip with no + * HTTP or authorization layer around it, and there is no concurrency. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * pnpm vitest run tests/integration/bi-p103-slo-baseline.integration.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { InMemoryResultCache, type CacheKeyComponents } from '@/src/analytics/cache/query-cache'; +import { compiledQueryHash } from '@/src/analytics/execution/query-execution-record'; +import { + biUsageEventsForExecution, + type Aud0BiEvent, + type ExecutionTelemetry, +} from '@/src/analytics/slo/usage-events'; +import { BI_SLO_CATALOGUE, evaluateBiSlos, summariseSloReport } from '@/src/analytics/slo/bi-slo'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_p103_${Date.now().toString(36)}`; +const TENANT = 't_alpha'; +const ROWS = 400_000; +const COLD_DASHBOARD_QUERIES = 120; +const EXPLORE_QUERIES = 220; + +const headers: Record = CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } + : {}; + +async function ch(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); + url.searchParams.set('query', sql); + const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`); + return text; +} + +interface Measured { + rows: number; + ms: number; +} + +/** One real round trip, wall-clock measured. */ +async function timed(sql: string): Promise { + const started = performance.now(); + const text = await ch(`${sql} FORMAT JSON`); + const ms = performance.now() - started; + const parsed = JSON.parse(text) as { data: unknown[] }; + return { rows: parsed.data.length, ms }; +} + +const events: Aud0BiEvent[] = []; + +function record(t: ExecutionTelemetry): void { + events.push(...biUsageEventsForExecution(t)); +} + +function keyFor(i: number, cachedShape: string): CacheKeyComponents { + return { + tenant_id: TENANT, + dataset_id: 'ds_slo', + semantic_model_version: 1, + metric_versions: { m_amount: 1 }, + compiled_query_hash: compiledQueryHash(cachedShape, { i: String(i) }), + data_snapshot: 'v1', + format: 'json', + }; +} + +let windowFrom: Date; +let windowTo: Date; + +run('BI-P103 — measured baselines, from queries that actually ran', () => { + beforeAll(async () => { + await new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { [TENANT]: { region: 'region' } }, + }).ensureSchema(); + + // A fact table large enough that a scan is not free. Generated inside + // ClickHouse so the fixture cost is not measured as query latency. + await ch( + `CREATE TABLE ${DB}.slo_fact (tenant_id LowCardinality(String), region LowCardinality(String), ` + + `bucket UInt16, ts DateTime, amount Float64) ENGINE = MergeTree ORDER BY (tenant_id, bucket, ts)`, + ); + await ch( + `INSERT INTO ${DB}.slo_fact SELECT 't_alpha', ['SE','NO','DK','FI'][(number % 4) + 1], ` + + `toUInt16(number % 500), toDateTime('2026-09-01 00:00:00') + (number % 86400), ` + + `toFloat64(number % 997) FROM numbers(${ROWS})`, + ); + await ch(`OPTIMIZE TABLE ${DB}.slo_fact FINAL`); + }, 300_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await ch(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it('measures a cold and a cached dashboard path, and an exploration path', async () => { + windowFrom = new Date(); + const cache = new InMemoryResultCache(); + const shape = `SELECT region, sum(amount) FROM ${DB}.slo_fact WHERE tenant_id = ? AND bucket = ? GROUP BY region`; + + // ── cold: every query a different bucket, so the cache genuinely misses ── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const components = keyFor(i, shape); + const started = new Date().toISOString(); + const before = await cache.get(components); + expect(before).toBeUndefined(); // a "cold" sample served from cache is not cold + const m = await timed( + `SELECT region, sum(amount) AS total FROM ${DB}.slo_fact ` + + `WHERE tenant_id = 't_alpha' AND bucket = ${i} GROUP BY region ORDER BY region`, + ); + await cache.set(components, { rows: m.rows }, { ttlSeconds: 300, rowCount: m.rows }); + record({ + query_id: `q_cold_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + + // ── cached: the same questions, through the same shipped cache ────────── + for (let i = 0; i < COLD_DASHBOARD_QUERIES; i++) { + const started = new Date().toISOString(); + const t0 = performance.now(); + const hit = await cache.get(keyFor(i, shape)); + const ms = performance.now() - t0; + expect(hit).toBeDefined(); // a miss recorded as a hit would flatter the baseline + record({ + query_id: `q_hot_${i}`, + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: ms, + row_count: hit?.row_count ?? 0, + cache_hit: true, + }); + } + + // ── exploration: wider, ad-hoc shapes, enough samples for a real p99 ──── + for (let i = 0; i < EXPLORE_QUERIES; i++) { + const started = new Date().toISOString(); + const m = await timed( + `SELECT region, toStartOfHour(ts) AS h, sum(amount) AS total, count() AS n ` + + `FROM ${DB}.slo_fact WHERE tenant_id = 't_alpha' AND bucket >= ${i % 400} ` + + `GROUP BY region, h ORDER BY region, h LIMIT 5000`, + ); + record({ + query_id: `q_explore_${i}`, + surface: 'explore', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'succeeded', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: m.ms, + row_count: m.rows, + cache_hit: false, + }); + } + windowTo = new Date(); + + expect(events.length).toBeGreaterThan(2 * (COLD_DASHBOARD_QUERIES * 2 + EXPLORE_QUERIES) - 1); + }, 600_000); + + it('a REAL engine error becomes a failed event and moves the error rate', async () => { + // Proves the availability and error-rate paths are not measuring silence. + // Kept OUT of the baseline window so the recorded baseline describes the + // system's normal behaviour rather than a deliberately broken query. + const started = new Date().toISOString(); + let code: string | undefined; + const t0 = performance.now(); + try { + await ch(`SELECT no_such_column FROM ${DB}.slo_fact LIMIT 1 FORMAT JSON`); + } catch { + code = 'ENGINE_ERROR'; + } + expect(code).toBe('ENGINE_ERROR'); + + const failed = biUsageEventsForExecution({ + query_id: 'q_fail_1', + surface: 'dashboard', + dataset_id: 'ds_slo', + engine: 'clickhouse', + state: 'failed', + started_at: started, + finished_at: new Date().toISOString(), + duration_ms: performance.now() - t0, + error_code: code, + error_message: 'Unknown expression identifier no_such_column', + }); + expect(failed.map((e) => e.event_type)).toContain('aud0.analytics_query_failed'); + // The engine's message never becomes telemetry. + expect(JSON.stringify(failed)).not.toContain('no_such_column'); + + const withFailure = evaluateBiSlos({ + events: [...events, ...failed], + from: windowFrom, + to: new Date(), + }); + const availability = withFailure.results.find((r) => r.objective_id === 'dash.availability'); + expect(availability?.sample_size).toBeGreaterThanOrEqual(100); + expect(availability?.observed).toBeLessThan(1); + }); + + it('writes the baseline, including every objective it could NOT measure', async () => { + const report = evaluateBiSlos({ events, from: windowFrom, to: windowTo }); + const summary = summariseSloReport(report); + + // The three latency objectives must be real numbers over real samples — + // an evidence file full of insufficient_data is not a baseline. + for (const id of ['dash.p95_first_query', 'dash.p95_cached', 'explore.p95', 'explore.p99']) { + const r = report.results.find((x) => x.objective_id === id); + expect(r?.observed, `${id} produced no number`).toBeTypeOf('number'); + expect(r?.low_confidence, `${id} was computed from too few samples`).not.toBe(true); + } + // ...and the cached path must actually be faster than the cold one, or the + // two were not measuring different things. + const cold = report.results.find((r) => r.objective_id === 'dash.p95_first_query')?.observed ?? 0; + const cached = report.results.find((r) => r.objective_id === 'dash.p95_cached')?.observed ?? 0; + expect(cached).toBeLessThan(cold); + + const dir = path.join(process.cwd(), 'evidence', 'bi-p103'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'baseline.json'), + `${JSON.stringify( + { + prompt: 'BI-P103', + title: 'First measured BI SLO baselines', + recorded_at: new Date().toISOString(), + host_note: + 'One idle host, local ClickHouse, warm page cache, single-threaded client. A floor for the ' + + 'shape of the distribution on this fixture — not a production SLA.', + clickhouse_version: JSON.parse(await ch('SELECT version() AS v FORMAT JSON')).data[0].v, + fixture: { + rows: ROWS, + table: 'slo_fact', + partitions: 'MergeTree ORDER BY (tenant_id, bucket, ts)', + }, + workload: { + cold_dashboard_queries: COLD_DASHBOARD_QUERIES, + cached_dashboard_queries: COLD_DASHBOARD_QUERIES, + exploration_queries: EXPLORE_QUERIES, + report_queries: 0, + }, + window: { from: windowFrom.toISOString(), to: windowTo.toISOString() }, + catalogue_size: BI_SLO_CATALOGUE.length, + fully_measured: report.fully_measured, + unresolved_queries: report.unresolved_queries, + summary, + results: report.results.map((r) => ({ + objective_id: r.objective_id, + workload: r.workload, + kind: r.kind, + status: r.status, + target: r.target, + unit: r.unit, + observed: r.observed, + sample_size: r.sample_size, + low_confidence: r.low_confidence ?? false, + detail: r.detail, + })), + measured_but_narrower_than_production: [ + 'dash.p95_cached is the IN-PROCESS reference cache (InMemoryResultCache). A Valkey ' + + 'deployment adds a network round trip, so 0.03 ms here is a floor and not the number a ' + + 'user will see.', + 'Every latency is the ENGINE round trip only: no HTTP layer, no CL0UD authorization, no ' + + 'serialization to the caller. The end-to-end p95 is larger by whatever those cost.', + 'One tenant, one client, no concurrency. Contention is not represented.', + ], + not_measured_and_why: [ + 'dash.freshness — a property of the DATASET (BI-P026 watermark), not of a usage event', + 'explore.timeout — the configured BI-P012 ceiling, not an observation', + 'report.completion_sla — nothing in 0pt emits aud0.report_generated yet, so the sample is 0', + "report.delivery_sla — delivery is N0TIFY's receipt; aud0 has no delivery event", + ], + }, + null, + 2, + )}\n`, + ); + + // The file must be honest about the unmeasured half. + expect(report.fully_measured).toBe(false); + }); +}); -- 2.54.0