From 84bb9faa4f4bfc3ade56092149a32bea7ca02e47 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Sun, 6 Sep 2026 23:01:27 +0200 Subject: [PATCH] =?UTF-8?q?feat(analytics):=20BI-P002..BI-P012=20analytics?= =?UTF-8?q?=20core=20=E2=80=94=20domain=20model,=20tiers,=20contracts,=20A?= =?UTF-8?q?rrow,=20ClickHouse=20engine,=20isolation,=20governance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the eight BI program prompts that make 0pt the canonical SIAX analytics capability. Every file is NEW: not one existing file is modified, and src/analytics/clickhouse-analytics.ts is byte-identical to main. BI-P002 — the nineteen canonical entities, verbatim, with tenant_id + project_id on every tenant-owned record. sql/008 gives each one a table. The four version entities BI-P002 names as immutable are enforced by BEFORE UPDATE/DELETE triggers, not by a TypeScript `readonly` — proven by direct SQL against a live Postgres. BI-P003 — RAW/NORMALIZED/CURATED/SERVING with the read rule enforced: assertReadableByBiSurface refuses a RAW dataset to a dashboard unless a recorded exception exists. All eight required declarations are NOT NULL. BI-P004 — DatasetContract and a fail-closed compatibility gate. Fifteen breaking-change classes are each tested twice: refused under a minor bump, accepted under a major one. BI-P005 — Arrow IPC alongside JSON, with a hand-written FlatBuffers writer so no npm dependency (or CVE surface) is added. Conformance comes from three independent implementations, not from self-agreement: pyarrow 25.0.1 and DuckDB 1.5.5 read identical values from both ClickHouse's Arrow and ours, and ClickHouse's own Arrow C++ ingests our 20,000-row batch with exact aggregates. BI-P009 — SiaxAnalyticsEngine with all eight capabilities. The existing ClickHouseAnalytics is COMPOSED, never replaced. executeQuery (analyst SQL) is declared and deliberately refused with CAPABILITY_UNAVAILABLE until BI-P017 and BI-P033 land, rather than opening the raw-SQL hole BI-P011 enumerates. BI-P010 — four purpose-built tables, not one giant one, and the difference is measured against the counterfactual: 48.8ms/40,960 rows vs 174.4ms/5,400,000 rows for the same question; 5/660 granules vs 660/1149. BI-P011 — layered isolation, each layer proven with the layer above removed. Without the forced predicate the same statement returns 50,050 (both tenants); with it, t_alpha gets 50. Without a row policy an unrestricted user sees 2 tenants; the restricted user running the IDENTICAL statement sees 1. BI-P012 — every named limit, fired individually and identified by ClickHouse error code, plus the half usually skipped: an unrelated tenant is timed while a real CPU bomb is in flight and must still return the CORRECT answer. Three defects found by writing tests that can fail: * `CHECK (array_length(purposes,1) >= 1)` can NEVER fail — array_length is NULL for '{}' and a NULL CHECK passes. Fixed with coalesce(); the test that caught it is pinned. * EXPLAIN ESTIMATE run under the caller's own max_rows_to_read fails with code 158 exactly when the estimate would be over budget. Planner calls now use a narrower envelope. * result_overflow_mode=break returns HTTP 200 with 65,409 of 100,000 rows and no indication it is short. Every SELECT ships `throw`. Verified rather than assumed: server7's ClickHouse holds `opt_scraped` (51 tables, 407,793,471 rows, `persons`/`cz_owners` included) — the b0ts plane this module is prohibited from touching — has no `opt_analytics` database, zero tenant_id columns and zero row policies. Nothing there was modified; every proof ran against disposable containers. Evidence: evidence/bi-p002-p012/{test-transcript.txt,bi-p010-benchmark.json, bi-p005-arrow-roundtrip.json,server7-clickhouse-probe.md} Docs: docs/analytics/BI_ANALYTICS_CORE.md Co-Authored-By: Claude Opus 5 --- docs/analytics/BI_ANALYTICS_CORE.md | 271 +++++++++ .../bi-p002-p012/bi-p005-arrow-roundtrip.json | 210 +++++++ evidence/bi-p002-p012/bi-p010-benchmark.json | 149 +++++ .../bi-p002-p012/server7-clickhouse-probe.md | 64 ++ evidence/bi-p002-p012/test-transcript.txt | 109 ++++ scripts/bi-p005-arrow-roundtrip.mjs | 161 +++++ scripts/bi-p010-benchmark.mjs | 311 ++++++++++ sql/008_analytics_domain_model.sql | 436 ++++++++++++++ src/analytics/arrow/flatbuffer-builder.ts | 237 ++++++++ src/analytics/arrow/ipc.ts | 547 +++++++++++++++++ src/analytics/domain/contract.ts | 474 +++++++++++++++ src/analytics/domain/entities.ts | 536 +++++++++++++++++ src/analytics/domain/tiers.ts | 83 +++ src/analytics/engine/clickhouse-engine.ts | 563 ++++++++++++++++++ src/analytics/engine/query-compiler.ts | 304 ++++++++++ src/analytics/engine/resource-governance.ts | 222 +++++++ src/analytics/engine/schema.ts | 169 ++++++ src/analytics/engine/siax-analytics-engine.ts | 180 ++++++ .../analytics-domain-repository.ts | 356 +++++++++++ .../analytics/analytics-domain-model.test.ts | 189 ++++++ ...nalytics-domain-postgres.realinfra.test.ts | 364 +++++++++++ .../analytics-engine-contract.test.ts | 124 ++++ tests/analytics/arrow-fixture.test.ts | 54 ++ tests/analytics/arrow-ipc.test.ts | 101 ++++ tests/analytics/dataset-contract.test.ts | 164 +++++ .../query-compiler-injection.test.ts | 224 +++++++ tests/analytics/resource-governance.test.ts | 118 ++++ .../tenant-predicate-regression.test.ts | 144 +++++ ...ytics-arrow-clickhouse.integration.test.ts | 183 ++++++ ...tics-engine-clickhouse.integration.test.ts | 252 ++++++++ ...cs-resource-governance.integration.test.ts | 237 ++++++++ ...ytics-tenant-isolation.integration.test.ts | 308 ++++++++++ 32 files changed, 7844 insertions(+) create mode 100644 docs/analytics/BI_ANALYTICS_CORE.md create mode 100644 evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json create mode 100644 evidence/bi-p002-p012/bi-p010-benchmark.json create mode 100644 evidence/bi-p002-p012/server7-clickhouse-probe.md create mode 100644 evidence/bi-p002-p012/test-transcript.txt create mode 100755 scripts/bi-p005-arrow-roundtrip.mjs create mode 100755 scripts/bi-p010-benchmark.mjs create mode 100644 sql/008_analytics_domain_model.sql create mode 100644 src/analytics/arrow/flatbuffer-builder.ts create mode 100644 src/analytics/arrow/ipc.ts create mode 100644 src/analytics/domain/contract.ts create mode 100644 src/analytics/domain/entities.ts create mode 100644 src/analytics/domain/tiers.ts create mode 100644 src/analytics/engine/clickhouse-engine.ts create mode 100644 src/analytics/engine/query-compiler.ts create mode 100644 src/analytics/engine/resource-governance.ts create mode 100644 src/analytics/engine/schema.ts create mode 100644 src/analytics/engine/siax-analytics-engine.ts create mode 100644 src/analytics/repositories/analytics-domain-repository.ts create mode 100644 tests/analytics/analytics-domain-model.test.ts create mode 100644 tests/analytics/analytics-domain-postgres.realinfra.test.ts create mode 100644 tests/analytics/analytics-engine-contract.test.ts create mode 100644 tests/analytics/arrow-fixture.test.ts create mode 100644 tests/analytics/arrow-ipc.test.ts create mode 100644 tests/analytics/dataset-contract.test.ts create mode 100644 tests/analytics/query-compiler-injection.test.ts create mode 100644 tests/analytics/resource-governance.test.ts create mode 100644 tests/analytics/tenant-predicate-regression.test.ts create mode 100644 tests/integration/analytics-arrow-clickhouse.integration.test.ts create mode 100644 tests/integration/analytics-engine-clickhouse.integration.test.ts create mode 100644 tests/integration/analytics-resource-governance.integration.test.ts create mode 100644 tests/integration/analytics-tenant-isolation.integration.test.ts diff --git a/docs/analytics/BI_ANALYTICS_CORE.md b/docs/analytics/BI_ANALYTICS_CORE.md new file mode 100644 index 0000000..eb19d0f --- /dev/null +++ b/docs/analytics/BI_ANALYTICS_CORE.md @@ -0,0 +1,271 @@ +# 0pt Analytics Core — BI-P002 … BI-P012 + +Status of this document: it describes what **is in this repository and has been +run**, and states separately what has **not** been deployed. Nothing here is a +plan written in the present tense. + +--- + +## 0. The ownership boundary (normative, not negotiable) + +``` +SIAX / 0pt owns the analytics engine, the query compiler, the tenant forcing, + the resource governance and the semantic/contract layer. + +Product repos (VehIQ and the rest) own DOMAIN MEANING — what a row means, + which metric a dealer cares about, how a screen reads. +``` + +Concretely, and enforced rather than asserted: + +* `BIApplicationBinding` (BI-P002 entity 19) has **no credential field**, in + TypeScript or in `sql/008_analytics_domain_model.sql`. A bound BI + application authenticates to the 0pt Analytics API with a CL0UD identity and + receives compiled, tenant-forced results. It never learns a ClickHouse DSN, + user, or password. Two tests assert the absence of such a column — one over + the migration text, one over `information_schema.columns` on a live + Postgres — so adding one later is a test failure. +* `AnalyticsIdentity` (BI-P009) is a **separate parameter** from the request, + never a field on it. This estate has already shipped a live cross-tenant + hole by reading a tenant out of a request body (INF0 Wave 2); the type + shape is what stops that recurring here. + +--- + +## 1. What each prompt produced + +| Prompt | Deliverable | Where | +|---|---|---| +| BI-P002 | 19 canonical entities + migrations + tenant-scoped persistence + restore drill | `src/analytics/domain/entities.ts`, `sql/008_analytics_domain_model.sql`, `src/analytics/repositories/analytics-domain-repository.ts` | +| BI-P003 | RAW / NORMALIZED / CURATED / SERVING, with the BI-surface read rule enforced | `src/analytics/domain/tiers.ts` | +| BI-P004 | `DatasetContract` + a fail-closed compatibility gate | `src/analytics/domain/contract.ts` | +| BI-P005 | Arrow IPC encode/decode + ClickHouse pass-through | `src/analytics/arrow/ipc.ts`, `src/analytics/arrow/flatbuffer-builder.ts` | +| BI-P009 | `SiaxAnalyticsEngine` interface + ClickHouse provider, existing implementation **preserved** | `src/analytics/engine/siax-analytics-engine.ts`, `src/analytics/engine/clickhouse-engine.ts` | +| BI-P010 | Four purpose-built ClickHouse tables + rollup MV, benchmarked | `src/analytics/engine/schema.ts`, `scripts/bi-p010-benchmark.mjs` | +| BI-P011 | Layered tenant isolation: verified identity, forced compiler predicate, restricted users, row policies | `src/analytics/engine/query-compiler.ts` | +| BI-P012 | Every named ClickHouse limit + CL0UD quota + per-tenant admission control | `src/analytics/engine/resource-governance.ts` | + +--- + +## 2. BI-P009 — what "preserve" means here + +`src/analytics/clickhouse-analytics.ts` is **byte-identical to main**. The new +engine composes it: + +* `health()` delegates to it; +* the pre-existing `opt_metric_events` path is reachable through + `executeLegacyMetric()`, still using its own compiler; +* everything else uses the new compiler for the BI-P010 tables. + +"Do not regress the existing forced tenant predicate" is enforced by +`tests/analytics/tenant-predicate-regression.test.ts`, which asserts the exact +string `tenant_id = {tenant:String}` on the URL the adapter builds. It has been +run with the forcing deliberately deleted from both compilers and goes RED +(2 of 6 tests fail); the transcript is +`evidence/bi-p002-p012/test-transcript.txt` §1–§2. + +The eight capabilities are all implemented, with one **deliberate refusal**: +`executeQuery` (analyst SQL) throws `CAPABILITY_UNAVAILABLE` until BI-P017 +(SQL dialect contract) and BI-P033 (analytics authorization) exist. Shipping a +raw-SQL surface first would create exactly the "raw SQL attempt" hole BI-P011 +enumerates. It is declared and refused, not silently missing. + +--- + +## 3. BI-P010 — why four tables, measured + +BI-P010 warns against "one giant unbounded table for every analytics use +case". `scripts/bi-p010-benchmark.mjs` builds both fixtures and the +counterfactual, and times them. Measured on a disposable +`clickhouse/clickhouse-server:24.8` (full transcript: +`evidence/bi-p002-p012/bi-p010-benchmark.json`): + +| Question | Purpose-built | One giant table | +|---|---|---| +| 90-day metric total, one tenant | **48.8 ms p50**, 40 960 rows read | 174.4 ms p50, **5 400 000** rows read | +| Vehicle point lookup by key, one tenant | **49.3 ms p50**, 8 192 rows read | 236.6 ms p50, **7 401 152** rows read | +| Avg asking price by make over 2M vehicles | **389.6 ms p50** | 680.8 ms p50 | +| Granules read for the metric total | **5 / 660** | 660 / 1 149 | + +Fixtures: 5 400 000 SaaS events (200 tenants × 12 metrics × 90 days) and +4 000 000 vehicle projections (2M for the tenant under test + 2M decoys owned +by 50 other tenants). + +**Honest finding that does not flatter the design:** at this scale the +`AggregatingMergeTree` rollup (37.98 ms p50, 21 416 rows read) was *not* faster +in wall time than scanning raw events (32.55 ms p50, 40 960 rows read). It +reads about half the rows, and its value is the long TTL — raw events expire +in 30 days, the rollup keeps three years — so the rollup earns its keep on +retention and on scale beyond this fixture, not on latency at 5.4M rows. Do +not quote it as a speed win. + +--- + +## 4. BI-P011 — the isolation proof + +Four layers, and each one is tested with the layer above it *removed*: + +1. **Verified CL0UD identity.** `AnalyticsIdentity` is not part of the + request; a blank tenant or principal is `UNAUTHENTICATED`, never a wildcard. +2. **The 0pt query compiler forces the tenant.** Every compiled statement + contains `tenant_id = {tenant:String}` as its first WHERE condition, with + the value bound as an HTTP parameter — the tenant string never appears in + the SQL text. +3. **Restricted database users + row policies.** + `restrictedTenantUserDdl()` renders a per-tenant ClickHouse user with + `readonly = 1` and a `CREATE ROW POLICY ... USING tenant_id = ''` + on every analytics table. +4. **Separate database/cluster tier** for very-high-isolation customers: + supported by construction — the engine takes a `database` and a `baseUrl` + per instance — but **not yet operated**, because there is no deployed + ClickHouse for 0pt analytics at all (see §7). + +The proof, run against a disposable ClickHouse with 50 rows for `t_alpha` +(value 1 each) and 50 for `t_beta` (value 1000 each): + +* **hole, layer 2 removed:** the same statement minus the forced predicate + returns **50 050** — both tenants in one number; +* **closed, layer 2 present:** `t_alpha` gets **50**, `t_beta` gets **50 000**; +* **hole, layer 3 absent:** an unrestricted user issuing a predicate-free + `uniqExact(tenant_id)` sees **2** tenants; +* **closed, layer 3 present:** the restricted user issuing the *identical* + statement sees **1**, and `sum(value)` returns 50 rather than 50 050. + +Adversarial cases from BI-P011's list — tenant A querying B's dataset, forged +dataset IDs, raw SQL, SQL injection, dimension identifier injection, filter +injection — are covered in +`tests/analytics/query-compiler-injection.test.ts` (compiler layer) and +`tests/integration/analytics-tenant-isolation.integration.test.ts` (live). + +Dimension names are **resolved through a per-tenant allow-list** +(`DimensionDefinition`), never escaped and never spliced. A tenant that has +declared no dimensions may reference none — fail closed. + +--- + +## 5. BI-P012 — governance, and the partial-result trap + +Every setting BI-P012 names is applied. Five ride on the statement itself +(`max_execution_time`, `max_result_rows`, `max_rows_to_read`, +`max_bytes_to_read`, `max_memory_usage`) under `readonly = 1`, so the query +text cannot re-raise them; `max_concurrent_queries` is a per-user quota plus a +local admission gate. + +Overflow modes are **`throw`, never `break`**. That is not a style preference. +Measured on ClickHouse 24.8: with `result_overflow_mode = break` and +`max_result_rows = 10`, a 100 000-row query returns **HTTP 200 with 65 409 +rows** and nothing in the payload says it is short. A truncated answer that +looks complete is the analytics form of a false green. + +Query-bomb outcomes, each fired individually and identified by ClickHouse +error code: 158 `TOO_MANY_ROWS`, 307 `TOO_MANY_BYTES`, 159 `TIMEOUT_EXCEEDED`, +396 `TOO_MANY_ROWS_OR_BYTES`, 241 `MEMORY_LIMIT_EXCEEDED`. + +The second half of the DoD — "without degrading unrelated tenant beyond +declared SLO" — is timed, not assumed: an unrelated tenant's interactive query +is run five times *while* an 8-second CPU bomb is in flight, must still return +the **correct** answer, and must stay under a declared 5000 ms ceiling (measured ~30 ms on an idle +server; the ceiling is set wide for a shared CI runner, and a genuinely +starved tenant would take seconds or fail). + +Two measured details worth keeping: + +* `EXPLAIN ESTIMATE` must **not** run under the caller's own + `max_rows_to_read`. It fails with code 158 exactly when the estimate would + be over budget — i.e. in the only case the estimate matters. Planner calls + use `planningSettings()`, a narrower envelope, instead. +* `SELECT count() FROM numbers(2e10)` is *not* a query bomb — ClickHouse + answers it in about 5 seconds because it can skip the values. The bomb in + the test hashes every row, which cannot be short-circuited. + +--- + +## 6. BI-P005 — Arrow, and why it is not self-certified + +Two paths: + +* **pass-through** — ClickHouse emits Arrow IPC itself (`FORMAT ArrowStream`); + 0pt streams those bytes unchanged. `output_format_arrow_string_as_string=1` + is required, or String columns arrive as Arrow **Binary** and both pyarrow + and DuckDB read them as bytes: silent semantic loss. +* **encoded** — for results 0pt assembles itself, `encodeArrowStream()` writes + the IPC stream, including a small hand-written FlatBuffers writer (no new + npm dependency, no new supply-chain surface). + +A hand-rolled encoder is only worth having if somebody else's parser agrees +with it, so conformance is established by three independent implementations: + +* **pyarrow 25.0.1** and **DuckDB 1.5.5** decode both ClickHouse's bytes and + 0pt's bytes to *identical* values and types + (`evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json`, verdict `PASS`); +* **ClickHouse's own Arrow C++** ingests 0pt-encoded Arrow — including a + 20 000-row batch whose aggregates come back exact — in + `tests/integration/analytics-arrow-clickhouse.integration.test.ts`, which + runs on every push. + +Supported types and their caveats are the table in `ARROW_TYPE_SUPPORT`; +`DateTime64(6|9)` and `UInt64` above 2^63 are deliberately **outside** the +supported set rather than silently lossy. + +Arrow is also not an escape hatch from BI-P012: +`assertArrowWithinResultBudget()` decodes the row count and refuses a payload +over the caller's `max_result_rows`. + +--- + +## 7. What is NOT done — verified, not assumed + +* **There is no deployed ClickHouse for 0pt analytics.** Probed 2026-09-06 on + server7, the only ClickHouse in the estate: databases are + `opt_scraped`, `default`, `system`, `INFORMATION_SCHEMA`, + `information_schema`. There is no `opt_analytics` database, so + `ClickHouseAnalytics` — the "existing working" provider — has never been + run against a production instance. Everything in this document was proven + against disposable containers. +* **server7's ClickHouse must not be used for this.** It holds + `opt_scraped` — 51 tables, 407 793 471 rows, including `persons` and + `cz_owners`. That is the b0ts scraping plane, which + `src/analytics/clickhouse-analytics.ts`'s own header already prohibits this + module from touching (Ratsit/person-PII legal case, written DPIA + prohibition on quarantined market data). It also has **zero** columns named + `tenant_id` or `project_id` anywhere on the server and **zero** row + policies, so it could not satisfy BI-P011 even if it were legally + available. Nothing on that server was modified by this work. +* **BI-P011 layer 5** (separate database/cluster tier per high-isolation + customer) is supported by construction but not operated. +* **`executeQuery`** is refused pending BI-P017/BI-P033 (§2). +* The domain repository covers `AnalyticsDataset`, `DatasetVersion`, + `MetricDefinition`, `MetricVersion` and `DimensionDefinition`. The other + fourteen entities have tables, constraints and types but no repository + methods yet; their consumers (dashboards, reports, quality runs, lineage) + arrive with BI-P025 … BI-P035. + +--- + +## 8. Running the proofs yourself + +```bash +# unit suites (no infrastructure) +pnpm vitest run tests/analytics + +# live ClickHouse suites +docker run -d --name opt-bi-ch -p 18123:8123 \ + -e CLICKHOUSE_USER=opt_admin -e CLICKHOUSE_PASSWORD= \ + -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 clickhouse/clickhouse-server:24.8 +CLICKHOUSE_URL=http://localhost:18123 CLICKHOUSE_AUTH=opt_admin: \ + pnpm vitest run tests/integration + +# BI-P002 restore drill +docker run -d --name opt-bi-pg -p 15455:5432 \ + -e POSTGRES_USER=opt -e POSTGRES_PASSWORD= -e POSTGRES_DB=opt_bi_test \ + postgres:16-alpine +ANALYTICS_TEST_DATABASE_URL=postgres://opt:@localhost:15455/opt_bi_test \ + pnpm vitest run tests/analytics/analytics-domain-postgres.realinfra.test.ts + +# BI-P010 benchmark and BI-P005 external roundtrip (rewrite the evidence files) +CLICKHOUSE_URL=... CLICKHOUSE_AUTH=... node scripts/bi-p010-benchmark.mjs +CLICKHOUSE_URL=... CLICKHOUSE_AUTH=... node scripts/bi-p005-arrow-roundtrip.mjs +``` + +Both scripts CREATE and DROP a database. Never point either at server7 or at +any ClickHouse holding b0ts or quarantined market data. diff --git a/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json b/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json new file mode 100644 index 0000000..6e065aa --- /dev/null +++ b/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json @@ -0,0 +1,210 @@ +{ + "generated_at": "2026-09-06T20:53:00.409Z", + "clickhouse_version": "24.8.14.39", + "query": "SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag,\n toDateTime64(ts, 3, 'UTC') AS ts\nFROM (\n SELECT arrayJoin([\n ('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'),\n ('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'),\n ('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789')\n ]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts\n)", + "directions": { + "clickhouse -> arrow -> pyarrow/duckdb": { + "pyarrow_version": "25.0.1", + "duckdb_version": "1.5.5", + "schema": [ + { + "name": "tenant_id", + "type": "string" + }, + { + "name": "n", + "type": "int64" + }, + { + "name": "v", + "type": "double" + }, + { + "name": "flag", + "type": "bool" + }, + { + "name": "ts", + "type": "timestamp[ms, tz=UTC]" + } + ], + "num_rows": 3, + "pyarrow_values": { + "tenant_id": [ + "t_alpha", + "t_beta", + "t_gamma" + ], + "n": [ + 1, + 2, + 3 + ], + "v": [ + 1.5, + 2.5, + 3.25 + ], + "flag": [ + true, + false, + true + ], + "ts": [ + "2026-01-01 00:00:00+00:00", + "2026-06-15 08:09:10.123000+00:00", + "2026-09-06 12:34:56.789000+00:00" + ] + }, + "duckdb_types": [ + [ + "tenant_id", + "VARCHAR" + ], + [ + "n", + "BIGINT" + ], + [ + "v", + "DOUBLE" + ], + [ + "flag", + "BOOLEAN" + ], + [ + "ts", + "TIMESTAMP WITH TIME ZONE" + ] + ], + "duckdb_rows": [ + [ + "t_alpha", + "1", + "1.5", + "True", + "2026-01-01 00:00:00+00:00" + ], + [ + "t_beta", + "2", + "2.5", + "False", + "2026-06-15 08:09:10.123000+00:00" + ], + [ + "t_gamma", + "3", + "3.25", + "True", + "2026-09-06 12:34:56.789000+00:00" + ] + ] + }, + "0pt encoder -> arrow -> pyarrow/duckdb": { + "pyarrow_version": "25.0.1", + "duckdb_version": "1.5.5", + "schema": [ + { + "name": "tenant_id", + "type": "string" + }, + { + "name": "n", + "type": "int64" + }, + { + "name": "v", + "type": "double" + }, + { + "name": "flag", + "type": "bool" + }, + { + "name": "ts", + "type": "timestamp[ms, tz=UTC]" + } + ], + "num_rows": 3, + "pyarrow_values": { + "tenant_id": [ + "t_alpha", + "t_beta", + "t_gamma" + ], + "n": [ + 1, + 2, + 3 + ], + "v": [ + 1.5, + 2.5, + 3.25 + ], + "flag": [ + true, + false, + true + ], + "ts": [ + "2026-01-01 00:00:00+00:00", + "2026-06-15 08:09:10.123000+00:00", + "2026-09-06 12:34:56.789000+00:00" + ] + }, + "duckdb_types": [ + [ + "tenant_id", + "VARCHAR" + ], + [ + "n", + "BIGINT" + ], + [ + "v", + "DOUBLE" + ], + [ + "flag", + "BOOLEAN" + ], + [ + "ts", + "TIMESTAMP WITH TIME ZONE" + ] + ], + "duckdb_rows": [ + [ + "t_alpha", + "1", + "1.5", + "True", + "2026-01-01 00:00:00+00:00" + ], + [ + "t_beta", + "2", + "2.5", + "False", + "2026-06-15 08:09:10.123000+00:00" + ], + [ + "t_gamma", + "3", + "3.25", + "True", + "2026-09-06 12:34:56.789000+00:00" + ] + ] + } + }, + "byte_sizes": { + "clickhouse_encoded": 968, + "opt_encoded": 864 + }, + "verdict": "PASS — pyarrow and DuckDB read identical values and types from BOTH encoders" +} diff --git a/evidence/bi-p002-p012/bi-p010-benchmark.json b/evidence/bi-p002-p012/bi-p010-benchmark.json new file mode 100644 index 0000000..6793dd4 --- /dev/null +++ b/evidence/bi-p002-p012/bi-p010-benchmark.json @@ -0,0 +1,149 @@ +{ + "generated_at": "2026-09-06T20:44:16.335Z", + "clickhouse_version": "24.8.14.39", + "database": "opt_bench_mtqa56j3", + "fixtures": { + "saas": { + "events": 5400000, + "tenants": 200, + "metrics": 12, + "days": 90, + "seed_ms": 22912 + }, + "vehiq": { + "vehicles_under_test": 2000000, + "decoy_vehicles": 2000000, + "seed_ms": 31960 + }, + "on_disk": [ + { + "table": "giant_everything", + "rows": 9400000, + "bytes": 84015753 + }, + { + "table": "opt_analytics_event", + "rows": 5400000, + "bytes": 60158827 + }, + { + "table": "opt_dataset_projection", + "rows": 4000000, + "bytes": 48936260 + }, + { + "table": "opt_metric_daily", + "rows": 37800, + "bytes": 86181 + } + ] + }, + "measurements": [ + { + "label": "SaaS dashboard panel: 90-day metric total for ONE tenant (purpose-built event table)", + "runs": 12, + "p50_ms": 48.8, + "p95_ms": 72.24, + "min_ms": 31.36, + "max_ms": 72.24, + "rows_read": 40960, + "bytes_read": 75216 + }, + { + "label": "SAME question against ONE GIANT TABLE (time-leading sort key)", + "runs": 12, + "p50_ms": 174.38, + "p95_ms": 320.61, + "min_ms": 142.41, + "max_ms": 320.61, + "rows_read": 5400000, + "bytes_read": 165330000 + }, + { + "label": "SaaS time series: daily series for ONE tenant from the AggregatingMergeTree rollup", + "runs": 12, + "p50_ms": 37.98, + "p95_ms": 60.34, + "min_ms": 22.25, + "max_ms": 60.34, + "rows_read": 21416, + "bytes_read": 42832 + }, + { + "label": "SAME time series computed from raw events instead of the rollup", + "runs": 12, + "p50_ms": 32.55, + "p95_ms": 85.96, + "min_ms": 22.03, + "max_ms": 85.96, + "rows_read": 40960, + "bytes_read": 75216 + }, + { + "label": "SaaS group-by: one tenant sliced by a dimension", + "runs": 12, + "p50_ms": 44.45, + "p95_ms": 317.18, + "min_ms": 22.92, + "max_ms": 317.18, + "rows_read": 40960, + "bytes_read": 75216 + }, + { + "label": "VehIQ point lookup: one vehicle by key, tenant-scoped (purpose-built projection)", + "runs": 12, + "p50_ms": 49.25, + "p95_ms": 89.99, + "min_ms": 28.31, + "max_ms": 89.99, + "rows_read": 8192, + "bytes_read": 761856 + }, + { + "label": "SAME point lookup against ONE GIANT TABLE", + "runs": 12, + "p50_ms": 236.6, + "p95_ms": 1227.93, + "min_ms": 174.21, + "max_ms": 1227.93, + "rows_read": 7401152, + "bytes_read": 196760496 + }, + { + "label": "VehIQ aggregate: average asking price by make for ONE tenant across 2M vehicles", + "runs": 8, + "p50_ms": 389.57, + "p95_ms": 16748.19, + "min_ms": 189.51, + "max_ms": 16748.19, + "rows_read": 2000000, + "bytes_read": 256400000 + }, + { + "label": "SAME aggregate against ONE GIANT TABLE", + "runs": 8, + "p50_ms": 680.82, + "p95_ms": 756.4, + "min_ms": 410.57, + "max_ms": 756.4, + "rows_read": 7401152, + "bytes_read": 447184368 + } + ], + "notes": [ + "Fixtures are synthetic and generated in-server with numbers(); no production or b0ts data was read or written.", + "Each timing is the p50/p95 of repeated runs after one warm-up, measured client-side over HTTP on the same host.", + "The 'ONE GIANT TABLE' rows are the counterfactual BI-P010 warns against, measured rather than asserted." + ], + "index_pruning": { + "purpose_built_event_table": { + "read": 5, + "total": 660 + }, + "one_giant_table": { + "read": 660, + "total": 1149 + }, + "note": "Granules read/total for the SAME question. The tenant-leading ORDER BY is what turns the forced tenant predicate (BI-P011) into a primary-key range scan instead of a full scan." + } +} diff --git a/evidence/bi-p002-p012/server7-clickhouse-probe.md b/evidence/bi-p002-p012/server7-clickhouse-probe.md new file mode 100644 index 0000000..957b9f8 --- /dev/null +++ b/evidence/bi-p002-p012/server7-clickhouse-probe.md @@ -0,0 +1,64 @@ +# Live probe: the only ClickHouse in the estate (2026-09-06) + +Read-only inspection over the tailnet (`root@100.100.215.20`, siax-server7). +Nothing was written, altered or deleted. Only server metadata +(`system.databases`, `system.tables`, `system.columns`, `system.users`, +`system.row_policies`) was read — **no table content, no person data**. + +## What is there + +``` +version() 26.7.1.1278 +process clickhouse-server (native, not a container) +listening 127.0.0.1:8123/9000 and 10.100.0.5:8123/9000 +databases INFORMATION_SCHEMA, default, information_schema, + opt_scraped, system +users default, b0ts, b0ts_writer, b0ts_reader +row policies 0 +settings profiles default, readonly +``` + +`opt_scraped`: 51 tables, 407 793 471 rows in total. Largest by row count: + +``` +vehicles ReplacingMergeTree 104 759 599 +cz_owners MergeTree 90 979 084 +vehicle_details_v2 ReplacingMergeTree 50 987 449 +cz_vehicles_full MergeTree 38 222 160 +vehicle_history MergeTree 28 421 783 +vehicle_price_history MergeTree 20 475 073 +... +persons ReplacingMergeTree 9 457 +persons_meta ReplacingMergeTree 9 457 +``` + +## Three findings that change what can be claimed + +1. **There is no `opt_analytics` database anywhere on this server.** The + `ClickHouseAnalytics` capability in this repo — described upstream as "an + existing working implementation" — has never been run against a deployed + instance. It works (it is exercised against real ClickHouse in CI and in + this branch's evidence), but it is not in production. + +2. **This server is the b0ts scraping plane, and this module is prohibited + from touching it.** `src/analytics/clickhouse-analytics.ts`'s own header + states the boundary: never the b0ts ClickHouse pipelines + (Ratsit / person-PII, separate legal case) and never quarantined + `market='sverige'`/`'norge'` data (written DPIA prohibition). `opt_scraped` + holds exactly that, `persons`/`cz_owners` included. The briefing note that + "ClickHouse runs on server7 and holds 40M+ real Swedish vehicle rows" is + accurate about the server and, on inspection, describes the plane 0pt + analytics may not use. + +3. **It could not satisfy BI-P011 even if it were available.** There is not a + single column named `tenant_id` or `project_id` anywhere on the server, and + there are zero row policies. There is nothing to force a predicate on. + +## Consequence for BI-P009 … BI-P012 + +The canonical OLAP provider needs a **new, separate** ClickHouse for +`opt_analytics` — its own instance or at minimum its own database with +restricted users — before any of this reaches production. Every proof in this +branch was therefore run against disposable +`clickhouse/clickhouse-server:24.8` containers created and torn down by the +test/benchmark scripts. diff --git a/evidence/bi-p002-p012/test-transcript.txt b/evidence/bi-p002-p012/test-transcript.txt new file mode 100644 index 0000000..f7a2dfd --- /dev/null +++ b/evidence/bi-p002-p012/test-transcript.txt @@ -0,0 +1,109 @@ +BI-P002..BI-P012 — measured test transcript +=========================================== +host: Linux 6.8.0-139-generic x86_64 node: v24.18.0 date(UTC): 2026-09-06T20:53:23Z +clickhouse under test: clickhouse/clickhouse-server:24.8 (disposable container, created and torn down by this run) +postgres under test: postgres:16-alpine (disposable container) + +### 1. BI-P009 regression guard WITH the forced tenant predicate REMOVED (must be RED) + × BI-P009 regression guard: the forced tenant predicate > PRESERVED ClickHouseAnalytics still emits tenant_id = {tenant:String} on the wire 163ms + ✓ BI-P009 regression guard: the forced tenant predicate > PRESERVED ClickHouseAnalytics still refuses a blank tenant_id 36ms + ✓ BI-P009 regression guard: the forced tenant predicate > PRESERVED ClickHouseAnalytics still refuses a hostile group_by identifier 1ms + × BI-P009 regression guard: the forced tenant predicate > every statement the NEW compiler produces contains the forced predicate 34ms + ✓ BI-P009 regression guard: the forced tenant predicate > the NEW compiler fails closed on a blank/absent tenant instead of emitting an unfiltered query 33ms + ✓ BI-P009 regression guard: the forced tenant predicate > the constant itself is the predicate, so deleting the forcing cannot pass silently 1ms +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 2 ⎯⎯⎯⎯⎯⎯⎯ + Test Files 1 failed (1) + Tests 2 failed | 4 passed (6) + +### 2. The same suite with the forcing RESTORED (must be GREEN) + ✓ tests/analytics/tenant-predicate-regression.test.ts (6 tests) 85ms + Test Files 1 passed (1) + Tests 6 passed (6) +(working tree restored: 0 lines of diff in the preserved file) + +### 3. BI-P011 tenant isolation against a LIVE disposable ClickHouse (hole shown OPEN, then CLOSED) + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > HOLE, layer 2 REMOVED: the same query WITHOUT the forced predicate returns t_beta's rows to t_alpha 49ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000 60ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > tenant A querying tenant B's dataset id gets ZERO rows, not B's rows 23ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > a forged dataset id yields an empty result, never an error that leaks existence 22ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > a tenant with no data at all sees zero, not everyone's aggregate 24ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > group-by results carry only this tenant's dimension values 66ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > SQL/dimension/filter injection through the live path changes nothing 74ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > raw SQL attempt: the analyst-SQL surface is refused, not silently executed 1ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant 42ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant 200ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > the restricted user cannot write, and cannot reach a table it was not granted 90ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > every analytics table's sorting key LEADS with tenant_id (what makes the predicate cheap) 143ms + ✓ tests/integration/analytics-tenant-isolation.integration.test.ts > BI-P011 tenant isolation against a LIVE ClickHouse > the tenant value never appears in the statement text the server receives 149ms + Test Files 1 passed (1) + Tests 13 passed (13) + +### 4. BI-P009/BI-P010 provider + schema against the same LIVE ClickHouse + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > health() reports the live server's version 30ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > BI-P010: four purpose-built tables exist, NOT one giant table 12ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > BI-P010: the tables use different engines and different sort keys — the split is real 30ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > BI-P010: TTL and LowCardinality are actually applied, not just written in a comment 22ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > BI-P010: the materialized view actually populates the rollup 92ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > executeMetric returns grouped rows keyed by the CALLER-FACING dimension name 48ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget 15ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > estimateQuery REFUSES a query whose estimate exceeds a tightened quota 11ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts 15ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > cancelQuery issues a real KILL QUERY the server accepts 33ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > ingestProjection writes the SERVING tier with the identity's tenant, ignoring any row-level claim 33ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL) 58ms + ✓ tests/integration/analytics-engine-clickhouse.integration.test.ts > BI-P009/BI-P010 ClickHouse provider against a LIVE server > the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine 35ms + Test Files 1 passed (1) + Tests 13 passed (13) + +### 5. BI-P012 resource governance / query bomb against the same LIVE ClickHouse + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS) 60ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES) 26ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED) 1014ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > max_result_rows terminates a result bomb and does NOT return a partial result 422ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED) 84ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > readonly=1 means the query text cannot re-raise its own limits 37ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > readonly=1 blocks writes through the query surface 26ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight 8163ms + ✓ tests/integration/analytics-resource-governance.integration.test.ts > BI-P012 resource governance against a LIVE ClickHouse > the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only 220ms + Test Files 1 passed (1) + Tests 9 passed (9) + +### 6. BI-P005 Arrow interop with ClickHouse's own Arrow C++ implementation + ✓ tests/integration/analytics-arrow-clickhouse.integration.test.ts > BI-P005 Arrow IPC against a LIVE ClickHouse > THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count 106ms + ✓ tests/integration/analytics-arrow-clickhouse.integration.test.ts > BI-P005 Arrow IPC against a LIVE ClickHouse > WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents 42ms + ✓ tests/integration/analytics-arrow-clickhouse.integration.test.ts > BI-P005 Arrow IPC against a LIVE ClickHouse > OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged 76ms + ✓ tests/integration/analytics-arrow-clickhouse.integration.test.ts > BI-P005 Arrow IPC against a LIVE ClickHouse > full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows 281ms + ✓ tests/integration/analytics-arrow-clickhouse.integration.test.ts > BI-P005 Arrow IPC against a LIVE ClickHouse > a large result survives the round trip (batching/offsets are not only correct for three rows) 690ms + Test Files 1 passed (1) + Tests 5 passed (5) + +### 7. BI-P002 domain model against a LIVE disposable Postgres (restore drill + immutability) + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > restore drill: sql/008 applies cleanly on top of the replayed chain 6ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > all nineteen entity tables really exist after the restore 125ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > replaying sql/008 a second time is a no-op, not an error 253ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > persists a dataset and reads it back through the repository 27ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > TENANT ISOLATION: t_beta cannot read t_alpha's dataset, even with the exact id 7ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > TENANT ISOLATION: a version cannot be minted against another tenant's dataset 4ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > dataset versions are monotonic and carry the contract hash 136ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > IMMUTABILITY is enforced by the DATABASE: UPDATE and DELETE on a version row are refused 27ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > all four immutable entities refuse an UPDATE, not just the one that was tested first 638ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > a metric version after the first must state WHY the number changed meaning 123ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > a DatasetGrant without a purpose is refused by the DATABASE (purpose limitation) 134ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > a BIApplicationBinding without a purpose is refused too, and carries no credential column 40ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > the dimension catalog is tenant-scoped and FAILS CLOSED for a tenant with none 111ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > a dimension whose physical field is not identifier-safe is refused by the DATABASE 3ms + ✓ tests/analytics/analytics-domain-postgres.realinfra.test.ts > BI-P002 analytics domain model against a LIVE Postgres > the repository refuses to act without a verified tenant scope 1ms + Test Files 1 passed (1) + Tests 15 passed (15) + +### 8. Repository gates (the same commands .gitea/workflows/ci.yml runs) +TYPECHECK_EXIT=0 + Test Files 54 passed | 9 skipped (63) + Tests 463 passed | 80 skipped (543) +TEST_EXIT=0 +BUILD_EXIT=0 +lint (ADVISORY in CI, continue-on-error): 12 pre-existing errors in tests/p0/* — ZERO in any file this branch adds. + +### 9. Scope check +src/analytics/clickhouse-analytics.ts diff vs main: 0 lines (0 = preserved untouched) +.gitea/workflows diff vs main: 0 lines (0 = no protected file touched) diff --git a/scripts/bi-p005-arrow-roundtrip.mjs b/scripts/bi-p005-arrow-roundtrip.mjs new file mode 100755 index 0000000..281e769 --- /dev/null +++ b/scripts/bi-p005-arrow-roundtrip.mjs @@ -0,0 +1,161 @@ +#!/usr/bin/env node +/** + * BI-P005 DoD, executed literally: + * + * "Same query result roundtrips ClickHouse -> Arrow -> DuckDB/Python + * without semantic loss for supported types." + * + * Pulls a result from a LIVE ClickHouse as Arrow IPC, hands the identical + * bytes to pyarrow and DuckDB, and records what each of them read back. Also + * runs the reverse direction — 0pt's own encoder's bytes decoded by pyarrow + + * DuckDB — because the encoder is hand-rolled and the only conformance + * evidence worth having comes from somebody else's parser. + * + * CLICKHOUSE_URL=http://localhost:18123 CLICKHOUSE_AUTH=user:pass \ + * node scripts/bi-p005-arrow-roundtrip.mjs + * + * pyarrow/DuckDB run in a throwaway `python:3.12-slim` container, so the host + * needs docker but no Python packages. Output: + * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json + * + * The 0pt-encoded fixture is produced by `pnpm vitest run + * tests/analytics/arrow-fixture.emit.ts`-style code inlined here via a tiny + * tsx-free path: the encoder is re-implemented ONLY in the sense that this + * script shells out to vitest to write the file, so the bytes tested are the + * bytes the library produces. + */ +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { execFileSync } from "node:child_process"; + +const URL_ = process.env.CLICKHOUSE_URL; +const AUTH = process.env.CLICKHOUSE_AUTH; +if (!URL_) { + console.error("CLICKHOUSE_URL is required — this script measures, it does not simulate."); + process.exit(2); +} + +const work = fs.mkdtempSync(path.join(os.tmpdir(), "bi-p005-")); +const headers = AUTH ? { Authorization: `Basic ${Buffer.from(AUTH).toString("base64")}` } : {}; + +const QUERY = ` +SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag, + toDateTime64(ts, 3, 'UTC') AS ts +FROM ( + SELECT arrayJoin([ + ('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'), + ('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'), + ('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789') + ]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts +)`; + +async function chArrow(sql, settings) { + const u = new URL(URL_); + u.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) u.searchParams.set(k, v); + const res = await fetch(u.toString(), { method: "POST", headers, body: "" }); + const buf = Buffer.from(await res.arrayBuffer()); + if (!res.ok) throw new Error(`CH ${res.status}: ${buf.toString("utf8").slice(0, 300)}`); + return buf; +} + +const PY = ` +import pyarrow as pa, pyarrow.ipc as ipc, duckdb, json, sys, os +out = {} +for name in sorted(os.listdir('/w')): + if not name.endswith('.arrow'): + continue + tbl = ipc.open_stream(pa.BufferReader(open('/w/'+name,'rb').read())).read_all() + con = duckdb.connect(); con.register('t', tbl) + out[name] = { + "pyarrow_version": pa.__version__, + "duckdb_version": duckdb.__version__, + "schema": [{"name": f.name, "type": str(f.type)} for f in tbl.schema], + "num_rows": tbl.num_rows, + "pyarrow_values": json.loads(json.dumps(tbl.to_pydict(), default=str)), + "duckdb_types": [(r[0], r[1]) for r in con.execute("describe select * from t").fetchall()], + "duckdb_rows": [[str(c) for c in row] for row in con.execute("select * from t order by n").fetchall()], + } +print(json.dumps(out)) +`; + +async function main() { + const version = (await (async () => { + const u = new URL(URL_); + u.searchParams.set("query", "SELECT version() FORMAT TSVRaw"); + const r = await fetch(u.toString(), { method: "POST", headers, body: "" }); + return (await r.text()).trim(); + })()); + + // Direction 1: ClickHouse's own Arrow C++ output. + const chBytes = await chArrow(`${QUERY} FORMAT ArrowStream`, { + output_format_arrow_string_as_string: "1", + output_format_arrow_low_cardinality_as_dictionary: "0", + }); + fs.writeFileSync(path.join(work, "01_clickhouse_encoded.arrow"), chBytes); + + // Direction 2: 0pt's own encoder. Driven through the SHIPPED module via + // tests/analytics/arrow-fixture.test.ts (ARROW_FIXTURE_DIR) so the bytes + // pyarrow sees are the library's, not a copy pasted into this script. + execFileSync( + "npx", + ["vitest", "run", "tests/analytics/arrow-fixture.test.ts"], + { stdio: "inherit", env: { ...process.env, ARROW_FIXTURE_DIR: work } }, + ); + + // The reader script goes in as a FILE, not as `python -c "..."`: passing a + // multi-line program through a shell argument mangles the newlines. + fs.writeFileSync(path.join(work, "read_arrow.py"), PY); + const raw = execFileSync( + "docker", + [ + "run", "--rm", "-v", `${work}:/w`, "-w", "/w", "python:3.12-slim", + "bash", "-lc", + "pip install --quiet pyarrow duckdb pytz >/dev/null 2>&1 && python /w/read_arrow.py", + ], + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ); + + const decoded = JSON.parse(raw); + const evidence = { + generated_at: new Date().toISOString(), + clickhouse_version: version, + query: QUERY.trim(), + directions: { + "clickhouse -> arrow -> pyarrow/duckdb": decoded["01_clickhouse_encoded.arrow"], + "0pt encoder -> arrow -> pyarrow/duckdb": decoded["02_0pt_encoded.arrow"], + }, + byte_sizes: { + clickhouse_encoded: chBytes.length, + opt_encoded: fs.statSync(path.join(work, "02_0pt_encoded.arrow")).size, + }, + verdict: null, + }; + + // The DoD is "without semantic loss": the two independent readers must agree + // on the SAME values for both encoders. + const a = evidence.directions["clickhouse -> arrow -> pyarrow/duckdb"]; + const b = evidence.directions["0pt encoder -> arrow -> pyarrow/duckdb"]; + const same = + JSON.stringify(a.pyarrow_values) === JSON.stringify(b.pyarrow_values) && + JSON.stringify(a.duckdb_rows) === JSON.stringify(b.duckdb_rows) && + JSON.stringify(a.duckdb_types) === JSON.stringify(b.duckdb_types); + evidence.verdict = same + ? "PASS — pyarrow and DuckDB read identical values and types from BOTH encoders" + : "FAIL — the two encoders do not round-trip to the same values"; + + const out = path.join(process.cwd(), "evidence", "bi-p002-p012", "bi-p005-arrow-roundtrip.json"); + fs.mkdirSync(path.dirname(out), { recursive: true }); + fs.writeFileSync(out, JSON.stringify(evidence, null, 2) + "\n"); + console.log(evidence.verdict); + console.log(`wrote ${out}`); + if (!same) process.exitCode = 1; +} + +main() + .catch((e) => { + console.error(e); + process.exitCode = 1; + }) + .finally(() => fs.rmSync(work, { recursive: true, force: true })); diff --git a/scripts/bi-p010-benchmark.mjs b/scripts/bi-p010-benchmark.mjs new file mode 100755 index 0000000..abbf462 --- /dev/null +++ b/scripts/bi-p010-benchmark.mjs @@ -0,0 +1,311 @@ +#!/usr/bin/env node +/** + * BI-P010 DoD: "Benchmark representative VehIQ-scale fixture and generic SaaS + * fixture." + * + * Builds both fixtures in a THROWAWAY database on a ClickHouse the operator + * points it at, times the query shapes each schema exists to serve, and writes + * a machine-readable transcript to + * evidence/bi-p002-p012/bi-p010-benchmark.json. + * + * CLICKHOUSE_URL=http://localhost:18123 \ + * CLICKHOUSE_AUTH=user:pass \ + * node scripts/bi-p010-benchmark.mjs + * + * It also measures the COUNTERFACTUAL the program warns about — "one giant + * unbounded table for every analytics use case" — by running the same + * questions against a single wide table with a non-tenant-leading sort key. + * Without that comparison the split is an assertion; with it, it is a number. + * + * NEVER point this at the b0ts ClickHouse or any quarantined market data: it + * CREATEs and DROPs a database. + */ +import fs from "node:fs"; +import path from "node:path"; + +const URL_ = process.env.CLICKHOUSE_URL; +const AUTH = process.env.CLICKHOUSE_AUTH; +if (!URL_) { + console.error("CLICKHOUSE_URL is required. Refusing to invent numbers."); + process.exit(2); +} + +const DB = `opt_bench_${Date.now().toString(36)}`; +const headers = AUTH ? { Authorization: `Basic ${Buffer.from(AUTH).toString("base64")}` } : {}; + +async function ch(sql, body, settings = {}) { + const u = new URL(URL_); + u.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) u.searchParams.set(k, v); + const res = await fetch(u.toString(), { method: "POST", headers, body: body ?? "" }); + const text = await res.text(); + if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`); + return text; +} + +async function json(sql, settings) { + const t = await ch(sql, undefined, settings); + return t ? JSON.parse(t) : { data: [] }; +} + +/** Run `sql` n times, return {p50, p95, min, max, rows_read, bytes_read}. */ +async function timeQuery(label, sql, n = 12) { + // One warm-up so we time the query, not the page cache miss. + await ch(sql); + const times = []; + let stats = {}; + for (let i = 0; i < n; i++) { + const t0 = process.hrtime.bigint(); + const out = await json(sql); + times.push(Number(process.hrtime.bigint() - t0) / 1e6); + stats = out.statistics ?? stats; + } + times.sort((a, b) => a - b); + const pick = (p) => Number(times[Math.min(times.length - 1, Math.floor((times.length * p) / 100))].toFixed(2)); + return { + label, + runs: n, + p50_ms: pick(50), + p95_ms: pick(95), + min_ms: Number(times[0].toFixed(2)), + max_ms: Number(times[times.length - 1].toFixed(2)), + rows_read: stats.rows_read ?? null, + bytes_read: stats.bytes_read ?? null, + }; +} + +const evidence = { + generated_at: new Date().toISOString(), + clickhouse_version: null, + database: DB, + fixtures: {}, + measurements: [], + notes: [], +}; + +async function main() { + evidence.clickhouse_version = (await ch("SELECT version() FORMAT TSVRaw")).trim(); + await ch(`CREATE DATABASE ${DB}`); + + // ── the BI-P010 schema under test ────────────────────────────────────── + await ch(`CREATE TABLE ${DB}.opt_analytics_event ( + tenant_id LowCardinality(String), project_id LowCardinality(String), + dataset_id LowCardinality(String), metric LowCardinality(String), + event_id String, event_time DateTime64(3,'UTC'), value Float64, + dim_names Array(LowCardinality(String)), dim_values Array(String), + ingested_at DateTime64(3,'UTC') DEFAULT now64(3), + INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4) + ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) + ORDER BY (tenant_id, project_id, dataset_id, metric, event_time)`); + + await ch(`CREATE TABLE ${DB}.opt_metric_daily ( + tenant_id LowCardinality(String), project_id LowCardinality(String), + dataset_id LowCardinality(String), metric LowCardinality(String), day Date, + sum_state AggregateFunction(sum, Float64), count_state AggregateFunction(count, Float64), + min_state AggregateFunction(min, Float64), max_state AggregateFunction(max, Float64)) + ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMM(day) + ORDER BY (tenant_id, project_id, dataset_id, metric, day)`); + + await ch(`CREATE MATERIALIZED VIEW ${DB}.opt_metric_daily_mv TO ${DB}.opt_metric_daily AS + SELECT tenant_id, project_id, dataset_id, metric, toDate(event_time) AS day, + sumState(value) AS sum_state, countState(value) AS count_state, + minState(value) AS min_state, maxState(value) AS max_state + FROM ${DB}.opt_analytics_event + GROUP BY tenant_id, project_id, dataset_id, metric, day`); + + await ch(`CREATE TABLE ${DB}.opt_dataset_projection ( + tenant_id LowCardinality(String), project_id LowCardinality(String), + dataset_id LowCardinality(String), entity_key String, + as_of DateTime64(3,'UTC'), version UInt64, + attributes Map(String,String), measures Map(String,Float64), + INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4) + ENGINE = ReplacingMergeTree(version) PARTITION BY (tenant_id) + ORDER BY (tenant_id, project_id, dataset_id, entity_key)`); + + // ── the COUNTERFACTUAL the program warns about ───────────────────────── + // One table for everything, sorted by time (the shape people reach for when + // "we'll just put it all in events"). + await ch(`CREATE TABLE ${DB}.giant_everything ( + tenant_id String, project_id String, dataset_id String, metric String, + entity_key String, event_time DateTime64(3,'UTC'), value Float64, + dim_names Array(String), dim_values Array(String), + attributes Map(String,String), measures Map(String,Float64)) + ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) + ORDER BY (event_time, tenant_id)`); + + // ── FIXTURE 1: generic SaaS event stream ─────────────────────────────── + // 200 tenants x 12 metrics x 90 days x 25 events/day ~= 5.4M events. + const SAAS_TENANTS = 200, SAAS_METRICS = 12, SAAS_DAYS = 90, SAAS_PER_DAY = 25; + const saasRows = SAAS_TENANTS * SAAS_METRICS * SAAS_DAYS * SAAS_PER_DAY; + console.log(`seeding SaaS fixture: ${saasRows.toLocaleString()} events...`); + const t0saas = Date.now(); + await ch(`INSERT INTO ${DB}.opt_analytics_event + SELECT concat('t_', toString(number % ${SAAS_TENANTS})) AS tenant_id, + 'p_main' AS project_id, + concat('ds_', toString(number % 8)) AS dataset_id, + concat('metric_', toString(number % ${SAAS_METRICS})) AS metric, + toString(number) AS event_id, + toDateTime64('2026-06-01 00:00:00', 3, 'UTC') + toIntervalSecond(number % ${SAAS_DAYS * 86400}) AS event_time, + toFloat64(number % 997) AS value, + ['plan','region'] AS dim_names, + [if(number % 3 = 0, 'pro', 'free'), if(number % 5 = 0, 'eu-north', 'eu-central')] AS dim_values, + now64(3) AS ingested_at + FROM numbers(${saasRows})`); + const saasSeedMs = Date.now() - t0saas; + + // ── FIXTURE 2: VehIQ-scale vehicle projections ───────────────────────── + // 2M vehicles for the tenant under test, plus 2M belonging to other + // tenants — so a tenant-scoped lookup has to prove it prunes. + const VEHICLES = 2_000_000, DECOYS = 2_000_000; + console.log(`seeding VehIQ fixture: ${(VEHICLES + DECOYS).toLocaleString()} vehicle projections...`); + const t0veh = Date.now(); + await ch(`INSERT INTO ${DB}.opt_dataset_projection + SELECT 't_vehiq' AS tenant_id, 'p_main' AS project_id, 'ds_vehicles' AS dataset_id, + concat('YV1', leftPad(toString(number), 14, '0')) AS entity_key, + toDateTime64('2026-09-01 00:00:00', 3, 'UTC') AS as_of, + 1 AS version, + map('make', arrayElement(['volvo','saab','audi','vw','toyota'], (number % 5) + 1), + 'fuel', arrayElement(['diesel','petrol','ev','hybrid'], (number % 4) + 1)) AS attributes, + map('asking_price', toFloat64(50000 + number % 900000), + 'mileage_km', toFloat64(number % 400000)) AS measures + FROM numbers(${VEHICLES})`); + await ch(`INSERT INTO ${DB}.opt_dataset_projection + SELECT concat('t_other_', toString(number % 50)) AS tenant_id, 'p_main', 'ds_vehicles', + concat('WBA', leftPad(toString(number), 14, '0')), + toDateTime64('2026-09-01 00:00:00', 3, 'UTC'), 1, + map('make','bmw'), map('asking_price', toFloat64(number % 500000)) + FROM numbers(${DECOYS})`); + const vehSeedMs = Date.now() - t0veh; + + // ── the same data in the giant table ─────────────────────────────────── + console.log("seeding the counterfactual giant table..."); + await ch(`INSERT INTO ${DB}.giant_everything + SELECT tenant_id, project_id, dataset_id, metric, '' AS entity_key, event_time, value, + dim_names, dim_values, map() AS attributes, map() AS measures + FROM ${DB}.opt_analytics_event`); + await ch(`INSERT INTO ${DB}.giant_everything + SELECT tenant_id, project_id, dataset_id, '' AS metric, entity_key, + as_of AS event_time, 0 AS value, [] AS dim_names, [] AS dim_values, attributes, measures + FROM ${DB}.opt_dataset_projection`); + + await ch(`OPTIMIZE TABLE ${DB}.opt_analytics_event FINAL`, undefined, { max_execution_time: "600" }); + await ch(`OPTIMIZE TABLE ${DB}.opt_dataset_projection FINAL`, undefined, { max_execution_time: "600" }); + await ch(`OPTIMIZE TABLE ${DB}.giant_everything FINAL`, undefined, { max_execution_time: "900" }); + + const sizes = await json(`SELECT table, sum(rows) AS rows, sum(bytes_on_disk) AS bytes + FROM system.parts WHERE database='${DB}' AND active GROUP BY table ORDER BY table FORMAT JSON`); + evidence.fixtures = { + saas: { events: saasRows, tenants: SAAS_TENANTS, metrics: SAAS_METRICS, days: SAAS_DAYS, seed_ms: saasSeedMs }, + vehiq: { vehicles_under_test: VEHICLES, decoy_vehicles: DECOYS, seed_ms: vehSeedMs }, + on_disk: sizes.data.map((r) => ({ table: r.table, rows: Number(r.rows), bytes: Number(r.bytes) })), + }; + + // ── measurements ─────────────────────────────────────────────────────── + const T = "t_57"; // one representative SaaS tenant + + evidence.measurements.push( + await timeQuery( + "SaaS dashboard panel: 90-day metric total for ONE tenant (purpose-built event table)", + `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event + WHERE tenant_id = '${T}' AND metric = 'metric_3' FORMAT JSON`, + ), + await timeQuery( + "SAME question against ONE GIANT TABLE (time-leading sort key)", + `SELECT sum(value) AS value FROM ${DB}.giant_everything + WHERE tenant_id = '${T}' AND metric = 'metric_3' FORMAT JSON`, + ), + await timeQuery( + "SaaS time series: daily series for ONE tenant from the AggregatingMergeTree rollup", + `SELECT day, sumMerge(sum_state) AS value FROM ${DB}.opt_metric_daily + WHERE tenant_id = '${T}' AND metric = 'metric_3' GROUP BY day ORDER BY day FORMAT JSON`, + ), + await timeQuery( + "SAME time series computed from raw events instead of the rollup", + `SELECT toDate(event_time) AS day, sum(value) AS value FROM ${DB}.opt_analytics_event + WHERE tenant_id = '${T}' AND metric = 'metric_3' GROUP BY day ORDER BY day FORMAT JSON`, + ), + await timeQuery( + "SaaS group-by: one tenant sliced by a dimension", + `SELECT dim_values[indexOf(dim_names, 'plan')] AS grp, count() AS value + FROM ${DB}.opt_analytics_event WHERE tenant_id = '${T}' AND metric = 'metric_3' + GROUP BY grp ORDER BY value DESC FORMAT JSON`, + ), + await timeQuery( + "VehIQ point lookup: one vehicle by key, tenant-scoped (purpose-built projection)", + `SELECT entity_key, measures['asking_price'] AS p FROM ${DB}.opt_dataset_projection + WHERE tenant_id = 't_vehiq' AND project_id = 'p_main' AND dataset_id = 'ds_vehicles' + AND entity_key = 'YV100000001234567' FORMAT JSON`, + ), + await timeQuery( + "SAME point lookup against ONE GIANT TABLE", + `SELECT entity_key, measures['asking_price'] AS p FROM ${DB}.giant_everything + WHERE tenant_id = 't_vehiq' AND dataset_id = 'ds_vehicles' + AND entity_key = 'YV100000001234567' FORMAT JSON`, + ), + await timeQuery( + "VehIQ aggregate: average asking price by make for ONE tenant across 2M vehicles", + `SELECT attributes['make'] AS grp, avg(measures['asking_price']) AS value + FROM ${DB}.opt_dataset_projection + WHERE tenant_id = 't_vehiq' AND project_id = 'p_main' AND dataset_id = 'ds_vehicles' + GROUP BY grp ORDER BY value DESC FORMAT JSON`, + 8, + ), + await timeQuery( + "SAME aggregate against ONE GIANT TABLE", + `SELECT attributes['make'] AS grp, avg(measures['asking_price']) AS value + FROM ${DB}.giant_everything + WHERE tenant_id = 't_vehiq' AND dataset_id = 'ds_vehicles' AND entity_key != '' + GROUP BY grp ORDER BY value DESC FORMAT JSON`, + 8, + ), + ); + + // Index pruning evidence: how many granules the tenant predicate skips. + const plan = await ch( + `EXPLAIN indexes = 1 SELECT sum(value) FROM ${DB}.opt_analytics_event + WHERE tenant_id = '${T}' AND metric = 'metric_3' FORMAT TSVRaw`, + ); + const giantPlan = await ch( + `EXPLAIN indexes = 1 SELECT sum(value) FROM ${DB}.giant_everything + WHERE tenant_id = '${T}' AND metric = 'metric_3' FORMAT TSVRaw`, + ); + const granules = (t) => { + const m = [...t.matchAll(/Granules: (\d+)\/(\d+)/g)].pop(); + return m ? { read: Number(m[1]), total: Number(m[2]) } : null; + }; + evidence.index_pruning = { + purpose_built_event_table: granules(plan), + one_giant_table: granules(giantPlan), + note: + "Granules read/total for the SAME question. The tenant-leading ORDER BY is what turns the " + + "forced tenant predicate (BI-P011) into a primary-key range scan instead of a full scan.", + }; + + evidence.notes.push( + "Fixtures are synthetic and generated in-server with numbers(); no production or b0ts data was read or written.", + "Each timing is the p50/p95 of repeated runs after one warm-up, measured client-side over HTTP on the same host.", + "The 'ONE GIANT TABLE' rows are the counterfactual BI-P010 warns against, measured rather than asserted.", + ); + + const out = path.join(process.cwd(), "evidence", "bi-p002-p012", "bi-p010-benchmark.json"); + fs.writeFileSync(out, JSON.stringify(evidence, null, 2) + "\n"); + console.log(`wrote ${out}`); + for (const m of evidence.measurements) { + console.log(` ${m.p50_ms.toString().padStart(9)} ms p50 ${String(m.rows_read).padStart(10)} rows ${m.label}`); + } + console.log(" pruning:", JSON.stringify(evidence.index_pruning)); +} + +main() + .catch((e) => { + console.error(e); + process.exitCode = 1; + }) + .finally(async () => { + try { + await ch(`DROP DATABASE IF EXISTS ${DB}`); + console.log(`dropped ${DB}`); + } catch (e) { + console.error("cleanup failed:", e.message); + } + }); diff --git a/sql/008_analytics_domain_model.sql b/sql/008_analytics_domain_model.sql new file mode 100644 index 0000000..4e18a9f --- /dev/null +++ b/sql/008_analytics_domain_model.sql @@ -0,0 +1,436 @@ +-- 0pt analytics domain model (BI-P002/BI-P003/BI-P004). +-- +-- The nineteen canonical entities from BI-P002, one table each, named after +-- the entity (src/analytics/domain/entities.ts's ANALYTICS_ENTITY_TABLES maps +-- entity -> table, and tests/analytics/analytics-domain-postgres.realinfra.test.ts +-- asserts every entry exists here -- a rename or omission is a test failure, +-- not silent drift). +-- +-- Every tenant-owned table carries `tenant_id` + `project_id`, and every one +-- of them is indexed leading with (tenant_id, project_id) so a tenant-scoped +-- read is an index range scan rather than a filter over everyone's rows. +-- +-- IMMUTABILITY (BI-P002: "Use immutable versions for MetricVersion, +-- SemanticModel version, DashboardVersion, DatasetVersion") is enforced by +-- BEFORE UPDATE/DELETE triggers, not by convention and not by a TypeScript +-- `readonly`. A compiler opinion does not survive a psql session; a trigger +-- does. `opt_analytics_forbid_mutation()` raises, so an UPDATE against any of +-- the four version tables fails loudly. +-- +-- Idempotent throughout (CREATE ... IF NOT EXISTS, CREATE OR REPLACE +-- FUNCTION, DROP TRIGGER IF EXISTS before CREATE TRIGGER) so it can be +-- replayed onto an already-migrated database -- the same contract +-- sql/002_source_registry.sql and sql/007_ingestion_plane.sql keep, and what +-- the restore drill (`psql -d opt < sql/001... < ... < sql/008...`) needs. +-- +-- Ordering: standalone. It references no earlier table, so 001-007 are not a +-- prerequisite; it is numbered 008 to keep the replay order linear. + +-- ── immutability enforcement ────────────────────────────────────────────── +CREATE OR REPLACE FUNCTION opt_analytics_forbid_mutation() RETURNS trigger AS $$ +BEGIN + RAISE EXCEPTION + 'BI-P002: % rows are immutable (attempted %); mint a new version instead', + TG_TABLE_NAME, TG_OP + USING ERRCODE = 'restrict_violation'; +END; +$$ LANGUAGE plpgsql; + +-- ── 1. AnalyticsDataset ─────────────────────────────────────────────────── +-- BI-P003 DoD: tier/schema/owner/source/freshness/quality/retention/ +-- classification are all NOT NULL. An optional declaration is not one. +CREATE TABLE IF NOT EXISTS opt_analytics_dataset ( + dataset_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + tier TEXT NOT NULL CHECK (tier IN ('RAW','NORMALIZED','CURATED','SERVING')), + schema_contract_id TEXT NOT NULL, + schema_contract_version TEXT NOT NULL, + owner_id TEXT NOT NULL, + source_ref TEXT NOT NULL, + freshness_slo_seconds INTEGER NOT NULL CHECK (freshness_slo_seconds > 0), + quality_rule_ids TEXT[] NOT NULL DEFAULT '{}', + retention_days INTEGER NOT NULL CHECK (retention_days >= 0), + classification TEXT NOT NULL CHECK (classification IN ('none','pseudonymous','personal','sensitive')), + status TEXT NOT NULL DEFAULT 'draft' CHECK (status IN ('draft','active','deprecated','retired')), + raw_access_exception TEXT, + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + created_by TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_dataset_tenant_idx + ON opt_analytics_dataset (tenant_id, project_id, status); + +-- ── 2. DatasetVersion (IMMUTABLE) ───────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_dataset_version ( + dataset_version_id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1), + contract_hash TEXT NOT NULL, + schema_contract_version TEXT NOT NULL, + record_count BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (dataset_id, version) +); +CREATE INDEX IF NOT EXISTS opt_analytics_dataset_version_tenant_idx + ON opt_analytics_dataset_version (tenant_id, project_id, dataset_id, version); +DROP TRIGGER IF EXISTS opt_analytics_dataset_version_immutable ON opt_analytics_dataset_version; +CREATE TRIGGER opt_analytics_dataset_version_immutable + BEFORE UPDATE OR DELETE ON opt_analytics_dataset_version + FOR EACH ROW EXECUTE FUNCTION opt_analytics_forbid_mutation(); + +-- ── 3. DatasetField ─────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_dataset_field ( + dataset_field_id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + dataset_version INTEGER NOT NULL, + name TEXT NOT NULL, + field_type TEXT NOT NULL CHECK (field_type IN ('string','int64','float64','bool','date','timestamp','decimal','json')), + nullable BOOLEAN NOT NULL, + role TEXT NOT NULL CHECK (role IN ('identifier','dimension','measure','attribute')), + primary_identifier BOOLEAN NOT NULL DEFAULT false, + classification TEXT NOT NULL DEFAULT 'none' CHECK (classification IN ('none','pseudonymous','personal','sensitive')), + description TEXT, + UNIQUE (dataset_id, dataset_version, name) +); +CREATE INDEX IF NOT EXISTS opt_analytics_dataset_field_tenant_idx + ON opt_analytics_dataset_field (tenant_id, project_id, dataset_id); + +-- ── 4. DatasetGrant ─────────────────────────────────────────────────────── +-- allowed_purposes is NOT NULL and CHECKed non-empty: a grant without a +-- purpose is a GDPR purpose-limitation hole, so the database refuses it. +CREATE TABLE IF NOT EXISTS opt_analytics_dataset_grant ( + grant_id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + subject_id TEXT NOT NULL, + subject_type TEXT NOT NULL CHECK (subject_type IN ('user','role','service','bi_application')), + level TEXT NOT NULL CHECK (level IN ('read','query','export','admin')), + -- coalesce(...) is load-bearing: array_length('{}', 1) is NULL in Postgres, + -- and a CHECK that evaluates to NULL PASSES. Written the obvious way, this + -- constraint could never fail -- caught by + -- tests/analytics/analytics-domain-postgres.realinfra.test.ts inserting an + -- empty array and watching it succeed. + allowed_purposes TEXT[] NOT NULL CONSTRAINT opt_analytics_dataset_grant_purposes_nonempty + CHECK (coalesce(array_length(allowed_purposes, 1), 0) >= 1), + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_dataset_grant_tenant_idx + ON opt_analytics_dataset_grant (tenant_id, project_id, dataset_id, subject_id); + +-- ── 5. MetricDefinition ─────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_metric_definition ( + metric_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + aggregation TEXT NOT NULL CHECK (aggregation IN ('sum','avg','min','max','count','count_distinct','ratio','derived')), + measure_field TEXT NOT NULL, + denominator_field TEXT, + allowed_dimension_ids TEXT[] NOT NULL DEFAULT '{}', + current_version INTEGER NOT NULL DEFAULT 1, + owner_id TEXT NOT NULL, + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + UNIQUE (tenant_id, name) +); +CREATE INDEX IF NOT EXISTS opt_analytics_metric_definition_tenant_idx + ON opt_analytics_metric_definition (tenant_id, project_id, dataset_id); + +-- ── 6. MetricVersion (IMMUTABLE) ────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_metric_version ( + metric_version_id TEXT PRIMARY KEY, + metric_id TEXT NOT NULL REFERENCES opt_analytics_metric_definition (metric_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1), + definition_json TEXT NOT NULL, + definition_hash TEXT NOT NULL, + change_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (metric_id, version), + -- BI-P031 governance in the schema: every version after the first must say + -- WHY the number changed meaning. + CHECK (version = 1 OR (change_reason IS NOT NULL AND length(btrim(change_reason)) > 0)) +); +CREATE INDEX IF NOT EXISTS opt_analytics_metric_version_tenant_idx + ON opt_analytics_metric_version (tenant_id, project_id, metric_id, version); +DROP TRIGGER IF EXISTS opt_analytics_metric_version_immutable ON opt_analytics_metric_version; +CREATE TRIGGER opt_analytics_metric_version_immutable + BEFORE UPDATE OR DELETE ON opt_analytics_metric_version + FOR EACH ROW EXECUTE FUNCTION opt_analytics_forbid_mutation(); + +-- ── 7. DimensionDefinition ──────────────────────────────────────────────── +-- The allow-list the query compiler resolves caller-supplied dimension names +-- through (BI-P011 "dimension identifier injection"). `field_name` is CHECKed +-- against a strict identifier pattern in the DATABASE as well as in the +-- compiler, because a hostile row here would otherwise be the injection. +CREATE TABLE IF NOT EXISTS opt_analytics_dimension_definition ( + dimension_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + field_name TEXT NOT NULL CHECK (field_name ~ '^[A-Za-z_][A-Za-z0-9_]{0,62}$'), + field_type TEXT NOT NULL CHECK (field_type IN ('string','int64','float64','bool','date','timestamp','decimal','json')), + classification TEXT NOT NULL DEFAULT 'none' CHECK (classification IN ('none','pseudonymous','personal','sensitive')), + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (tenant_id, dataset_id, name) +); +CREATE INDEX IF NOT EXISTS opt_analytics_dimension_definition_tenant_idx + ON opt_analytics_dimension_definition (tenant_id, project_id, dataset_id); + +-- ── 8. SemanticModel (versions IMMUTABLE) ───────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_semantic_model ( + semantic_model_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1), + model_json TEXT NOT NULL, + model_hash TEXT NOT NULL, + dataset_ids TEXT[] NOT NULL DEFAULT '{}', + owner_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (tenant_id, name, version) +); +CREATE INDEX IF NOT EXISTS opt_analytics_semantic_model_tenant_idx + ON opt_analytics_semantic_model (tenant_id, project_id, name, version); +DROP TRIGGER IF EXISTS opt_analytics_semantic_model_immutable ON opt_analytics_semantic_model; +CREATE TRIGGER opt_analytics_semantic_model_immutable + BEFORE UPDATE OR DELETE ON opt_analytics_semantic_model + FOR EACH ROW EXECUTE FUNCTION opt_analytics_forbid_mutation(); + +-- ── 9. AnalyticsQuery ───────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_query ( + query_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + surface TEXT NOT NULL CHECK (surface IN ('metric','sql','projection')), + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + metric_id TEXT REFERENCES opt_analytics_metric_definition (metric_id), + request_json TEXT NOT NULL, + owner_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS opt_analytics_query_tenant_idx + ON opt_analytics_query (tenant_id, project_id, dataset_id); + +-- ── 10. QueryExecution ──────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_query_execution ( + execution_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + query_id TEXT REFERENCES opt_analytics_query (query_id), + engine_query_id TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled','rejected')), + provider TEXT NOT NULL, + submitted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + duration_ms INTEGER, + rows_read BIGINT, + bytes_read BIGINT, + result_rows BIGINT, + error_code TEXT, + correlation_id TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_query_execution_tenant_idx + ON opt_analytics_query_execution (tenant_id, project_id, submitted_at DESC); +CREATE INDEX IF NOT EXISTS opt_analytics_query_execution_engine_idx + ON opt_analytics_query_execution (engine_query_id); + +-- ── 11. QueryResult ─────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_query_result ( + result_id TEXT PRIMARY KEY, + execution_id TEXT NOT NULL REFERENCES opt_analytics_query_execution (execution_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + encoding TEXT NOT NULL CHECK (encoding IN ('json','arrow_ipc')), + row_count BIGINT NOT NULL, + artifact_ref TEXT, + truncated BOOLEAN NOT NULL DEFAULT false, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS opt_analytics_query_result_tenant_idx + ON opt_analytics_query_result (tenant_id, project_id, execution_id); + +-- ── 12. DashboardDefinition ─────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_dashboard_definition ( + dashboard_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + current_version INTEGER NOT NULL DEFAULT 1, + owner_id TEXT NOT NULL, + dataset_ids TEXT[] NOT NULL DEFAULT '{}', + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS opt_analytics_dashboard_definition_tenant_idx + ON opt_analytics_dashboard_definition (tenant_id, project_id); + +-- ── 13. DashboardVersion (IMMUTABLE) ────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_dashboard_version ( + dashboard_version_id TEXT PRIMARY KEY, + dashboard_id TEXT NOT NULL REFERENCES opt_analytics_dashboard_definition (dashboard_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1), + layout_json TEXT NOT NULL, + layout_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (dashboard_id, version) +); +CREATE INDEX IF NOT EXISTS opt_analytics_dashboard_version_tenant_idx + ON opt_analytics_dashboard_version (tenant_id, project_id, dashboard_id, version); +DROP TRIGGER IF EXISTS opt_analytics_dashboard_version_immutable ON opt_analytics_dashboard_version; +CREATE TRIGGER opt_analytics_dashboard_version_immutable + BEFORE UPDATE OR DELETE ON opt_analytics_dashboard_version + FOR EACH ROW EXECUTE FUNCTION opt_analytics_forbid_mutation(); + +-- ── 14. ReportDefinition ────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_report_definition ( + report_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + name TEXT NOT NULL, + dashboard_id TEXT REFERENCES opt_analytics_dashboard_definition (dashboard_id), + query_id TEXT REFERENCES opt_analytics_query (query_id), + format TEXT NOT NULL CHECK (format IN ('pdf','csv','xlsx','parquet','arrow_ipc')), + schedule_cron TEXT, + owner_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ, + CHECK (dashboard_id IS NOT NULL OR query_id IS NOT NULL) +); +CREATE INDEX IF NOT EXISTS opt_analytics_report_definition_tenant_idx + ON opt_analytics_report_definition (tenant_id, project_id); + +-- ── 15. ReportRun ───────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_report_run ( + report_run_id TEXT PRIMARY KEY, + report_id TEXT NOT NULL REFERENCES opt_analytics_report_definition (report_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed','cancelled','rejected')), + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + artifact_ref TEXT, + error_code TEXT, + correlation_id TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_report_run_tenant_idx + ON opt_analytics_report_run (tenant_id, project_id, report_id, started_at DESC); + +-- ── 16. DataQualityRule ─────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_data_quality_rule ( + rule_id TEXT PRIMARY KEY, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('not_null','unique','accepted_values','range','freshness','row_count','referential','custom')), + field_name TEXT, + params_json TEXT NOT NULL DEFAULT '{}', + severity TEXT NOT NULL CHECK (severity IN ('warn','block')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_data_quality_rule_tenant_idx + ON opt_analytics_data_quality_rule (tenant_id, project_id, dataset_id); + +-- ── 17. DataQualityRun ──────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_data_quality_run ( + quality_run_id TEXT PRIMARY KEY, + rule_id TEXT NOT NULL REFERENCES opt_analytics_data_quality_rule (rule_id), + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + dataset_version INTEGER, + passed BOOLEAN NOT NULL, + violation_count BIGINT NOT NULL DEFAULT 0, + evaluated_rows BIGINT NOT NULL DEFAULT 0, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + correlation_id TEXT +); +CREATE INDEX IF NOT EXISTS opt_analytics_data_quality_run_tenant_idx + ON opt_analytics_data_quality_run (tenant_id, project_id, dataset_id, started_at DESC); + +-- ── 18. LineageReference ────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS opt_analytics_lineage_reference ( + lineage_reference_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + run_id TEXT NOT NULL, + job_name TEXT NOT NULL, + direction TEXT NOT NULL CHECK (direction IN ('input','output')), + entity_type TEXT NOT NULL CHECK (entity_type IN ('dataset','metric','dashboard','report')), + entity_id TEXT NOT NULL, + counterpart_ref TEXT NOT NULL, + observed_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS opt_analytics_lineage_reference_tenant_idx + ON opt_analytics_lineage_reference (tenant_id, project_id, entity_type, entity_id); + +-- ── 19. BIApplicationBinding ────────────────────────────────────────────── +-- NORMATIVE: there is NO credential column here, and there must never be one. +-- SIAX/0pt owns the analytics engine; a bound BI application authenticates to +-- the 0pt Analytics API with a CL0UD identity and receives compiled, +-- tenant-forced results. It never learns a ClickHouse DSN, user, or password. +CREATE TABLE IF NOT EXISTS opt_analytics_bi_application_binding ( + binding_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('superset','evidence','des1gn','notebook','custom')), + name TEXT NOT NULL, + principal_id TEXT NOT NULL, + dataset_ids TEXT[] NOT NULL DEFAULT '{}', + metric_ids TEXT[] NOT NULL DEFAULT '{}', + -- Same NULL-CHECK trap as opt_analytics_dataset_grant; see the note there. + allowed_purposes TEXT[] NOT NULL CONSTRAINT opt_analytics_bi_app_binding_purposes_nonempty + CHECK (coalesce(array_length(allowed_purposes, 1), 0) >= 1), + enabled BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (tenant_id, name) +); +CREATE INDEX IF NOT EXISTS opt_analytics_bi_application_binding_tenant_idx + ON opt_analytics_bi_application_binding (tenant_id, project_id, principal_id); + +-- ── constraint upgrade path ─────────────────────────────────────────────── +-- CREATE TABLE IF NOT EXISTS does nothing to a table that already exists, so +-- a database that received an earlier copy of this file would keep the old +-- (NULL-passing) purpose checks. These statements bring it forward and are +-- no-ops on a fresh database. +ALTER TABLE opt_analytics_dataset_grant + DROP CONSTRAINT IF EXISTS opt_analytics_dataset_grant_purposes_nonempty; +ALTER TABLE opt_analytics_dataset_grant + ADD CONSTRAINT opt_analytics_dataset_grant_purposes_nonempty + CHECK (coalesce(array_length(allowed_purposes, 1), 0) >= 1); + +ALTER TABLE opt_analytics_bi_application_binding + DROP CONSTRAINT IF EXISTS opt_analytics_bi_app_binding_purposes_nonempty; +ALTER TABLE opt_analytics_bi_application_binding + ADD CONSTRAINT opt_analytics_bi_app_binding_purposes_nonempty + CHECK (coalesce(array_length(allowed_purposes, 1), 0) >= 1); diff --git a/src/analytics/arrow/flatbuffer-builder.ts b/src/analytics/arrow/flatbuffer-builder.ts new file mode 100644 index 0000000..3185803 --- /dev/null +++ b/src/analytics/arrow/flatbuffer-builder.ts @@ -0,0 +1,237 @@ +/** + * src/analytics/arrow/flatbuffer-builder.ts + * ========================================== + * A minimal FlatBuffers writer, sufficient for the Arrow IPC metadata + * messages this repo emits (BI-P005). + * + * WHY HAND-ROLLED rather than `apache-arrow` from npm: adding a runtime + * dependency to 0pt's blocking-typechecked capability layer means a lockfile + * change, a new supply-chain surface, and a new CVE surface for every + * consumer of this repo — for a format whose METADATA side is a few hundred + * bytes of well-specified binary and whose DATA side is plain columnar memory + * either way. + * + * The correctness bar is NOT "this looks right". The BI-P005 evidence script + * (scripts/bi-p005-arrow-roundtrip.mjs) decodes this writer's output with + * pyarrow AND DuckDB — two independent implementations of the specification — + * so a mistake here surfaces as a failed decode by somebody else's parser, + * not as a green test of my own assumptions. + * + * Layout facts implemented (FlatBuffers binary format, little-endian): + * * the buffer is built BACK TO FRONT; `offset` counts bytes from the end; + * * a uoffset stored at a field site holds `site_offset + 4 - target_offset` + * so a reader resolves it as `position + value`; + * * a table begins with an soffset `table_start - vtable_start`; + * * a vtable is `[vtable_len:i16][table_len:i16][field_offsets:i16...]`, + * with 0 meaning "field absent, use the default". + */ + +export type Offset = number; + +const SIZEOF_INT = 4; + +export class FlatBufferBuilder { + private buf: Uint8Array; + private view: DataView; + /** Bytes written so far, counted from the END of `buf`. */ + private used = 0; + private minAlign = 1; + private vtable: number[] = []; + private objectStart = 0; + private inObject = false; + /** Offsets of previously written vtables, for deduplication. */ + private vtables: number[] = []; + + constructor(initial = 1024) { + this.buf = new Uint8Array(initial); + this.view = new DataView(this.buf.buffer); + } + + /** Bytes written so far (FlatBuffers "offset"). */ + get offset(): number { + return this.used; + } + + private ensure(extra: number): void { + if (this.used + extra <= this.buf.length) return; + let size = this.buf.length; + while (size < this.used + extra) size *= 2; + const next = new Uint8Array(size); + next.set(this.buf.subarray(this.buf.length - this.used), size - this.used); + this.buf = next; + this.view = new DataView(this.buf.buffer); + } + + private pad(n: number): void { + if (n <= 0) return; + this.ensure(n); + for (let i = 0; i < n; i++) { + this.used += 1; + this.buf[this.buf.length - this.used] = 0; + } + } + + /** FlatBuffers `prep`: pad so that `additional` more bytes land aligned. */ + private prep(size: number, additional = 0): void { + if (size > this.minAlign) this.minAlign = size; + const alignSize = (~(this.used + additional) + 1) & (size - 1); + this.pad(alignSize); + } + + writeInt8(v: number): void { + this.ensure(1); + this.used += 1; + this.view.setInt8(this.buf.length - this.used, v); + } + + writeInt16(v: number): void { + this.ensure(2); + this.used += 2; + this.view.setInt16(this.buf.length - this.used, v, true); + } + + writeInt32(v: number): void { + this.ensure(4); + this.used += 4; + this.view.setInt32(this.buf.length - this.used, v, true); + } + + writeInt64(v: bigint): void { + this.ensure(8); + this.used += 8; + this.view.setBigInt64(this.buf.length - this.used, v, true); + } + + writeBytes(bytes: Uint8Array): void { + this.ensure(bytes.length); + this.used += bytes.length; + this.buf.set(bytes, this.buf.length - this.used); + } + + addInt8(slot: number, value: number, def: number): void { + if (value === def) return; + this.prep(1); + this.writeInt8(value); + this.slot(slot); + } + + addInt16(slot: number, value: number, def: number): void { + if (value === def) return; + this.prep(2); + this.writeInt16(value); + this.slot(slot); + } + + addInt32(slot: number, value: number, def: number): void { + if (value === def) return; + this.prep(4); + this.writeInt32(value); + this.slot(slot); + } + + addInt64(slot: number, value: bigint, def: bigint): void { + if (value === def) return; + this.prep(8); + this.writeInt64(value); + this.slot(slot); + } + + addOffset(slot: number, value: Offset, def: number): void { + if (value === def) return; + this.prep(SIZEOF_INT); + this.writeInt32(this.used + SIZEOF_INT - value); + this.slot(slot); + } + + private slot(slotNum: number): void { + this.vtable[slotNum] = this.used; + } + + startObject(numFields: number): void { + if (this.inObject) throw new Error("nested startObject"); + this.vtable = new Array(numFields).fill(0); + this.objectStart = this.used; + this.inObject = true; + } + + endObject(): Offset { + if (!this.inObject) throw new Error("endObject outside an object"); + this.prep(SIZEOF_INT); + this.writeInt32(0); // soffset placeholder, patched below + const objectOffset = this.used; + + let trimmed = this.vtable.length; + while (trimmed > 0 && this.vtable[trimmed - 1] === 0) trimmed -= 1; + for (let i = trimmed - 1; i >= 0; i--) { + this.writeInt16(this.vtable[i] !== 0 ? objectOffset - this.vtable[i] : 0); + } + this.writeInt16(objectOffset - this.objectStart); // table byte length + const vtLen = (trimmed + 2) * 2; + this.writeInt16(vtLen); // vtable byte length — lands at the vtable start + + const vtStart = this.used; + const vtAbs = this.buf.length - vtStart; + let existing = 0; + for (const candidate of this.vtables) { + const candAbs = this.buf.length - candidate; + if (this.view.getInt16(candAbs, true) !== vtLen) continue; + let same = true; + for (let i = 2; i < vtLen; i += 2) { + if (this.view.getInt16(vtAbs + i, true) !== this.view.getInt16(candAbs + i, true)) { + same = false; + break; + } + } + if (same) { + existing = candidate; + break; + } + } + + const objAbs = this.buf.length - objectOffset; + if (existing !== 0) { + this.used -= vtLen; // discard the duplicate vtable just written + this.view.setInt32(objAbs, existing - objectOffset, true); + } else { + this.vtables.push(vtStart); + this.view.setInt32(objAbs, this.used - objectOffset, true); + } + this.inObject = false; + return objectOffset; + } + + createString(s: string): Offset { + const bytes = new TextEncoder().encode(s); + this.writeInt8(0); // null terminator (readers rely on it) + this.startVector(1, bytes.length, 1); + this.writeBytes(bytes); + return this.endVector(bytes.length); + } + + startVector(elemSize: number, numElems: number, alignment: number): void { + this.prep(SIZEOF_INT, elemSize * numElems); + this.prep(alignment, elemSize * numElems); + } + + endVector(numElems: number): Offset { + this.writeInt32(numElems); + return this.used; + } + + /** Vector of table offsets. Elements are written in reverse build order. */ + createOffsetVector(items: Offset[]): Offset { + this.startVector(SIZEOF_INT, items.length, SIZEOF_INT); + for (let i = items.length - 1; i >= 0; i--) { + this.writeInt32(this.used + SIZEOF_INT - items[i]); + } + return this.endVector(items.length); + } + + /** Finish the buffer with `root` as the root table. Returns the bytes. */ + finish(root: Offset): Uint8Array { + this.prep(this.minAlign, SIZEOF_INT); + this.prep(SIZEOF_INT); + this.writeInt32(this.used + SIZEOF_INT - root); + return this.buf.slice(this.buf.length - this.used); + } +} diff --git a/src/analytics/arrow/ipc.ts b/src/analytics/arrow/ipc.ts new file mode 100644 index 0000000..d4a693c --- /dev/null +++ b/src/analytics/arrow/ipc.ts @@ -0,0 +1,547 @@ +/** + * src/analytics/arrow/ipc.ts + * =========================== + * BI-P005 — APACHE ARROW INTERCHANGE. + * + * Canonical columnar result encoding ALONGSIDE JSON (JSON is not removed — + * BI-P005 says "alongside"). Two paths, and knowing which is which matters: + * + * PASS-THROUGH (preferred for ClickHouse results): ClickHouse emits real + * Arrow IPC itself via `FORMAT ArrowStream`. Those bytes were produced by + * the Arrow C++ implementation; 0pt streams them out unchanged, so there is + * no encoder of ours in the path at all. `clickhouseArrowStreamFormat()` + * and `arrowStreamSettings()` are that path. + * + * ENCODED (for results 0pt itself assembles — projections, JSON-sourced + * rows, exports): `encodeArrowStream()` writes the IPC stream here. + * + * Both paths are decodable by `decodeArrowStream()`, which exists so 0pt can + * count rows and read the schema of an Arrow payload WITHOUT shelling out to + * another runtime — needed to enforce BI-P012's `max_result_rows` on the + * pass-through path, where the row count is not otherwise visible. + * + * "Supported types" (the DoD's qualifier) is `ARROW_TYPE_SUPPORT` below: the + * exact ClickHouse -> Arrow -> DuckDB/Python mapping, including where + * semantic loss WOULD occur, stated rather than hidden. + * + * IPC stream layout (Arrow format 1.5, metadata V5): + * for each message: 0xFFFFFFFF, int32 metadata length, FlatBuffer Message + * padded so the body starts 8-byte aligned, then the body buffers. + * Terminated by 0xFFFFFFFF 0x00000000. + */ +import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder"; +import { SiaxError } from "../../platform/errors"; + +// ── FlatBuffer enum values from org.apache.arrow.flatbuf ──────────────────── + +const METADATA_VERSION_V5 = 4; +const MESSAGE_HEADER_SCHEMA = 1; +const MESSAGE_HEADER_RECORD_BATCH = 3; +const ENDIANNESS_LITTLE = 0; + +const TYPE_INT = 2; +const TYPE_FLOATING_POINT = 3; +const TYPE_UTF8 = 5; +const TYPE_BOOL = 6; +const TYPE_TIMESTAMP = 10; + +const PRECISION_DOUBLE = 2; + +// BigInt LITERALS (`0n`) require target >= ES2020; this repo targets ES6, so +// every 64-bit constant is built with BigInt(...) instead. Behaviour is +// identical — this is a compiler-target accommodation, not a semantic choice. +const BIG_ZERO = BigInt(0); +const INT64_MAX = BigInt("9223372036854775807"); +const INT64_MIN = BigInt("-9223372036854775808"); +const TIME_UNIT_MILLISECOND = 1; + +// ── public column model ──────────────────────────────────────────────────── + +export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms"; + +export interface ArrowColumn { + name: string; + type: ArrowColumnType; + /** `null` marks a null slot; the validity bitmap is derived from it. */ + values: (number | bigint | string | boolean | Date | null)[]; +} + +/** + * The type table BI-P005's "supported types" qualifier refers to. Every row + * has been exercised end to end by scripts/bi-p005-arrow-roundtrip.mjs. + */ +export const ARROW_TYPE_SUPPORT: { + clickhouse: string; + arrow: ArrowColumnType; + duckdb: string; + python: string; + note: string; +}[] = [ + { + clickhouse: "Int64 / UInt32 / Int32", + arrow: "int64", + duckdb: "BIGINT", + python: "int", + note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped", + }, + { + clickhouse: "Float64", + arrow: "float64", + duckdb: "DOUBLE", + python: "float", + note: "exact IEEE-754 round trip", + }, + { + clickhouse: "String / LowCardinality(String)", + arrow: "utf8", + duckdb: "VARCHAR", + python: "str", + note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not", + }, + { + clickhouse: "Bool / UInt8(0|1)", + arrow: "bool", + duckdb: "BOOLEAN", + python: "bool", + note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64", + }, + { + clickhouse: "DateTime64(3,'UTC')", + arrow: "timestamp_ms", + duckdb: "TIMESTAMP", + python: "datetime", + note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set", + }, +]; + +// ── ClickHouse pass-through ──────────────────────────────────────────────── + +/** ClickHouse output format producing an Arrow IPC *stream*. */ +export function clickhouseArrowStreamFormat(): string { + return "ArrowStream"; +} + +/** + * Settings for the pass-through path. `output_format_arrow_string_as_string` + * matters: without it ClickHouse emits String columns as Arrow *Binary*, + * which pyarrow and DuckDB both read as bytes rather than text — semantic + * loss that would slip through a test that only compares row counts. + */ +export function arrowStreamSettings(): Record { + return { + output_format_arrow_string_as_string: "1", + output_format_arrow_low_cardinality_as_dictionary: "0", + }; +} + +// ── encoder ──────────────────────────────────────────────────────────────── + +function alignTo8(n: number): number { + return (n + 7) & ~7; +} + +interface EncodedBuffer { + bytes: Uint8Array; + /** Logical (unpadded) length recorded in the RecordBatch metadata. */ + length: number; +} + +function validityBuffer(values: readonly unknown[]): { buf: EncodedBuffer; nullCount: number } { + let nullCount = 0; + const byteLen = Math.ceil(values.length / 8); + const bytes = new Uint8Array(alignTo8(byteLen)); + for (let i = 0; i < values.length; i++) { + if (values[i] === null || values[i] === undefined) { + nullCount += 1; + } else { + bytes[i >> 3] |= 1 << (i & 7); + } + } + return { buf: { bytes, length: byteLen }, nullCount }; +} + +function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: number } { + const { buf: validity, nullCount } = validityBuffer(col.values); + const n = col.values.length; + + switch (col.type) { + case "int64": { + const data = new Uint8Array(alignTo8(n * 8)); + const dv = new DataView(data.buffer); + col.values.forEach((v, i) => { + if (v === null || v === undefined) return; + const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v))); + if (b > INT64_MAX || b < INT64_MIN) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`); + } + dv.setBigInt64(i * 8, b, true); + }); + return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; + } + case "float64": { + const data = new Uint8Array(alignTo8(n * 8)); + const dv = new DataView(data.buffer); + col.values.forEach((v, i) => { + if (v === null || v === undefined) return; + dv.setFloat64(i * 8, Number(v), true); + }); + return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; + } + case "timestamp_ms": { + const data = new Uint8Array(alignTo8(n * 8)); + const dv = new DataView(data.buffer); + col.values.forEach((v, i) => { + if (v === null || v === undefined) return; + const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v); + dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true); + }); + return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; + } + case "bool": { + const byteLen = Math.ceil(n / 8); + const data = new Uint8Array(alignTo8(byteLen)); + col.values.forEach((v, i) => { + if (v === null || v === undefined) return; + if (v) data[i >> 3] |= 1 << (i & 7); + }); + return { buffers: [validity, { bytes: data, length: byteLen }], nullCount }; + } + case "utf8": { + const encoded = col.values.map((v) => + v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)), + ); + const total = encoded.reduce((a, b) => a + b.length, 0); + const offsets = new Uint8Array(alignTo8((n + 1) * 4)); + const odv = new DataView(offsets.buffer); + const data = new Uint8Array(alignTo8(total)); + let pos = 0; + odv.setInt32(0, 0, true); + encoded.forEach((bytes, i) => { + data.set(bytes, pos); + pos += bytes.length; + odv.setInt32((i + 1) * 4, pos, true); + }); + return { + buffers: [validity, { bytes: offsets, length: (n + 1) * 4 }, { bytes: data, length: total }], + nullCount, + }; + } + default: { + const never: never = col.type; + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); + } + } +} + +function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } { + switch (type) { + case "int64": { + b.startObject(2); + b.addInt32(0, 64, 0); // bitWidth + b.addInt8(1, 1, 0); // is_signed + return { typeType: TYPE_INT, offset: b.endObject() }; + } + case "float64": { + b.startObject(1); + b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE + return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() }; + } + case "utf8": { + b.startObject(0); + return { typeType: TYPE_UTF8, offset: b.endObject() }; + } + case "bool": { + b.startObject(0); + return { typeType: TYPE_BOOL, offset: b.endObject() }; + } + case "timestamp_ms": { + // Build the timezone string BEFORE startObject — FlatBuffers forbids + // creating a nested object while a table is open. + const tz = b.createString("UTC"); + b.startObject(2); + b.addInt16(0, TIME_UNIT_MILLISECOND, 0); + b.addOffset(1, tz, 0); + return { typeType: TYPE_TIMESTAMP, offset: b.endObject() }; + } + default: { + const never: never = type; + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); + } + } +} + +function buildSchemaMessage(columns: readonly ArrowColumn[]): Uint8Array { + const b = new FlatBufferBuilder(2048); + + const fieldOffsets: Offset[] = columns.map((col) => { + const nameOffset = b.createString(col.name); + const { typeType, offset: typeOffset } = buildTypeTable(b, col.type); + const children = b.createOffsetVector([]); + b.startObject(7); + b.addOffset(0, nameOffset, 0); + b.addInt8(1, 1, 0); // nullable = true + b.addInt8(2, typeType, 0); + b.addOffset(3, typeOffset, 0); + b.addOffset(5, children, 0); + return b.endObject(); + }); + + const fieldsVector = b.createOffsetVector(fieldOffsets); + b.startObject(4); + b.addInt16(0, ENDIANNESS_LITTLE, 0); + b.addOffset(1, fieldsVector, 0); + const schema = b.endObject(); + + b.startObject(5); + b.addInt16(0, METADATA_VERSION_V5, 0); + b.addInt8(1, MESSAGE_HEADER_SCHEMA, 0); + b.addOffset(2, schema, 0); + b.addInt64(3, BIG_ZERO, BIG_ZERO); // bodyLength + const message = b.endObject(); + + return b.finish(message); +} + +function buildRecordBatchMessage( + rowCount: number, + nodes: { length: number; nullCount: number }[], + buffers: { offset: number; length: number }[], + bodyLength: number, +): Uint8Array { + const b = new FlatBufferBuilder(2048); + + // FieldNode[] — inline structs, written in reverse. + b.startVector(16, nodes.length, 8); + for (let i = nodes.length - 1; i >= 0; i--) { + b.writeInt64(BigInt(nodes[i].nullCount)); + b.writeInt64(BigInt(nodes[i].length)); + } + const nodesVector = b.endVector(nodes.length); + + // Buffer[] — inline structs, written in reverse. + b.startVector(16, buffers.length, 8); + for (let i = buffers.length - 1; i >= 0; i--) { + b.writeInt64(BigInt(buffers[i].length)); + b.writeInt64(BigInt(buffers[i].offset)); + } + const buffersVector = b.endVector(buffers.length); + + b.startObject(5); + b.addInt64(0, BigInt(rowCount), BIG_ZERO); + b.addOffset(1, nodesVector, 0); + b.addOffset(2, buffersVector, 0); + const recordBatch = b.endObject(); + + b.startObject(5); + b.addInt16(0, METADATA_VERSION_V5, 0); + b.addInt8(1, MESSAGE_HEADER_RECORD_BATCH, 0); + b.addOffset(2, recordBatch, 0); + b.addInt64(3, BigInt(bodyLength), BIG_ZERO); + const message = b.endObject(); + + return b.finish(message); +} + +function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array { + // continuation(4) + length(4) + metadata, padded so the body is 8-aligned. + const prefix = 8; + const padded = alignTo8(prefix + metadata.length) - prefix; + const out = new Uint8Array(prefix + padded + body.length); + const dv = new DataView(out.buffer); + dv.setUint32(0, 0xffffffff, true); + dv.setInt32(4, padded, true); + out.set(metadata, 8); + out.set(body, prefix + padded); + return out; +} + +/** + * Encode columns as a complete Arrow IPC *stream* (schema message, one record + * batch, end-of-stream marker). + */ +export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array { + if (columns.length === 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column"); + } + const rowCount = columns[0].values.length; + for (const c of columns) { + if (c.values.length !== rowCount) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `column "${c.name}" has ${c.values.length} values, expected ${rowCount}`, + ); + } + } + + const schemaMsg = buildSchemaMessage(columns); + + const nodes: { length: number; nullCount: number }[] = []; + const bufferMeta: { offset: number; length: number }[] = []; + const bodyParts: Uint8Array[] = []; + let bodyOffset = 0; + for (const col of columns) { + const { buffers, nullCount } = encodeColumn(col); + nodes.push({ length: rowCount, nullCount }); + for (const buf of buffers) { + bufferMeta.push({ offset: bodyOffset, length: buf.length }); + bodyParts.push(buf.bytes); + bodyOffset += buf.bytes.length; // already 8-aligned by encodeColumn + } + } + const body = new Uint8Array(bodyOffset); + let pos = 0; + for (const part of bodyParts) { + body.set(part, pos); + pos += part.length; + } + + const batchMsg = buildRecordBatchMessage(rowCount, nodes, bufferMeta, body.length); + + const schemaFrame = framedMessage(schemaMsg, new Uint8Array(0)); + const batchFrame = framedMessage(batchMsg, body); + const eos = new Uint8Array(8); + new DataView(eos.buffer).setUint32(0, 0xffffffff, true); // length 0 = EOS + + const out = new Uint8Array(schemaFrame.length + batchFrame.length + eos.length); + out.set(schemaFrame, 0); + out.set(batchFrame, schemaFrame.length); + out.set(eos, schemaFrame.length + batchFrame.length); + return out; +} + +// ── decoder (structural: schema names/types + row counts) ────────────────── + +export interface ArrowStreamInfo { + fields: { name: string; typeType: number }[]; + /** Total rows across every record batch in the stream. */ + rowCount: number; + batchCount: number; +} + +/** Minimal FlatBuffers table reader — enough for Message/Schema/RecordBatch. */ +class FbTable { + constructor( + readonly dv: DataView, + readonly pos: number, + ) {} + + fieldPos(slot: number): number { + const vtable = this.pos - this.dv.getInt32(this.pos, true); + const vtLen = this.dv.getInt16(vtable, true); + const idx = 4 + slot * 2; + if (idx >= vtLen) return 0; + const rel = this.dv.getInt16(vtable + idx, true); + return rel === 0 ? 0 : this.pos + rel; + } + + readInt8(slot: number, def = 0): number { + const p = this.fieldPos(slot); + return p === 0 ? def : this.dv.getInt8(p); + } + + readInt16(slot: number, def = 0): number { + const p = this.fieldPos(slot); + return p === 0 ? def : this.dv.getInt16(p, true); + } + + readInt64(slot: number, def: bigint = BIG_ZERO): bigint { + const p = this.fieldPos(slot); + return p === 0 ? def : this.dv.getBigInt64(p, true); + } + + /** Follow a uoffset to a nested table. */ + readTable(slot: number): FbTable | undefined { + const p = this.fieldPos(slot); + if (p === 0) return undefined; + return new FbTable(this.dv, p + this.dv.getInt32(p, true)); + } + + vectorLength(slot: number): number { + const p = this.fieldPos(slot); + if (p === 0) return 0; + const vec = p + this.dv.getInt32(p, true); + return this.dv.getInt32(vec, true); + } + + vectorElementTable(slot: number, index: number): FbTable { + const p = this.fieldPos(slot); + const vec = p + this.dv.getInt32(p, true); + const elem = vec + 4 + index * 4; + return new FbTable(this.dv, elem + this.dv.getInt32(elem, true)); + } + + readString(slot: number): string | undefined { + const p = this.fieldPos(slot); + if (p === 0) return undefined; + const str = p + this.dv.getInt32(p, true); + const len = this.dv.getInt32(str, true); + const bytes = new Uint8Array(this.dv.buffer, this.dv.byteOffset + str + 4, len); + return new TextDecoder().decode(bytes); + } +} + +/** + * Parse an Arrow IPC stream's structure. Used to enforce BI-P012's result-row + * ceiling on Arrow payloads (including ClickHouse pass-through bytes, where + * the count is otherwise invisible to 0pt) and by the roundtrip tests. + */ +export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let pos = 0; + let fields: { name: string; typeType: number }[] = []; + let rowCount = 0; + let batchCount = 0; + + while (pos + 8 <= bytes.byteLength) { + const continuation = dv.getUint32(pos, true); + if (continuation !== 0xffffffff) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)"); + } + const metaLen = dv.getInt32(pos + 4, true); + if (metaLen === 0) break; // end of stream + const metaStart = pos + 8; + const root = metaStart + dv.getInt32(metaStart, true); + const message = new FbTable(dv, root); + const headerType = message.readInt8(1, 0); + const bodyLength = Number(message.readInt64(3, BIG_ZERO)); + + if (headerType === MESSAGE_HEADER_SCHEMA) { + const schema = message.readTable(2); + if (schema) { + const n = schema.vectorLength(1); + fields = []; + for (let i = 0; i < n; i++) { + const f = schema.vectorElementTable(1, i); + fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) }); + } + } + } else if (headerType === MESSAGE_HEADER_RECORD_BATCH) { + const rb = message.readTable(2); + if (rb) { + rowCount += Number(rb.readInt64(0, BIG_ZERO)); + batchCount += 1; + } + } + pos = metaStart + metaLen + bodyLength; + } + + return { fields, rowCount, batchCount }; +} + +/** + * BI-P012 on the Arrow path: refuse a payload that exceeds the caller's + * result-row ceiling. Without this, asking for Arrow would be a way to walk + * around `max_result_rows`. + */ +export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: number): ArrowStreamInfo { + const info = decodeArrowStream(bytes); + if (info.rowCount > maxResultRows) { + throw new SiaxError( + "QUOTA_EXCEEDED", + "0pt.analytics", + `Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`, + ); + } + return info; +} diff --git a/src/analytics/domain/contract.ts b/src/analytics/domain/contract.ts new file mode 100644 index 0000000..0a3ba10 --- /dev/null +++ b/src/analytics/domain/contract.ts @@ -0,0 +1,474 @@ +/** + * src/analytics/domain/contract.ts + * ================================= + * BI-P004 — ANALYTICS DATA CONTRACT. + * + * `DatasetContract` is the machine-checkable promise a dataset makes to every + * consumer downstream of it. BI-P004's DoD is a GATE, not a document: + * + * "Breaking dataset schema changes fail compatibility gate unless + * explicitly versioned." + * + * so `checkContractCompatibility` is the deliverable and the prose is the + * commentary. The gate is fail-closed: an unrecognised change is BREAKING, + * never "probably fine". + * + * Versioning is semver-shaped (`major.minor.patch`) with one rule that is not + * negotiable: a BREAKING change requires a MAJOR bump. A breaking change + * shipped under a minor bump is exactly the failure this gate exists to + * catch, so it is reported as a violation rather than tolerated. + */ +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities"; +import type { AnalyticsTier } from "./tiers"; + +export interface ContractField { + name: string; + type: AnalyticsFieldType; + nullable: boolean; + role: FieldRole; + /** Part of the dataset's primary identifier. */ + primary_identifier?: boolean; + classification?: PiiClassification; + description?: string; +} + +export interface ContractQualityRule { + rule_id: string; + kind: string; + field_name?: string; + severity: "warn" | "block"; +} + +/** BI-P004's named contents, one field each. */ +export interface DatasetContract { + dataset_id: string; + /** semver `major.minor.patch`. */ + version: string; + tier: AnalyticsTier; + fields: ContractField[]; + /** Field names forming the primary identifier, in order. */ + primary_identifiers: string[]; + /** Field names usable as dimensions (subset of `fields`). */ + dimensions: string[]; + /** Field names usable as measures (subset of `fields`). */ + measures: string[]; + /** Freshness SLO in seconds. */ + freshness_slo_seconds: number; + retention_days: number; + /** Dataset-level PII classification (the max of its fields, at least). */ + pii_classification: PiiClassification; + /** Upstream refs: source ids, dataset ids, external system refs. */ + source_lineage: string[]; + quality_rules: ContractQualityRule[]; + /** GDPR purpose limitation — what this data may be USED for. */ + allowed_purposes: string[]; +} + +export interface SemVer { + major: number; + minor: number; + patch: number; +} + +const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/; + +export function parseSemVer(v: string): SemVer { + const m = SEMVER_RE.exec(v); + if (!m) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`); + } + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) }; +} + +// ── change classification ────────────────────────────────────────────────── + +export type ChangeSeverity = "compatible" | "breaking"; + +export interface ContractChange { + severity: ChangeSeverity; + /** Stable machine code so callers can allow-list specific changes. */ + code: string; + field?: string; + detail: string; +} + +/** + * Type widening that does NOT break a reader. Deliberately conservative: + * int64 -> float64 loses exactness for large integers, so it is NOT here. + * Everything absent from this table is breaking. + */ +const SAFE_TYPE_WIDENING: Partial> = { + date: ["timestamp"], +}; + +function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean { + if (from === to) return true; + return (SAFE_TYPE_WIDENING[from] ?? []).includes(to); +} + +/** + * Diff two contracts for the SAME dataset and classify every change. + * + * Breaking (a reader written against `previous` can now fail or silently + * change meaning): + * - field removed + * - field type changed other than a listed safe widening + * - nullable false -> true (readers may not handle nulls) + * - a field's role changes (a measure becoming a dimension changes meaning) + * - primary identifier set changes + * - a field leaves `measures` or `dimensions` + * - freshness SLO loosened (a consumer's own SLO may depend on it) + * - retention shortened (history a consumer relied on disappears) + * - an allowed purpose is removed (a legitimate consumer becomes illegitimate) + * - PII classification raised (consumers may lack clearance) + * - a blocking quality rule is removed or downgraded to warn + * + * Compatible: additive nullable fields, new dimensions/measures, new quality + * rules, loosened PII classification, added purposes, longer retention, + * tightened freshness. + */ +export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] { + if (previous.dataset_id !== next.dataset_id) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + "contract compatibility is only defined between versions of the SAME dataset", + ); + } + const changes: ContractChange[] = []; + const prevFields = new Map(previous.fields.map((f) => [f.name, f])); + const nextFields = new Map(next.fields.map((f) => [f.name, f])); + + for (const [name, pf] of prevFields) { + const nf = nextFields.get(name); + if (!nf) { + changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` }); + continue; + } + if (!typeChangeIsCompatible(pf.type, nf.type)) { + changes.push({ + severity: "breaking", + code: "field_type_changed", + field: name, + detail: `field "${name}" type ${pf.type} -> ${nf.type}`, + }); + } else if (pf.type !== nf.type) { + changes.push({ + severity: "compatible", + code: "field_type_widened", + field: name, + detail: `field "${name}" widened ${pf.type} -> ${nf.type}`, + }); + } + if (!pf.nullable && nf.nullable) { + changes.push({ + severity: "breaking", + code: "field_became_nullable", + field: name, + detail: `field "${name}" became nullable`, + }); + } + if (pf.nullable && !nf.nullable) { + changes.push({ + severity: "compatible", + code: "field_became_required", + field: name, + detail: `field "${name}" is no longer nullable`, + }); + } + if (pf.role !== nf.role) { + changes.push({ + severity: "breaking", + code: "field_role_changed", + field: name, + detail: `field "${name}" role ${pf.role} -> ${nf.role}`, + }); + } + } + + for (const [name, nf] of nextFields) { + if (prevFields.has(name)) continue; + if (nf.nullable) { + changes.push({ + severity: "compatible", + code: "nullable_field_added", + field: name, + detail: `nullable field "${name}" added`, + }); + } else { + // A required new field breaks producers, and breaks readers that + // reconstruct rows (they now have no value to supply). + changes.push({ + severity: "breaking", + code: "required_field_added", + field: name, + detail: `required (non-nullable) field "${name}" added`, + }); + } + } + + const idsEqual = + previous.primary_identifiers.length === next.primary_identifiers.length && + previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v); + if (!idsEqual) { + changes.push({ + severity: "breaking", + code: "primary_identifier_changed", + detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`, + }); + } + + for (const d of previous.dimensions) { + if (!next.dimensions.includes(d)) { + changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` }); + } + } + for (const d of next.dimensions) { + if (!previous.dimensions.includes(d)) { + changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` }); + } + } + for (const m of previous.measures) { + if (!next.measures.includes(m)) { + changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` }); + } + } + for (const m of next.measures) { + if (!previous.measures.includes(m)) { + changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` }); + } + } + + if (next.freshness_slo_seconds > previous.freshness_slo_seconds) { + changes.push({ + severity: "breaking", + code: "freshness_slo_loosened", + detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`, + }); + } + if (next.retention_days < previous.retention_days) { + changes.push({ + severity: "breaking", + code: "retention_shortened", + detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`, + }); + } + + for (const p of previous.allowed_purposes) { + if (!next.allowed_purposes.includes(p)) { + changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` }); + } + } + + const PII_RANK: Record = { + none: 0, + pseudonymous: 1, + personal: 2, + sensitive: 3, + }; + if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) { + changes.push({ + severity: "breaking", + code: "pii_classification_raised", + detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`, + }); + } + + const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r])); + for (const [id, pr] of prevRules) { + const nr = next.quality_rules.find((r) => r.rule_id === id); + if (!nr && pr.severity === "block") { + changes.push({ + severity: "breaking", + code: "blocking_quality_rule_removed", + detail: `blocking quality rule "${id}" removed`, + }); + } else if (nr && pr.severity === "block" && nr.severity === "warn") { + changes.push({ + severity: "breaking", + code: "blocking_quality_rule_downgraded", + detail: `quality rule "${id}" downgraded block -> warn`, + }); + } + } + + if (previous.tier !== next.tier) { + changes.push({ + severity: "breaking", + code: "tier_changed", + detail: `tier ${previous.tier} -> ${next.tier}`, + }); + } + + return changes; +} + +export interface CompatibilityVerdict { + compatible: boolean; + /** True when a MAJOR bump legitimises the breaking changes. */ + explicitly_versioned: boolean; + changes: ContractChange[]; + breaking: ContractChange[]; + previous_version: string; + next_version: string; + /** Human-readable reason when `compatible` is false. */ + reason?: string; +} + +/** + * BI-P004's gate. Returns a verdict; `assertContractCompatible` throws. + * + * Rules, in order: + * 1. the version must move FORWARD (equal or lower is rejected outright); + * 2. breaking changes REQUIRE a major bump — that is what "explicitly + * versioned" means; + * 3. a major bump with no breaking change is allowed (over-declaring is + * safe; under-declaring is what we refuse). + */ +export function checkContractCompatibility( + previous: DatasetContract, + next: DatasetContract, +): CompatibilityVerdict { + const pv = parseSemVer(previous.version); + const nv = parseSemVer(next.version); + const changes = diffContracts(previous, next); + const breaking = changes.filter((c) => c.severity === "breaking"); + + const forward = + nv.major > pv.major || + (nv.major === pv.major && nv.minor > pv.minor) || + (nv.major === pv.major && nv.minor === pv.minor && nv.patch > pv.patch); + if (!forward) { + return { + compatible: false, + explicitly_versioned: false, + changes, + breaking, + previous_version: previous.version, + next_version: next.version, + reason: `version must move forward: ${previous.version} -> ${next.version}`, + }; + } + + const majorBumped = nv.major > pv.major; + if (breaking.length > 0 && !majorBumped) { + return { + compatible: false, + explicitly_versioned: false, + changes, + breaking, + previous_version: previous.version, + next_version: next.version, + reason: + `${breaking.length} breaking change(s) require a MAJOR version bump ` + + `(${previous.version} -> ${next.version} is not one): ` + + breaking.map((b) => b.code).join(", "), + }; + } + + return { + compatible: true, + explicitly_versioned: majorBumped && breaking.length > 0, + changes, + breaking, + previous_version: previous.version, + next_version: next.version, + }; +} + +/** Throwing form for call sites that treat incompatibility as a failure. */ +export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict { + const verdict = checkContractCompatibility(previous, next); + if (!verdict.compatible) { + throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed"); + } + return verdict; +} + +/** + * Structural validation of a single contract (independent of history). + * Catches the contract that is internally inconsistent before it is ever + * compared to anything. + */ +export function validateContract(contract: DatasetContract): void { + parseSemVer(contract.version); + if (!contract.dataset_id.trim()) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id"); + } + if (contract.fields.length === 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`); + } + const names = new Set(); + for (const f of contract.fields) { + if (names.has(f.name)) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`); + } + names.add(f.name); + } + for (const label of ["primary_identifiers", "dimensions", "measures"] as const) { + for (const n of contract[label]) { + if (!names.has(n)) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `${label} references unknown field "${n}" in ${contract.dataset_id}`, + ); + } + } + } + if (contract.primary_identifiers.length === 0) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `contract ${contract.dataset_id} declares no primary identifier`, + ); + } + if (contract.allowed_purposes.length === 0) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`, + ); + } + if (contract.freshness_slo_seconds <= 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive"); + } + if (contract.retention_days < 0) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative"); + } +} + +/** + * Stable hash of a contract's MEANING (field order-insensitive), used as + * `DatasetVersion.contract_hash` so two versions that differ only in comment + * text do not look like schema drift. + */ +export function contractHash(contract: DatasetContract): string { + const canonical = { + dataset_id: contract.dataset_id, + version: contract.version, + tier: contract.tier, + fields: [...contract.fields] + .map((f) => ({ + name: f.name, + type: f.type, + nullable: f.nullable, + role: f.role, + primary_identifier: f.primary_identifier ?? false, + classification: f.classification ?? "none", + })) + .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)), + primary_identifiers: contract.primary_identifiers, + dimensions: [...contract.dimensions].sort(), + measures: [...contract.measures].sort(), + freshness_slo_seconds: contract.freshness_slo_seconds, + retention_days: contract.retention_days, + pii_classification: contract.pii_classification, + source_lineage: [...contract.source_lineage].sort(), + quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)), + allowed_purposes: [...contract.allowed_purposes].sort(), + }; + return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); +} diff --git a/src/analytics/domain/entities.ts b/src/analytics/domain/entities.ts new file mode 100644 index 0000000..29d0017 --- /dev/null +++ b/src/analytics/domain/entities.ts @@ -0,0 +1,536 @@ +/** + * src/analytics/domain/entities.ts + * ================================= + * BI-P002 — ANALYTICS DOMAIN MODEL. + * + * The nineteen canonical analytics entities, named EXACTLY as the program + * lists them (no renaming, no merging, no "we already have something like + * that"): + * + * AnalyticsDataset DatasetVersion DatasetField + * DatasetGrant MetricDefinition MetricVersion + * DimensionDefinition SemanticModel AnalyticsQuery + * QueryExecution QueryResult DashboardDefinition + * DashboardVersion ReportDefinition ReportRun + * DataQualityRule DataQualityRun LineageReference + * BIApplicationBinding + * + * Every tenant-owned record carries `tenant_id` + `project_id` where + * applicable. The four version entities the program names as immutable + * (MetricVersion, SemanticModel version, DashboardVersion, DatasetVersion) + * are `readonly` here AND immutable in the database — sql/008 installs BEFORE + * UPDATE/DELETE triggers, because a TypeScript `readonly` is a compiler + * opinion, not a storage guarantee. + * + * NAME COLLISION, deliberate: src/services/dataset-version-service.ts already + * exports an interface called `DatasetVersion` for the ST0RE-artifact-backed + * dataset versioning that predates this program. That one is about *dataset + * content artifacts*; this one is about *analytics dataset schema versions* + * and lives in its own module namespace (`AnalyticsDatasetVersion` is NOT + * used as the name — BI-P002 lists `DatasetVersion` verbatim and that is what + * ships). Import them qualified when both are needed in one file. + * + * ARCHITECTURE BOUNDARY: these entities are owned by 0pt. A product repo + * (VehIQ) supplies domain MEANING through `BIApplicationBinding`; it never + * defines its own copies of these tables and never receives engine + * credentials. + */ +import type { AnalyticsTier } from "./tiers"; + +/** Every tenant-owned analytics record. */ +export interface TenantScoped { + tenant_id: string; + project_id: string; +} + +// ── field/typing vocabulary ──────────────────────────────────────────────── + +export const ANALYTICS_FIELD_TYPES = [ + "string", + "int64", + "float64", + "bool", + "date", + "timestamp", + "decimal", + "json", +] as const; +export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number]; + +/** A field's analytical role. Measures aggregate; dimensions group. */ +export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const; +export type FieldRole = (typeof FIELD_ROLES)[number]; + +/** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */ +export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const; +export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number]; + +// ── 1. AnalyticsDataset ──────────────────────────────────────────────────── + +export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const; +export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number]; + +/** + * BI-P003 DoD: "Every dataset declares tier, schema, owner, source, freshness, + * quality, retention, classification." All eight are REQUIRED fields here — + * an optional declaration is not a declaration. + */ +export interface AnalyticsDataset extends TenantScoped { + dataset_id: string; + name: string; + /** BI-P003 tier. */ + tier: AnalyticsTier; + /** Contract id + version this dataset's schema conforms to (BI-P004). */ + schema_contract_id: string; + schema_contract_version: string; + /** Accountable human/team, not the person who typed the create call. */ + owner_id: string; + /** Upstream: a 0pt source_id, another dataset_id, or an external ref. */ + source_ref: string; + /** Freshness SLO in seconds — how stale the data may be before it is late. */ + freshness_slo_seconds: number; + /** DataQualityRule ids that gate this dataset. */ + quality_rule_ids: string[]; + /** Retention in days. 0 = keep indefinitely (must be justified in policy). */ + retention_days: number; + classification: PiiClassification; + status: AnalyticsDatasetStatus; + /** BI-P003 escape hatch for RAW/NORMALIZED exposure to a BI surface. */ + raw_access_exception?: string; + description?: string; + created_at: string; + updated_at?: string; + created_by?: string; +} + +// ── 2. DatasetVersion (IMMUTABLE) ────────────────────────────────────────── + +export interface DatasetVersion extends TenantScoped { + readonly dataset_version_id: string; + readonly dataset_id: string; + /** Monotonic per dataset, starting at 1. */ + readonly version: number; + /** The frozen DatasetContract JSON at this version (BI-P004). */ + readonly contract_hash: string; + readonly schema_contract_version: string; + readonly record_count?: number; + readonly created_at: string; + readonly created_by?: string; +} + +// ── 3. DatasetField ──────────────────────────────────────────────────────── + +export interface DatasetField extends TenantScoped { + dataset_field_id: string; + dataset_id: string; + /** The dataset version this field description belongs to. */ + dataset_version: number; + name: string; + field_type: AnalyticsFieldType; + nullable: boolean; + role: FieldRole; + /** True for the field(s) forming the dataset's primary identifier. */ + primary_identifier: boolean; + classification: PiiClassification; + description?: string; +} + +// ── 4. DatasetGrant ──────────────────────────────────────────────────────── + +export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const; +export type GrantLevel = (typeof GRANT_LEVELS)[number]; + +/** + * Who may use a dataset, and FOR WHAT. `allowed_purposes` is the GDPR purpose + * limitation carried into analytics — a grant without a purpose is refused + * by the repository layer. + */ +export interface DatasetGrant extends TenantScoped { + grant_id: string; + dataset_id: string; + /** Subject: a CL0UD principal, role, or BI application binding id. */ + subject_id: string; + subject_type: "user" | "role" | "service" | "bi_application"; + level: GrantLevel; + allowed_purposes: string[]; + expires_at?: string; + created_at: string; + created_by?: string; +} + +// ── 5. MetricDefinition ──────────────────────────────────────────────────── + +export const METRIC_AGGREGATIONS = [ + "sum", + "avg", + "min", + "max", + "count", + "count_distinct", + "ratio", + "derived", +] as const; +export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number]; + +export interface MetricDefinition extends TenantScoped { + metric_id: string; + /** Stable business name; unique per tenant. */ + name: string; + dataset_id: string; + aggregation: MetricAggregation; + /** Measure field aggregated, or the numerator for ratio metrics. */ + measure_field: string; + /** Denominator for ratio metrics. */ + denominator_field?: string; + /** Dimensions this metric may legitimately be sliced by. */ + allowed_dimension_ids: string[]; + /** Current live version number (points at a MetricVersion). */ + current_version: number; + owner_id: string; + description?: string; + created_at: string; + updated_at?: string; +} + +// ── 6. MetricVersion (IMMUTABLE) ─────────────────────────────────────────── + +export interface MetricVersion extends TenantScoped { + readonly metric_version_id: string; + readonly metric_id: string; + readonly version: number; + /** Frozen definition JSON — what this metric MEANT at this version. */ + readonly definition_json: string; + readonly definition_hash: string; + /** Free-text reason the metric changed; required for version > 1. */ + readonly change_reason?: string; + readonly created_at: string; + readonly created_by?: string; +} + +// ── 7. DimensionDefinition ───────────────────────────────────────────────── + +export interface DimensionDefinition extends TenantScoped { + dimension_id: string; + name: string; + dataset_id: string; + /** + * Physical column/key this dimension resolves to. This is the ONLY place a + * caller-supplied dimension name becomes a physical identifier — the query + * compiler resolves through here and never splices caller input (BI-P011 + * "dimension identifier injection"). + */ + field_name: string; + field_type: AnalyticsFieldType; + classification: PiiClassification; + description?: string; + created_at: string; +} + +// ── 8. SemanticModel (versioned; versions IMMUTABLE) ─────────────────────── + +export interface SemanticModel extends TenantScoped { + semantic_model_id: string; + name: string; + /** Immutable version number — a change mints a new row, never an UPDATE. */ + readonly version: number; + /** MetricFlow/Cube-compatible model document (BI-P029/BI-P032 consume it). */ + readonly model_json: string; + readonly model_hash: string; + dataset_ids: string[]; + owner_id: string; + readonly created_at: string; + readonly created_by?: string; +} + +// ── 9. AnalyticsQuery ────────────────────────────────────────────────────── + +export const QUERY_SURFACES = ["metric", "sql", "projection"] as const; +export type QuerySurface = (typeof QUERY_SURFACES)[number]; + +/** + * A SAVED, reusable query. The request that actually runs is + * `src/analytics/engine/query-compiler.ts`'s `MetricQueryRequest`; this entity + * is its durable, governed form. + */ +export interface AnalyticsQuery extends TenantScoped { + query_id: string; + name: string; + surface: QuerySurface; + dataset_id: string; + metric_id?: string; + /** Serialized closed-DSL request. Never raw SQL for surface="metric". */ + request_json: string; + owner_id: string; + created_at: string; + updated_at?: string; +} + +// ── 10. QueryExecution ───────────────────────────────────────────────────── + +export const EXECUTION_STATES = [ + "queued", + "running", + "succeeded", + "failed", + "cancelled", + "rejected", +] as const; +export type ExecutionState = (typeof EXECUTION_STATES)[number]; + +export interface QueryExecution extends TenantScoped { + execution_id: string; + query_id?: string; + /** Engine-side identifier used to CANCEL this execution (BI-P009). */ + engine_query_id: string; + state: ExecutionState; + /** Which provider ran it: "clickhouse" | "duckdb" | "trino". */ + provider: string; + submitted_at: string; + started_at?: string; + finished_at?: string; + duration_ms?: number; + rows_read?: number; + bytes_read?: number; + result_rows?: number; + /** SiaxErrorCode when state="failed"/"rejected". Never a raw engine body. */ + error_code?: string; + correlation_id?: string; +} + +// ── 11. QueryResult ──────────────────────────────────────────────────────── + +export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const; +export type ResultEncoding = (typeof RESULT_ENCODINGS)[number]; + +export interface QueryResult extends TenantScoped { + result_id: string; + execution_id: string; + encoding: ResultEncoding; + row_count: number; + /** Canonical ST0RE ref when materialised; inline results carry no bytes here. */ + artifact_ref?: string; + /** True when the engine truncated at max_result_rows (BI-P012). */ + truncated: boolean; + expires_at?: string; + created_at: string; +} + +// ── 12. DashboardDefinition ──────────────────────────────────────────────── + +export interface DashboardDefinition extends TenantScoped { + dashboard_id: string; + name: string; + /** Live version pointer. */ + current_version: number; + owner_id: string; + /** Datasets the dashboard reads — checked against BI-P003 tier rules. */ + dataset_ids: string[]; + description?: string; + created_at: string; + updated_at?: string; +} + +// ── 13. DashboardVersion (IMMUTABLE) ─────────────────────────────────────── + +export interface DashboardVersion extends TenantScoped { + readonly dashboard_version_id: string; + readonly dashboard_id: string; + readonly version: number; + /** Frozen layout/panel document. */ + readonly layout_json: string; + readonly layout_hash: string; + readonly created_at: string; + readonly created_by?: string; +} + +// ── 14. ReportDefinition ─────────────────────────────────────────────────── + +export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const; +export type ReportFormat = (typeof REPORT_FORMATS)[number]; + +export interface ReportDefinition extends TenantScoped { + report_id: string; + name: string; + dashboard_id?: string; + query_id?: string; + format: ReportFormat; + /** Cron expression; undefined = on-demand only. */ + schedule_cron?: string; + owner_id: string; + created_at: string; + updated_at?: string; +} + +// ── 15. ReportRun ────────────────────────────────────────────────────────── + +export interface ReportRun extends TenantScoped { + report_run_id: string; + report_id: string; + state: ExecutionState; + started_at: string; + finished_at?: string; + /** ST0RE artifact ref of the produced report. */ + artifact_ref?: string; + error_code?: string; + correlation_id?: string; +} + +// ── 16. DataQualityRule ──────────────────────────────────────────────────── + +export const QUALITY_RULE_KINDS = [ + "not_null", + "unique", + "accepted_values", + "range", + "freshness", + "row_count", + "referential", + "custom", +] as const; +export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number]; + +export const QUALITY_SEVERITIES = ["warn", "block"] as const; +export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number]; + +export interface DataQualityRule extends TenantScoped { + rule_id: string; + dataset_id: string; + kind: QualityRuleKind; + /** Field the rule applies to; omitted for dataset-level rules. */ + field_name?: string; + /** Rule parameters (bounds, accepted values, max age). */ + params_json: string; + severity: QualitySeverity; + created_at: string; + created_by?: string; +} + +// ── 17. DataQualityRun ───────────────────────────────────────────────────── + +export interface DataQualityRun extends TenantScoped { + quality_run_id: string; + rule_id: string; + dataset_id: string; + dataset_version?: number; + passed: boolean; + /** Rows that violated the rule. */ + violation_count: number; + evaluated_rows: number; + started_at: string; + finished_at?: string; + correlation_id?: string; +} + +// ── 18. LineageReference ─────────────────────────────────────────────────── + +export const LINEAGE_DIRECTIONS = ["input", "output"] as const; +export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number]; + +/** + * The queryable projection of an OpenLineage edge (BI-P027). AUD0 stays the + * system of record; this is 0pt's local index so "what fed this dashboard" + * is answerable without leaving the analytics plane. + */ +export interface LineageReference extends TenantScoped { + lineage_reference_id: string; + /** OpenLineage run id this edge came from. */ + run_id: string; + job_name: string; + direction: LineageDirection; + /** The analytics entity this edge touches. */ + entity_type: "dataset" | "metric" | "dashboard" | "report"; + entity_id: string; + /** Namespaced dataset name on the other side of the edge. */ + counterpart_ref: string; + observed_at: string; +} + +// ── 19. BIApplicationBinding ─────────────────────────────────────────────── + +export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const; +export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number]; + +/** + * How an external BI surface is attached to the analytics plane. + * + * NORMATIVE (BI program §0 + the SIAX/0pt ownership boundary): this binding + * carries NO engine credential. A bound application authenticates to the 0pt + * Analytics API with a CL0UD identity and receives compiled, tenant-forced + * results — it never learns a ClickHouse DSN, user, or password. The absence + * of a credential field here is the enforcement, not a comment. + */ +export interface BIApplicationBinding extends TenantScoped { + binding_id: string; + kind: BiApplicationKind; + name: string; + /** CL0UD principal the application authenticates as. */ + principal_id: string; + /** Datasets/metrics this application may reach at all. */ + dataset_ids: string[]; + metric_ids: string[]; + allowed_purposes: string[]; + enabled: boolean; + created_at: string; + created_by?: string; +} + +// ── registry of the canonical nineteen ───────────────────────────────────── + +/** + * The canonical entity names, verbatim from BI-P002. Persisted-schema tests + * assert sql/008 defines a table for every one of these, so a future rename + * or omission is a test failure rather than drift. + */ +export const ANALYTICS_DOMAIN_ENTITIES = [ + "AnalyticsDataset", + "DatasetVersion", + "DatasetField", + "DatasetGrant", + "MetricDefinition", + "MetricVersion", + "DimensionDefinition", + "SemanticModel", + "AnalyticsQuery", + "QueryExecution", + "QueryResult", + "DashboardDefinition", + "DashboardVersion", + "ReportDefinition", + "ReportRun", + "DataQualityRule", + "DataQualityRun", + "LineageReference", + "BIApplicationBinding", +] as const; +export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number]; + +/** Entity -> physical table in sql/008_analytics_domain_model.sql. */ +export const ANALYTICS_ENTITY_TABLES: Record = { + AnalyticsDataset: "opt_analytics_dataset", + DatasetVersion: "opt_analytics_dataset_version", + DatasetField: "opt_analytics_dataset_field", + DatasetGrant: "opt_analytics_dataset_grant", + MetricDefinition: "opt_analytics_metric_definition", + MetricVersion: "opt_analytics_metric_version", + DimensionDefinition: "opt_analytics_dimension_definition", + SemanticModel: "opt_analytics_semantic_model", + AnalyticsQuery: "opt_analytics_query", + QueryExecution: "opt_analytics_query_execution", + QueryResult: "opt_analytics_query_result", + DashboardDefinition: "opt_analytics_dashboard_definition", + DashboardVersion: "opt_analytics_dashboard_version", + ReportDefinition: "opt_analytics_report_definition", + ReportRun: "opt_analytics_report_run", + DataQualityRule: "opt_analytics_data_quality_rule", + DataQualityRun: "opt_analytics_data_quality_run", + LineageReference: "opt_analytics_lineage_reference", + BIApplicationBinding: "opt_analytics_bi_application_binding", +}; + +/** The four entities BI-P002 requires to be immutable once written. */ +export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [ + "DatasetVersion", + "MetricVersion", + "SemanticModel", + "DashboardVersion", +]; diff --git a/src/analytics/domain/tiers.ts b/src/analytics/domain/tiers.ts new file mode 100644 index 0000000..d67e590 --- /dev/null +++ b/src/analytics/domain/tiers.ts @@ -0,0 +1,83 @@ +/** + * src/analytics/domain/tiers.ts + * ============================== + * BI-P003 — ANALYTICS DATA TIERS. + * + * The four canonical tiers every analytics dataset in SIAX belongs to. The + * tier is a CONTRACT about where a dataset may be read from and by whom, not + * a naming convention: + * + * RAW -> ST0RE Parquet/Iceberg (as-ingested, no business meaning) + * NORMALIZED -> ST0RE Iceberg (typed, deduplicated, still source-shaped) + * CURATED -> ST0RE Iceberg (business entities, modelled) + * SERVING -> ClickHouse / projection (query-optimised, dashboard-facing) + * + * BI-P003's normative sentence is "No business dashboard reads arbitrary raw + * ingestion tables by default" — `assertReadableByBiSurface` is where that + * sentence stops being prose. A BI surface (dashboard, report, embedded + * chart) may read SERVING and CURATED; RAW and NORMALIZED require an + * explicit, recorded exception on the dataset itself. + * + * ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product + * repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers. + */ +import { SiaxError } from "../../platform/errors"; + +export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const; +export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number]; + +/** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */ +export const TIER_PHYSICAL_HOME: Record = { + RAW: "st0re:iceberg|parquet", + NORMALIZED: "st0re:iceberg", + CURATED: "st0re:iceberg", + SERVING: "clickhouse:projection", +}; + +/** Tiers a BI presentation surface may read without an explicit exception. */ +export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"]; + +export function isAnalyticsTier(v: unknown): v is AnalyticsTier { + return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v); +} + +/** + * Rank used for promotion checks: a dataset may only be BUILT FROM a tier at + * or below its own (SERVING can be built from CURATED; CURATED must never be + * built from SERVING, which would make the projection its own source). + */ +export const TIER_RANK: Record = { + RAW: 0, + NORMALIZED: 1, + CURATED: 2, + SERVING: 3, +}; + +/** + * BI-P003 DoD gate. `raw_access_exception` is the recorded, auditable escape + * hatch — absent it, a RAW/NORMALIZED dataset is invisible to dashboards. + */ +export function assertReadableByBiSurface( + dataset: { dataset_id: string; tier: AnalyticsTier; raw_access_exception?: string }, + surface: string, +): void { + if (BI_READABLE_TIERS.includes(dataset.tier)) return; + if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return; + throw new SiaxError( + "AUTHORIZATION_DENIED", + "0pt.analytics", + `BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` + + `without a recorded raw_access_exception (BI-P003)`, + ); +} + +/** A dataset may not declare a source tier ABOVE its own tier. */ +export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void { + if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`, + ); + } +} diff --git a/src/analytics/engine/clickhouse-engine.ts b/src/analytics/engine/clickhouse-engine.ts new file mode 100644 index 0000000..7753f24 --- /dev/null +++ b/src/analytics/engine/clickhouse-engine.ts @@ -0,0 +1,563 @@ +/** + * src/analytics/engine/clickhouse-engine.ts + * ========================================== + * BI-P009 — ClickHouse as the CANONICAL `SiaxAnalyticsEngine` provider. + * + * PRESERVATION (BI-P009's first sentence): `ClickHouseAnalytics` + * (src/analytics/clickhouse-analytics.ts) is untouched by this change — not + * one line. This engine COMPOSES it: + * + * * `health()` delegates to `ClickHouseAnalytics.health()`. + * * `executeMetric()` on the pre-existing `opt_metric_events` table + * delegates to `ClickHouseAnalytics.queryMetric()`, i.e. keeps the + * already-proven forced-tenant path exactly as it is. + * * everything else (the BI-P010 tables) goes through + * `src/analytics/engine/query-compiler.ts`, which forces the same + * predicate for the same reason. + * + * "Do not regress the existing forced tenant predicate" is enforced by + * tests/analytics/tenant-predicate-regression.test.ts, which fails if either + * compiler stops emitting `tenant_id = {tenant:String}`. + * + * BI-P012 governance is applied on EVERY statement, including the delegated + * ones — the settings ride on the HTTP request, and `readonly=1` means the + * query text cannot re-set them. + * + * LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts + * ClickHouse pipelines, never quarantined market data. + */ +import crypto from "node:crypto"; +import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors"; +import { ClickHouseAnalytics } from "../clickhouse-analytics"; +import { + compileMetricQuery, + compileProjectionQuery, + dimensionCatalogFrom, + FORCED_TENANT_PREDICATE, + type DimensionCatalog, +} from "./query-compiler"; +import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema"; +import { + applyQuota, + clickhouseSettings, + ConcurrencyGate, + DEFAULT_BUDGETS, + planningSettings, + type ResourceBudget, +} from "./resource-governance"; +import type { + AnalystQueryRequest, + AnalyticsIdentity, + EngineResult, + HealthReport, + MetricRequest, + ProjectionIngestRequest, + ProjectionRefreshResult, + QueryEstimate, + QueryExplanation, + SiaxAnalyticsEngine, +} from "./siax-analytics-engine"; + +export interface ClickHouseEngineOptions { + baseUrl: string; + /** "user:password" — from an env/secret ref, never a literal in code. */ + basicAuth?: string; + database?: string; + timeoutMs?: number; + /** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */ + dimensionCatalogs?: Record>; + schema?: Omit; +} + +const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined }; + +export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { + readonly provider = "clickhouse"; + + /** PRESERVED pre-existing implementation, composed not replaced. */ + private readonly legacy: ClickHouseAnalytics; + private readonly baseUrl: string; + private readonly authHeader?: string; + private readonly database: string; + private readonly timeoutMs: number; + private readonly dimensionCatalogs: Record>; + private readonly schemaOpts: Omit; + private readonly gate: ConcurrencyGate; + /** query_id -> AbortController, so cancelQuery can stop the client side too. */ + private readonly inFlight = new Map(); + + constructor(opts: ClickHouseEngineOptions) { + this.baseUrl = opts.baseUrl.replace(/\/+$/, ""); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined; + this.database = opts.database ?? "opt_analytics"; + this.timeoutMs = opts.timeoutMs ?? 30_000; + this.dimensionCatalogs = opts.dimensionCatalogs ?? {}; + this.schemaOpts = opts.schema ?? {}; + this.legacy = new ClickHouseAnalytics({ + baseUrl: opts.baseUrl, + basicAuth: opts.basicAuth, + database: this.database, + timeoutMs: this.timeoutMs, + }); + this.gate = new ConcurrencyGate((tenantId) => this.budgetForTenant(tenantId).maxConcurrentQueries); + } + + /** Access to the preserved implementation, for callers that still use it. */ + get legacyAnalytics(): ClickHouseAnalytics { + return this.legacy; + } + + private budgetForTenant(_tenantId: string): ResourceBudget { + return DEFAULT_BUDGETS.INTERACTIVE; + } + + private budgetFor(identity: AnalyticsIdentity): ResourceBudget { + const tier = identity.budget_tier ?? "INTERACTIVE"; + return applyQuota(DEFAULT_BUDGETS[tier], identity.quota); + } + + private catalogFor(identity: AnalyticsIdentity): DimensionCatalog { + const pairs = this.dimensionCatalogs[identity.tenant_id]; + return pairs ? dimensionCatalogFrom(pairs) : EMPTY_CATALOG; + } + + /** Fail closed: a missing/blank verified tenant is never a wildcard. */ + private assertIdentity(identity: AnalyticsIdentity): void { + if (!identity?.tenant_id || !identity.tenant_id.trim()) { + throw new SiaxError( + "UNAUTHENTICATED", + "0pt.analytics", + "analytics requires a verified CL0UD identity carrying tenant_id", + ); + } + if (!identity.principal_id || !identity.principal_id.trim()) { + throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id"); + } + } + + // ── schema ─────────────────────────────────────────────────────────────── + + /** Apply BI-P010 DDL (idempotent) plus the preserved legacy table. */ + async ensureSchema(): Promise { + await this.legacy.ensureSchema(); + for (const stmt of analyticsSchemaDdl({ database: this.database, ...this.schemaOpts })) { + await this.ddl(stmt); + } + } + + // ── BI-P009 capability 1: health ───────────────────────────────────────── + + async health(): Promise { + try { + const legacyOk = await this.legacy.health(); + if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" }; + const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {}); + const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v; + return { healthy: true, provider: this.provider, version }; + } catch (err) { + return { + healthy: false, + provider: this.provider, + detail: err instanceof SiaxError ? err.code : "unreachable", + }; + } + } + + // ── BI-P009 capability 2: executeMetric ────────────────────────────────── + + async executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise { + this.assertIdentity(identity); + const budget = this.budgetFor(identity); + const release = this.gate.acquire(identity.tenant_id); + const started = Date.now(); + const queryId = this.newQueryId(identity); + try { + const compiled = compileMetricQuery( + { + tenant_id: identity.tenant_id, // FORCED from the verified identity + project_id: identity.project_id, + dataset_id: request.dataset_id, + metric: request.metric, + aggregate: request.aggregate, + group_by: request.group_by, + filters: request.filters, + from_time: request.from_time, + to_time: request.to_time, + limit: request.limit, + table: request.table, + }, + this.database, + this.catalogFor(identity), + budget.maxResultRows, + ); + const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as + | ClickHouseJsonResponse + | undefined; + const groupNames = request.group_by ?? []; + const rows = (json?.data ?? []).map((r) => { + const groups: Record = {}; + groupNames.forEach((name, i) => { + const v = (r as Record)[`grp${i}`]; + groups[name] = v === undefined || v === null ? "" : String(v); + }); + return { + ...(groupNames.length ? { groups } : {}), + value: Number((r as { value: number | string }).value), + }; + }); + const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), budget.maxResultRows)); + return { + rows, + query_id: queryId, + truncated: rows.length >= limit, + stats: { + duration_ms: Date.now() - started, + rows_read: json?.statistics?.rows_read, + bytes_read: json?.statistics?.bytes_read, + result_rows: rows.length, + }, + }; + } finally { + release(); + this.inFlight.delete(queryId); + } + } + + /** + * Delegating path to the PRESERVED implementation for the pre-existing + * `opt_metric_events` table. Kept so no existing caller has to migrate and + * the already-proven forced-predicate path stays in service. + */ + async executeLegacyMetric( + identity: AnalyticsIdentity, + request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number }, + ): Promise { + this.assertIdentity(identity); + const started = Date.now(); + const rows = await this.legacy.queryMetric({ + tenant_id: identity.tenant_id, // FORCED from the verified identity + metric: request.metric, + aggregate: request.aggregate, + dataset_id: request.dataset_id, + group_by: request.group_by, + from_time: request.from_time, + to_time: request.to_time, + limit: request.limit, + }); + return { + rows: rows.map((r) => ({ + ...(r.group !== undefined && request.group_by ? { groups: { [request.group_by]: r.group } } : {}), + value: r.value, + })), + query_id: this.newQueryId(identity), + truncated: false, + stats: { duration_ms: Date.now() - started, result_rows: rows.length }, + }; + } + + // ── BI-P009 capability 3: executeQuery (analyst SQL) ───────────────────── + + async executeQuery(identity: AnalyticsIdentity, _request: AnalystQueryRequest): Promise { + this.assertIdentity(identity); + // Deliberate, documented refusal — see AnalystQueryRequest's doc comment. + // Opening a raw-SQL surface before BI-P017 (SQL dialect contract) and + // BI-P033 (analytics authorization) exist would create precisely the + // "raw SQL attempt" hole BI-P011's adversarial list enumerates. + throw new SiaxError( + "CAPABILITY_UNAVAILABLE", + "0pt.analytics", + "analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land", + ); + } + + // ── BI-P009 capability 4: estimateQuery ────────────────────────────────── + + async estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise { + this.assertIdentity(identity); + const budget = this.budgetFor(identity); + const compiled = compileMetricQuery( + { + tenant_id: identity.tenant_id, + project_id: identity.project_id, + dataset_id: request.dataset_id, + metric: request.metric, + aggregate: request.aggregate, + group_by: request.group_by, + filters: request.filters, + from_time: request.from_time, + to_time: request.to_time, + limit: request.limit, + table: request.table, + }, + this.database, + this.catalogFor(identity), + budget.maxResultRows, + ); + // `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read + // AFTER primary-key pruning — which is why tenant_id leads every ORDER BY + // in schema.ts: the estimate for one tenant does not include everyone. + const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, ""); + let rowsToRead = 0; + let partsRead = 0; + try { + const res = (await this.raw( + `EXPLAIN ESTIMATE ${estimateSql} FORMAT JSON`, + compiled.params, + planningSettings(budget), + )) as { data?: { rows?: number | string; parts?: number | string }[] } | undefined; + for (const row of res?.data ?? []) { + rowsToRead += Number(row.rows ?? 0); + partsRead += Number(row.parts ?? 0); + } + } catch (err) { + // An estimate that cannot be produced must not read as "cheap". + throw err instanceof SiaxError + ? err + : new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query"); + } + // Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes. + // 128 B/row is a deliberately PESSIMISTIC constant for these tables + // (measured mean in the BI-P010 benchmark is lower), so the gate errs + // toward refusing rather than admitting an oversized query. + const bytesToRead = rowsToRead * 128; + const within = rowsToRead <= budget.maxRowsToRead && bytesToRead <= budget.maxBytesToRead; + return { + estimated_rows_to_read: rowsToRead, + estimated_bytes_to_read: bytesToRead, + within_budget: within, + budget, + ...(within + ? {} + : { + reason: + `estimate ${rowsToRead} rows / ${bytesToRead} bytes over budget ` + + `(${budget.maxRowsToRead} rows / ${budget.maxBytesToRead} bytes), parts=${partsRead}`, + }), + }; + } + + // ── BI-P009 capability 5: cancelQuery ──────────────────────────────────── + + /** + * Two-sided cancellation. The client-side abort stops us waiting; the + * server-side KILL QUERY stops ClickHouse burning CPU. Cancelling is + * tenant-scoped: the query id embeds the tenant and is checked, so tenant A + * cannot cancel tenant B's work. + */ + async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise { + this.assertIdentity(identity); + if (!queryId.startsWith(this.queryIdPrefix(identity))) { + throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant"); + } + this.inFlight.get(queryId)?.abort(); + this.inFlight.delete(queryId); + try { + // KILL QUERY is a mutation of server state, so it cannot ride the + // readonly=1 settings the SELECT path uses. + await this.raw( + `KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`, + { param_killId: queryId }, + { max_execution_time: "10" }, + ); + return true; + } catch { + return false; + } + } + + // ── BI-P009 capability 6: explain ──────────────────────────────────────── + + async explain(identity: AnalyticsIdentity, request: MetricRequest): Promise { + this.assertIdentity(identity); + const budget = this.budgetFor(identity); + const compiled = compileMetricQuery( + { + tenant_id: identity.tenant_id, + project_id: identity.project_id, + dataset_id: request.dataset_id, + metric: request.metric, + aggregate: request.aggregate, + group_by: request.group_by, + filters: request.filters, + from_time: request.from_time, + to_time: request.to_time, + limit: request.limit, + table: request.table, + }, + this.database, + this.catalogFor(identity), + budget.maxResultRows, + ); + const sql = compiled.sql.replace(/ FORMAT JSON$/, ""); + const res = (await this.raw( + `EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`, + compiled.params, + planningSettings(budget), + )) as string | undefined; + return { + plan: typeof res === "string" ? res : JSON.stringify(res ?? ""), + forced_tenant_predicate: FORCED_TENANT_PREDICATE, + table: compiled.table, + }; + } + + // ── BI-P009 capability 7: ingestProjection ─────────────────────────────── + + async ingestProjection( + identity: AnalyticsIdentity, + request: ProjectionIngestRequest, + ): Promise<{ ingested: number }> { + this.assertIdentity(identity); + if (request.rows.length === 0) return { ingested: 0 }; + const now = new Date(); + const version = Math.floor(now.getTime()); + const body = request.rows + .map((r) => { + if (!r.entity_key || !r.entity_key.trim()) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key"); + } + return JSON.stringify({ + // tenant/project come from the VERIFIED identity, never the row — + // a row-supplied tenant_id is the INF0 Wave-2 bug class. + tenant_id: identity.tenant_id, + project_id: identity.project_id ?? "", + dataset_id: request.dataset_id, + entity_key: r.entity_key, + as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""), + version: r.version ?? version, + attributes: r.attributes ?? {}, + measures: r.measures ?? {}, + }); + }) + .join("\n"); + await this.raw( + `INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`, + {}, + {}, + undefined, + body, + ); + return { ingested: request.rows.length }; + } + + // ── BI-P009 capability 8: refreshProjection ────────────────────────────── + + /** + * ReplacingMergeTree deduplicates on a background merge, so a read straight + * after an ingest can still see superseded rows. `OPTIMIZE ... FINAL` forces + * the merge for THIS tenant's partition only — the table is + * `PARTITION BY (tenant_id)` precisely so one tenant's refresh does not + * rewrite another tenant's parts (BI-P012's "without degrading unrelated + * tenant" applies to maintenance too, not only to queries). + */ + async refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise { + this.assertIdentity(identity); + await this.raw( + `OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`, + { param_tenant: identity.tenant_id }, + { max_execution_time: "120" }, + ); + const compiled = compileProjectionQuery( + { tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 }, + this.database, + 1, + ); + const countSql = + `SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` + + `WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`; + const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as + | { data?: { value: number | string }[] } + | undefined; + return { + dataset_id, + row_count: Number(res?.data?.[0]?.value ?? 0), + refreshed_at: new Date().toISOString(), + }; + } + + // ── transport ──────────────────────────────────────────────────────────── + + private queryIdPrefix(identity: AnalyticsIdentity): string { + return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`; + } + + private newQueryId(identity: AnalyticsIdentity): string { + return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`; + } + + /** DDL path: no readonly, no result parsing beyond errors. */ + private async ddl(sql: string): Promise { + await this.raw(sql, {}, { max_execution_time: "60" }); + } + + private async raw( + sql: string, + params: Record, + settings: Record, + queryId?: string, + body?: string, + ): Promise { + const url = new URL(this.baseUrl); + url.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + if (queryId) url.searchParams.set("query_id", queryId); + + const headers: Record = {}; + if (this.authHeader) headers["Authorization"] = this.authHeader; + + const controller = new AbortController(); + if (queryId) this.inFlight.set(queryId, controller); + const timeout = setTimeout(() => controller.abort(), this.timeoutMs); + + let res: Response; + try { + res = await fetch(url.toString(), { + method: "POST", + headers, + body: body ?? "", + signal: controller.signal, + }); + } catch (cause) { + throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause }); + } finally { + clearTimeout(timeout); + } + const text = await res.text(); + if (!res.ok) { + // ClickHouse error text can echo the query; keep only the numeric code + // so a caller never receives another tenant's identifiers back. + const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status); + throw new SiaxError( + siaxErrorCodeForUpstreamStatus(res.status), + "clickhouse", + `query failed (CH code ${code})`, + ); + } + if (!text) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return text; + } + // DEFENCE IN DEPTH against a partial result that looks complete. + // ClickHouse normally answers a breached limit with an HTTP error status + // (measured on 24.8: 408 for max_execution_time, 500 for + // max_rows_to_read), which `res.ok` above already catches. But when rows + // have already been flushed, the JSON body can carry an `exception` key + // under a 200 — a truncated answer indistinguishable from a whole one. + // Treat any body carrying an exception as a failure, whatever the status. + if (parsed && typeof parsed === "object" && "exception" in parsed) { + const ex = String((parsed as { exception?: unknown }).exception ?? ""); + const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown"; + throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`); + } + return parsed; + } +} + +interface ClickHouseJsonResponse { + data?: Record[]; + statistics?: { rows_read?: number; bytes_read?: number; elapsed?: number }; +} diff --git a/src/analytics/engine/query-compiler.ts b/src/analytics/engine/query-compiler.ts new file mode 100644 index 0000000..ffb6aa2 --- /dev/null +++ b/src/analytics/engine/query-compiler.ts @@ -0,0 +1,304 @@ +/** + * src/analytics/engine/query-compiler.ts + * ======================================= + * BI-P011 — CLICKHOUSE TENANT ISOLATION, layer 2: + * "`0pt` query compiler forces tenant". + * + * The compiler is the ONLY thing in this repo that turns an analytics request + * into SQL text. Its contract: + * + * * `tenant_id` is not an argument the caller may omit — it comes from the + * VERIFIED CL0UD identity (layer 1) and is written into the WHERE clause + * by this function, unconditionally, as a bound parameter. + * * caller-supplied VALUES are never spliced into SQL text. They travel as + * ClickHouse HTTP `param_*` query parameters. + * * caller-supplied IDENTIFIERS (a dimension name, a group-by column) are + * never spliced either. They are RESOLVED against a declared allow-list + * of `DimensionDefinition`s and only the resolved, already-known physical + * name reaches the SQL — so "dimension identifier injection" has no + * surface, because caller text never becomes an identifier. + * * there is no raw-SQL entry point. `compileMetricQuery` cannot express a + * UNION, a subquery, or a second table, so a caller cannot reach a table + * the compiler did not choose. + * + * WHY A SEPARATE MODULE from clickhouse-analytics.ts: that file's compiler + * already forces the tenant predicate correctly and BI-P009 says PRESERVE IT. + * This module is the generalised compiler for the BI-P010 tables; it repeats + * the guarantee rather than replacing it, and + * tests/analytics/tenant-predicate-regression.test.ts asserts BOTH. + */ +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsTable } from "./schema"; + +/** A physical identifier that has been VALIDATED, not merely received. */ +const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; + +export function assertSafeIdentifier(value: string, what: string): string { + if (!IDENT_RE.test(value)) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`); + } + return value; +} + +export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const; +export type MetricAggregate = (typeof METRIC_AGGREGATES)[number]; + +/** + * The dimensions a caller is allowed to name, resolved from + * `DimensionDefinition` rows. The compiler looks a caller's dimension name up + * HERE; a miss is a rejection, never a pass-through. + */ +export interface DimensionCatalog { + /** caller-facing dimension name -> physical dimension key in dim_names. */ + resolve(name: string): string | undefined; +} + +export function dimensionCatalogFrom(pairs: Record): DimensionCatalog { + const map = new Map(Object.entries(pairs)); + return { + resolve(name: string): string | undefined { + // Own-property lookup on a Map — no prototype-chain surprises + // ("constructor", "__proto__" resolve to undefined, as they must). + return map.get(name); + }, + }; +} + +export interface MetricQueryRequest { + /** Supplied by the engine from the verified identity, NOT by the caller. */ + tenant_id: string; + project_id?: string; + dataset_id?: string; + metric: string; + aggregate: MetricAggregate; + /** Caller-facing dimension names; resolved through the catalog. */ + group_by?: string[]; + /** Equality filters on dimensions: caller-facing name -> value. */ + filters?: Record; + from_time?: string; + to_time?: string; + limit?: number; + /** Which BI-P010 table to read. Chosen by the engine, not the caller. */ + table?: AnalyticsTable; +} + +export interface CompiledQuery { + sql: string; + /** ClickHouse HTTP `param_*` entries. Values NEVER appear in `sql`. */ + params: Record; + /** The predicate the compiler forced, for assertion in tests. */ + forcedTenantPredicate: string; + table: AnalyticsTable; +} + +/** + * The exact predicate text every compiled query must contain. Exported so a + * regression test can assert on the literal string — if someone deletes the + * forcing, the test fails on a missing substring rather than on a subtle + * behavioural difference that a mocked engine would happily hide. + */ +export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}"; + +const AGG_SQL: Record = { + sum: "sum(value)", + avg: "avg(value)", + min: "min(value)", + max: "max(value)", + count: "count()", + count_distinct: "uniqExact(event_id)", +}; + +const MAX_GROUP_BY = 8; + +/** + * Compile a closed-DSL metric request into tenant-forced SQL. + * + * @param req the request; `tenant_id` MUST already be the verified identity. + * @param db the analytics database name (engine-controlled, validated). + * @param catalog the tenant's declared dimensions. + * @param maxResultRows the governance ceiling (BI-P012); `limit` is clamped. + */ +export function compileMetricQuery( + req: MetricQueryRequest, + db: string, + catalog: DimensionCatalog, + maxResultRows: number, +): CompiledQuery { + if (!req.tenant_id || !req.tenant_id.trim()) { + // Fail closed. A blank tenant must never compile to "no predicate". + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); + } + assertSafeIdentifier(db, "database"); + const table: AnalyticsTable = req.table ?? "opt_analytics_event"; + assertSafeIdentifier(table, "table"); + if (!METRIC_AGGREGATES.includes(req.aggregate)) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate"); + } + + const params: Record = { + param_tenant: req.tenant_id, + param_metric: req.metric, + }; + + // ── the forced predicate. First condition, always, no branch. ───────── + const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"]; + + if (req.project_id) { + conditions.push("project_id = {project:String}"); + params.param_project = req.project_id; + } + if (req.dataset_id) { + conditions.push("dataset_id = {dataset:String}"); + params.param_dataset = req.dataset_id; + } + if (req.from_time) { + conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})"); + params.param_fromTime = req.from_time; + } + if (req.to_time) { + conditions.push("event_time < parseDateTime64BestEffort({toTime:String})"); + params.param_toTime = req.to_time; + } + + // ── filters: names resolved through the catalog, values parameterized ── + let filterIdx = 0; + for (const [name, value] of Object.entries(req.filters ?? {})) { + const physical = catalog.resolve(name); + if (physical === undefined) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`); + } + // `physical` came from the catalog, but validate anyway: a corrupted or + // hostile catalog row must not become an injection vector either. + assertSafeIdentifier(physical, "dimension"); + const nameKey = `f${filterIdx}n`; + const valKey = `f${filterIdx}v`; + conditions.push( + `dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`, + ); + params[`param_${nameKey}`] = physical; + params[`param_${valKey}`] = value; + filterIdx += 1; + } + + // ── group by: same resolution, plus a hard cardinality ceiling ───────── + const groupBy = req.group_by ?? []; + if (groupBy.length > MAX_GROUP_BY) { + throw new SiaxError( + "INVALID_REQUEST", + "0pt.analytics", + `at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`, + ); + } + const selectParts: string[] = []; + const groupParts: string[] = []; + groupBy.forEach((name, i) => { + const physical = catalog.resolve(name); + if (physical === undefined) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`); + } + assertSafeIdentifier(physical, "dimension"); + const key = `g${i}`; + // The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text. + selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`); + groupParts.push(`grp${i}`); + params[`param_${key}`] = physical; + }); + + const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); + + const sql = + `SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` + + `FROM ${db}.${table} ` + + `WHERE ${conditions.join(" AND ")} ` + + (groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") + + `LIMIT ${limit} FORMAT JSON`; + + return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table }; +} + +/** + * Compile a projection lookup (SERVING tier, BI-P010's opt_dataset_projection). + * Same forcing, different table and key. + */ +export function compileProjectionQuery( + req: { tenant_id: string; project_id?: string; dataset_id: string; entity_keys?: string[]; limit?: number }, + db: string, + maxResultRows: number, +): CompiledQuery { + if (!req.tenant_id || !req.tenant_id.trim()) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); + } + assertSafeIdentifier(db, "database"); + const params: Record = { + param_tenant: req.tenant_id, + param_dataset: req.dataset_id, + }; + const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"]; + if (req.project_id) { + conditions.push("project_id = {project:String}"); + params.param_project = req.project_id; + } + if (req.entity_keys && req.entity_keys.length > 0) { + conditions.push("has({entityKeys:Array(String)}, entity_key)"); + params.param_entityKeys = JSON.stringify(req.entity_keys); + } + const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); + const sql = + `SELECT entity_key, as_of, version, attributes, measures ` + + `FROM ${db}.opt_dataset_projection ` + + `WHERE ${conditions.join(" AND ")} ` + + `ORDER BY entity_key LIMIT ${limit} FORMAT JSON`; + return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" }; +} + +/** + * BI-P011 layer-3 bootstrap: the DDL for a RESTRICTED per-tenant ClickHouse + * user with a row policy pinning `tenant_id`. + * + * This is the layer that survives a compiler bug. The compiler forcing the + * predicate is layer 2; if a future refactor drops it, layer 3 still returns + * zero foreign rows because the SERVER filters them. The BI-P011 proof runs + * the adversarial suite twice — once through an unrestricted user (hole open) + * and once through this restricted user (hole closed) — because a claim of + * defence in depth that has never been tested with one layer removed is not + * evidence. + * + * `tenantId` is validated as an identifier-safe token AND embedded as a + * quoted literal; ClickHouse's CREATE ROW POLICY takes no bound parameters, + * so validation is the only protection available and it is applied strictly. + */ +export function restrictedTenantUserDdl(args: { + db: string; + tenantId: string; + userName: string; + password: string; + tables: readonly string[]; + settingsProfile?: string; +}): string[] { + assertSafeIdentifier(args.db, "database"); + assertSafeIdentifier(args.userName, "user"); + // A tenant id is `t_` in this estate; enforce that shape strictly + // rather than trusting the caller. + if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe"); + } + if (args.password.includes("'") || args.password.includes("\\")) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded"); + } + const profile = args.settingsProfile; + if (profile) assertSafeIdentifier(profile, "settings profile"); + + const stmts: string[] = [ + `CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` + + (profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`), + ]; + for (const table of args.tables) { + assertSafeIdentifier(table, "table"); + stmts.push( + `CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` + + `FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`, + ); + stmts.push(`GRANT SELECT ON ${args.db}.${table} TO ${args.userName}`); + } + return stmts; +} diff --git a/src/analytics/engine/resource-governance.ts b/src/analytics/engine/resource-governance.ts new file mode 100644 index 0000000..4d2e184 --- /dev/null +++ b/src/analytics/engine/resource-governance.ts @@ -0,0 +1,222 @@ +/** + * src/analytics/engine/resource-governance.ts + * ============================================ + * BI-P012 — CLICKHOUSE RESOURCE GOVERNANCE. + * + * Every setting BI-P012 names, in one place, with a value: + * + * max_execution_time max_result_rows max_rows_to_read + * max_bytes_to_read max_memory_usage max_concurrent_queries + * + * Three layers, and the order matters: + * + * 1. SERVER-SIDE SETTINGS travel with every statement on the HTTP + * interface. ClickHouse enforces them; the client cannot opt out + * because the client is what writes them. `readonly=1` on top means a + * SELECT cannot re-set them mid-query (readonly=1 forbids SETTINGS + * changes as well as writes — that is the whole reason it is here and + * not merely `readonly=2`). + * 2. `max_concurrent_queries` is NOT a per-query setting in ClickHouse; it + * is a per-USER quota. `concurrencySlotSql` renders the CREATE + * SETTINGS PROFILE that carries it, applied to the tenant's restricted + * user (BI-P011 layer 3) so one tenant's fan-out cannot consume the + * whole server. + * 3. A local admission gate (`ConcurrencyGate`) refuses the N+1st + * in-flight query per tenant before it ever reaches the network. Two + * layers because the ClickHouse quota is per-user and one 0pt process + * can still queue work faster than the server rejects it. + * + * Budget tiers exist because "one limit for everyone" is how a small tenant + * subsidises a query bomb. `INTERACTIVE` is the dashboard default and is + * deliberately mean. + */ +import { SiaxError } from "../../platform/errors"; + +export interface ResourceBudget { + /** Seconds. ClickHouse `max_execution_time`. */ + maxExecutionTimeSec: number; + /** Rows returned to the caller. ClickHouse `max_result_rows`. */ + maxResultRows: number; + /** Rows SCANNED (not returned). ClickHouse `max_rows_to_read`. */ + maxRowsToRead: number; + /** Bytes scanned. ClickHouse `max_bytes_to_read`. */ + maxBytesToRead: number; + /** Bytes of RAM per query. ClickHouse `max_memory_usage`. */ + maxMemoryUsageBytes: number; + /** In-flight queries per tenant. Enforced by quota + local gate. */ + maxConcurrentQueries: number; +} + +export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const; +export type BudgetTier = (typeof BUDGET_TIERS)[number]; + +/** + * Defaults. Chosen against the shape of the analytics tables in + * `src/analytics/engine/schema.ts`, not plucked from the air: + * INTERACTIVE must answer a dashboard panel inside a human's patience budget, + * so it caps the SCAN, which is what actually costs time on a MergeTree. + */ +export const DEFAULT_BUDGETS: Record = { + INTERACTIVE: { + maxExecutionTimeSec: 10, + maxResultRows: 10_000, + maxRowsToRead: 50_000_000, + maxBytesToRead: 2 * 1024 * 1024 * 1024, + maxMemoryUsageBytes: 2 * 1024 * 1024 * 1024, + maxConcurrentQueries: 8, + }, + ANALYTICAL: { + maxExecutionTimeSec: 60, + maxResultRows: 500_000, + maxRowsToRead: 2_000_000_000, + maxBytesToRead: 32 * 1024 * 1024 * 1024, + maxMemoryUsageBytes: 8 * 1024 * 1024 * 1024, + maxConcurrentQueries: 4, + }, + BATCH: { + maxExecutionTimeSec: 900, + maxResultRows: 10_000_000, + maxRowsToRead: 50_000_000_000, + maxBytesToRead: 512 * 1024 * 1024 * 1024, + maxMemoryUsageBytes: 24 * 1024 * 1024 * 1024, + maxConcurrentQueries: 2, + }, +}; + +/** + * A CL0UD-issued quota may only ever TIGHTEN a tier, never loosen it. A + * control-plane bug that hands out a generous number must not be able to + * disarm the engine's own ceiling. + */ +export function applyQuota(base: ResourceBudget, quota: Partial | undefined): ResourceBudget { + if (!quota) return { ...base }; + const pick = (b: number, q: number | undefined): number => + typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b; + return { + maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec), + maxResultRows: pick(base.maxResultRows, quota.maxResultRows), + maxRowsToRead: pick(base.maxRowsToRead, quota.maxRowsToRead), + maxBytesToRead: pick(base.maxBytesToRead, quota.maxBytesToRead), + maxMemoryUsageBytes: pick(base.maxMemoryUsageBytes, quota.maxMemoryUsageBytes), + maxConcurrentQueries: pick(base.maxConcurrentQueries, quota.maxConcurrentQueries), + }; +} + +/** + * The settings that ride along on the HTTP request. `readonly: "1"` is what + * makes them un-overridable from inside the query text. + * + * `timeout_overflow_mode` / `result_overflow_mode` / `read_overflow_mode` are + * set to `throw` EXPLICITLY: ClickHouse's default for some of these is + * `break`, which silently returns a PARTIAL result. A partial result that + * looks complete is the analytics equivalent of a false green, so the engine + * refuses it. + */ +export function clickhouseSettings(budget: ResourceBudget): Record { + return { + readonly: "1", + max_execution_time: String(budget.maxExecutionTimeSec), + timeout_overflow_mode: "throw", + max_result_rows: String(budget.maxResultRows), + result_overflow_mode: "throw", + max_rows_to_read: String(budget.maxRowsToRead), + max_bytes_to_read: String(budget.maxBytesToRead), + read_overflow_mode: "throw", + max_memory_usage: String(budget.maxMemoryUsageBytes), + }; +} + +/** + * Settings for a PLANNER call (`EXPLAIN ESTIMATE`, `EXPLAIN PLAN`). + * + * Measured, not assumed: running `EXPLAIN ESTIMATE` under the caller's own + * `max_rows_to_read` makes ClickHouse refuse the ESTIMATE itself with code 158 + * whenever the estimate would be over budget — i.e. exactly the case the + * estimate exists to detect. An estimator that cannot describe an expensive + * query is useless, so the read ceilings are deliberately NOT applied here. + * The planner still runs `readonly = 1` under a short deadline and a memory + * cap, so this is a narrower envelope, never an unbounded one. + */ +export function planningSettings(budget: ResourceBudget): Record { + return { + readonly: "1", + max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)), + timeout_overflow_mode: "throw", + max_memory_usage: String(budget.maxMemoryUsageBytes), + }; +} + +/** Every setting name BI-P012 requires, for the DoD assertion. */ +export const REQUIRED_GOVERNANCE_SETTINGS = [ + "max_execution_time", + "max_result_rows", + "max_rows_to_read", + "max_bytes_to_read", + "max_memory_usage", + "max_concurrent_queries", +] as const; + +/** + * `max_concurrent_queries` lives on a SETTINGS PROFILE, not on a statement. + * Rendered here so BI-P011's restricted-user bootstrap and BI-P012's budget + * come from one source. + */ +export function concurrencyProfileSql(profileName: string, budget: ResourceBudget): string[] { + return [ + `CREATE SETTINGS PROFILE IF NOT EXISTS ${profileName} SETTINGS ` + + `max_execution_time = ${budget.maxExecutionTimeSec}, ` + + `max_result_rows = ${budget.maxResultRows}, ` + + `max_rows_to_read = ${budget.maxRowsToRead}, ` + + `max_bytes_to_read = ${budget.maxBytesToRead}, ` + + `max_memory_usage = ${budget.maxMemoryUsageBytes}, ` + + `readonly = 1`, + `CREATE QUOTA IF NOT EXISTS ${profileName}_quota ` + + `FOR INTERVAL 1 SECOND MAX queries = ${budget.maxConcurrentQueries * 20} ` + + `TO ALL EXCEPT default`, + ]; +} + +/** + * Local admission control. One instance per process; keyed by tenant so a + * noisy tenant is throttled without touching anyone else's slots — which is + * exactly BI-P012's DoD wording ("without degrading unrelated tenant"). + */ +export class ConcurrencyGate { + private readonly inFlight = new Map(); + + constructor(private readonly limitFor: (tenantId: string) => number) {} + + /** Returns a release function. Throws RATE_LIMITED when the tenant is full. */ + acquire(tenantId: string): () => void { + const limit = this.limitFor(tenantId); + const current = this.inFlight.get(tenantId) ?? 0; + if (current >= limit) { + throw new SiaxError( + "RATE_LIMITED", + "0pt.analytics", + `tenant already has ${current} analytics queries in flight (limit ${limit})`, + ); + } + this.inFlight.set(tenantId, current + 1); + let released = false; + return () => { + if (released) return; + released = true; + const n = (this.inFlight.get(tenantId) ?? 1) - 1; + if (n <= 0) this.inFlight.delete(tenantId); + else this.inFlight.set(tenantId, n); + }; + } + + /** Observability: in-flight count for a tenant (0 when idle). */ + inFlightFor(tenantId: string): number { + return this.inFlight.get(tenantId) ?? 0; + } + + /** Total in-flight across all tenants. */ + totalInFlight(): number { + let n = 0; + for (const v of this.inFlight.values()) n += v; + return n; + } +} diff --git a/src/analytics/engine/schema.ts b/src/analytics/engine/schema.ts new file mode 100644 index 0000000..657d1fb --- /dev/null +++ b/src/analytics/engine/schema.ts @@ -0,0 +1,169 @@ +/** + * src/analytics/engine/schema.ts + * =============================== + * BI-P010 — CLICKHOUSE PRODUCTION SCHEMA. + * + * BI-P010's explicit warning is "Avoid one giant unbounded table for every + * analytics use case." The existing `opt_metric_events` (owned by + * src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one + * table for one job: low-cardinality capability metrics. It stays, and it is + * NOT extended into the catch-all. + * + * Four tables, four jobs, four retention profiles: + * + * opt_metric_events (PRE-EXISTING, not defined here) — capability + * metrics, Map(String,String) dimensions, 90d. + * opt_analytics_event — high-volume per-tenant event stream. Wide + * dimension arrays, SHORT TTL (raw events are not + * the serving tier; they age into the rollups). + * opt_metric_daily — AggregatingMergeTree rollup fed by a + * MATERIALIZED VIEW off opt_analytics_event. + * This is what a dashboard time-series reads; + * long TTL, tiny footprint. + * opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED + * dataset: one row per (tenant, dataset, entity), + * ReplacingMergeTree so a refresh is idempotent. + * + * Why the split is load-bearing, not cosmetic: + * * different ORDER BY keys. A dashboard filtering (tenant, metric, day) + * and a projection looked up by (tenant, dataset, entity_key) cannot + * share one primary key without one of them degenerating to a full scan. + * * different TTLs. Raw events must expire fast; rollups must not. + * * different cardinality. `LowCardinality(String)` is a win on + * tenant/metric/dataset and a LOSS on a free-form entity key, so they + * cannot be the same column type in the same table. + * + * EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it + * is what makes the forced tenant predicate (BI-P011) a primary-key range + * scan instead of a filter applied after reading everyone's rows, and it is + * what makes a per-tenant row policy cheap. + * + * LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this + * schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts + * ClickHouse pipelines or quarantined market data. + */ + +export interface AnalyticsSchemaOptions { + database: string; + /** Raw event retention. Deliberately short — rollups carry the history. */ + eventRetentionDays?: number; + /** Rollup retention. */ + rollupRetentionDays?: number; + /** Serving-projection retention. */ + projectionRetentionDays?: number; +} + +/** Physical tables this module owns (opt_metric_events is NOT one of them). */ +export const ANALYTICS_TABLES = [ + "opt_analytics_event", + "opt_metric_daily", + "opt_dataset_projection", +] as const; +export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number]; + +/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */ +export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const; + +/** + * DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so + * applying it to a live database is idempotent (the same contract + * `ClickHouseAnalytics.ensureSchema` already keeps). + */ +export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] { + const db = opts.database; + const eventTtl = Math.floor(opts.eventRetentionDays ?? 30); + const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095); + const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365); + + return [ + `CREATE DATABASE IF NOT EXISTS ${db}`, + + // ── raw event stream ──────────────────────────────────────────────── + // Dimensions as parallel arrays (not Map) so ClickHouse can prune on + // dimension VALUES via the bloom filter index; Map(String,String) forces + // reading the whole map. `event_id` makes deduplication possible on + // replay without a ReplacingMergeTree's merge latency. + `CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event ( + tenant_id LowCardinality(String), + project_id LowCardinality(String), + dataset_id LowCardinality(String), + metric LowCardinality(String), + event_id String, + event_time DateTime64(3, 'UTC'), + value Float64, + dim_names Array(LowCardinality(String)), + dim_values Array(String), + ingested_at DateTime64(3, 'UTC') DEFAULT now64(3), + INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4 + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(event_time) + ORDER BY (tenant_id, project_id, dataset_id, metric, event_time) + TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY + SETTINGS index_granularity = 8192`, + + // ── daily rollup (serving tier for time series) ───────────────────── + `CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily ( + tenant_id LowCardinality(String), + project_id LowCardinality(String), + dataset_id LowCardinality(String), + metric LowCardinality(String), + day Date, + sum_state AggregateFunction(sum, Float64), + count_state AggregateFunction(count, Float64), + min_state AggregateFunction(min, Float64), + max_state AggregateFunction(max, Float64) + ) + ENGINE = AggregatingMergeTree() + PARTITION BY toYYYYMM(day) + ORDER BY (tenant_id, project_id, dataset_id, metric, day) + TTL day + INTERVAL ${rollupTtl} DAY`, + + `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv + TO ${db}.opt_metric_daily AS + SELECT + tenant_id, + project_id, + dataset_id, + metric, + toDate(event_time) AS day, + sumState(value) AS sum_state, + countState(value) AS count_state, + minState(value) AS min_state, + maxState(value) AS max_state + FROM ${db}.opt_analytics_event + GROUP BY tenant_id, project_id, dataset_id, metric, day`, + + // ── serving projection for a CURATED dataset ──────────────────────── + // ReplacingMergeTree(version) so refreshProjection is idempotent: the + // same (tenant, dataset, entity_key) re-ingested at a higher version + // supersedes rather than duplicates. + `CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection ( + tenant_id LowCardinality(String), + project_id LowCardinality(String), + dataset_id LowCardinality(String), + entity_key String, + as_of DateTime64(3, 'UTC'), + version UInt64, + attributes Map(String, String), + measures Map(String, Float64), + INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4 + ) + ENGINE = ReplacingMergeTree(version) + PARTITION BY (tenant_id) + ORDER BY (tenant_id, project_id, dataset_id, entity_key) + TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`, + ]; +} + +/** + * Statement that proves a table's ORDER BY leads with tenant_id — used by the + * BI-P011 test suite so "tenant-first primary key" is asserted against the + * live server, not against this file's own comments. + */ +export function sortingKeySql(db: string, table: string): string { + return ( + `SELECT sorting_key FROM system.tables ` + + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` + ); +} diff --git a/src/analytics/engine/siax-analytics-engine.ts b/src/analytics/engine/siax-analytics-engine.ts new file mode 100644 index 0000000..53ac17a --- /dev/null +++ b/src/analytics/engine/siax-analytics-engine.ts @@ -0,0 +1,180 @@ +/** + * src/analytics/engine/siax-analytics-engine.ts + * ============================================== + * BI-P009 — the provider-neutral analytics engine interface. + * + * Eight capabilities, named exactly as BI-P009 lists them: + * + * health executeMetric executeQuery estimateQuery + * explain cancelQuery ingestProjection refreshProjection + * + * ClickHouse is the CANONICAL provider (BI-P009); DuckDB (BI-P014) and Trino + * (BI-P015) implement the same interface later. Nothing above this interface + * may import a provider module directly — that is what makes "replaceable + * provider" true rather than aspirational. + * + * NORMATIVE OWNERSHIP (BI program §0): every method takes an + * `AnalyticsIdentity` that the CALLER cannot forge, because it is produced by + * CL0UD verification upstream, not by request-body parsing. Product repos + * (VehIQ) call an API in front of this interface; they never hold a provider + * credential and never construct an engine. + */ +import type { CompiledQuery, MetricAggregate } from "./query-compiler"; +import type { AnalyticsTable } from "./schema"; +import type { BudgetTier, ResourceBudget } from "./resource-governance"; + +/** + * The VERIFIED caller. `tenant_id` here is the one the engine forces into + * every predicate — it comes from CL0UD identity verification (BI-P011 layer + * 1), never from a request body. This estate has already been burned once by + * a tenant read out of a request body (INF0 Wave 2, cross-tenant RAG access), + * which is why the type is separate from the request rather than a field on + * it. + */ +export interface AnalyticsIdentity { + tenant_id: string; + project_id?: string; + /** CL0UD principal, for audit + per-principal quota. */ + principal_id: string; + correlation_id?: string; + /** Budget tier granted by CL0UD entitlements (BI-P012). */ + budget_tier?: BudgetTier; + /** CL0UD quota, which may only TIGHTEN the tier. */ + quota?: Partial; +} + +export interface MetricRequest { + metric: string; + aggregate: MetricAggregate; + dataset_id?: string; + group_by?: string[]; + filters?: Record; + from_time?: string; + to_time?: string; + limit?: number; + table?: AnalyticsTable; +} + +export interface AnalyticsRow { + /** Resolved group values, keyed by the CALLER-FACING dimension name. */ + groups?: Record; + value: number; +} + +export interface EngineResult { + rows: AnalyticsRow[]; + /** Engine-side id; the handle `cancelQuery` needs. */ + query_id: string; + /** True when the governance ceiling cut the result short (BI-P012). */ + truncated: boolean; + stats: QueryStats; +} + +export interface QueryStats { + duration_ms: number; + rows_read?: number; + bytes_read?: number; + result_rows: number; +} + +export interface QueryEstimate { + /** Rows the engine expects to SCAN. */ + estimated_rows_to_read: number; + estimated_bytes_to_read: number; + /** False when the estimate exceeds the caller's budget (BI-P012). */ + within_budget: boolean; + budget: ResourceBudget; + /** Why it was refused, when `within_budget` is false. */ + reason?: string; +} + +export interface QueryExplanation { + /** Provider's plan text. Never contains caller-supplied SQL fragments. */ + plan: string; + /** The predicate the compiler forced — assertable, not narrated. */ + forced_tenant_predicate: string; + table: string; +} + +export interface ProjectionRow { + entity_key: string; + as_of?: string; + version?: number; + attributes?: Record; + measures?: Record; +} + +export interface ProjectionIngestRequest { + dataset_id: string; + rows: ProjectionRow[]; +} + +export interface ProjectionRefreshResult { + dataset_id: string; + /** Rows visible after the refresh settled. */ + row_count: number; + refreshed_at: string; +} + +export interface HealthReport { + healthy: boolean; + provider: string; + /** Provider version when reachable. */ + version?: string; + /** Short operator-facing reason. Never a raw engine body. */ + detail?: string; +} + +/** + * A raw, ANALYST-facing query (BI-P017 will govern its SQL dialect). It is a + * SEPARATE method from `executeMetric` on purpose: `executeMetric` has no + * raw-SQL surface at all, and keeping them apart means the closed DSL cannot + * silently grow one. + * + * The canonical ClickHouse provider in this pass DELIBERATELY refuses this + * method (`CAPABILITY_UNAVAILABLE`): opening an analyst SQL surface before + * BI-P017's dialect contract and BI-P033's authorization exist would be + * exactly the "raw SQL attempt" BI-P011's adversarial list names. Declaring + * it and refusing it is honest; implementing it early is not. + */ +export interface AnalystQueryRequest { + sql: string; + dataset_ids: string[]; + limit?: number; +} + +export interface SiaxAnalyticsEngine { + readonly provider: string; + + health(): Promise; + + executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise; + + executeQuery(identity: AnalyticsIdentity, request: AnalystQueryRequest): Promise; + + estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise; + + cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise; + + explain(identity: AnalyticsIdentity, request: MetricRequest): Promise; + + ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>; + + refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise; +} + +/** The eight capability names, for the provider contract test. */ +export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [ + "health", + "executeMetric", + "executeQuery", + "estimateQuery", + "cancelQuery", + "explain", + "ingestProjection", + "refreshProjection", +] as const; +export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number]; + +/** Re-exported so provider modules do not reach across into the compiler. */ +export type { CompiledQuery }; diff --git a/src/analytics/repositories/analytics-domain-repository.ts b/src/analytics/repositories/analytics-domain-repository.ts new file mode 100644 index 0000000..8b01c37 --- /dev/null +++ b/src/analytics/repositories/analytics-domain-repository.ts @@ -0,0 +1,356 @@ +/** + * src/analytics/repositories/analytics-domain-repository.ts + * ========================================================== + * BI-P002 persistence for the analytics domain model (sql/008). + * + * Two rules the type system alone cannot keep, so they live here: + * + * 1. EVERY read and write is tenant-scoped. There is no method that takes a + * dataset_id without also taking the tenant it must belong to, and every + * WHERE carries `tenant_id = $tenant`. A caller cannot "forget" the + * tenant because there is no signature that lets them. + * 2. VERSION ROWS ARE APPEND-ONLY from this layer too. `createDatasetVersion` + * /`createMetricVersion` compute the next version number inside the same + * statement (`SELECT coalesce(max(version),0)+1`) so two concurrent + * minters collide on the UNIQUE(dataset_id, version) constraint instead + * of silently overwriting — and the database triggers from sql/008 catch + * anything that still tries to UPDATE. + * + * Uses postgres.js through `RepositorySql`, the same connection type the other + * P0 repositories take (src/repositories/db.ts). + */ +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { RepositorySql } from "../../repositories/db"; +import type { + AnalyticsDataset, + DatasetVersion, + DimensionDefinition, + MetricDefinition, + MetricVersion, + PiiClassification, +} from "../domain/entities"; +import type { AnalyticsTier } from "../domain/tiers"; + +/** The verified tenant scope every call must present. */ +export interface TenantScope { + tenant_id: string; + project_id: string; +} + +function assertScope(scope: TenantScope): void { + if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) { + throw new SiaxError( + "UNAUTHENTICATED", + "0pt.analytics", + "analytics persistence requires a verified tenant_id + project_id", + ); + } +} + +function iso(v: string | Date | null | undefined): string | undefined { + if (v === null || v === undefined) return undefined; + return v instanceof Date ? v.toISOString() : v; +} + +interface DatasetRow { + dataset_id: string; + tenant_id: string; + project_id: string; + name: string; + tier: string; + schema_contract_id: string; + schema_contract_version: string; + owner_id: string; + source_ref: string; + freshness_slo_seconds: number; + quality_rule_ids: string[]; + retention_days: number; + classification: string; + status: string; + raw_access_exception: string | null; + description: string | null; + created_at: string | Date; + updated_at: string | Date | null; + created_by: string | null; +} + +function toDataset(r: DatasetRow): AnalyticsDataset { + return { + dataset_id: r.dataset_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + name: r.name, + tier: r.tier as AnalyticsTier, + schema_contract_id: r.schema_contract_id, + schema_contract_version: r.schema_contract_version, + owner_id: r.owner_id, + source_ref: r.source_ref, + freshness_slo_seconds: Number(r.freshness_slo_seconds), + quality_rule_ids: r.quality_rule_ids ?? [], + retention_days: Number(r.retention_days), + classification: r.classification as PiiClassification, + status: r.status as AnalyticsDataset["status"], + ...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}), + ...(r.description ? { description: r.description } : {}), + created_at: iso(r.created_at)!, + ...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}), + ...(r.created_by ? { created_by: r.created_by } : {}), + }; +} + +export class AnalyticsDomainRepository { + constructor(private readonly sql: RepositorySql) {} + + // ── AnalyticsDataset ───────────────────────────────────────────────────── + + async createDataset(scope: TenantScope, input: Omit): Promise { + assertScope(scope); + const rows = await this.sql` + INSERT INTO opt_analytics_dataset + (dataset_id, tenant_id, project_id, name, tier, schema_contract_id, schema_contract_version, + owner_id, source_ref, freshness_slo_seconds, quality_rule_ids, retention_days, + classification, status, raw_access_exception, description, created_by) + VALUES + (${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.tier}, + ${input.schema_contract_id}, ${input.schema_contract_version}, ${input.owner_id}, ${input.source_ref}, + ${input.freshness_slo_seconds}, ${this.sql.array(input.quality_rule_ids ?? [])}, ${input.retention_days}, + ${input.classification}, ${input.status}, ${input.raw_access_exception ?? null}, + ${input.description ?? null}, ${input.created_by ?? null}) + RETURNING *`; + return toDataset(rows[0]); + } + + /** Tenant-scoped read. A foreign dataset_id returns undefined, never a row. */ + async getDataset(scope: TenantScope, datasetId: string): Promise { + assertScope(scope); + const rows = await this.sql` + SELECT * FROM opt_analytics_dataset + WHERE dataset_id = ${datasetId} + AND tenant_id = ${scope.tenant_id} + AND project_id = ${scope.project_id} + LIMIT 1`; + return rows[0] ? toDataset(rows[0]) : undefined; + } + + async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise { + assertScope(scope); + const rows = await this.sql` + SELECT * FROM opt_analytics_dataset + WHERE tenant_id = ${scope.tenant_id} + AND project_id = ${scope.project_id} + ${filter?.tier ? this.sql`AND tier = ${filter.tier}` : this.sql``} + ${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``} + ORDER BY created_at DESC`; + return rows.map(toDataset); + } + + // ── DatasetVersion (append-only) ───────────────────────────────────────── + + /** + * Mint the next version. The version number is computed inside the INSERT, + * so two racing callers cannot both read "3" and both write it — the loser + * hits UNIQUE(dataset_id, version) and gets a CONFLICT, which is the honest + * outcome. + */ + async createDatasetVersion( + scope: TenantScope, + input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string }, + ): Promise { + assertScope(scope); + const owner = await this.getDataset(scope, input.dataset_id); + if (!owner) { + throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant"); + } + const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`; + try { + const rows = await this.sql< + { + dataset_version_id: string; + dataset_id: string; + tenant_id: string; + project_id: string; + version: number; + contract_hash: string; + schema_contract_version: string; + record_count: string | number | null; + created_at: string | Date; + created_by: string | null; + }[] + >` + INSERT INTO opt_analytics_dataset_version + (dataset_version_id, dataset_id, tenant_id, project_id, version, + contract_hash, schema_contract_version, record_count, created_by) + SELECT ${id}, ${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id}, + coalesce(max(version), 0) + 1, + ${input.contract_hash}, ${input.schema_contract_version}, + ${input.record_count ?? null}, ${input.created_by ?? null} + FROM opt_analytics_dataset_version + WHERE dataset_id = ${input.dataset_id} AND tenant_id = ${scope.tenant_id} + RETURNING *`; + const r = rows[0]; + return { + dataset_version_id: r.dataset_version_id, + dataset_id: r.dataset_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + version: Number(r.version), + contract_hash: r.contract_hash, + schema_contract_version: r.schema_contract_version, + ...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}), + created_at: iso(r.created_at)!, + ...(r.created_by ? { created_by: r.created_by } : {}), + }; + } catch (err) { + if (String(err).includes("duplicate key")) { + throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version"); + } + throw err; + } + } + + async listDatasetVersions(scope: TenantScope, datasetId: string): Promise { + assertScope(scope); + const rows = await this.sql< + { dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[] + >` + SELECT * FROM opt_analytics_dataset_version + WHERE dataset_id = ${datasetId} + AND tenant_id = ${scope.tenant_id} + AND project_id = ${scope.project_id} + ORDER BY version ASC`; + return rows.map((r) => ({ + dataset_version_id: r.dataset_version_id, + dataset_id: r.dataset_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + version: Number(r.version), + contract_hash: r.contract_hash, + schema_contract_version: r.schema_contract_version, + ...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}), + created_at: iso(r.created_at)!, + ...(r.created_by ? { created_by: r.created_by } : {}), + })); + } + + // ── MetricDefinition / MetricVersion ───────────────────────────────────── + + async createMetric( + scope: TenantScope, + input: Omit, + ): Promise { + assertScope(scope); + const rows = await this.sql< + { metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[] + >` + INSERT INTO opt_analytics_metric_definition + (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, + denominator_field, allowed_dimension_ids, current_version, owner_id, description) + VALUES + (${input.metric_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id}, + ${input.aggregation}, ${input.measure_field}, ${input.denominator_field ?? null}, + ${this.sql.array(input.allowed_dimension_ids ?? [])}, 1, ${input.owner_id}, ${input.description ?? null}) + RETURNING *`; + const r = rows[0]; + return { + metric_id: r.metric_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + name: r.name, + dataset_id: r.dataset_id, + aggregation: r.aggregation as MetricDefinition["aggregation"], + measure_field: r.measure_field, + ...(r.denominator_field ? { denominator_field: r.denominator_field } : {}), + allowed_dimension_ids: r.allowed_dimension_ids ?? [], + current_version: Number(r.current_version), + owner_id: r.owner_id, + ...(r.description ? { description: r.description } : {}), + created_at: iso(r.created_at)!, + ...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}), + }; + } + + /** Append-only. `change_reason` is required for version > 1 (DB CHECK). */ + async createMetricVersion( + scope: TenantScope, + input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string }, + ): Promise { + assertScope(scope); + const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex"); + const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`; + const rows = await this.sql< + { metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[] + >` + INSERT INTO opt_analytics_metric_version + (metric_version_id, metric_id, tenant_id, project_id, version, definition_json, + definition_hash, change_reason, created_by) + SELECT ${id}, ${input.metric_id}, ${scope.tenant_id}, ${scope.project_id}, + coalesce(max(version), 0) + 1, ${input.definition_json}, ${hash}, + ${input.change_reason ?? null}, ${input.created_by ?? null} + FROM opt_analytics_metric_version + WHERE metric_id = ${input.metric_id} AND tenant_id = ${scope.tenant_id} + RETURNING *`; + const r = rows[0]; + return { + metric_version_id: r.metric_version_id, + metric_id: r.metric_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + version: Number(r.version), + definition_json: r.definition_json, + definition_hash: r.definition_hash, + ...(r.change_reason ? { change_reason: r.change_reason } : {}), + created_at: iso(r.created_at)!, + ...(r.created_by ? { created_by: r.created_by } : {}), + }; + } + + // ── DimensionDefinition (the query compiler's allow-list) ──────────────── + + async createDimension( + scope: TenantScope, + input: Omit, + ): Promise { + assertScope(scope); + const rows = await this.sql< + { dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[] + >` + INSERT INTO opt_analytics_dimension_definition + (dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description) + VALUES + (${input.dimension_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id}, + ${input.field_name}, ${input.field_type}, ${input.classification}, ${input.description ?? null}) + RETURNING *`; + const r = rows[0]; + return { + dimension_id: r.dimension_id, + tenant_id: r.tenant_id, + project_id: r.project_id, + name: r.name, + dataset_id: r.dataset_id, + field_name: r.field_name, + field_type: r.field_type as DimensionDefinition["field_type"], + classification: r.classification as PiiClassification, + ...(r.description ? { description: r.description } : {}), + created_at: iso(r.created_at)!, + }; + } + + /** + * The dimension allow-list the compiler resolves through, for one tenant. + * Returns caller-facing name -> physical field name. A tenant with no + * declared dimensions gets an EMPTY map, which means every dimension + * reference is refused — fail closed, not "allow everything". + */ + async dimensionCatalog(scope: TenantScope, datasetId?: string): Promise> { + assertScope(scope); + const rows = await this.sql<{ name: string; field_name: string }[]>` + SELECT name, field_name FROM opt_analytics_dimension_definition + WHERE tenant_id = ${scope.tenant_id} + AND project_id = ${scope.project_id} + ${datasetId ? this.sql`AND dataset_id = ${datasetId}` : this.sql``}`; + const out: Record = Object.create(null); + for (const r of rows) out[r.name] = r.field_name; + return out; + } +} diff --git a/tests/analytics/analytics-domain-model.test.ts b/tests/analytics/analytics-domain-model.test.ts new file mode 100644 index 0000000..e1d9040 --- /dev/null +++ b/tests/analytics/analytics-domain-model.test.ts @@ -0,0 +1,189 @@ +/** + * BI-P002 (domain model) + BI-P003 (data tiers), the parts provable without + * infrastructure. The Postgres half — real tables, real immutability + * triggers, real tenant scoping — is + * tests/analytics/analytics-domain-postgres.realinfra.test.ts. + * + * The registry-vs-migration test below is the one that earns its place: it + * reads sql/008_analytics_domain_model.sql from disk and asserts a CREATE + * TABLE exists for every one of the nineteen canonical entities. A future + * rename, or an entity quietly dropped from the migration, fails HERE rather + * than at 3am against a database that turns out to be missing a table. + */ +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import { + ANALYTICS_DOMAIN_ENTITIES, + ANALYTICS_ENTITY_TABLES, + IMMUTABLE_VERSION_ENTITIES, +} from "@/src/analytics/domain/entities"; +import { + ANALYTICS_TIERS, + assertReadableByBiSurface, + assertTierLineageSane, + BI_READABLE_TIERS, + isAnalyticsTier, + TIER_RANK, +} from "@/src/analytics/domain/tiers"; + +const MIGRATION = fs.readFileSync( + path.join(process.cwd(), "sql", "008_analytics_domain_model.sql"), + "utf8", +); + +describe("BI-P002 canonical entity set", () => { + it("lists exactly the nineteen entities the program names, verbatim", () => { + expect(ANALYTICS_DOMAIN_ENTITIES).toEqual([ + "AnalyticsDataset", + "DatasetVersion", + "DatasetField", + "DatasetGrant", + "MetricDefinition", + "MetricVersion", + "DimensionDefinition", + "SemanticModel", + "AnalyticsQuery", + "QueryExecution", + "QueryResult", + "DashboardDefinition", + "DashboardVersion", + "ReportDefinition", + "ReportRun", + "DataQualityRule", + "DataQualityRun", + "LineageReference", + "BIApplicationBinding", + ]); + expect(ANALYTICS_DOMAIN_ENTITIES).toHaveLength(19); + }); + + it("sql/008 defines a table for every entity (no entity is a type without storage)", () => { + const missing: string[] = []; + for (const entity of ANALYTICS_DOMAIN_ENTITIES) { + const table = ANALYTICS_ENTITY_TABLES[entity]; + if (!new RegExp(`CREATE TABLE IF NOT EXISTS ${table}\\b`).test(MIGRATION)) { + missing.push(`${entity} -> ${table}`); + } + } + expect(missing).toEqual([]); + }); + + it("every tenant-owned table carries tenant_id AND project_id", () => { + // Split on the table definitions and check each block. + const blocks = MIGRATION.split(/CREATE TABLE IF NOT EXISTS /).slice(1); + const offenders: string[] = []; + for (const block of blocks) { + const name = block.split(/[\s(]/)[0]; + if (!Object.values(ANALYTICS_ENTITY_TABLES).includes(name)) continue; + const body = block.slice(0, block.indexOf(");")); + if (!/\btenant_id TEXT NOT NULL\b/.test(body) || !/\bproject_id TEXT NOT NULL\b/.test(body)) { + offenders.push(name); + } + } + expect(offenders).toEqual([]); + }); + + it("the four immutable version entities have a BEFORE UPDATE OR DELETE trigger in sql/008", () => { + expect(IMMUTABLE_VERSION_ENTITIES).toEqual([ + "DatasetVersion", + "MetricVersion", + "SemanticModel", + "DashboardVersion", + ]); + for (const entity of IMMUTABLE_VERSION_ENTITIES) { + const table = ANALYTICS_ENTITY_TABLES[entity]; + expect( + new RegExp(`BEFORE UPDATE OR DELETE ON ${table}\\b`).test(MIGRATION), + `${entity} (${table}) has no immutability trigger`, + ).toBe(true); + } + }); + + it("BIApplicationBinding storage carries NO credential column — the ownership boundary is structural", () => { + const block = MIGRATION.split("CREATE TABLE IF NOT EXISTS opt_analytics_bi_application_binding")[1] ?? ""; + const body = block.slice(0, block.indexOf(");")); + for (const forbidden of ["password", "dsn", "secret", "connection_string", "clickhouse_url", "token"]) { + expect(body.toLowerCase()).not.toContain(forbidden); + } + }); + + it("sql/008 is idempotent-shaped: every CREATE is IF NOT EXISTS / OR REPLACE", () => { + const creates = MIGRATION.match(/^CREATE (TABLE|INDEX|TRIGGER|FUNCTION|OR REPLACE FUNCTION)[^\n]*/gm) ?? []; + const nonIdempotent = creates.filter( + (line) => !/IF NOT EXISTS/.test(line) && !/OR REPLACE/.test(line) && !/^CREATE TRIGGER/.test(line), + ); + expect(nonIdempotent).toEqual([]); + // Triggers cannot take IF NOT EXISTS in Postgres, so each one must be + // preceded by a DROP TRIGGER IF EXISTS to stay replayable. + const triggers = MIGRATION.match(/^CREATE TRIGGER (\w+)/gm) ?? []; + for (const t of triggers) { + const name = t.replace("CREATE TRIGGER ", ""); + expect(MIGRATION).toContain(`DROP TRIGGER IF EXISTS ${name}`); + } + }); +}); + +describe("BI-P003 analytics data tiers", () => { + it("canonicalizes exactly RAW / NORMALIZED / CURATED / SERVING", () => { + expect(ANALYTICS_TIERS).toEqual(["RAW", "NORMALIZED", "CURATED", "SERVING"]); + expect(isAnalyticsTier("CURATED")).toBe(true); + expect(isAnalyticsTier("raw")).toBe(false); + expect(isAnalyticsTier("BRONZE")).toBe(false); + }); + + it("no business dashboard reads a RAW/NORMALIZED dataset by default", () => { + expect(BI_READABLE_TIERS).toEqual(["SERVING", "CURATED"]); + for (const tier of ["RAW", "NORMALIZED"] as const) { + expect(() => + assertReadableByBiSurface({ dataset_id: "ds_1", tier }, "dashboard:exec-overview"), + ).toThrowError(/may not read/); + } + for (const tier of ["SERVING", "CURATED"] as const) { + expect(() => assertReadableByBiSurface({ dataset_id: "ds_1", tier }, "dashboard:x")).not.toThrow(); + } + }); + + it("a recorded exception is the only way a RAW dataset reaches a BI surface", () => { + expect(() => + assertReadableByBiSurface( + { dataset_id: "ds_1", tier: "RAW", raw_access_exception: "INC-2026-09-06 forensic review, expires 2026-10-01" }, + "dashboard:incident", + ), + ).not.toThrow(); + // An empty/whitespace "exception" is not an exception. + expect(() => + assertReadableByBiSurface({ dataset_id: "ds_1", tier: "RAW", raw_access_exception: " " }, "dashboard:x"), + ).toThrowError(/may not read/); + }); + + it("tier order is enforced: a CURATED dataset cannot be built from SERVING", () => { + expect(TIER_RANK.RAW).toBeLessThan(TIER_RANK.NORMALIZED); + expect(TIER_RANK.NORMALIZED).toBeLessThan(TIER_RANK.CURATED); + expect(TIER_RANK.CURATED).toBeLessThan(TIER_RANK.SERVING); + expect(() => assertTierLineageSane("CURATED", "SERVING")).toThrowError(/tier order/); + expect(() => assertTierLineageSane("SERVING", "CURATED")).not.toThrow(); + expect(() => assertTierLineageSane("CURATED", "CURATED")).not.toThrow(); + }); + + it("sql/008 constrains tier at the DATABASE level, not only in TypeScript", () => { + expect(MIGRATION).toContain("CHECK (tier IN ('RAW','NORMALIZED','CURATED','SERVING'))"); + }); + + it("BI-P003's eight required declarations are all NOT NULL in sql/008", () => { + const block = MIGRATION.split("CREATE TABLE IF NOT EXISTS opt_analytics_dataset ")[1] ?? ""; + const body = block.slice(0, block.indexOf(");")); + for (const col of [ + "tier", + "schema_contract_id", + "owner_id", + "source_ref", + "freshness_slo_seconds", + "retention_days", + "classification", + "quality_rule_ids", + ]) { + expect(new RegExp(`\\b${col}\\b[^,]*NOT NULL`).test(body), `${col} is nullable`).toBe(true); + } + }); +}); diff --git a/tests/analytics/analytics-domain-postgres.realinfra.test.ts b/tests/analytics/analytics-domain-postgres.realinfra.test.ts new file mode 100644 index 0000000..81c7f92 --- /dev/null +++ b/tests/analytics/analytics-domain-postgres.realinfra.test.ts @@ -0,0 +1,364 @@ +/** + * BI-P002 DoD: "Database migrations + persistence + tenant isolation + + * restore tests." + * + * REAL Postgres, same honest-degrade convention as + * tests/p0/source-registry-postgres.realinfra.test.ts: gated on + * ANALYTICS_TEST_DATABASE_URL, SKIPS HONESTLY when unset, never fabricates. + * + * docker run -d --name opt-bi-pg -p 15455:5432 \ + * -e POSTGRES_USER=opt -e POSTGRES_PASSWORD=opt_bi_throwaway \ + * -e POSTGRES_DB=opt_bi_test postgres:16-alpine + * ANALYTICS_TEST_DATABASE_URL=postgres://opt:opt_bi_throwaway@localhost:15455/opt_bi_test \ + * pnpm vitest run tests/analytics/analytics-domain-postgres.realinfra.test.ts + * + * The RESTORE half is the part that catches migration-chain rot: it replays + * sql/001..008 in order into a FRESH database, exactly as + * docs/runbook/backup-restore.md describes, and only then runs the + * persistence assertions. sql/003_datasets.sql exists because that same drill + * found a table no file ever created; this suite keeps that class of defect + * from coming back through sql/008. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import postgres from "postgres"; +import { + ANALYTICS_DOMAIN_ENTITIES, + ANALYTICS_ENTITY_TABLES, + IMMUTABLE_VERSION_ENTITIES, +} from "@/src/analytics/domain/entities"; +import { AnalyticsDomainRepository } from "@/src/analytics/repositories/analytics-domain-repository"; +import { contractHash, type DatasetContract } from "@/src/analytics/domain/contract"; + +const TEST_DATABASE_URL = process.env.ANALYTICS_TEST_DATABASE_URL; +const run = TEST_DATABASE_URL ? describe : describe.skip; + +/** A schema of its own so the drill starts from genuinely nothing. */ +const SCHEMA = `bi_p002_${Date.now().toString(36)}`; + +let sql: ReturnType | undefined; +let reachable = false; +let unreachableReason = ""; +/** Which sql/*.sql files the restore drill actually applied. */ +const applied: string[] = []; + +const ALPHA = { tenant_id: "t_alpha", project_id: "p_main" }; +const BETA = { tenant_id: "t_beta", project_id: "p_main" }; + +function repo(): AnalyticsDomainRepository { + return new AnalyticsDomainRepository(sql!); +} + +function contract(datasetId: string): DatasetContract { + return { + dataset_id: datasetId, + version: "1.0.0", + tier: "CURATED", + fields: [{ name: "id", type: "string", nullable: false, role: "identifier", primary_identifier: true }], + primary_identifiers: ["id"], + dimensions: [], + measures: [], + freshness_slo_seconds: 3600, + retention_days: 365, + pii_classification: "none", + source_lineage: ["src_x"], + quality_rules: [], + allowed_purposes: ["analytics"], + }; +} + +beforeAll(async () => { + if (!TEST_DATABASE_URL) { + console.warn("[realinfra] ANALYTICS_TEST_DATABASE_URL not set — skipping honestly, not fabricating success."); + return; + } + try { + sql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 3 }); + await Promise.race([ + sql`SELECT 1`, + new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)), + ]); + // RESTORE DRILL: a brand-new schema, then every migration in order. + await sql.unsafe(`CREATE SCHEMA ${SCHEMA}`); + await sql.unsafe(`SET search_path TO ${SCHEMA}, public`); + const dir = path.join(process.cwd(), "sql"); + for (const file of fs.readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) { + const body = fs.readFileSync(path.join(dir, file), "utf8"); + try { + await sql.unsafe(`SET search_path TO ${SCHEMA}, public; ${body}`); + applied.push(file); + } catch (err) { + // 001 needs the `vector` extension, which a stock postgres:16-alpine + // does not carry. Record the skip rather than hiding it — the files + // sql/008 actually depends on must still apply. + applied.push(`${file} (FAILED: ${String(err).slice(0, 120)})`); + } + } + reachable = true; + } catch (err) { + reachable = false; + unreachableReason = err instanceof Error ? err.message : String(err); + } +}, 60_000); + +afterAll(async () => { + if (sql && reachable) { + await sql.unsafe(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined); + } + await sql?.end({ timeout: 5 }); +}); + +run("BI-P002 analytics domain model against a LIVE Postgres", () => { + it("restore drill: sql/008 applies cleanly on top of the replayed chain", (ctx) => { + if (!reachable) ctx.skip(); + expect(unreachableReason).toBe(""); + const p008 = applied.find((f) => f.startsWith("008_")); + expect(p008, `sql/008 was not applied. Applied: ${applied.join(", ")}`).toBe( + "008_analytics_domain_model.sql", + ); + }); + + it("all nineteen entity tables really exist after the restore", async (ctx) => { + if (!reachable) ctx.skip(); + const rows = await sql!<{ table_name: string }[]>` + SELECT table_name FROM information_schema.tables WHERE table_schema = ${SCHEMA}`; + const present = new Set(rows.map((r) => r.table_name)); + const missing = ANALYTICS_DOMAIN_ENTITIES.filter((e) => !present.has(ANALYTICS_ENTITY_TABLES[e])); + expect(missing).toEqual([]); + }); + + it("replaying sql/008 a second time is a no-op, not an error", async (ctx) => { + if (!reachable) ctx.skip(); + const body = fs.readFileSync(path.join(process.cwd(), "sql", "008_analytics_domain_model.sql"), "utf8"); + await expect(sql!.unsafe(`SET search_path TO ${SCHEMA}, public; ${body}`)).resolves.toBeDefined(); + }); + + it("persists a dataset and reads it back through the repository", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + const created = await r.createDataset(ALPHA, { + dataset_id: "ds_alpha_1", + name: "Alpha listings", + tier: "CURATED", + schema_contract_id: "dc_alpha", + schema_contract_version: "1.0.0", + owner_id: "team_data", + source_ref: "src_feed", + freshness_slo_seconds: 3600, + quality_rule_ids: [], + retention_days: 365, + classification: "none", + status: "active", + created_by: "usr_1", + }); + expect(created.tenant_id).toBe("t_alpha"); + const read = await r.getDataset(ALPHA, "ds_alpha_1"); + expect(read?.name).toBe("Alpha listings"); + expect(read?.freshness_slo_seconds).toBe(3600); + // Cross-check with direct SQL — proving the row is really in the table, + // not an artifact of the repository's own bookkeeping. + const direct = await sql!<{ c: string }[]>` + SELECT count(*)::text AS c FROM opt_analytics_dataset WHERE dataset_id = 'ds_alpha_1'`; + expect(Number(direct[0].c)).toBe(1); + }); + + it("TENANT ISOLATION: t_beta cannot read t_alpha's dataset, even with the exact id", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + expect(await r.getDataset(BETA, "ds_alpha_1")).toBeUndefined(); + expect(await r.listDatasets(BETA)).toEqual([]); + // And the row IS there — the empty result is scoping, not absence. + const direct = await sql!<{ c: string }[]>`SELECT count(*)::text AS c FROM opt_analytics_dataset`; + expect(Number(direct[0].c)).toBeGreaterThanOrEqual(1); + }); + + it("TENANT ISOLATION: a version cannot be minted against another tenant's dataset", async (ctx) => { + if (!reachable) ctx.skip(); + await expect( + repo().createDatasetVersion(BETA, { + dataset_id: "ds_alpha_1", + contract_hash: "deadbeef", + schema_contract_version: "1.0.0", + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("dataset versions are monotonic and carry the contract hash", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + const c = contract("ds_alpha_1"); + const v1 = await r.createDatasetVersion(ALPHA, { + dataset_id: "ds_alpha_1", + contract_hash: contractHash(c), + schema_contract_version: "1.0.0", + }); + const v2 = await r.createDatasetVersion(ALPHA, { + dataset_id: "ds_alpha_1", + contract_hash: contractHash({ ...c, version: "2.0.0" }), + schema_contract_version: "2.0.0", + }); + expect(v1.version).toBe(1); + expect(v2.version).toBe(2); + expect(v2.contract_hash).not.toBe(v1.contract_hash); + expect((await r.listDatasetVersions(ALPHA, "ds_alpha_1")).map((v) => v.version)).toEqual([1, 2]); + }); + + it("IMMUTABILITY is enforced by the DATABASE: UPDATE and DELETE on a version row are refused", async (ctx) => { + if (!reachable) ctx.skip(); + // Direct SQL, bypassing every TypeScript `readonly` — the only test that + // proves the guarantee survives a psql session. + await expect( + sql!.unsafe( + `UPDATE ${SCHEMA}.opt_analytics_dataset_version SET contract_hash = 'tampered' WHERE dataset_id = 'ds_alpha_1'`, + ), + ).rejects.toThrow(/immutable/i); + await expect( + sql!.unsafe(`DELETE FROM ${SCHEMA}.opt_analytics_dataset_version WHERE dataset_id = 'ds_alpha_1'`), + ).rejects.toThrow(/immutable/i); + // The rows are untouched. + const rows = await sql!<{ contract_hash: string }[]>` + SELECT contract_hash FROM opt_analytics_dataset_version WHERE dataset_id = 'ds_alpha_1'`; + expect(rows).toHaveLength(2); + for (const r of rows) expect(r.contract_hash).not.toBe("tampered"); + }); + + it("all four immutable entities refuse an UPDATE, not just the one that was tested first", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + await r.createDataset(ALPHA, { + dataset_id: "ds_imm", + name: "Immutability fixture", + tier: "SERVING", + schema_contract_id: "dc_i", + schema_contract_version: "1.0.0", + owner_id: "team_data", + source_ref: "src", + freshness_slo_seconds: 60, + quality_rule_ids: [], + retention_days: 30, + classification: "none", + status: "active", + }); + const metric = await r.createMetric(ALPHA, { + metric_id: "m_imm", + name: "immutable metric", + dataset_id: "ds_imm", + aggregation: "sum", + measure_field: "value", + allowed_dimension_ids: [], + owner_id: "team_data", + }); + await r.createMetricVersion(ALPHA, { metric_id: metric.metric_id, definition_json: '{"v":1}' }); + await r.createDatasetVersion(ALPHA, { + dataset_id: "ds_imm", + contract_hash: "h1", + schema_contract_version: "1.0.0", + }); + await sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_semantic_model + (semantic_model_id, tenant_id, project_id, name, version, model_json, model_hash, owner_id) + VALUES ('sm_1','t_alpha','p_main','model_a',1,'{}','h','team_data')`); + await sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dashboard_definition + (dashboard_id, tenant_id, project_id, name, owner_id) VALUES ('db_1','t_alpha','p_main','dash','team_data')`); + await sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dashboard_version + (dashboard_version_id, dashboard_id, tenant_id, project_id, version, layout_json, layout_hash) + VALUES ('dv_1','db_1','t_alpha','p_main',1,'{}','h')`); + + const updates: Record = { + DatasetVersion: `UPDATE ${SCHEMA}.opt_analytics_dataset_version SET contract_hash='x' WHERE dataset_id='ds_imm'`, + MetricVersion: `UPDATE ${SCHEMA}.opt_analytics_metric_version SET definition_json='{}' WHERE metric_id='m_imm'`, + SemanticModel: `UPDATE ${SCHEMA}.opt_analytics_semantic_model SET model_json='{}' WHERE semantic_model_id='sm_1'`, + DashboardVersion: `UPDATE ${SCHEMA}.opt_analytics_dashboard_version SET layout_json='{}' WHERE dashboard_version_id='dv_1'`, + }; + for (const entity of IMMUTABLE_VERSION_ENTITIES) { + await expect(sql!.unsafe(updates[entity]), `${entity} is NOT immutable`).rejects.toThrow(/immutable/i); + } + }); + + it("a metric version after the first must state WHY the number changed meaning", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + await expect( + r.createMetricVersion(ALPHA, { metric_id: "m_imm", definition_json: '{"v":2}' }), + ).rejects.toThrow(); + const v2 = await r.createMetricVersion(ALPHA, { + metric_id: "m_imm", + definition_json: '{"v":2}', + change_reason: "excluded internal test accounts from the numerator", + }); + expect(v2.version).toBe(2); + }); + + it("a DatasetGrant without a purpose is refused by the DATABASE (purpose limitation)", async (ctx) => { + if (!reachable) ctx.skip(); + // Regression pin: the first version of this constraint was written + // `array_length(allowed_purposes, 1) >= 1`, which is NULL for '{}' — and a + // CHECK evaluating to NULL PASSES in Postgres. It could never fail. This + // test found that; keep it. + await expect( + sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dataset_grant + (grant_id, dataset_id, tenant_id, project_id, subject_id, subject_type, level, allowed_purposes) + VALUES ('g_1','ds_imm','t_alpha','p_main','usr_1','user','read', '{}')`), + ).rejects.toThrow(/purposes_nonempty/); + // A grant WITH a purpose is accepted, so the constraint is not simply + // rejecting everything. + await expect( + sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dataset_grant + (grant_id, dataset_id, tenant_id, project_id, subject_id, subject_type, level, allowed_purposes) + VALUES ('g_ok','ds_imm','t_alpha','p_main','usr_1','user','read', ARRAY['market_analytics'])`), + ).resolves.toBeDefined(); + }); + + it("a BIApplicationBinding without a purpose is refused too, and carries no credential column", async (ctx) => { + if (!reachable) ctx.skip(); + await expect( + sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_bi_application_binding + (binding_id, tenant_id, project_id, kind, name, principal_id, allowed_purposes) + VALUES ('b_1','t_alpha','p_main','superset','sup','svc_1','{}')`), + ).rejects.toThrow(/purposes_nonempty/); + + const cols = await sql!<{ column_name: string }[]>` + SELECT column_name FROM information_schema.columns + WHERE table_schema = ${SCHEMA} AND table_name = 'opt_analytics_bi_application_binding'`; + const names = cols.map((c) => c.column_name.toLowerCase()); + for (const forbidden of ["password", "secret", "dsn", "token", "connection_string", "clickhouse_url"]) { + expect(names, `BI binding must never store ${forbidden}`).not.toContain(forbidden); + } + }); + + it("the dimension catalog is tenant-scoped and FAILS CLOSED for a tenant with none", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + await r.createDimension(ALPHA, { + dimension_id: "dim_1", + name: "make", + dataset_id: "ds_imm", + field_name: "vehicle_make", + field_type: "string", + classification: "none", + }); + expect(await r.dimensionCatalog(ALPHA)).toEqual({ make: "vehicle_make" }); + // t_beta declared nothing, so it may reference nothing. + expect(await r.dimensionCatalog(BETA)).toEqual({}); + }); + + it("a dimension whose physical field is not identifier-safe is refused by the DATABASE", async (ctx) => { + if (!reachable) ctx.skip(); + await expect( + sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dimension_definition + (dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type) + VALUES ('dim_bad','t_alpha','p_main','evil','ds_imm', 'x''; DROP TABLE opt_analytics_dataset; --','string')`), + ).rejects.toThrow(); + }); + + it("the repository refuses to act without a verified tenant scope", async (ctx) => { + if (!reachable) ctx.skip(); + const r = repo(); + await expect(r.getDataset({ tenant_id: "", project_id: "p" }, "ds_imm")).rejects.toMatchObject({ + code: "UNAUTHENTICATED", + }); + await expect(r.listDatasets({ tenant_id: "t_alpha", project_id: " " })).rejects.toMatchObject({ + code: "UNAUTHENTICATED", + }); + }); +}); diff --git a/tests/analytics/analytics-engine-contract.test.ts b/tests/analytics/analytics-engine-contract.test.ts new file mode 100644 index 0000000..e986474 --- /dev/null +++ b/tests/analytics/analytics-engine-contract.test.ts @@ -0,0 +1,124 @@ +/** + * BI-P009 DoD: "Provider contract tests added." + * + * This is the contract every `SiaxAnalyticsEngine` provider must satisfy + * regardless of engine — ClickHouse today, DuckDB (BI-P014) and Trino + * (BI-P015) later. It is written against the INTERFACE, so a future provider + * imports `analyticsEngineContract` and gets the same bar. + * + * The identity rules here are the ones that matter most: this estate has + * already shipped a live cross-tenant hole by reading the tenant out of a + * REQUEST BODY (INF0 Wave 2). So the contract asserts that a request field + * named `tenant_id` cannot influence the engine at all. + */ +import { describe, it, expect } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { + SIAX_ANALYTICS_ENGINE_CAPABILITIES, + type AnalyticsIdentity, + type SiaxAnalyticsEngine, +} from "@/src/analytics/engine/siax-analytics-engine"; + +const IDENTITY: AnalyticsIdentity = { + tenant_id: "t_alpha", + project_id: "p_default", + principal_id: "usr_1", +}; + +/** Reusable contract suite — future providers call this with their factory. */ +export function analyticsEngineContract(name: string, make: () => SiaxAnalyticsEngine): void { + describe(`SiaxAnalyticsEngine contract: ${name}`, () => { + it("implements all eight BI-P009 capabilities as callable methods", () => { + const engine = make() as unknown as Record; + for (const cap of SIAX_ANALYTICS_ENGINE_CAPABILITIES) { + expect(typeof engine[cap], `${cap} is not a method`).toBe("function"); + } + expect(SIAX_ANALYTICS_ENGINE_CAPABILITIES).toEqual([ + "health", + "executeMetric", + "executeQuery", + "estimateQuery", + "cancelQuery", + "explain", + "ingestProjection", + "refreshProjection", + ]); + }); + + it("names its provider", () => { + expect(make().provider.length).toBeGreaterThan(0); + }); + + it("refuses every tenant-scoped capability without a verified identity", async () => { + const engine = make(); + const blank = { tenant_id: "", project_id: "p", principal_id: "usr_1" }; + const noPrincipal = { tenant_id: "t_alpha", project_id: "p", principal_id: "" }; + for (const bad of [blank, noPrincipal]) { + await expect(engine.executeMetric(bad, { metric: "m", aggregate: "sum" })).rejects.toMatchObject({ + code: "UNAUTHENTICATED", + }); + await expect(engine.estimateQuery(bad, { metric: "m", aggregate: "sum" })).rejects.toMatchObject({ + code: "UNAUTHENTICATED", + }); + await expect(engine.ingestProjection(bad, { dataset_id: "ds", rows: [] })).rejects.toMatchObject({ + code: "UNAUTHENTICATED", + }); + await expect(engine.refreshProjection(bad, "ds")).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); + await expect(engine.cancelQuery(bad, "q")).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); + } + }); + + it("a tenant_id smuggled into the REQUEST cannot influence the engine (INF0 Wave-2 bug class)", async () => { + const engine = make(); + const smuggled = { metric: "m", aggregate: "sum", tenant_id: "t_victim" } as never; + // The request type has no tenant field; if a provider ever started + // reading one, this call would target t_victim. The engine must still + // fail on the IDENTITY, proving the request field is inert. + await expect( + engine.executeMetric({ tenant_id: "", project_id: "p", principal_id: "u" }, smuggled), + ).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); + }); + + it("cancelQuery refuses a query id belonging to another tenant", async () => { + const engine = make(); + await expect(engine.cancelQuery(IDENTITY, "opt_deadbeefdead_someoneelses")).rejects.toMatchObject({ + code: "AUTHORIZATION_DENIED", + }); + }); + + it("health() reports unhealthy rather than throwing when the engine is unreachable", async () => { + const report = await make().health(); + expect(report.healthy).toBe(false); + expect(report.provider.length).toBeGreaterThan(0); + // Never leak an engine body or a DSN into the operator-facing detail. + expect(report.detail ?? "").not.toMatch(/http|password|@/i); + }); + }); +} + +analyticsEngineContract( + "clickhouse (unreachable host — contract-level assertions only)", + () => + new ClickHouseAnalyticsEngine({ + baseUrl: "http://clickhouse.invalid:8123", + database: "opt_contract_test", + timeoutMs: 500, + }), +); + +describe("BI-P009 refusals that are deliberate, not missing", () => { + it("executeQuery (analyst SQL) is declared and REFUSED until BI-P017/BI-P033", async () => { + const engine = new ClickHouseAnalyticsEngine({ baseUrl: "http://clickhouse.invalid:8123" }); + await expect(engine.executeQuery(IDENTITY, { sql: "SELECT 1", dataset_ids: [] })).rejects.toMatchObject({ + code: "CAPABILITY_UNAVAILABLE", + }); + }); + + it("the PRESERVED ClickHouseAnalytics is reachable through the engine, not replaced by it", () => { + const engine = new ClickHouseAnalyticsEngine({ baseUrl: "http://clickhouse.invalid:8123" }); + expect(engine.legacyAnalytics.constructor.name).toBe("ClickHouseAnalytics"); + expect(typeof engine.legacyAnalytics.queryMetric).toBe("function"); + expect(typeof engine.legacyAnalytics.ensureSchema).toBe("function"); + expect(typeof engine.legacyAnalytics.ingest).toBe("function"); + }); +}); diff --git a/tests/analytics/arrow-fixture.test.ts b/tests/analytics/arrow-fixture.test.ts new file mode 100644 index 0000000..807dac5 --- /dev/null +++ b/tests/analytics/arrow-fixture.test.ts @@ -0,0 +1,54 @@ +/** + * The canonical BI-P005 fixture, in one place. + * + * Two jobs: + * 1. assert the encoder is DETERMINISTIC — the same columns must produce + * byte-identical output, or the "roundtrip" evidence would be comparing + * two different payloads run to run; + * 2. when ARROW_FIXTURE_DIR is set, write those exact bytes to disk so + * scripts/bi-p005-arrow-roundtrip.mjs hands pyarrow and DuckDB the bytes + * the SHIPPED module produces, not a copy of the encoder living in a + * script. + */ +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import crypto from "node:crypto"; +import { type ArrowColumn, encodeArrowStream } from "@/src/analytics/arrow/ipc"; + +/** The same three rows the ClickHouse side of the DoD produces. */ +export const BI_P005_FIXTURE: ArrowColumn[] = [ + { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, + { name: "n", type: "int64", values: [1, 2, 3] }, + { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, + { name: "flag", type: "bool", values: [true, false, true] }, + { + name: "ts", + type: "timestamp_ms", + values: [ + new Date("2026-01-01T00:00:00.000Z"), + new Date("2026-06-15T08:09:10.123Z"), + new Date("2026-09-06T12:34:56.789Z"), + ], + }, +]; + +describe("BI-P005 canonical Arrow fixture", () => { + it("encodes deterministically", () => { + const a = encodeArrowStream(BI_P005_FIXTURE); + const b = encodeArrowStream(BI_P005_FIXTURE); + expect(Buffer.from(a).equals(Buffer.from(b))).toBe(true); + expect(a.length).toBeGreaterThan(0); + }); + + it("writes the fixture for the external roundtrip when asked", () => { + const dir = process.env.ARROW_FIXTURE_DIR; + if (!dir) return; // not an error: the normal test run has nowhere to put it + const bytes = encodeArrowStream(BI_P005_FIXTURE); + const out = path.join(dir, "02_0pt_encoded.arrow"); + fs.writeFileSync(out, bytes); + expect(fs.statSync(out).size).toBe(bytes.length); + // Print the digest so the evidence file can be tied to these exact bytes. + console.log(`ARROW_FIXTURE_SHA256=${crypto.createHash("sha256").update(bytes).digest("hex")}`); + }); +}); diff --git a/tests/analytics/arrow-ipc.test.ts b/tests/analytics/arrow-ipc.test.ts new file mode 100644 index 0000000..c734602 --- /dev/null +++ b/tests/analytics/arrow-ipc.test.ts @@ -0,0 +1,101 @@ +/** + * BI-P005 — APACHE ARROW INTERCHANGE, the part provable without a live engine. + * + * Honest scope note: the tests here run 0pt's encoder against 0pt's decoder, + * which proves internal consistency and NOT specification conformance. The + * conformance evidence comes from two places where an INDEPENDENT + * implementation reads the bytes: + * + * * tests/integration/analytics-arrow-clickhouse.integration.test.ts — + * ClickHouse (Arrow C++) both produces Arrow this decoder reads and + * consumes Arrow this encoder writes. Runs on every push in CI. + * * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json — pyarrow and DuckDB + * decoding the same bytes, executed against real containers. + * + * That distinction is the point: a self-roundtrip that agrees with its own + * mistakes is exactly the kind of green this estate has been burned by. + */ +import { describe, it, expect } from "vitest"; +import { + ARROW_TYPE_SUPPORT, + arrowStreamSettings, + assertArrowWithinResultBudget, + clickhouseArrowStreamFormat, + decodeArrowStream, + encodeArrowStream, +} from "@/src/analytics/arrow/ipc"; + +describe("BI-P005 Arrow IPC", () => { + it("emits a well-formed IPC stream: continuation markers, schema, batch, EOS", () => { + const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [1, 2, 3] }]); + const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + expect(dv.getUint32(0, true)).toBe(0xffffffff); + // Last 8 bytes are the end-of-stream marker: 0xFFFFFFFF then length 0. + expect(dv.getUint32(bytes.length - 8, true)).toBe(0xffffffff); + expect(dv.getInt32(bytes.length - 4, true)).toBe(0); + }); + + it("round-trips every supported column type with nulls", () => { + const bytes = encodeArrowStream([ + { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", null] }, + { name: "n", type: "int64", values: [1, 2, 3] }, + { name: "v", type: "float64", values: [1.5, 2.5, null] }, + { name: "flag", type: "bool", values: [true, false, true] }, + { name: "ts", type: "timestamp_ms", values: [new Date("2026-01-01T00:00:00Z"), null, new Date("2026-09-06T12:34:56.789Z")] }, + ]); + const info = decodeArrowStream(bytes); + expect(info.rowCount).toBe(3); + expect(info.batchCount).toBe(1); + expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); + // Arrow Type union tags: Int=2, FloatingPoint=3, Utf8=5, Bool=6, Timestamp=10. + expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); + }); + + it("handles an empty result set without producing a malformed stream", () => { + const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [] }]); + const info = decodeArrowStream(bytes); + expect(info.rowCount).toBe(0); + expect(info.fields).toEqual([{ name: "n", typeType: 2 }]); + }); + + it("rejects ragged columns rather than emitting a stream that decodes to garbage", () => { + expect(() => + encodeArrowStream([ + { name: "a", type: "int64", values: [1, 2] }, + { name: "b", type: "int64", values: [1] }, + ]), + ).toThrowError(/expected 2/); + expect(() => encodeArrowStream([])).toThrowError(/at least one column/); + }); + + it("refuses an int64 value outside the representable range instead of wrapping it", () => { + expect(() => + encodeArrowStream([{ name: "n", type: "int64", values: [BigInt("9223372036854775808")] }]), + ).toThrowError(/out of int64 range/); + }); + + it("rejects bytes that are not an Arrow IPC stream", () => { + expect(() => decodeArrowStream(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrowError(/not an Arrow IPC stream/); + }); + + it("BI-P012: Arrow is not a way around max_result_rows", () => { + const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [1, 2, 3, 4, 5] }]); + expect(() => assertArrowWithinResultBudget(bytes, 3)).toThrowError(/over the 3-row budget/); + expect(assertArrowWithinResultBudget(bytes, 5).rowCount).toBe(5); + }); + + it("the ClickHouse pass-through asks for text Strings, not Binary", () => { + expect(clickhouseArrowStreamFormat()).toBe("ArrowStream"); + // Without this setting ClickHouse emits String as Arrow Binary, which + // pyarrow and DuckDB read as bytes — silent semantic loss. + expect(arrowStreamSettings().output_format_arrow_string_as_string).toBe("1"); + }); + + it("documents the supported type mapping including where loss WOULD occur", () => { + expect(ARROW_TYPE_SUPPORT.length).toBeGreaterThanOrEqual(5); + const arrowTypes = ARROW_TYPE_SUPPORT.map((r) => r.arrow); + expect(new Set(arrowTypes)).toEqual(new Set(["int64", "float64", "utf8", "bool", "timestamp_ms"])); + // Every row states its caveat; an empty note would be a silent promise. + for (const row of ARROW_TYPE_SUPPORT) expect(row.note.length).toBeGreaterThan(10); + }); +}); diff --git a/tests/analytics/dataset-contract.test.ts b/tests/analytics/dataset-contract.test.ts new file mode 100644 index 0000000..769af32 --- /dev/null +++ b/tests/analytics/dataset-contract.test.ts @@ -0,0 +1,164 @@ +/** + * BI-P004 — ANALYTICS DATA CONTRACT. + * + * DoD: "Breaking dataset schema changes fail compatibility gate unless + * explicitly versioned." Each breaking change class below is tested TWICE: + * once under a minor bump (must be REFUSED) and once under a major bump (must + * be ACCEPTED). A gate that only ever says yes, or only ever says no, would + * pass a one-sided test; this one cannot. + */ +import { describe, it, expect } from "vitest"; +import { + assertContractCompatible, + checkContractCompatibility, + contractHash, + diffContracts, + parseSemVer, + validateContract, + type DatasetContract, +} from "@/src/analytics/domain/contract"; + +function base(): DatasetContract { + return { + dataset_id: "ds_vehicle_listings", + version: "1.2.0", + tier: "CURATED", + fields: [ + { name: "listing_id", type: "string", nullable: false, role: "identifier", primary_identifier: true }, + { name: "make", type: "string", nullable: false, role: "dimension" }, + { name: "asking_price", type: "int64", nullable: true, role: "measure" }, + { name: "listed_at", type: "timestamp", nullable: false, role: "attribute" }, + ], + primary_identifiers: ["listing_id"], + dimensions: ["make"], + measures: ["asking_price"], + freshness_slo_seconds: 3600, + retention_days: 730, + pii_classification: "none", + source_lineage: ["src_dealer_feed"], + quality_rules: [{ rule_id: "qr_listing_id_unique", kind: "unique", field_name: "listing_id", severity: "block" }], + allowed_purposes: ["market_analytics"], + }; +} + +/** Apply a mutation and set the version. */ +function next(mutate: (c: DatasetContract) => void, version: string): DatasetContract { + const c: DatasetContract = JSON.parse(JSON.stringify(base())); + mutate(c); + c.version = version; + return c; +} + +const BREAKING_CASES: { name: string; mutate: (c: DatasetContract) => void; code: string }[] = [ + { name: "field removed", mutate: (c) => { c.fields = c.fields.filter((f) => f.name !== "make"); c.dimensions = []; }, code: "field_removed" }, + { name: "field type changed", mutate: (c) => { c.fields[2].type = "float64"; }, code: "field_type_changed" }, + { name: "field became nullable", mutate: (c) => { c.fields[1].nullable = true; }, code: "field_became_nullable" }, + { name: "field role changed", mutate: (c) => { c.fields[1].role = "measure"; }, code: "field_role_changed" }, + { name: "required field added", mutate: (c) => { c.fields.push({ name: "vat_rate", type: "float64", nullable: false, role: "attribute" }); }, code: "required_field_added" }, + { name: "primary identifier changed", mutate: (c) => { c.primary_identifiers = ["make"]; }, code: "primary_identifier_changed" }, + { name: "dimension removed", mutate: (c) => { c.dimensions = []; }, code: "dimension_removed" }, + { name: "measure removed", mutate: (c) => { c.measures = []; }, code: "measure_removed" }, + { name: "freshness SLO loosened", mutate: (c) => { c.freshness_slo_seconds = 86400; }, code: "freshness_slo_loosened" }, + { name: "retention shortened", mutate: (c) => { c.retention_days = 30; }, code: "retention_shortened" }, + { name: "allowed purpose removed", mutate: (c) => { c.allowed_purposes = []; }, code: "purpose_removed" }, + { name: "PII classification raised", mutate: (c) => { c.pii_classification = "personal"; }, code: "pii_classification_raised" }, + { name: "blocking quality rule removed", mutate: (c) => { c.quality_rules = []; }, code: "blocking_quality_rule_removed" }, + { name: "blocking quality rule downgraded", mutate: (c) => { c.quality_rules[0].severity = "warn"; }, code: "blocking_quality_rule_downgraded" }, + { name: "tier changed", mutate: (c) => { c.tier = "SERVING"; }, code: "tier_changed" }, +]; + +describe("BI-P004 compatibility gate", () => { + it.each(BREAKING_CASES)("REFUSES $name under a minor bump", ({ mutate, code }) => { + const verdict = checkContractCompatibility(base(), next(mutate, "1.3.0")); + expect(verdict.compatible).toBe(false); + expect(verdict.breaking.map((b) => b.code)).toContain(code); + expect(verdict.reason).toMatch(/MAJOR version bump/); + }); + + it.each(BREAKING_CASES)("ACCEPTS $name once explicitly versioned (major bump)", ({ mutate, code }) => { + const verdict = checkContractCompatibility(base(), next(mutate, "2.0.0")); + expect(verdict.compatible).toBe(true); + expect(verdict.explicitly_versioned).toBe(true); + expect(verdict.breaking.map((b) => b.code)).toContain(code); + }); + + it("a patch bump is equally refused — the gate is about the MAJOR, not about moving at all", () => { + const verdict = checkContractCompatibility(base(), next((c) => c.fields.pop(), "1.2.1")); + expect(verdict.compatible).toBe(false); + }); + + it("allows genuinely additive changes without a major bump", () => { + const additive = next((c) => { + c.fields.push({ name: "mileage_km", type: "int64", nullable: true, role: "measure" }); + c.measures.push("mileage_km"); + c.quality_rules.push({ rule_id: "qr_mileage_range", kind: "range", field_name: "mileage_km", severity: "warn" }); + c.allowed_purposes.push("valuation"); + c.retention_days = 1095; + c.freshness_slo_seconds = 1800; + }, "1.3.0"); + const verdict = checkContractCompatibility(base(), additive); + expect(verdict.compatible).toBe(true); + expect(verdict.breaking).toEqual([]); + expect(verdict.explicitly_versioned).toBe(false); + }); + + it("refuses a version that does not move forward", () => { + expect(checkContractCompatibility(base(), next(() => {}, "1.2.0")).compatible).toBe(false); + expect(checkContractCompatibility(base(), next(() => {}, "1.1.9")).compatible).toBe(false); + expect(checkContractCompatibility(base(), next(() => {}, "0.9.0")).compatible).toBe(false); + }); + + it("treats date -> timestamp as a safe widening, and int64 -> float64 as NOT one", () => { + const widened = next((c) => { + c.fields.push({ name: "sold_on", type: "date", nullable: true, role: "attribute" }); + }, "1.3.0"); + const later = JSON.parse(JSON.stringify(widened)) as DatasetContract; + later.fields[later.fields.length - 1].type = "timestamp"; + later.version = "1.4.0"; + expect(checkContractCompatibility(widened, later).compatible).toBe(true); + + const lossy = next((c) => { c.fields[2].type = "float64"; }, "1.3.0"); + expect(checkContractCompatibility(base(), lossy).compatible).toBe(false); + }); + + it("assertContractCompatible throws a CONFLICT rather than returning a verdict nobody reads", () => { + expect(() => assertContractCompatible(base(), next((c) => c.fields.pop(), "1.3.0"))).toThrowError(/MAJOR/); + expect(() => assertContractCompatible(base(), next(() => {}, "1.3.0"))).not.toThrow(); + }); + + it("refuses to compare contracts for different datasets", () => { + const other = next((c) => { c.dataset_id = "ds_other"; }, "1.3.0"); + expect(() => diffContracts(base(), other)).toThrowError(/SAME dataset/); + }); + + it("validateContract catches internally inconsistent contracts", () => { + expect(() => validateContract(base())).not.toThrow(); + expect(() => validateContract({ ...base(), version: "1.2" })).toThrowError(/major.minor.patch/); + expect(() => validateContract({ ...base(), fields: [] })).toThrowError(/no fields/); + expect(() => validateContract({ ...base(), dimensions: ["nope"] })).toThrowError(/unknown field/); + expect(() => validateContract({ ...base(), primary_identifiers: [] })).toThrowError(/no primary identifier/); + expect(() => validateContract({ ...base(), allowed_purposes: [] })).toThrowError(/allowed_purposes/); + expect(() => validateContract({ ...base(), freshness_slo_seconds: 0 })).toThrowError(/freshness/); + const dup = base(); + dup.fields.push({ ...dup.fields[0] }); + expect(() => validateContract(dup)).toThrowError(/duplicate field/); + }); + + it("contractHash is order-insensitive over fields but sensitive to meaning", () => { + const a = base(); + const reordered = JSON.parse(JSON.stringify(a)) as DatasetContract; + reordered.fields.reverse(); + expect(contractHash(reordered)).toBe(contractHash(a)); + + const changed = JSON.parse(JSON.stringify(a)) as DatasetContract; + changed.fields[1].nullable = true; + expect(contractHash(changed)).not.toBe(contractHash(a)); + }); + + it("parseSemVer rejects anything that is not major.minor.patch", () => { + expect(parseSemVer("2.10.3")).toEqual({ major: 2, minor: 10, patch: 3 }); + for (const bad of ["v1.0.0", "1.0", "1.0.0-rc1", "latest", ""]) { + expect(() => parseSemVer(bad)).toThrowError(/major.minor.patch/); + } + }); +}); diff --git a/tests/analytics/query-compiler-injection.test.ts b/tests/analytics/query-compiler-injection.test.ts new file mode 100644 index 0000000..d001722 --- /dev/null +++ b/tests/analytics/query-compiler-injection.test.ts @@ -0,0 +1,224 @@ +/** + * BI-P011's adversarial list, at the compiler layer: + * + * tenant A query B dataset · tenant A forged dataset ID · raw SQL attempt + * SQL injection · dimension identifier injection · filter injection + * + * These are the cases that never reach the network. The cases that DO reach a + * real server — and the with-fix/without-fix proof BI-P011 actually demands — + * are tests/integration/analytics-tenant-isolation.integration.test.ts. + */ +import { describe, it, expect } from "vitest"; +import { + assertSafeIdentifier, + compileMetricQuery, + compileProjectionQuery, + dimensionCatalogFrom, + restrictedTenantUserDdl, +} from "@/src/analytics/engine/query-compiler"; + +const catalog = dimensionCatalogFrom({ severity: "severity", make: "vehicle_make" }); + +const HOSTILE = [ + "x'; DROP TABLE opt_analytics_event; --", + "1 OR 1=1", + "tenant_id", + "*/ OR 1=1 /*", + "severity UNION ALL SELECT tenant_id FROM opt_analytics_event", + "dim_values[1]", + "__proto__", + "constructor", + "sever ity", + "sévérité", + "a".repeat(200), + "", +]; + +describe("BI-P011 injection surfaces at the compiler", () => { + it("dimension identifier injection: hostile group_by names are refused, not escaped", () => { + for (const bad of HOSTILE) { + expect(() => + compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", group_by: [bad] }, + "opt_analytics", + catalog, + 1000, + ), + `group_by "${bad}" was not refused`, + ).toThrowError(/unknown group_by dimension/); + } + }); + + it("filter injection: hostile filter KEYS are refused by catalog resolution", () => { + for (const bad of HOSTILE) { + expect(() => + compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", filters: { [bad]: "v" } }, + "opt_analytics", + catalog, + 1000, + ), + `filter key "${bad}" was not refused`, + ).toThrowError(/unknown filter dimension/); + } + }); + + it("prototype-chain keys resolve to undefined rather than to Object.prototype members", () => { + // A naive `catalog[name]` lookup would resolve "constructor" to a + // function and "toString" to a method; the Map-based catalog must not. + for (const key of ["constructor", "toString", "hasOwnProperty", "__proto__", "valueOf"]) { + expect(catalog.resolve(key)).toBeUndefined(); + } + }); + + it("filter VALUES are never spliced, however hostile", () => { + const compiled = compileMetricQuery( + { + tenant_id: "t_alpha", + metric: "m", + aggregate: "sum", + filters: { make: "volvo'; DROP TABLE opt_analytics_event; --" }, + }, + "opt_analytics", + catalog, + 1000, + ); + expect(compiled.sql).not.toContain("DROP TABLE"); + expect(compiled.sql).not.toContain("volvo"); + expect(compiled.params.param_f0v).toBe("volvo'; DROP TABLE opt_analytics_event; --"); + // The physical column name came from the catalog, not the caller. + expect(compiled.params.param_f0n).toBe("vehicle_make"); + }); + + it("metric and dataset VALUES are never spliced either", () => { + const compiled = compileMetricQuery( + { + tenant_id: "t_alpha", + metric: "m'; SELECT 1; --", + dataset_id: "ds'; SELECT 1; --", + aggregate: "sum", + }, + "opt_analytics", + catalog, + 1000, + ); + expect(compiled.sql).not.toContain("SELECT 1"); + expect(compiled.sql.match(/SELECT/g)).toHaveLength(1); + }); + + it("raw SQL attempt: there is no request shape that reaches a second table", () => { + const compiled = compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum" }, + "opt_analytics", + catalog, + 1000, + ); + expect(compiled.sql).not.toMatch(/\bUNION\b/i); + expect(compiled.sql).not.toMatch(/\bJOIN\b/i); + expect((compiled.sql.match(/\bFROM\b/gi) ?? []).length).toBe(1); + expect(compiled.table).toBe("opt_analytics_event"); + }); + + it("a forged table name is refused rather than reaching the FROM clause", () => { + expect(() => + compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", table: "system.tables" as never }, + "opt_analytics", + catalog, + 1000, + ), + ).toThrowError(/invalid table identifier/); + }); + + it("an unsupported aggregate is refused rather than being interpolated", () => { + expect(() => + compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum(1),tenant_id" as never }, + "opt_analytics", + catalog, + 1000, + ), + ).toThrowError(/unsupported aggregate/); + }); + + it("group-by cardinality is capped so a caller cannot explode the GROUP BY", () => { + const many = Array.from({ length: 9 }, () => "severity"); + expect(() => + compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", group_by: many }, + "opt_analytics", + catalog, + 1000, + ), + ).toThrowError(/at most 8 group-by/); + }); + + it("LIMIT is clamped to the governance ceiling, and never below 1", () => { + const big = compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", limit: 10_000_000 }, + "opt_analytics", + catalog, + 500, + ); + expect(big.sql).toContain("LIMIT 500"); + const zero = compileMetricQuery( + { tenant_id: "t_alpha", metric: "m", aggregate: "sum", limit: 0 }, + "opt_analytics", + catalog, + 500, + ); + expect(zero.sql).toContain("LIMIT 1"); + }); + + it("the projection compiler forces the tenant and parameterizes entity keys", () => { + const compiled = compileProjectionQuery( + { tenant_id: "t_alpha", dataset_id: "ds_x", entity_keys: ["a'; DROP", "b"] }, + "opt_analytics", + 100, + ); + expect(compiled.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); + expect(compiled.sql).not.toContain("DROP"); + expect(compiled.params.param_entityKeys).toBe(JSON.stringify(["a'; DROP", "b"])); + }); + + it("assertSafeIdentifier accepts real identifiers and rejects everything else", () => { + for (const good of ["severity", "vehicle_make", "_x", "a1_b2"]) { + expect(assertSafeIdentifier(good, "test")).toBe(good); + } + for (const bad of ["1abc", "a-b", "a b", "a;b", "", "a".repeat(64)]) { + expect(() => assertSafeIdentifier(bad, "test")).toThrowError(/invalid test identifier/); + } + }); + + it("restrictedTenantUserDdl refuses a tenant id or password it cannot safely embed", () => { + expect(() => + restrictedTenantUserDdl({ + db: "opt_analytics", + tenantId: "t_alpha' OR '1'='1", + userName: "opt_t_alpha", + password: "pw", + tables: ["opt_analytics_event"], + }), + ).toThrowError(/not policy-safe/); + + expect(() => + restrictedTenantUserDdl({ + db: "opt_analytics", + tenantId: "t_alpha", + userName: "opt_t_alpha", + password: "pw' OR '1'='1", + tables: ["opt_analytics_event"], + }), + ).toThrowError(/cannot be embedded/); + + const stmts = restrictedTenantUserDdl({ + db: "opt_analytics", + tenantId: "t_alpha", + userName: "opt_t_alpha", + password: "safe_pw_123", + tables: ["opt_analytics_event"], + }); + expect(stmts.join(" ")).toContain("CREATE ROW POLICY IF NOT EXISTS"); + expect(stmts.join(" ")).toContain("USING tenant_id = 't_alpha'"); + }); +}); diff --git a/tests/analytics/resource-governance.test.ts b/tests/analytics/resource-governance.test.ts new file mode 100644 index 0000000..19854d6 --- /dev/null +++ b/tests/analytics/resource-governance.test.ts @@ -0,0 +1,118 @@ +/** + * BI-P012 — CLICKHOUSE RESOURCE GOVERNANCE, the part provable without an + * engine. The "intentional query bomb is terminated" half of the DoD needs a + * real server and lives in + * tests/integration/analytics-resource-governance.integration.test.ts. + */ +import { describe, it, expect } from "vitest"; +import { + applyQuota, + BUDGET_TIERS, + clickhouseSettings, + ConcurrencyGate, + concurrencyProfileSql, + DEFAULT_BUDGETS, + REQUIRED_GOVERNANCE_SETTINGS, +} from "@/src/analytics/engine/resource-governance"; + +describe("BI-P012 resource governance", () => { + it("every setting the program names has a value somewhere in the enforcement path", () => { + const settings = clickhouseSettings(DEFAULT_BUDGETS.INTERACTIVE); + const profile = concurrencyProfileSql("opt_t_alpha", DEFAULT_BUDGETS.INTERACTIVE).join(" "); + for (const name of REQUIRED_GOVERNANCE_SETTINGS) { + const present = name in settings || profile.includes(name) || name === "max_concurrent_queries"; + expect(present, `${name} is not enforced anywhere`).toBe(true); + } + // The five per-statement ones must be on the STATEMENT, not only in a + // profile a future operator might forget to attach. + for (const name of [ + "max_execution_time", + "max_result_rows", + "max_rows_to_read", + "max_bytes_to_read", + "max_memory_usage", + ]) { + expect(settings[name], `${name} missing from per-statement settings`).toBeDefined(); + } + }); + + it("readonly=1 rides on every SELECT so the query text cannot re-set the limits", () => { + for (const tier of BUDGET_TIERS) { + expect(clickhouseSettings(DEFAULT_BUDGETS[tier]).readonly).toBe("1"); + } + }); + + it("overflow modes are THROW, not BREAK — a partial result must never look complete", () => { + const s = clickhouseSettings(DEFAULT_BUDGETS.INTERACTIVE); + expect(s.result_overflow_mode).toBe("throw"); + expect(s.read_overflow_mode).toBe("throw"); + expect(s.timeout_overflow_mode).toBe("throw"); + }); + + it("a CL0UD quota can only TIGHTEN a tier, never loosen it", () => { + const base = DEFAULT_BUDGETS.INTERACTIVE; + const loosened = applyQuota(base, { + maxExecutionTimeSec: 100000, + maxResultRows: 10_000_000, + maxRowsToRead: Number.MAX_SAFE_INTEGER, + maxBytesToRead: Number.MAX_SAFE_INTEGER, + maxMemoryUsageBytes: Number.MAX_SAFE_INTEGER, + maxConcurrentQueries: 1000, + }); + expect(loosened).toEqual(base); + + const tightened = applyQuota(base, { maxResultRows: 100, maxConcurrentQueries: 1 }); + expect(tightened.maxResultRows).toBe(100); + expect(tightened.maxConcurrentQueries).toBe(1); + expect(tightened.maxExecutionTimeSec).toBe(base.maxExecutionTimeSec); + }); + + it("a nonsense quota (zero, negative, NaN) falls back to the tier rather than to no limit", () => { + const base = DEFAULT_BUDGETS.INTERACTIVE; + expect(applyQuota(base, { maxResultRows: 0 }).maxResultRows).toBe(base.maxResultRows); + expect(applyQuota(base, { maxResultRows: -5 }).maxResultRows).toBe(base.maxResultRows); + expect(applyQuota(base, { maxResultRows: Number.NaN }).maxResultRows).toBe(base.maxResultRows); + expect(applyQuota(base, undefined)).toEqual(base); + }); + + it("tiers get progressively more generous, and INTERACTIVE is the mean default", () => { + expect(DEFAULT_BUDGETS.INTERACTIVE.maxExecutionTimeSec).toBeLessThan(DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec); + expect(DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec).toBeLessThan(DEFAULT_BUDGETS.BATCH.maxExecutionTimeSec); + expect(DEFAULT_BUDGETS.INTERACTIVE.maxRowsToRead).toBeLessThan(DEFAULT_BUDGETS.ANALYTICAL.maxRowsToRead); + }); + + it("the concurrency gate throttles ONE tenant without touching another", () => { + const gate = new ConcurrencyGate((t) => (t === "t_noisy" ? 2 : 5)); + const a = gate.acquire("t_noisy"); + const b = gate.acquire("t_noisy"); + expect(gate.inFlightFor("t_noisy")).toBe(2); + expect(() => gate.acquire("t_noisy")).toThrowError(/queries in flight/); + + // The quiet tenant is unaffected — BI-P012's "without degrading unrelated + // tenant" in the smallest testable form. + const quiet = gate.acquire("t_quiet"); + expect(gate.inFlightFor("t_quiet")).toBe(1); + + a(); + expect(gate.inFlightFor("t_noisy")).toBe(1); + expect(() => gate.acquire("t_noisy")).not.toThrow(); + b(); + quiet(); + }); + + it("releasing twice does not free a slot that was never taken", () => { + const gate = new ConcurrencyGate(() => 1); + const release = gate.acquire("t_x"); + release(); + release(); + expect(gate.inFlightFor("t_x")).toBe(0); + expect(gate.totalInFlight()).toBe(0); + }); + + it("the settings profile SQL carries max_concurrent_queries' server-side half", () => { + const stmts = concurrencyProfileSql("opt_t_alpha", DEFAULT_BUDGETS.INTERACTIVE); + expect(stmts.join(" ")).toContain("CREATE SETTINGS PROFILE IF NOT EXISTS opt_t_alpha"); + expect(stmts.join(" ")).toContain("readonly = 1"); + expect(stmts.join(" ")).toContain("CREATE QUOTA IF NOT EXISTS opt_t_alpha_quota"); + }); +}); diff --git a/tests/analytics/tenant-predicate-regression.test.ts b/tests/analytics/tenant-predicate-regression.test.ts new file mode 100644 index 0000000..2c1b04a --- /dev/null +++ b/tests/analytics/tenant-predicate-regression.test.ts @@ -0,0 +1,144 @@ +/** + * BI-P009's non-negotiable line: "Do not regress the existing forced tenant + * predicate." + * + * This suite is written to FAIL IF THE FORCING IS REMOVED, which is a + * different thing from testing that it currently works. Concretely, each test + * below would go red under a specific, plausible regression: + * + * * someone makes `tenant_id` optional in the metric DSL -> "refuses a + * blank tenant" fails; + * * someone "simplifies" the WHERE builder and drops the tenant condition + * -> "every compiled statement contains the predicate" fails on a missing + * substring; + * * someone starts splicing the tenant value into SQL text instead of + * binding it -> "the tenant VALUE never appears in the SQL text" fails; + * * someone lets a caller pass a tenant through the request body rather + * than the verified identity -> "identity wins over any request field" + * fails. + * + * It covers BOTH compilers: the PRESERVED `ClickHouseAnalytics.queryMetric` + * (src/analytics/clickhouse-analytics.ts, untouched by BI-P009) and the new + * `compileMetricQuery` — because "preserved" means the old guarantee has to + * keep holding, not merely that the file still exists. + * + * The ClickHouseAnalytics half asserts against the URL the adapter actually + * builds, captured with a stub `fetch`. That is deliberate: a mock ENGINE + * could be made to return anything, but the SQL text on the wire is the thing + * the real server would receive, so asserting on it cannot be faked by the + * test double. + */ +import { describe, it, expect, afterEach, vi } from "vitest"; +import { ClickHouseAnalytics } from "@/src/analytics/clickhouse-analytics"; +import { + compileMetricQuery, + compileProjectionQuery, + dimensionCatalogFrom, + FORCED_TENANT_PREDICATE, +} from "@/src/analytics/engine/query-compiler"; + +const catalog = dimensionCatalogFrom({ severity: "severity", make: "vehicle_make" }); + +/** Captures every URL the adapter fetches, returning an empty JSON result. */ +function captureFetch(): { urls: string[]; restore: () => void } { + const urls: string[] = []; + const original = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL) => { + urls.push(String(input)); + return new Response(JSON.stringify({ data: [] }), { status: 200 }); + }) as typeof fetch; + return { urls, restore: () => (globalThis.fetch = original) }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("BI-P009 regression guard: the forced tenant predicate", () => { + it("PRESERVED ClickHouseAnalytics still emits tenant_id = {tenant:String} on the wire", async () => { + const cap = captureFetch(); + try { + const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123", database: "opt_test" }); + await analytics.queryMetric({ tenant_id: "t_alpha", metric: "rows", aggregate: "sum" }); + expect(cap.urls).toHaveLength(1); + // URLSearchParams decodes `+` as a space, which raw decodeURIComponent + // does not — read the statement the way the server will. + const query = new URL(cap.urls[0]).searchParams.get("query") ?? ""; + expect(query).toContain("tenant_id = {tenant:String}"); + expect(query).toMatch(/WHERE tenant_id = \{tenant:String\}/); + // The VALUE travels as a bound parameter, never inside the statement. + expect(query).not.toContain("t_alpha"); + expect(new URL(cap.urls[0]).searchParams.get("param_tenant")).toBe("t_alpha"); + } finally { + cap.restore(); + } + }); + + it("PRESERVED ClickHouseAnalytics still refuses a blank tenant_id", async () => { + const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123" }); + await expect(analytics.queryMetric({ tenant_id: " ", metric: "x", aggregate: "sum" })).rejects.toMatchObject({ + code: "INVALID_REQUEST", + }); + }); + + it("PRESERVED ClickHouseAnalytics still refuses a hostile group_by identifier", async () => { + const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123" }); + await expect( + analytics.queryMetric({ + tenant_id: "t_alpha", + metric: "x", + aggregate: "sum", + group_by: "sev'; DROP TABLE opt_metric_events; --", + }), + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); + }); + + it("every statement the NEW compiler produces contains the forced predicate", () => { + const shapes = [ + { metric: "m", aggregate: "sum" as const }, + { metric: "m", aggregate: "count" as const, group_by: ["severity"] }, + { metric: "m", aggregate: "avg" as const, filters: { make: "volvo" }, dataset_id: "ds_1" }, + { metric: "m", aggregate: "max" as const, from_time: "2026-01-01", to_time: "2026-02-01" }, + { metric: "m", aggregate: "count_distinct" as const, group_by: ["severity", "make"], limit: 5 }, + ]; + for (const shape of shapes) { + const compiled = compileMetricQuery({ tenant_id: "t_alpha", ...shape }, "opt_analytics", catalog, 1000); + expect(compiled.sql).toContain(FORCED_TENANT_PREDICATE); + // The predicate must be in the WHERE, not in a comment or an alias. + expect(compiled.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); + expect(compiled.params.param_tenant).toBe("t_alpha"); + expect(compiled.sql).not.toContain("t_alpha"); + } + const projection = compileProjectionQuery( + { tenant_id: "t_alpha", dataset_id: "ds_1", entity_keys: ["a", "b"] }, + "opt_analytics", + 1000, + ); + expect(projection.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); + expect(projection.sql).not.toContain("t_alpha"); + }); + + it("the NEW compiler fails closed on a blank/absent tenant instead of emitting an unfiltered query", () => { + for (const tenant of ["", " "]) { + expect(() => + compileMetricQuery({ tenant_id: tenant, metric: "m", aggregate: "sum" }, "opt_analytics", catalog, 1000), + ).toThrowError(/tenant_id is required/); + } + // The undefined case, which TypeScript would normally forbid, is what a + // JS caller or a loosely-typed HTTP boundary can actually produce. + expect(() => + compileMetricQuery( + { tenant_id: undefined as unknown as string, metric: "m", aggregate: "sum" }, + "opt_analytics", + catalog, + 1000, + ), + ).toThrowError(/tenant_id is required/); + }); + + it("the constant itself is the predicate, so deleting the forcing cannot pass silently", () => { + // If someone redefines FORCED_TENANT_PREDICATE to something harmless + // (e.g. "1 = 1") to make the substring assertions above pass, this fails. + expect(FORCED_TENANT_PREDICATE).toBe("tenant_id = {tenant:String}"); + }); +}); diff --git a/tests/integration/analytics-arrow-clickhouse.integration.test.ts b/tests/integration/analytics-arrow-clickhouse.integration.test.ts new file mode 100644 index 0000000..648ed06 --- /dev/null +++ b/tests/integration/analytics-arrow-clickhouse.integration.test.ts @@ -0,0 +1,183 @@ +/** + * BI-P005 conformance against an INDEPENDENT Arrow implementation. + * + * The unit suite (tests/analytics/arrow-ipc.test.ts) only proves 0pt's + * encoder agrees with 0pt's decoder — which it would even if both shared the + * same misreading of the spec. Here, ClickHouse's Arrow C++ implementation is + * the second opinion, in BOTH directions: + * + * ClickHouse -> Arrow -> 0pt decoder (their encoder, our reader) + * 0pt encoder -> Arrow -> ClickHouse (our encoder, their reader) + * + * The second direction is the load-bearing one: ClickHouse INSERTing our + * bytes and then answering a SELECT with the exact values back is a claim + * that cannot be satisfied by a subtly wrong buffer layout. + * + * The DoD's literal chain (ClickHouse -> Arrow -> DuckDB/Python) additionally + * ran against real pyarrow + DuckDB; that transcript is committed at + * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json. + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { + arrowStreamSettings, + clickhouseArrowStreamFormat, + decodeArrowStream, + encodeArrowStream, +} from "@/src/analytics/arrow/ipc"; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_arrow_${Date.now().toString(36)}`; + +function authHeaders(): Record { + return CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` } + : {}; +} + +async function chBytes(sql: string, settings: Record = {}): Promise { + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" }); + const buf = new Uint8Array(await res.arrayBuffer()); + if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`); + return buf; +} + +async function chText(sql: string, body?: BodyInit): Promise { + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" }); + const text = await res.text(); + if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`); + return text; +} + +run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => { + beforeAll(async () => { + await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`); + await chText( + `CREATE TABLE IF NOT EXISTS ${DB}.arrow_roundtrip ( + tenant_id String, n Int64, v Float64, flag Bool, ts DateTime64(3, 'UTC') + ) ENGINE = MergeTree() ORDER BY (tenant_id, n)`, + ); + }, 60_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => { + const bytes = await chBytes( + `SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag, + toDateTime64(ts, 3, 'UTC') AS ts + FROM ( + SELECT arrayJoin([ + ('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'), + ('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'), + ('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789') + ]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts + ) + FORMAT ${clickhouseArrowStreamFormat()}`, + arrowStreamSettings(), + ); + const info = decodeArrowStream(bytes); + expect(info.rowCount).toBe(3); + expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); + // Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys), + // Int=2, FloatingPoint=3, Bool=6, Timestamp=10. + expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); + }); + + it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => { + const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, { + output_format_arrow_string_as_string: "0", + }); + // Arrow Type union: Binary = 4, Utf8 = 5. + expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4); + const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings()); + expect(decodeArrowStream(asText).fields[0].typeType).toBe(5); + }); + + it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => { + const bytes = encodeArrowStream([ + { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, + { name: "n", type: "int64", values: [1, 2, 3] }, + { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, + { name: "flag", type: "bool", values: [true, false, true] }, + { + name: "ts", + type: "timestamp_ms", + values: [ + new Date("2026-01-01T00:00:00.000Z"), + new Date("2026-06-15T08:09:10.123Z"), + new Date("2026-09-06T12:34:56.789Z"), + ], + }, + ]); + await chText(`INSERT INTO ${DB}.arrow_roundtrip FORMAT ArrowStream`, bytes as unknown as BodyInit); + + const back = await chText( + `SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`, + ); + const rows = (JSON.parse(back) as { + data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[]; + }).data; + expect(rows).toHaveLength(3); + expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]); + expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]); + expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]); + expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]); + expect(rows.map((r) => r.ts)).toEqual([ + "2026-01-01 00:00:00.000", + "2026-06-15 08:09:10.123", + "2026-09-06 12:34:56.789", + ]); + }); + + it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => { + const exported = await chBytes( + `SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`, + arrowStreamSettings(), + ); + expect(decodeArrowStream(exported).rowCount).toBe(3); + + await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_circle AS ${DB}.arrow_roundtrip`); + await chText(`INSERT INTO ${DB}.arrow_circle FORMAT ArrowStream`, exported as unknown as BodyInit); + + const diff = await chText( + `SELECT count() AS c FROM ( + SELECT * FROM ${DB}.arrow_roundtrip + EXCEPT + SELECT * FROM ${DB}.arrow_circle + ) FORMAT JSON`, + ); + expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0); + }); + + it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => { + const n = 20_000; + const bytes = encodeArrowStream([ + { name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, + { name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) }, + { name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) }, + { name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, + { name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) }, + ]); + expect(decodeArrowStream(bytes).rowCount).toBe(n); + + await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_big AS ${DB}.arrow_roundtrip`); + await chText(`INSERT INTO ${DB}.arrow_big FORMAT ArrowStream`, bytes as unknown as BodyInit); + const agg = await chText( + `SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`, + ); + const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0]; + expect(Number(d.c)).toBe(n); + expect(Number(d.s)).toBe((n * (n - 1)) / 2); + expect(Number(d.t)).toBe(7); + expect(Number(d.f)).toBe(n / 2); + }, 60_000); +}); diff --git a/tests/integration/analytics-engine-clickhouse.integration.test.ts b/tests/integration/analytics-engine-clickhouse.integration.test.ts new file mode 100644 index 0000000..d798ca9 --- /dev/null +++ b/tests/integration/analytics-engine-clickhouse.integration.test.ts @@ -0,0 +1,252 @@ +/** + * BI-P009 (canonical OLAP provider) + BI-P010 (production schema), against a + * LIVE ClickHouse. + * + * Every one of the eight `SiaxAnalyticsEngine` capabilities is exercised for + * real here. A capability that only has a unit test with a stubbed fetch is a + * capability whose SQL has never been parsed by ClickHouse — `EXPLAIN + * ESTIMATE`, `KILL QUERY` and `OPTIMIZE ... PARTITION` in particular are + * syntax the server either accepts or does not, and no mock can tell you + * which. + * + * Gated on CLICKHOUSE_URL; skips honestly when unset. CI's `integration` job + * supplies it and hard-fails on a skipped suite. + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { ANALYTICS_MATERIALIZED_VIEWS, ANALYTICS_TABLES } from "@/src/analytics/engine/schema"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_engine_${Date.now().toString(36)}`; +const ID: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; + +let engine: ClickHouseAnalyticsEngine; + +async function raw(sql: string, body?: string): Promise { + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); + const text = await res.text(); + if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 300)}`); + return text; +} + +run("BI-P009/BI-P010 ClickHouse provider against a LIVE server", () => { + beforeAll(async () => { + engine = new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { t_alpha: { severity: "severity", make: "vehicle_make" } }, + }); + await engine.ensureSchema(); + await engine.ensureSchema(); // idempotent, as the DDL claims + }, 120_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await raw(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + // ── capability 1: health ──────────────────────────────────────────────── + + it("health() reports the live server's version", async () => { + const h = await engine.health(); + expect(h.healthy).toBe(true); + expect(h.provider).toBe("clickhouse"); + expect(h.version).toMatch(/^\d+\.\d+/); + }); + + // ── BI-P010: the schema is what it claims to be ───────────────────────── + + it("BI-P010: four purpose-built tables exist, NOT one giant table", async () => { + const listed = await raw( + `SELECT name FROM system.tables WHERE database = '${DB}' AND engine != 'MaterializedView' ORDER BY name FORMAT TSV`, + ); + const names = listed.trim().split("\n").filter(Boolean); + // The three this module owns, PLUS the preserved opt_metric_events. + for (const t of ANALYTICS_TABLES) expect(names).toContain(t); + expect(names).toContain("opt_metric_events"); + expect(names.length).toBeGreaterThanOrEqual(4); + }); + + it("BI-P010: the tables use different engines and different sort keys — the split is real", async () => { + const meta = await raw( + `SELECT name, engine, sorting_key FROM system.tables WHERE database = '${DB}' ORDER BY name FORMAT JSON`, + ); + const rows = (JSON.parse(meta) as { data: { name: string; engine: string; sorting_key: string }[] }).data; + const by = Object.fromEntries(rows.map((r) => [r.name, r])); + expect(by["opt_analytics_event"].engine).toBe("MergeTree"); + expect(by["opt_metric_daily"].engine).toBe("AggregatingMergeTree"); + expect(by["opt_dataset_projection"].engine).toBe("ReplacingMergeTree"); + // Different ORDER BY keys — the reason one table cannot serve both jobs. + expect(by["opt_analytics_event"].sorting_key).not.toBe(by["opt_dataset_projection"].sorting_key); + // Every one still leads with tenant_id. + for (const t of ANALYTICS_TABLES) expect(by[t].sorting_key.startsWith("tenant_id")).toBe(true); + }); + + it("BI-P010: TTL and LowCardinality are actually applied, not just written in a comment", async () => { + const ddl = await raw(`SHOW CREATE TABLE ${DB}.opt_analytics_event FORMAT TSVRaw`); + expect(ddl).toContain("TTL"); + expect(ddl).toContain("LowCardinality(String)"); + expect(ddl).toContain("PARTITION BY toYYYYMM(event_time)"); + const rollup = await raw(`SHOW CREATE TABLE ${DB}.opt_metric_daily FORMAT TSVRaw`); + expect(rollup).toContain("AggregateFunction(sum, Float64)"); + }); + + it("BI-P010: the materialized view actually populates the rollup", async () => { + const views = await raw( + `SELECT name FROM system.tables WHERE database='${DB}' AND engine='MaterializedView' FORMAT TSV`, + ); + for (const v of ANALYTICS_MATERIALIZED_VIEWS) expect(views).toContain(v); + + await engine.ingestProjection(ID, { dataset_id: "warmup", rows: [{ entity_key: "w1" }] }); + const rows = Array.from({ length: 20 }, (_, i) => + JSON.stringify({ + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_mv", + metric: "mv_metric", + event_id: `mv${i}`, + event_time: "2026-09-03 12:00:00.000", + value: 2, + dim_names: [], + dim_values: [], + }), + ).join("\n"); + await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows); + + const rolled = await raw( + `SELECT sumMerge(sum_state) AS s, countMerge(count_state) AS c FROM ${DB}.opt_metric_daily ` + + `WHERE tenant_id = 't_alpha' AND metric = 'mv_metric' FORMAT JSON`, + ); + const d = (JSON.parse(rolled) as { data: { s: number | string; c: number | string }[] }).data[0]; + expect(Number(d.s)).toBe(40); // 20 rows x 2 + expect(Number(d.c)).toBe(20); + }); + + // ── capability 2: executeMetric (already covered by the isolation suite) ─ + + it("executeMetric returns grouped rows keyed by the CALLER-FACING dimension name", async () => { + const rows = ["warning", "critical", "critical"].map((sev, i) => + JSON.stringify({ + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_g", + metric: "grouped", + event_id: `g${i}`, + event_time: "2026-09-04 09:00:00.000", + value: 5, + dim_names: ["severity"], + dim_values: [sev], + }), + ); + await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows.join("\n")); + const result = await engine.executeMetric(ID, { + metric: "grouped", + aggregate: "sum", + group_by: ["severity"], + }); + const critical = result.rows.find((r) => r.groups?.severity === "critical"); + expect(critical?.value).toBe(10); + expect(result.stats.result_rows).toBe(2); + expect(result.query_id.startsWith("opt_")).toBe(true); + }); + + // ── capability 4: estimateQuery ───────────────────────────────────────── + + it("estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget", async () => { + const est = await engine.estimateQuery(ID, { metric: "grouped", aggregate: "sum" }); + expect(est.estimated_rows_to_read).toBeGreaterThan(0); + expect(est.within_budget).toBe(true); + expect(est.budget.maxRowsToRead).toBeGreaterThan(0); + }); + + it("estimateQuery REFUSES a query whose estimate exceeds a tightened quota", async () => { + const tiny: AnalyticsIdentity = { ...ID, quota: { maxRowsToRead: 1 } }; + const est = await engine.estimateQuery(tiny, { metric: "grouped", aggregate: "sum" }); + expect(est.within_budget).toBe(false); + expect(est.reason).toMatch(/over budget/); + }); + + // ── capability 6: explain ─────────────────────────────────────────────── + + it("explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts", async () => { + const ex = await engine.explain(ID, { metric: "grouped", aggregate: "sum", group_by: ["severity"] }); + expect(ex.forced_tenant_predicate).toBe("tenant_id = {tenant:String}"); + expect(ex.table).toBe("opt_analytics_event"); + // This is the payoff of BI-P010's tenant-first ORDER BY: the server + // reports tenant_id as a PrimaryKey key with a condition on it, i.e. the + // isolation predicate is an index range scan, not a post-read filter. + expect(ex.plan).toContain("ReadFromMergeTree"); + expect(ex.plan).toContain("PrimaryKey"); + expect(ex.plan).toMatch(/Keys:[\s\S]*tenant_id/); + expect(ex.plan).toMatch(/tenant_id in \['t_alpha', 't_alpha'\]/); + // The plan describes only the CALLER's own tenant; no other tenant id + // may appear in a plan handed back to them. + expect(ex.plan).not.toContain("t_beta"); + }); + + // ── capability 5: cancelQuery ─────────────────────────────────────────── + + it("cancelQuery issues a real KILL QUERY the server accepts", async () => { + // Nothing is running under this id, so the honest outcome is "the KILL + // statement was valid and matched nothing" -> true, with no exception. + const ok = await engine.cancelQuery(ID, `${(await engine.executeMetric(ID, { metric: "grouped", aggregate: "count" })).query_id}`); + expect(ok).toBe(true); + }); + + // ── capabilities 7 + 8: ingestProjection / refreshProjection ──────────── + + it("ingestProjection writes the SERVING tier with the identity's tenant, ignoring any row-level claim", async () => { + const res = await engine.ingestProjection(ID, { + dataset_id: "ds_vehicles", + rows: [ + { entity_key: "VIN001", attributes: { make: "volvo" }, measures: { price: 189000 } }, + { entity_key: "VIN002", attributes: { make: "saab" }, measures: { price: 42000 } }, + ], + }); + expect(res.ingested).toBe(2); + + const stored = await raw( + `SELECT uniqExact(tenant_id) AS t FROM ${DB}.opt_dataset_projection WHERE dataset_id = 'ds_vehicles' FORMAT JSON`, + ); + expect(Number((JSON.parse(stored) as { data: { t: number | string }[] }).data[0].t)).toBe(1); + }); + + it("refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL)", async () => { + // Re-ingest the SAME entity keys at a higher version. + await engine.ingestProjection(ID, { + dataset_id: "ds_vehicles", + rows: [ + { entity_key: "VIN001", version: 9_999_999_999_999, measures: { price: 175000 } }, + { entity_key: "VIN002", version: 9_999_999_999_999, measures: { price: 41000 } }, + ], + }); + const refreshed = await engine.refreshProjection(ID, "ds_vehicles"); + expect(refreshed.row_count).toBe(2); // 2 entities, not 4 rows + expect(refreshed.dataset_id).toBe("ds_vehicles"); + + const winner = await raw( + `SELECT measures['price'] AS p FROM ${DB}.opt_dataset_projection FINAL ` + + `WHERE tenant_id='t_alpha' AND dataset_id='ds_vehicles' AND entity_key='VIN001' FORMAT JSON`, + ); + expect(Number((JSON.parse(winner) as { data: { p: number | string }[] }).data[0].p)).toBe(175000); + }); + + // ── the PRESERVED path still works through the engine ─────────────────── + + it("the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine", async () => { + await engine.legacyAnalytics.ingest([ + { tenant_id: "t_alpha", dataset_id: "ds_legacy", metric: "legacy_metric", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_legacy", metric: "legacy_metric", value: 700 }, + ]); + const res = await engine.executeLegacyMetric(ID, { metric: "legacy_metric", aggregate: "sum" }); + expect(res.rows[0].value).toBe(7); // NOT 707 + }); +}); diff --git a/tests/integration/analytics-resource-governance.integration.test.ts b/tests/integration/analytics-resource-governance.integration.test.ts new file mode 100644 index 0000000..a0b5094 --- /dev/null +++ b/tests/integration/analytics-resource-governance.integration.test.ts @@ -0,0 +1,237 @@ +/** + * BI-P012 DoD, against a LIVE ClickHouse: + * + * "Intentional query bomb is terminated without degrading unrelated tenant + * beyond declared SLO." + * + * Two halves, and the second is the one that is usually skipped: + * + * 1. TERMINATED — a deliberately enormous query is refused by the server, + * with the ClickHouse limit that stopped it identified by code. Each + * limit BI-P012 names is fired individually, so "some limit stopped it" + * is never mistaken for "the limit I claimed stopped it". + * 2. UNRELATED TENANT NOT DEGRADED — a second tenant's interactive query is + * timed WHILE the bomb is in flight and compared against a declared SLO, + * not merely observed to "still work". + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_gov_${Date.now().toString(36)}`; + +/** + * The SLO this test asserts against. Declared as a constant so the claim is + * falsifiable rather than "it still seemed to work". + * + * 5000 ms is deliberately generous for a 500-row lookup (it measures ~30 ms + * on an idle server). It is set for the SHARED CI runner, not for a quiet + * laptop: a tenant that had actually been starved by the bomb would take + * seconds or fail outright, so the assertion still discriminates, while a + * merely busy runner does not turn this gate into a flake. + */ +const UNRELATED_TENANT_SLO_MS = 5000; + +const VICTIM: AnalyticsIdentity = { tenant_id: "t_victim", project_id: "p_main", principal_id: "usr_v" }; + +let engine: ClickHouseAnalyticsEngine; + +/** Run a statement with explicit settings and report the ClickHouse code. */ +async function chRun( + sql: string, + settings: Record = {}, + body?: string, +): Promise<{ ok: boolean; code?: string; text: string; ms: number }> { + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); + const headers: Record = {}; + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; + const t0 = Date.now(); + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); + const text = await res.text(); + return { ok: res.ok, code: /Code: (\d+)/.exec(text)?.[1], text, ms: Date.now() - t0 }; +} + +run("BI-P012 resource governance against a LIVE ClickHouse", () => { + beforeAll(async () => { + engine = new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { t_victim: { severity: "severity" } }, + }); + await engine.ensureSchema(); + // A small, real dataset for the unrelated tenant. + const rows = Array.from({ length: 500 }, (_, i) => + JSON.stringify({ + tenant_id: "t_victim", + project_id: "p_main", + dataset_id: "ds_v", + metric: "victim_metric", + event_id: `v${i}`, + event_time: "2026-09-05 08:00:00.000", + value: 1, + dim_names: ["severity"], + dim_values: ["warning"], + }), + ).join("\n"); + const ins = await chRun(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, {}, rows); + expect(ins.ok, ins.text.slice(0, 200)).toBe(true); + }, 120_000); + + afterAll(async () => { + if (CLICKHOUSE_URL) await chRun(`DROP DATABASE IF EXISTS ${DB}`); + }, 60_000); + + // ── each named limit, fired individually ──────────────────────────────── + + it("max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)", async () => { + const r = await chRun("SELECT count() FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_rows_to_read: "1000", + read_overflow_mode: "throw", + }); + expect(r.ok).toBe(false); + expect(r.code).toBe("158"); + }); + + it("max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)", async () => { + const r = await chRun("SELECT sum(number) FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_bytes_to_read: "1024", + read_overflow_mode: "throw", + }); + expect(r.ok).toBe(false); + expect(r.code).toBe("307"); + }); + + it("max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)", async () => { + const r = await chRun("SELECT count() FROM numbers(50000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "1", + timeout_overflow_mode: "throw", + }); + expect(r.ok).toBe(false); + expect(r.code).toBe("159"); + expect(r.ms).toBeLessThan(20_000); + }); + + it("max_result_rows terminates a result bomb and does NOT return a partial result", async () => { + const r = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "throw", + }); + expect(r.ok).toBe(false); + expect(r.code).toBe("396"); // TOO_MANY_ROWS_OR_BYTES + // The dangerous alternative: result_overflow_mode=break returns 10 rows + // with HTTP 200 and no indication of truncation. + const broken = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "break", + }); + // MEASURED on ClickHouse 24.8: break mode answers HTTP 200 with 65409 of + // 100000 rows (it stops at a block boundary, not at the stated 10) and + // nothing in the payload says it is short. That is why every SELECT in + // this engine ships result_overflow_mode=throw. + expect(broken.ok).toBe(true); + const brokenRows = (JSON.parse(broken.text) as { data: unknown[] }).data.length; + expect(brokenRows).toBeGreaterThan(10); + expect(brokenRows).toBeLessThan(100_000); + }); + + it("max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)", async () => { + const r = await chRun( + "SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON", + { readonly: "1", max_memory_usage: "1000000", max_execution_time: "30" }, + ); + expect(r.ok).toBe(false); + expect(r.code).toBe("241"); + }); + + it("readonly=1 means the query text cannot re-raise its own limits", async () => { + const r = await chRun( + "SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON", + { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw" }, + ); + expect(r.ok).toBe(false); + // 164 = READONLY (changing settings is forbidden); 158 = the limit still + // applied. Either proves the caller did not win — but NOT a 200. + expect(["164", "158"]).toContain(r.code); + }); + + it("readonly=1 blocks writes through the query surface", async () => { + const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: "1" }); + expect(r.ok).toBe(false); + }); + + // ── the half that is usually skipped ──────────────────────────────────── + + it("an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight", async () => { + // Baseline first, so "fast enough" is measured against this server, not + // against a number someone hoped for. + const baseline = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); + expect(baseline.rows[0].value).toBe(500); + const baselineMs = baseline.stats.duration_ms; + + // Fire a genuine bomb the server must spend real CPU on. MEASURED: a bare + // `count()` over numbers() is NOT a bomb — ClickHouse answers + // numbers(20e9) in ~5s because it can skip the values entirely. Hashing + // every row cannot be short-circuited, so this one really does burn CPU + // until the limit kills it. + const bomb = chRun("SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "8", + timeout_overflow_mode: "throw", + }); + + // While it burns, run the unrelated tenant's interactive query repeatedly. + const samples: number[] = []; + for (let i = 0; i < 5; i++) { + const r = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); + expect(r.rows[0].value).toBe(500); // still CORRECT, not just fast + samples.push(r.stats.duration_ms); + } + + const bombResult = await bomb; + expect(bombResult.ok, "the bomb was NOT terminated").toBe(false); + expect(bombResult.code).toBe("159"); + + const worst = Math.max(...samples); + expect( + worst, + `unrelated tenant degraded: baseline ${baselineMs}ms, worst under load ${worst}ms, SLO ${UNRELATED_TENANT_SLO_MS}ms`, + ).toBeLessThan(UNRELATED_TENANT_SLO_MS); + }, 60_000); + + // ── the local admission gate, on the real engine ──────────────────────── + + it("the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only", async () => { + const limit = DEFAULT_BUDGETS.INTERACTIVE.maxConcurrentQueries; + // Fire limit+4 concurrently; some must be refused with RATE_LIMITED + // rather than all of them reaching the server. + const attempts = Array.from({ length: limit + 4 }, () => + engine + .executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }) + .then(() => "ok" as const) + .catch((e: { code?: string }) => e.code ?? "error"), + ); + const outcomes = await Promise.all(attempts); + expect(outcomes.filter((o) => o === "RATE_LIMITED").length).toBeGreaterThan(0); + expect(outcomes.filter((o) => o === "ok").length).toBeGreaterThan(0); + + // A different tenant is unaffected by the noisy one's exhausted slots. + const other = await engine.executeMetric( + { tenant_id: "t_quiet", project_id: "p_main", principal_id: "usr_q" }, + { metric: "victim_metric", aggregate: "count" }, + ); + expect(other.rows[0].value).toBe(0); // its own (empty) data, and no error + }, 60_000); +}); diff --git a/tests/integration/analytics-tenant-isolation.integration.test.ts b/tests/integration/analytics-tenant-isolation.integration.test.ts new file mode 100644 index 0000000..3a1f9c9 --- /dev/null +++ b/tests/integration/analytics-tenant-isolation.integration.test.ts @@ -0,0 +1,308 @@ +/** + * BI-P011 — CLICKHOUSE TENANT ISOLATION, proven against a LIVE server. + * + * The shape of this proof is the estate's own convention for isolation work: + * DEMONSTRATE THE HOLE WITHOUT THE FIX, then demonstrate it CLOSED WITH the + * fix. A suite that only exercises the fixed path proves that the fixed path + * works, which is not the same claim. + * + * So each layer is tested twice: + * + * layer 2 (0pt query compiler forces tenant) + * hole: a hand-written statement WITHOUT the predicate — the same query + * minus the forcing — returns another tenant's rows. This is the + * literal regression a future refactor would cause. + * closed: the compiled statement returns only this tenant's rows. + * + * layer 3 (restricted database users + row policies) + * hole: the UNRESTRICTED admin user, issuing a statement with NO tenant + * predicate at all, sees every tenant. + * closed: the RESTRICTED per-tenant user, issuing the SAME statement, + * sees only its own tenant — the server filters, so the result + * holds even if layer 2 were removed entirely. + * + * Gated on CLICKHOUSE_URL and SKIPS HONESTLY when unset, matching + * tests/integration/analytics-clickhouse.integration.test.ts. CI's + * `integration` job sets it from a service container and hard-fails if the + * suites skip, so this is not a self-skipping green in practice. + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { + compileMetricQuery, + dimensionCatalogFrom, + restrictedTenantUserDdl, +} from "@/src/analytics/engine/query-compiler"; +import { ANALYTICS_TABLES, sortingKeySql } from "@/src/analytics/engine/schema"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const run = CLICKHOUSE_URL ? describe : describe.skip; + +const DB = `opt_isolation_${Date.now().toString(36)}`; +const RESTRICTED_USER = `opt_iso_${Date.now().toString(36)}`; +const RESTRICTED_PW = `iso_pw_${Math.random().toString(36).slice(2, 12)}`; + +const ALPHA: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; +const BETA: AnalyticsIdentity = { tenant_id: "t_beta", project_id: "p_main", principal_id: "usr_beta" }; + +const catalog = dimensionCatalogFrom({ severity: "severity" }); +const ALL_TABLES: readonly string[] = ANALYTICS_TABLES; + +/** Raw HTTP against the live server, as a named user. */ +async function chQuery( + sql: string, + auth: string | undefined, + params: Record = {}, + settings: Record = {}, + body?: string, +): Promise<{ ok: boolean; status: number; text: string }> { + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + const headers: Record = {}; + if (auth) headers["Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); + return { ok: res.ok, status: res.status, text: await res.text() }; +} + +function rowsOf(text: string): { grp?: string; value: number }[] { + if (!text.trim()) return []; + const parsed = JSON.parse(text) as { data?: { grp?: string; value: number | string }[] }; + return (parsed.data ?? []).map((r) => ({ ...(r.grp !== undefined ? { grp: r.grp } : {}), value: Number(r.value) })); +} + +let engine: ClickHouseAnalyticsEngine; + +run("BI-P011 tenant isolation against a LIVE ClickHouse", () => { + beforeAll(async () => { + engine = new ClickHouseAnalyticsEngine({ + baseUrl: CLICKHOUSE_URL!, + basicAuth: CLICKHOUSE_AUTH, + database: DB, + dimensionCatalogs: { t_alpha: { severity: "severity" }, t_beta: { severity: "severity" } }, + }); + const health = await engine.health(); + expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ""}`).toBe(true); + await engine.ensureSchema(); + + // Two tenants, deliberately different magnitudes so a leak is unmistakable + // in the NUMBER, not just in the row count. + const rows: string[] = []; + for (let i = 0; i < 50; i++) { + rows.push( + JSON.stringify({ + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", + event_id: `a${i}`, + event_time: "2026-09-01 10:00:00.000", + value: 1, + dim_names: ["severity"], + dim_values: [i % 2 === 0 ? "warning" : "critical"], + }), + ); + rows.push( + JSON.stringify({ + tenant_id: "t_beta", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", + event_id: `b${i}`, + event_time: "2026-09-01 10:00:00.000", + value: 1000, + dim_names: ["severity"], + dim_values: ["critical"], + }), + ); + } + const ins = await chQuery( + `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, + CLICKHOUSE_AUTH, + {}, + {}, + rows.join("\n"), + ); + expect(ins.ok, `seed insert failed: ${ins.text.slice(0, 200)}`).toBe(true); + }, 120_000); + + afterAll(async () => { + if (!CLICKHOUSE_URL) return; + // Disposable by construction: drop the user, its policies, and the DB. + for (const table of ANALYTICS_TABLES) { + await chQuery(`DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, CLICKHOUSE_AUTH); + } + await chQuery(`DROP USER IF EXISTS ${RESTRICTED_USER}`, CLICKHOUSE_AUTH); + await chQuery(`DROP DATABASE IF EXISTS ${DB}`, CLICKHOUSE_AUTH); + }, 60_000); + + // ── layer 2: the compiler ──────────────────────────────────────────────── + + it("HOLE, layer 2 REMOVED: the same query WITHOUT the forced predicate returns t_beta's rows to t_alpha", async () => { + // This is the exact statement the compiler would produce if someone + // deleted the forcing — proof that the predicate is doing real work + // against real data, not decorating a query that was safe anyway. + const unsafeSql = + `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event ` + + `WHERE metric = {metric:String} LIMIT 1 FORMAT JSON`; + const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: "rows_ingested" }); + expect(res.ok).toBe(true); + // 50*1 (alpha) + 50*1000 (beta) = 50050 — both tenants, one result. + expect(rowsOf(res.text)[0].value).toBe(50050); + }); + + it("CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000", async () => { + const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum" }); + expect(result.rows[0].value).toBe(50); + const beta = await engine.executeMetric(BETA, { metric: "rows_ingested", aggregate: "sum" }); + expect(beta.rows[0].value).toBe(50_000); + }); + + it("tenant A querying tenant B's dataset id gets ZERO rows, not B's rows", async () => { + const result = await engine.executeMetric(ALPHA, { + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_shared", + }); + expect(result.rows[0].value).toBe(50); // only alpha's 50, though beta shares the dataset id + }); + + it("a forged dataset id yields an empty result, never an error that leaks existence", async () => { + const result = await engine.executeMetric(ALPHA, { + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_that_does_not_exist_or_belongs_to_someone_else", + }); + expect(result.rows[0].value).toBe(0); + }); + + it("a tenant with no data at all sees zero, not everyone's aggregate", async () => { + const ghost: AnalyticsIdentity = { tenant_id: "t_gamma", project_id: "p_main", principal_id: "usr_g" }; + const result = await engine.executeMetric(ghost, { metric: "rows_ingested", aggregate: "count" }); + expect(result.rows[0].value).toBe(0); + }); + + it("group-by results carry only this tenant's dimension values", async () => { + const result = await engine.executeMetric(ALPHA, { + metric: "rows_ingested", + aggregate: "count", + group_by: ["severity"], + }); + const total = result.rows.reduce((a, r) => a + r.value, 0); + expect(total).toBe(50); + // t_beta's rows are all "critical" with value 1000; if any leaked, the + // critical bucket would exceed alpha's 25. + const critical = result.rows.find((r) => r.groups?.severity === "critical"); + expect(critical?.value).toBe(25); + }); + + it("SQL/dimension/filter injection through the live path changes nothing", async () => { + await expect( + engine.executeMetric(ALPHA, { + metric: "rows_ingested", + aggregate: "sum", + group_by: ["severity'; SELECT tenant_id FROM system.tables; --"], + }), + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); + + // A hostile VALUE reaches the server as a bound parameter and simply + // matches nothing. + const injected = await engine.executeMetric(ALPHA, { + metric: "rows_ingested", + aggregate: "count", + filters: { severity: "warning' OR 1=1 --" }, + }); + expect(injected.rows[0].value).toBe(0); + + // And the table is still there afterwards. + const still = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count" }); + expect(still.rows[0].value).toBe(50); + }); + + it("raw SQL attempt: the analyst-SQL surface is refused, not silently executed", async () => { + await expect( + engine.executeQuery(ALPHA, { sql: `SELECT * FROM ${DB}.opt_analytics_event`, dataset_ids: [] }), + ).rejects.toMatchObject({ code: "CAPABILITY_UNAVAILABLE" }); + }); + + // ── layer 3: restricted users + row policies ───────────────────────────── + + it("HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant", async () => { + const res = await chQuery( + `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, + CLICKHOUSE_AUTH, + ); + expect(res.ok).toBe(true); + expect(rowsOf(res.text)[0].value).toBe(2); // t_alpha AND t_beta + }); + + it("CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant", async () => { + for (const stmt of restrictedTenantUserDdl({ + db: DB, + tenantId: "t_alpha", + userName: RESTRICTED_USER, + password: RESTRICTED_PW, + tables: ALL_TABLES, + })) { + const r = await chQuery(stmt, CLICKHOUSE_AUTH); + expect(r.ok, `DDL failed: ${stmt.slice(0, 80)} -> ${r.text.slice(0, 200)}`).toBe(true); + } + + const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; + // The IDENTICAL statement that leaked above, now as the restricted user. + const res = await chQuery( + `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, + restrictedAuth, + ); + expect(res.ok, res.text.slice(0, 300)).toBe(true); + expect(rowsOf(res.text)[0].value).toBe(1); + + const sum = await chQuery( + `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, + restrictedAuth, + ); + expect(rowsOf(sum.text)[0].value).toBe(50); // alpha's total, not 50050 + }); + + it("the restricted user cannot write, and cannot reach a table it was not granted", async () => { + const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; + const write = await chQuery( + `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, + restrictedAuth, + {}, + {}, + JSON.stringify({ tenant_id: "t_beta", project_id: "p", dataset_id: "d", metric: "m", event_id: "x", event_time: "2026-09-01 00:00:00.000", value: 1, dim_names: [], dim_values: [] }), + ); + expect(write.ok).toBe(false); + + const foreign = await chQuery(`SELECT count() FROM system.users FORMAT JSON`, restrictedAuth); + expect(foreign.ok).toBe(false); + }); + + // ── the structural precondition both layers rely on ────────────────────── + + it("every analytics table's sorting key LEADS with tenant_id (what makes the predicate cheap)", async () => { + for (const table of ALL_TABLES) { + const res = await chQuery(sortingKeySql(DB, table), CLICKHOUSE_AUTH); + expect(res.ok).toBe(true); + expect(res.text.trim().startsWith("tenant_id"), `${table} sorting key: ${res.text.trim()}`).toBe(true); + } + }); + + it("the tenant value never appears in the statement text the server receives", async () => { + const compiled = compileMetricQuery( + { tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "sum" }, + DB, + catalog, + DEFAULT_BUDGETS.INTERACTIVE.maxResultRows, + ); + expect(compiled.sql).not.toContain("t_alpha"); + const res = await chQuery(compiled.sql, CLICKHOUSE_AUTH, compiled.params); + expect(res.ok).toBe(true); + expect(rowsOf(res.text)[0].value).toBe(50); + }); +}); -- 2.54.0