diff --git a/analytics/dbt/README.md b/analytics/dbt/README.md new file mode 100644 index 0000000..1f7e9c8 --- /dev/null +++ b/analytics/dbt/README.md @@ -0,0 +1,51 @@ +# `analytics/dbt/` — BI-P022 + +The canonical SIAX transformation project. **Nothing in this repository runs +it.** Execution lives in N0D (`n0d#38`), against the contract in +`src/analytics/transform/dbt-contract.ts`. + +## Layers, and why they map onto BI-P003 + +| directory | tier (BI-P003) | may read | must not contain | +| ---------------------- | -------------- | --------------------- | --------------------- | +| `models/sources/` | `RAW` | — (declarations only) | any SQL | +| `models/staging/` | `NORMALIZED` | sources | joins, business logic | +| `models/intermediate/` | `CURATED` | staging | presentation shaping | +| `models/marts/` | `SERVING` | intermediate | direct source reads | + +`assertLayerDependenciesLegal` enforces the "may read" column against the +dependency graph in `manifest.json`. A mart reading a source directly is +refused — that is raw data reaching a dashboard, which is precisely what +BI-P003's tier rule exists to prevent, and a convention nobody checks would not +prevent it. + +## What a run must produce + +`manifest.json` and `run_results.json`, both registered as ST0RE artifacts. +`assertValidDbtJobResult` refuses a result missing either: without them the run +cannot be audited and its tests cannot be ingested by BI-P025's +`DbtQualityProvider`. + +It also refuses: + +- a result whose revision differs from the request — the run is not reproducible; +- a result whose observed dbt version differs from the pin — a runtime that is + not the pinned one silently changes what the models mean; +- `status: success` with **zero nodes executed** — a dbt selector that matches + nothing exits 0, which is the dbt-shaped false green. + +## Version posture + +`DBT_RUNTIME_PIN` records dbt Core 1.10.13 (stable line) with +`dbt-clickhouse` 1.9.2. **Both review flags are `false`.** No licence or +supply-chain review has been performed in this repo, and none is claimed. +`assertDbtRuntimeCertified` refuses a `production` run until they are true, so +BI-P022's "after current license/security verification" is a gate rather than a +sentence in a document. + +## Credentials + +A dbt run never receives a tenant's warehouse credential from this repo. N0D +resolves a credential whose write scope matches `target_layer` and +`expected_target_schema`, and the contract refuses a request whose expected +schema is not scoped to the job's tenant. diff --git a/analytics/dbt/dbt_project.yml b/analytics/dbt/dbt_project.yml new file mode 100644 index 0000000..59c8a95 --- /dev/null +++ b/analytics/dbt/dbt_project.yml @@ -0,0 +1,43 @@ +# BI-P022 — the canonical SIAX analytics transformation project. +# +# Execution happens in N0D (n0d#38), never here. This project is the MODELS and +# the CONVENTIONS; src/analytics/transform/dbt-contract.ts is the job contract +# N0D consumes, and it is what refuses a run that is not reproducible. +name: siax_analytics +version: 1.0.0 +config-version: 2 +profile: siax_analytics + +model-paths: ['models'] +test-paths: ['tests'] +snapshot-paths: ['snapshots'] +macro-paths: ['macros'] + +# The four layers map onto BI-P003's tiers rather than inventing a fifth +# vocabulary. `+tags` carries the tier so a manifest reader can assign one +# without parsing paths, and dbt-contract.ts asserts the mapping. +models: + siax_analytics: + +materialized: view + staging: + +schema: staging + +materialized: view + +tags: ['tier:NORMALIZED'] + intermediate: + +schema: intermediate + +materialized: table + +tags: ['tier:CURATED'] + marts: + +schema: marts + +materialized: table + +tags: ['tier:SERVING'] + +snapshots: + siax_analytics: + +target_schema: snapshots + +# `on_schema_change: fail` rather than `sync_all_columns`: a silently widened +# table is how a downstream metric starts averaging a column that did not exist +# yesterday. BI-P004's contract gate is the same rule one layer up. +vars: + siax_tenant_id: '__set_by_n0d__' diff --git a/analytics/dbt/models/intermediate/int_sales_enriched.sql b/analytics/dbt/models/intermediate/int_sales_enriched.sql new file mode 100644 index 0000000..22efc0b --- /dev/null +++ b/analytics/dbt/models/intermediate/int_sales_enriched.sql @@ -0,0 +1,16 @@ +-- CURATED (BI-P003). Joins and conformed entities. Reads STAGING only. +{{ config(materialized='table', tags=['tier:CURATED']) }} + +select + e.event_id, + e.tenant_id, + e.customer_id, + e.amount_sek, + e.channel, + coalesce(c.country_code, e.country_code) as country_code, + c.segment, + e.invoiced_at +from {{ ref('stg_sales_events') }} e +left join {{ ref('stg_customers') }} c + on e.customer_id = c.customer_id + and e.tenant_id = c.tenant_id -- the join itself stays tenant-scoped diff --git a/analytics/dbt/models/marts/mart_sales_daily.sql b/analytics/dbt/models/marts/mart_sales_daily.sql new file mode 100644 index 0000000..229caf7 --- /dev/null +++ b/analytics/dbt/models/marts/mart_sales_daily.sql @@ -0,0 +1,17 @@ +-- SERVING (BI-P003). What a dashboard or a BI-P028 metric reads. +-- +-- Reads INTERMEDIATE only. Reading a source here would skip two layers, and +-- assertLayerDependenciesLegal refuses that against the manifest graph. +{{ config(materialized='table', tags=['tier:SERVING']) }} + +select + tenant_id, + date_trunc('day', invoiced_at) as day, + country_code, + channel, + segment, + count(*) as order_count, + sum(amount_sek) as net_revenue_sek +from {{ ref('int_sales_enriched') }} +where channel != 'internal' -- matches the net_revenue metric's baked filter +group by 1, 2, 3, 4, 5 diff --git a/analytics/dbt/models/marts/schema.yml b/analytics/dbt/models/marts/schema.yml new file mode 100644 index 0000000..a84817d --- /dev/null +++ b/analytics/dbt/models/marts/schema.yml @@ -0,0 +1,19 @@ +version: 2 + +models: + - name: mart_sales_daily + description: > + SERVING projection behind the `net_revenue` and `orders` metrics. + A blocking test failure here quarantines the serving projection + (BI-P025); the quarantine is enforced on the READ side. + columns: + - name: tenant_id + tests: [not_null] + - name: day + tests: [not_null] + - name: net_revenue_sek + tests: [not_null] + - name: order_count + tests: + - not_null + - dbt_utils.accepted_range: { min_value: 1, inclusive: true } diff --git a/analytics/dbt/models/sources/sources.yml b/analytics/dbt/models/sources/sources.yml new file mode 100644 index 0000000..34c152c --- /dev/null +++ b/analytics/dbt/models/sources/sources.yml @@ -0,0 +1,34 @@ +version: 2 + +# RAW. Declared, never modelled. A source has no SQL of its own — anything that +# transforms belongs in staging, where BI-P003 assigns it NORMALIZED. +sources: + - name: opt_raw + description: > + Landed, untransformed data as delivered by the 0pt sourcing engine. + RAW tier: no BI surface may read this directly without a recorded + BI-P003 exception. + schema: "{{ var('siax_tenant_id') }}_raw" + loaded_at_field: ingested_at + freshness: + # BI-P026's DELAYED/STALE thresholds, expressed where dbt can act on them. + warn_after: { count: 90, period: minute } + error_after: { count: 180, period: minute } + tables: + - name: sales_events + description: One row per invoiced sales line, as landed. + columns: + - name: event_id + tests: [not_null, unique] + - name: tenant_id + description: Present on every raw row; the tier boundary depends on it. + tests: [not_null] + - name: ingested_at + tests: [not_null] + - name: customers + description: Customer master as landed. + columns: + - name: customer_id + tests: [not_null, unique] + - name: tenant_id + tests: [not_null] diff --git a/analytics/dbt/models/staging/schema.yml b/analytics/dbt/models/staging/schema.yml new file mode 100644 index 0000000..dd4216c --- /dev/null +++ b/analytics/dbt/models/staging/schema.yml @@ -0,0 +1,26 @@ +version: 2 + +models: + - name: stg_sales_events + description: Normalized sales events — one row per event_id, latest landing wins. + columns: + - name: event_id + tests: [not_null, unique] + - name: tenant_id + tests: [not_null] + - name: amount_sek + tests: + - not_null + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + - name: channel + tests: + - accepted_values: + values: ['web', 'retail', 'partner', 'internal'] + - name: stg_customers + columns: + - name: customer_id + tests: [not_null, unique] + - name: tenant_id + tests: [not_null] diff --git a/analytics/dbt/models/staging/stg_customers.sql b/analytics/dbt/models/staging/stg_customers.sql new file mode 100644 index 0000000..2ef66df --- /dev/null +++ b/analytics/dbt/models/staging/stg_customers.sql @@ -0,0 +1,11 @@ +-- NORMALIZED (BI-P003). +{{ config(materialized='view', tags=['tier:NORMALIZED']) }} + +select + cast(customer_id as varchar) as customer_id, + cast(tenant_id as varchar) as tenant_id, + cast(name as varchar) as customer_name, + cast(segment as varchar) as segment, + cast(country as varchar) as country_code +from {{ source('opt_raw', 'customers') }} +where tenant_id = '{{ var("siax_tenant_id") }}' diff --git a/analytics/dbt/models/staging/stg_sales_events.sql b/analytics/dbt/models/staging/stg_sales_events.sql new file mode 100644 index 0000000..e1147e7 --- /dev/null +++ b/analytics/dbt/models/staging/stg_sales_events.sql @@ -0,0 +1,30 @@ +-- NORMALIZED (BI-P003). Rename, cast, deduplicate. No joins, no business logic. +-- +-- The tenant predicate is applied HERE, at the first modelled layer, so every +-- downstream model inherits it and no later model has to remember. It is the +-- same forced-predicate rule as the query compiler's, one layer up. +{{ config(materialized='view', tags=['tier:NORMALIZED']) }} + +with source as ( + select * from {{ source('opt_raw', 'sales_events') }} + where tenant_id = '{{ var("siax_tenant_id") }}' +), + +deduplicated as ( + select + *, + row_number() over (partition by event_id order by ingested_at desc) as _rn + from source +) + +select + cast(event_id as varchar) as event_id, + cast(tenant_id as varchar) as tenant_id, + cast(customer_id as varchar) as customer_id, + cast(amount as double) as amount_sek, + cast(channel as varchar) as channel, + cast(country as varchar) as country_code, + cast(invoiced_at as timestamp) as invoiced_at, + cast(ingested_at as timestamp) as ingested_at +from deduplicated +where _rn = 1 diff --git a/analytics/dbt/snapshots/snap_customers.sql b/analytics/dbt/snapshots/snap_customers.sql new file mode 100644 index 0000000..d712410 --- /dev/null +++ b/analytics/dbt/snapshots/snap_customers.sql @@ -0,0 +1,15 @@ +-- Slowly-changing customer attributes, so a historical metric can be +-- recomputed against what a customer's segment WAS, not what it is now. +{% snapshot snap_customers %} +{{ + config( + target_schema='snapshots', + unique_key='customer_id', + strategy='check', + check_cols=['segment', 'country_code'], + invalidate_hard_deletes=True + ) +}} +select customer_id, tenant_id, customer_name, segment, country_code +from {{ ref('stg_customers') }} +{% endsnapshot %} diff --git a/analytics/dbt/tests/assert_mart_is_not_empty.sql b/analytics/dbt/tests/assert_mart_is_not_empty.sql new file mode 100644 index 0000000..ead1f98 --- /dev/null +++ b/analytics/dbt/tests/assert_mart_is_not_empty.sql @@ -0,0 +1,7 @@ +-- Fails when the serving mart is EMPTY. +-- +-- Without this, every column test passes trivially over zero rows and the +-- whole suite reports green while the pipeline is dead — the same false green +-- BI-P025's NativeQualityProvider refuses. +select 1 as empty_mart +having (select count(*) from {{ ref('mart_sales_daily') }}) = 0 diff --git a/analytics/dbt/tests/assert_no_cross_tenant_rows.sql b/analytics/dbt/tests/assert_no_cross_tenant_rows.sql new file mode 100644 index 0000000..5e5c679 --- /dev/null +++ b/analytics/dbt/tests/assert_no_cross_tenant_rows.sql @@ -0,0 +1,11 @@ +-- A singular test that FAILS if the serving mart ever contains a row belonging +-- to a tenant other than the one the run was scoped to. +-- +-- The tenant predicate is applied in staging and inherited downstream, so this +-- test should be impossible to fail. That is precisely why it exists: an +-- isolation guarantee nobody tests is the one that turns out to have four +-- holes rather than the one reported. +select tenant_id, count(*) as rows_from_another_tenant +from {{ ref('mart_sales_daily') }} +where tenant_id != '{{ var("siax_tenant_id") }}' +group by tenant_id diff --git a/docs/analytics/BI_QUERY_METRIC_QUALITY.md b/docs/analytics/BI_QUERY_METRIC_QUALITY.md new file mode 100644 index 0000000..07c8040 --- /dev/null +++ b/docs/analytics/BI_QUERY_METRIC_QUALITY.md @@ -0,0 +1,160 @@ +# BI-P014, P016–P022, P025–P026, P028–P031 — query, metric, quality + +Extends the analytics core merged in `0pt#34` (BI-P002..P012). Nothing there is +rebuilt. + +This document is organised around what is **proven** and what is **not**, +because the second list is the one that gets dropped. + +--- + +## Where things live + +| prompt | module | live proof | +| -------------------------------------- | ----------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| BI-P014 DuckDB provider | `src/analytics/providers/duckdb-provider.ts` | `tests/duckdb/duckdb-provider.live.test.ts` | +| BI-P016 query request contract | `src/analytics/contract/query-request.ts`, `compile-clickhouse.ts`, `compile-semantic.ts` | unit only (pure compilation) | +| BI-P017 analyst SQL contract | `src/analytics/sql/` | `tests/integration/analyst-sql-clickhouse.integration.test.ts` | +| BI-P018 execution model | `src/analytics/execution/query-execution-record.ts` | `sql/009` + Postgres realinfra | +| BI-P019 cancellation | `src/analytics/execution/cancellation.ts` | `tests/integration/query-cancellation-budget.integration.test.ts` | +| BI-P020 query cache | `src/analytics/cache/query-cache.ts` | unit only (no engine involved) | +| BI-P021 cost / budget | `src/analytics/cost/query-budget.ts` | same live suite as BI-P019 | +| BI-P022 dbt contract | `src/analytics/transform/dbt-contract.ts`, `analytics/dbt/` | contract only — execution is `n0d#38` | +| BI-P025 quality provider | `src/analytics/quality/` | `tests/integration/data-quality-clickhouse.integration.test.ts` | +| BI-P026 freshness | `src/analytics/quality/freshness.ts` | unit + `sql/009` defaults | +| BI-P028/P031 metric model + governance | `src/analytics/metrics/metric-model.ts` | Postgres trigger, realinfra | +| BI-P029 semantic provider | `src/analytics/metrics/semantic-provider.ts` | **partially — see below** | +| BI-P030 semantic registry | `src/analytics/metrics/semantic-registry.ts` | unit + Postgres constraints | + +--- + +## Five findings that changed the implementation + +Each was found by writing a test that could fail, and each is pinned so it +cannot quietly stop being true. + +### 1. ClickHouse `readonly = 1` on a user profile forbids **every** request setting + +Measured on 24.8.14.39. A user whose profile carries `readonly = 1` answers +`Code: 164. Cannot modify 'max_rows_to_read' setting in readonly mode` to any +setting sent with the statement. So BI-P012's ceilings **cannot travel with an +analyst statement** — they are baked into the user's `SETTINGS PROFILE` at +provisioning. Proven both ways: an over-budget scan fails with code 158 from +the profile with no setting sent, and an attempt to raise it from the request +fails with 164. + +### 2. `CREATE ROW POLICY IF NOT EXISTS` is a silent no-op on re-provisioning + +Dropping a user empties the policy's apply-to list. The next provisioning run +sees the policy exists and does nothing, so it survives with +`apply_to_list = []` — applying to **nobody** — while `system.row_policies` +still shows the correct `select_filter`. The restricted user read 100 of 100 +rows, **both tenants**, from the base table. + +Fixed with `OR REPLACE`, plus `rowPolicyVerificationSql()` which turns "the +policy is applied" from an assumption into a query, and a test asserting +`applied_to_user = 1` per object. `CREATE SETTINGS PROFILE IF NOT EXISTS` has +the identical shape and is fixed the same way. + +### 3. DuckDB's `allowed_directories` **alone** is a no-op + +Measured on 1.5.5. With `allowed_directories` set to one mount directory and +nothing else, `read_csv('/etc/passwd')` **returned the file** and an `https://` +URL was fetched over the network. Only `enable_external_access = false` +alongside it makes the allow-list bind. + +The obvious spelling of this control appears in `duckdb_settings()`, looks like +a sandbox, and permits everything. The measurement is pinned by a test that +never skips, so simplifying the bootstrap breaks the build rather than the +guarantee. + +### 4. `pg_constraint.conname` is not schema-scoped + +`SELECT 1 FROM pg_constraint WHERE conname = '...'` matches a same-named +constraint in **any** schema, so `sql/009` applied into a second schema +silently skipped every guarded `CHECK`. Every guard now scopes via +`conrelid = ''::regclass`. + +### 5. `describe.skipIf(flagSetInBeforeAll)` skips **always** + +Vitest evaluates `skipIf` at collection time. Ten tests reported green while +skipping against a live, reachable ClickHouse. Replaced with the repo's +env-var gate, which **fails** when the variable is set and the server is down. + +--- + +## What is NOT proven + +- **BI-P029 / MetricFlow was never executed.** No Python runtime is + provisioned for this repo's CI, and BI-P029's "pin a currently verified + version" needs a licence and supply-chain review that has not happened. + `MetricFlowProvider.providerReadiness()` returns `verified: false` and names + all four open items. What _is_ proven is that the engine-neutral spec + compiles to a MetricFlow-shaped query and that canonical SIAX metric ids + survive a provider swap — tested against a second, deliberately + different-named provider so the equality is not trivially true. +- **The Superset integration path and the AI Analyst comparison** (BI-P029's + other two DoD surfaces) are not implemented. +- **dbt is not run here.** `DBT_RUNTIME_PIN` has both review flags `false`, and + `assertDbtRuntimeCertified` refuses a production run until they are true. +- **DuckDB Arrow IPC** is unproven: the `arrow` extension is a network install + the offline environment could not download. `readiness()` says so. +- **BI-P021's budget is a bound, not a hard cap.** In-flight enforcement is + granular to the poll interval, so the query overshoots between polls — and on + a columnar engine the overshoot is large. In the committed run + (`evidence/bi-p019-p021/`) a 1,000,000-row window cancelled the query at + **153,318,696 rows**, 55 ms in: a **153×** overshoot. The figure is + machine-dependent and moves between runs (an earlier run on the same box + recorded 131 M / 130×), which is itself the point — the bound is "roughly one + poll interval of engine throughput", not a number. It is still a bound: the + same query, unwatched, read **4,227,318,261 rows** before cancellation in the + BI-P019 case. Anyone quoting it as a hard cap is wrong. +- **The per-process window store does not bind across replicas.** + `describeEnforcement()` reports `"with N replicas a tenant can spend up to N +times this budget"`. A shared store (Valkey, or the CL0UD quota service) + behind `SharedWindowStore` is required for it to bind. + +--- + +## Running the live suites + +```bash +docker run -d --name ch -p 28123:8123 -e CLICKHOUSE_SKIP_USER_SETUP=1 \ + clickhouse/clickhouse-server:24.8 +docker run -d --name pg -p 25433:5432 -e POSTGRES_PASSWORD=test \ + -e POSTGRES_DB=opt postgres:16-alpine + +CLICKHOUSE_URL=http://127.0.0.1:28123 pnpm vitest run tests/integration +ANALYTICS_TEST_DATABASE_URL=postgres://postgres:test@127.0.0.1:25433/opt \ + pnpm vitest run tests/analytics/analytics-governance-postgres.realinfra.test.ts + +python -m venv .venv && .venv/bin/pip install duckdb pyarrow 'pyiceberg[sql-sqlite]' +DUCKDB_PYTHON=.venv/bin/python pnpm vitest run tests/duckdb +``` + +Every one of these **fails** rather than skipping when its variable is set and +the backend is unreachable. + +--- + +## One note for whoever maintains `.gitea/workflows/ci.yml` + +Not changed here — it is a protected file, and this branch touched none. + +The integration job's guard is: + +```sh +if grep -qE "skipped" /tmp/itest.log && ! grep -qE "Tests [0-9]+ passed" /tmp/itest.log +``` + +Vitest writes that summary with ANSI colour codes, so the literal text is +`Tests [22m [1m[32m81 passed[39m…`. The second pattern therefore **never +matches**, and the guard is effectively "any skip fails the job" rather than +the "skipped _and_ nothing passed" it reads as. That is stricter than intended, +not looser, so nothing is currently unguarded — but the escape hatch does not +work, and someone will eventually be puzzled by it. Piping through +`sed 's/\x1b\[[0-9;]*m//g'` before the grep would restore the intended +behaviour. + +This is how the DuckDB suite's placement was caught, so the guard earned its +keep either way. diff --git a/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json b/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json index 6e065aa..91cfd8b 100644 --- a/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json +++ b/evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json @@ -30,26 +30,10 @@ ], "num_rows": 3, "pyarrow_values": { - "tenant_id": [ - "t_alpha", - "t_beta", - "t_gamma" - ], - "n": [ - 1, - 2, - 3 - ], - "v": [ - 1.5, - 2.5, - 3.25 - ], - "flag": [ - true, - false, - true - ], + "tenant_id": ["t_alpha", "t_beta", "t_gamma"], + "n": [1, 2, 3], + "v": [1.5, 2.5, 3.25], + "flag": [true, false, true], "ts": [ "2026-01-01 00:00:00+00:00", "2026-06-15 08:09:10.123000+00:00", @@ -57,49 +41,16 @@ ] }, "duckdb_types": [ - [ - "tenant_id", - "VARCHAR" - ], - [ - "n", - "BIGINT" - ], - [ - "v", - "DOUBLE" - ], - [ - "flag", - "BOOLEAN" - ], - [ - "ts", - "TIMESTAMP WITH TIME ZONE" - ] + ["tenant_id", "VARCHAR"], + ["n", "BIGINT"], + ["v", "DOUBLE"], + ["flag", "BOOLEAN"], + ["ts", "TIMESTAMP WITH TIME ZONE"] ], "duckdb_rows": [ - [ - "t_alpha", - "1", - "1.5", - "True", - "2026-01-01 00:00:00+00:00" - ], - [ - "t_beta", - "2", - "2.5", - "False", - "2026-06-15 08:09:10.123000+00:00" - ], - [ - "t_gamma", - "3", - "3.25", - "True", - "2026-09-06 12:34:56.789000+00:00" - ] + ["t_alpha", "1", "1.5", "True", "2026-01-01 00:00:00+00:00"], + ["t_beta", "2", "2.5", "False", "2026-06-15 08:09:10.123000+00:00"], + ["t_gamma", "3", "3.25", "True", "2026-09-06 12:34:56.789000+00:00"] ] }, "0pt encoder -> arrow -> pyarrow/duckdb": { @@ -129,26 +80,10 @@ ], "num_rows": 3, "pyarrow_values": { - "tenant_id": [ - "t_alpha", - "t_beta", - "t_gamma" - ], - "n": [ - 1, - 2, - 3 - ], - "v": [ - 1.5, - 2.5, - 3.25 - ], - "flag": [ - true, - false, - true - ], + "tenant_id": ["t_alpha", "t_beta", "t_gamma"], + "n": [1, 2, 3], + "v": [1.5, 2.5, 3.25], + "flag": [true, false, true], "ts": [ "2026-01-01 00:00:00+00:00", "2026-06-15 08:09:10.123000+00:00", @@ -156,49 +91,16 @@ ] }, "duckdb_types": [ - [ - "tenant_id", - "VARCHAR" - ], - [ - "n", - "BIGINT" - ], - [ - "v", - "DOUBLE" - ], - [ - "flag", - "BOOLEAN" - ], - [ - "ts", - "TIMESTAMP WITH TIME ZONE" - ] + ["tenant_id", "VARCHAR"], + ["n", "BIGINT"], + ["v", "DOUBLE"], + ["flag", "BOOLEAN"], + ["ts", "TIMESTAMP WITH TIME ZONE"] ], "duckdb_rows": [ - [ - "t_alpha", - "1", - "1.5", - "True", - "2026-01-01 00:00:00+00:00" - ], - [ - "t_beta", - "2", - "2.5", - "False", - "2026-06-15 08:09:10.123000+00:00" - ], - [ - "t_gamma", - "3", - "3.25", - "True", - "2026-09-06 12:34:56.789000+00:00" - ] + ["t_alpha", "1", "1.5", "True", "2026-01-01 00:00:00+00:00"], + ["t_beta", "2", "2.5", "False", "2026-06-15 08:09:10.123000+00:00"], + ["t_gamma", "3", "3.25", "True", "2026-09-06 12:34:56.789000+00:00"] ] } }, diff --git a/evidence/bi-p014/duckdb-sandbox.json b/evidence/bi-p014/duckdb-sandbox.json new file mode 100644 index 0000000..7366789 --- /dev/null +++ b/evidence/bi-p014/duckdb-sandbox.json @@ -0,0 +1,107 @@ +{ + "duckdb_version": "1.5.5", + "recorded_at": "2026-09-07T01:15:40.775Z", + "parquet": { + "SE": 394.5, + "NO": 105.5 + }, + "csv": { + "rows": 3, + "includes_quoted_value": true + }, + "injection_attempt": { + "matched_rows": 0, + "note": "escaped as data; 0 rows, not 3" + }, + "iceberg": { + "read_by": "iceberg_scan on the exact metadata JSON", + "note": "an exact metadata file is a SNAPSHOT reference; DuckDB refuses to glob for the latest version by default, and that refusal is correct — globbing can read uncommitted data", + "totals": { + "NO": 20, + "SE": 40 + } + }, + "symlink_escape_refused": true, + "sandbox_bypass_attempts": [ + { + "id": "read-etc-passwd", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/etc/passwd\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * FROM read_c" + }, + { + "id": "read-etc-shadow", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/etc/shadow\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * FROM read_c" + }, + { + "id": "traversal-from-mount", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/tmp/claude-1000/-/987b696c-9462-4fac-b1c0-4e07bad73920/scratchpad/w-0pt-bi/tests/fixtures/duckdb/.." + }, + { + "id": "glob-root", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/etc/*\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * FROM read_csv('/" + }, + { + "id": "http-exfiltration", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"https://example.com/a.csv\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT" + }, + { + "id": "s3-read", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"s3://bucket/a.parquet\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * F" + }, + { + "id": "attach-external-db", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/tmp/duckdb_escape.db\\\" - file system operations are disabled by configuration\"" + }, + { + "id": "attach-etc", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/etc/passwd\\\" - file system operations are disabled by configuration\"" + }, + { + "id": "copy-out-of-mount", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/tmp/duckdb_escape.csv\\\" - file system operations are disabled by configuration\"" + }, + { + "id": "read-parquet-outside", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/root/x.parquet\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * FROM re" + }, + { + "id": "json-outside", + "succeeded": false, + "engine_message": "\"Permission Error: Cannot access file \\\"/etc/hosts\\\" - file system operations are disabled by configuration\\n\\nLINE 1: SELECT * FROM read_js" + } + ], + "counterfactual_mounted_file_readable": true, + "naive_allowed_directories_is_a_noop": { + "statements": [ + "SET allowed_directories=['/tmp/claude-1000/-/987b696c-9462-4fac-b1c0-4e07bad73920/scratchpad/w-0pt-bi/tests/fixtures/duckdb']" + ], + "read_etc_passwd_succeeded": true, + "note": "allowed_directories alone appears in duckdb_settings() and looks like a sandbox. It permits /etc/passwd and outbound HTTP. Only enable_external_access=false binds it." + }, + "reconfiguration_refused": [ + { + "statement": "SET allowed_directories=['/etc']", + "refused": true, + "engine_message": "\"Invalid Input Error: Cannot change configuration option \\\"allowed_directories\\\" - the configuration has been locked\"" + }, + { + "statement": "SET enable_external_access=true", + "refused": true, + "engine_message": "\"Invalid Input Error: Cannot change configuration option \\\"enable_external_access\\\" - the configuration has been locked\"" + }, + { + "statement": "SET lock_configuration=false", + "refused": true, + "engine_message": "\"Invalid Input Error: Cannot change configuration option \\\"lock_configuration\\\" - the configuration has been locked\"" + } + ] +} diff --git a/evidence/bi-p017/analyst-sql-live-bypass.json b/evidence/bi-p017/analyst-sql-live-bypass.json new file mode 100644 index 0000000..b92328a --- /dev/null +++ b/evidence/bi-p017/analyst-sql-live-bypass.json @@ -0,0 +1,844 @@ +{ + "prompt": "BI-P017", + "claim": "Every bypass attempt, executed directly against a live ClickHouse as the tenant's restricted user WITH THE STATEMENT GUARD REMOVED, fails to return a foreign-tenant row, an ungranted-table row, or server metadata.", + "clickhouse_version": "24.8.14.39", + "recorded_at": "2026-09-07T01:15:43.364Z", + "base_table_rows_both_tenants": 100, + "tenant_view_rows_alpha": 50, + "attempts_total": 64, + "attempts_that_leaked": 0, + "legitimate_statements_that_succeeded": 10, + "legitimate_statements_total": 10, + "attempts": [ + { + "id": "cross-catalog-direct", + "goal": "read the other tenant's catalog", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer, amount, event_time)" + }, + { + "id": "cross-catalog-union", + "goal": "union in the other tenant", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer) ON t_beta_sql.sales." + }, + { + "id": "cross-catalog-subquery", + "goal": "subquery on the other tenant", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT for at least one column on t_b" + }, + { + "id": "cross-catalog-join", + "goal": "join the other tenant", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer) ON t_beta_sql.sales." + }, + { + "id": "cross-catalog-cte", + "goal": "hide the other tenant in a CTE", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer) ON t_beta_sql.sales." + }, + { + "id": "cross-catalog-quoted", + "goal": "quote the identifier to dodge the scan", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer, amount, event_time)" + }, + { + "id": "cross-catalog-backtick", + "goal": "backtick the identifier", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"t_beta_sql\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(customer, amount, event_time)" + }, + { + "id": "base-table-direct", + "goal": "read the shared base table, which holds every tenant", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"opt_analytics\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(tenant_id, project_id, dataset" + }, + { + "id": "base-table-in-cte", + "goal": "base table behind a CTE", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"opt_analytics\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(tenant_id) ON opt_analytics.op" + }, + { + "id": "base-table-union", + "goal": "union the base table in", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"opt_analytics\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(tenant_id) ON opt_analytics.op" + }, + { + "id": "system-tables", + "goal": "enumerate every table on the server", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "opt_analytics events opt_analytics sales t_alpha_sql events t_alpha_sql sales" + }, + { + "id": "system-columns", + "goal": "enumerate columns", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "opt_analytics events tenant_id String 1 0 0 0 0 1 1 0 \\N \\N \\N \\N \\N opt_analytics events customer String 2 0 0 0 0 0 0 0 \\N \\N \\N \\N \\N opt_analytics e" + }, + { + "id": "system-users", + "goal": "enumerate users", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(name, id, storage, auth_type," + }, + { + "id": "system-processes", + "goal": "read other tenants' running queries", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(query) ON system.processes. (A" + }, + { + "id": "system-query-log", + "goal": "read other tenants' statement text out of the log", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(query) ON system.query_log. (A" + }, + { + "id": "information-schema", + "goal": "metadata via the ANSI schema", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"information_schema\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(table_catalog, table_schema, t" + }, + { + "id": "one-table", + "goal": "the always-present one-row table", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "0" + }, + { + "id": "tf-url", + "goal": "exfiltrate over HTTP", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"url\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE, URL ON *.*. (" + }, + { + "id": "tf-file", + "goal": "read the server filesystem", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"file\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE, FILE ON *.*." + }, + { + "id": "tf-s3", + "goal": "read object storage directly", + "engine_http_status": 400, + "engine_clickhouse_code": "36", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"s3\" is not permitted in an analyst session", + "engine_response_head": "Code: 36. DB::Exception: Bucket name length is out of bounds in virtual hosted style S3 URI: 'b' (http://minio/b/%2A). (BAD_ARGUMENTS) (version 24.8.14.39 (offi" + }, + { + "id": "tf-remote", + "goal": "query another ClickHouse server", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"remote\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE, REMOTE ON *.*" + }, + { + "id": "tf-mysql", + "goal": "reach an external database", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"mysql\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE, MYSQL ON *.*." + }, + { + "id": "tf-executable", + "goal": "run a program", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"executable\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE ON *.*. (ACCES" + }, + { + "id": "tf-numbers", + "goal": "a CPU bomb that touches no table", + "engine_http_status": 500, + "engine_clickhouse_code": "158", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"numbers_mt\" is not permitted in an analyst session", + "engine_response_head": "Code: 158. DB::Exception: Limit for rows (controlled by 'max_rows_to_read' setting) exceeded, max rows: 100.00 thousand, current rows: 100.00 billion. (TOO_MANY" + }, + { + "id": "tf-generate", + "goal": "unbounded synthetic scan", + "engine_http_status": 500, + "engine_clickhouse_code": "158", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"generaterandom\" is not permitted in an analyst session", + "engine_response_head": "Code: 158. DB::Exception: Limit for rows or bytes to read exceeded, max rows: 100.00 thousand, current rows: 315.79 thousand: While executing GenerateRandom. (T" + }, + { + "id": "tf-merge", + "goal": "merge() glob across a database", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"merge\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE TEMPORARY TABLE ON *.*. (ACCES" + }, + { + "id": "tf-cluster", + "goal": "fan out across the cluster", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"cluster\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant REMOTE ON *.*. (ACCESS_DENIED) (versi" + }, + { + "id": "tf-view", + "goal": "wrap a subquery in view()", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"view\" is not permitted in an analyst session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(tenant_id, project_id, dataset" + }, + { + "id": "fn-dictget", + "goal": "read a dictionary, bypassing table grants", + "engine_http_status": 400, + "engine_clickhouse_code": "36", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"dictget\" is not permitted in an analyst session", + "engine_response_head": "Code: 36. DB::Exception: Dictionary (`d`) not found: In scope SELECT dictGet('d', 'v', toUInt64(1)) AS x FROM t_alpha_sql.sales. (BAD_ARGUMENTS) (version 24.8.1" + }, + { + "id": "stacked-drop", + "goal": "stacked statement", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: multiple statements are not allowed in an analyst SQL session", + "engine_response_head": "Code: 62. DB::Exception: Syntax error (Multi-statements are not allowed): failed at position 32 (end of query): ; DROP TABLE opt_analytics.opt_analytics_event ." + }, + { + "id": "insert", + "goal": "write", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant INSERT(customer, amount, event_time)" + }, + { + "id": "insert-select", + "goal": "write from a read", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant INSERT(customer, amount, event_time)" + }, + { + "id": "alter-delete", + "goal": "mutate", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant ALTER DELETE ON opt_analytics.opt_ana" + }, + { + "id": "create-view-escape", + "goal": "create an unfiltered view to read later", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant CREATE VIEW ON t_alpha_sql.escape. (A" + }, + { + "id": "grant-self", + "goal": "grant itself more", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT ON opt_analytics.* WITH GRANT" + }, + { + "id": "system-command", + "goal": "operate the server", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SYSTEM RELOAD DICTIONARY ON *.*. (ACC" + }, + { + "id": "kill-query", + "goal": "kill another tenant's query", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(query_id, user, query) ON syst" + }, + { + "id": "truncate", + "goal": "destroy data", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant TRUNCATE ON t_alpha_sql.sales. (ACCES" + }, + { + "id": "optimize", + "goal": "trigger heavy background work", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant OPTIMIZE ON opt_analytics.opt_analyti" + }, + { + "id": "outfile", + "goal": "write to the filesystem", + "engine_http_status": 500, + "engine_clickhouse_code": "358", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"OUTFILE\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 358. DB::Exception: INTO OUTFILE is not allowed. (INTO_OUTFILE_NOT_ALLOWED) (version 24.8.14.39 (official build))" + }, + { + "id": "settings-raise-rows", + "goal": "raise max_rows_to_read", + "engine_http_status": 500, + "engine_clickhouse_code": "164", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SETTINGS\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 164. DB::Exception: Cannot modify 'max_rows_to_read' setting in readonly mode. (READONLY) (version 24.8.14.39 (official build))" + }, + { + "id": "settings-readonly-off", + "goal": "turn readonly off", + "engine_http_status": 500, + "engine_clickhouse_code": "164", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SETTINGS\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 164. DB::Exception: Cannot modify 'readonly' setting in readonly mode. (READONLY) (version 24.8.14.39 (official build))" + }, + { + "id": "set-statement", + "goal": "SET before the query", + "engine_http_status": 500, + "engine_clickhouse_code": "164", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: analyst SQL sessions accept SELECT and WITH statements only", + "engine_response_head": "Code: 164. DB::Exception: Cannot modify 'max_execution_time' setting in readonly mode. (READONLY) (version 24.8.14.39 (official build))" + }, + { + "id": "format-override", + "goal": "choose the wire format", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"FORMAT\" is not permitted in a read-only analyst session", + "engine_response_head": "\u00032\bcustomer\u0006String ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 ALPHA_RO" + }, + { + "id": "comment-split-name", + "goal": "hide a catalog boundary behind a comment", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: unqualified table \"opt_analytics\" — an analyst statement must name t_alpha_sql., or a CTE it defined itself", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT(tenant_id, project_id, dataset" + }, + { + "id": "comment-line-hide", + "goal": "hide the tail after a -- comment", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 ALPHA_ROW_11 ALPHA_ROW_12 A" + }, + { + "id": "comment-hash-hide", + "goal": "hide the tail after a # comment", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 ALPHA_ROW_11 ALPHA_ROW_12 A" + }, + { + "id": "unterminated-comment", + "goal": "run off the end of a comment", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [INVALID_REQUEST] 0pt.analytics.sql: unterminated block comment", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 33 ('/* x '): /* x . Multiline comment is not closed: '/* x '. (SYNTAX_ERROR) (version 24.8.14.39 (off" + }, + { + "id": "unterminated-string", + "goal": "run off the end of a literal", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [INVALID_REQUEST] 0pt.analytics.sql: unterminated string literal", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 50 (''x '): 'x . Single quoted string is not closed: ''x '. (SYNTAX_ERROR) (version 24.8.14.39 (offici" + }, + { + "id": "unicode-homoglyph", + "goal": "cyrillic homoglyph inside the catalog name", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: statement contains a disallowed character (U+0430) outside a string literal", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 17 ('�'): аlpha_sql.sales . Unrecognized token: '�'. (SYNTAX_ERROR) (version 24.8.14.39 (official buil" + }, + { + "id": "zero-width", + "goal": "zero-width space inside a reserved catalog name", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: statement contains a disallowed character (U+200B) outside a string literal", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 24 ('.'): .tables . Expected one of: FINAL, SAMPLE, table, table function, subquery or list of joined" + }, + { + "id": "nul-byte", + "goal": "NUL truncation between two halves of a statement", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: statement contains a disallowed character (U+0000) outside a string literal", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 40 ('\u0000'): \u0000 UNION ALL SELECT name FROM system.tables . Unrecognized token: '\u0000'. (SYNTAX_ERROR) (versio" + }, + { + "id": "case-mixing", + "goal": "mixed case keywords and names", + "engine_http_status": 404, + "engine_clickhouse_code": "81", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "Code: 81. DB::Exception: Database SyStEm does not exist. Maybe you meant system?. (UNKNOWN_DATABASE) (version 24.8.14.39 (official build))" + }, + { + "id": "whitespace-newlines", + "goal": "newlines and tabs between tokens", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", + "engine_response_head": "opt_analytics events 204f5e8f-9134-417f-b7f5-569500c15059 MergeTree 0 ['/var/lib/clickhouse/store/204/204f5e8f-9134-417f-b7f5-569500c15059/'] /var/lib/clickhous" + }, + { + "id": "describe-subquery", + "goal": "DESCRIBE a base table from inside a subquery, where the FROM-scan sees only a paren", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: DESCRIBE is not permitted in a read-only analyst session", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 16 ('DESCRIBE'): DESCRIBE TABLE opt_analytics.opt_analytics_event) . Expected one of: SELECT query, po" + }, + { + "id": "desc-subquery-short", + "goal": "the same, using the DESC abbreviation", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: DESCRIBE is not permitted in a read-only analyst session", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 16 ('DESC'): DESC TABLE opt_analytics.opt_analytics_event) . Expected one of: SELECT query, possibly w" + }, + { + "id": "current-user", + "goal": "identify the session's ClickHouse principal", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"currentuser\" is not permitted in an analyst session", + "engine_response_head": "t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha_sql_user t_alpha" + }, + { + "id": "three-part-name", + "goal": "a three-part name", + "engine_http_status": 400, + "engine_clickhouse_code": "62", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"a.b.c\" is not a valid . reference", + "engine_response_head": "Code: 62. DB::Exception: Syntax error: failed at position 21 (end of query): . Expected one of: ParserArrayOfJSONIdentifierDelimiter, token sequence, OpeningSqu" + }, + { + "id": "unqualified-table", + "goal": "unqualified name resolved by the connection's default database", + "engine_http_status": 404, + "engine_clickhouse_code": "60", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: unqualified table \"opt_analytics_event\" — an analyst statement must name t_alpha_sql., or a CTE it defined itself", + "engine_response_head": "Code: 60. DB::Exception: Unknown table expression identifier 'opt_analytics_event' in scope SELECT * FROM opt_analytics_event. (UNKNOWN_TABLE) (version 24.8.14." + }, + { + "id": "cte-shadow", + "goal": "shadow a legitimate name with a CTE while also reading the real base table", + "engine_http_status": 500, + "engine_clickhouse_code": "497", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: catalog \"opt_analytics\" is not readable by this session", + "engine_response_head": "Code: 497. DB::Exception: t_alpha_sql_user: Not enough privileges. To execute this query, it's necessary to have the grant SELECT for at least one column on opt" + }, + { + "id": "no-from", + "goal": "a statement that reads no table at all", + "engine_http_status": 200, + "engine_clickhouse_code": null, + "engine_succeeded": true, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: statement references no readable object in the tenant catalog", + "engine_response_head": "1" + }, + { + "id": "object-not-granted", + "goal": "an object in the right catalog that the session was not given", + "engine_http_status": 404, + "engine_clickhouse_code": "60", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: object \"t_alpha_sql.secret_ledger\" is not readable by this session", + "engine_response_head": "Code: 60. DB::Exception: Unknown table expression identifier 't_alpha_sql.secret_ledger' in scope SELECT * FROM t_alpha_sql.secret_ledger. (UNKNOWN_TABLE) (vers" + }, + { + "id": "hidden-tenant-column", + "goal": "select the tenant column the view does not project", + "engine_http_status": 404, + "engine_clickhouse_code": "47", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "admitted", + "engine_response_head": "Code: 47. DB::Exception: Unknown expression identifier 'tenant_id' in scope SELECT tenant_id FROM t_alpha_sql.sales. (UNKNOWN_IDENTIFIER) (version 24.8.14.39 (o" + }, + { + "id": "hidden-column-star-then-filter", + "goal": "filter on a column the view does not expose", + "engine_http_status": 404, + "engine_clickhouse_code": "47", + "engine_succeeded": false, + "leaked_foreign_tenant_row": false, + "leaked_ungranted_table_row": false, + "leaked_server_metadata": false, + "guard_verdict_for_reference": "admitted", + "engine_response_head": "Code: 47. DB::Exception: Unknown expression or function identifier 'tenant_id' in scope SELECT count() AS n FROM t_alpha_sql.sales WHERE tenant_id = 't_beta'. (" + } + ], + "legitimate": [ + { + "id": "simple", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "aggregate", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "cte", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "self-join", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "literal-with-quotes", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "literal-containing-sql-text", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "comment-then-query", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "subquery", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "union-own-catalog", + "ok": true, + "status": 200, + "contains_beta_canary": false + }, + { + "id": "trailing-semicolon", + "ok": true, + "status": 200, + "contains_beta_canary": false + } + ] +} diff --git a/evidence/bi-p019-p021/cancellation-and-budget.json b/evidence/bi-p019-p021/cancellation-and-budget.json new file mode 100644 index 0000000..809350f --- /dev/null +++ b/evidence/bi-p019-p021/cancellation-and-budget.json @@ -0,0 +1,29 @@ +{ + "clickhouse_version": "24.8.14.39", + "recorded_at": "2026-09-07T01:15:40.234Z", + "counterfactual_still_running_after_2s": true, + "cancellation": { + "rows_read_before_cancel": 4227318261, + "elapsed_ms_to_verified_stop": 162, + "verified_stopped": true, + "clickhouse_query_log_type": "ExceptionWhileProcessing", + "clickhouse_exception_code": 394 + }, + "budget_in_flight": { + "budget_max_rows_scanned": 1000000, + "observed_rows_read_at_cancel": 153318696, + "observed_elapsed_ms_at_cancel": 55, + "polls_before_breach": 1, + "overshoot_factor": 153, + "overshoot_note": "in-flight enforcement is granular to the poll interval; the query overshot the window by this factor before the poll that caught it. A bound, not a hard cap.", + "cancellation_verified": true, + "clickhouse_exception_code": 394, + "charged_for_work_done_before_cancel": true + }, + "advisory_posture_is_declared": { + "admission": "estimate is compared to the remaining window before the query starts", + "in_flight": "NOT ENFORCED — this budget is advisory. A query that exceeds its estimate runs to completion.", + "across_replicas": "window spend is per-process — with N replicas a tenant can spend up to N times this budget", + "binding": false + } +} diff --git a/evidence/bi-p025/native-quality-live.json b/evidence/bi-p025/native-quality-live.json new file mode 100644 index 0000000..c2d296b --- /dev/null +++ b/evidence/bi-p025/native-quality-live.json @@ -0,0 +1,33 @@ +{ + "clickhouse_version": "24.8.14.39", + "recorded_at": "2026-09-07T01:15:45.657Z", + "not_null": { + "evaluated": 118, + "violations": 5 + }, + "range": { + "violations": 3 + }, + "distribution": { + "violations": 4 + }, + "unique": { + "violations": 11, + "breakdown": "6 rows on dup_0/dup_1, plus 5 rows sharing the empty-string key" + }, + "empty_dataset": { + "raw_counts_from_clickhouse": { + "violations": 0, + "evaluated": 0 + }, + "verdict_passed": false, + "note": "a real engine returns zero violations over zero rows; the provider refuses to call that a pass" + }, + "tenant_isolation": { + "base_table_rows": 618, + "alpha_evaluated": 118, + "alpha_violations": 5, + "beta_evaluated": 500, + "beta_violations": 500 + } +} diff --git a/sql/009_analytics_execution_metrics_governance.sql b/sql/009_analytics_execution_metrics_governance.sql new file mode 100644 index 0000000..6b78200 --- /dev/null +++ b/sql/009_analytics_execution_metrics_governance.sql @@ -0,0 +1,346 @@ +-- 0pt analytics: execution metadata, metric governance, semantic registry, +-- quarantine (BI-P018, BI-P021, BI-P025, BI-P028, BI-P030, BI-P031). +-- +-- Extends sql/008. Every statement is ADDITIVE and idempotent: ALTER TABLE ... +-- ADD COLUMN IF NOT EXISTS, CREATE TABLE IF NOT EXISTS, CREATE OR REPLACE +-- FUNCTION, DROP TRIGGER IF EXISTS before CREATE TRIGGER. It can be replayed +-- onto an already-migrated database, which is what the restore drill needs. +-- +-- Ordering: requires sql/008 (it alters two of its tables). +-- +-- WHY COLUMNS RATHER THAN NEW TABLES for the metric fields: BI-P028 names +-- fifteen fields and BI-P002 already persists nine of them. A second metric +-- table would be exactly the failure BI-P028's DoD is about ("no production +-- metric exists only inside a dashboard tool" generalises to "one metric, one +-- home"), so the six missing fields are added where the metric already lives. + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P018 — QUERY EXECUTION MODEL +-- ═══════════════════════════════════════════════════════════════════════════ +-- The fields BI-P018 lists that sql/008's QueryExecution did not carry. +-- +-- `compiled_query_hash` and NOT the statement: BI-P018 says "without +-- unnecessarily storing sensitive raw SQL". There is deliberately no +-- `query_text` column, and the CHECK below makes adding one by accident +-- visible — a hash is 64 hex characters and nothing else. +ALTER TABLE opt_analytics_query_execution + ADD COLUMN IF NOT EXISTS principal_id TEXT, + ADD COLUMN IF NOT EXISTS workload TEXT, + ADD COLUMN IF NOT EXISTS compiled_query_hash TEXT, + ADD COLUMN IF NOT EXISTS datasets TEXT[] NOT NULL DEFAULT '{}', + ADD COLUMN IF NOT EXISTS metrics TEXT[] NOT NULL DEFAULT '{}', + ADD COLUMN IF NOT EXISTS cache_status TEXT NOT NULL DEFAULT 'miss', + ADD COLUMN IF NOT EXISTS cost_units NUMERIC(20, 6); + +-- NOTE ON THE GUARD SHAPE. `pg_constraint.conname` is NOT unique across +-- schemas: a bare `WHERE conname = '...'` matches a same-named constraint in +-- ANY schema and silently skips the ADD here. Found by running this file into +-- a second schema on a database that already had it in `public` — every CHECK +-- below was quietly absent, and the tests that should have proven them passed +-- because the INSERTs they expected to fail succeeded. Every guard therefore +-- scopes to the table via `conrelid = '
'::regclass`, which resolves +-- through search_path. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'opt_analytics_query_execution_hash_shape' + AND conrelid = 'opt_analytics_query_execution'::regclass + ) THEN + ALTER TABLE opt_analytics_query_execution + ADD CONSTRAINT opt_analytics_query_execution_hash_shape + CHECK (compiled_query_hash IS NULL OR compiled_query_hash ~ '^[0-9a-f]{64}$'); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'opt_analytics_query_execution_cache_status' + AND conrelid = 'opt_analytics_query_execution'::regclass + ) THEN + ALTER TABLE opt_analytics_query_execution + ADD CONSTRAINT opt_analytics_query_execution_cache_status + CHECK (cache_status IN ('hit','miss','bypass','stale_refused','not_cacheable')); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'opt_analytics_query_execution_workload' + AND conrelid = 'opt_analytics_query_execution'::regclass + ) THEN + ALTER TABLE opt_analytics_query_execution + ADD CONSTRAINT opt_analytics_query_execution_workload + CHECK (workload IS NULL OR workload IN + ('dashboard','report','api','analyst_sql','ai_analyst','scheduled','internal')); + END IF; +END $$; + +-- Cost reporting is per tenant per window; index for it rather than making +-- every billing query a sequential scan of every tenant's executions. +CREATE INDEX IF NOT EXISTS opt_analytics_query_execution_cost_idx + ON opt_analytics_query_execution (tenant_id, submitted_at DESC) + WHERE cost_units IS NOT NULL; + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P017 — ANALYST SQL STATEMENT HISTORY +-- ═══════════════════════════════════════════════════════════════════════════ +-- The ONE place raw SQL is retained, and it is a separate table with a +-- separate retention posture precisely so that decision is visible. +-- +-- `statement_normalized` (literals masked) is NOT NULL: it is always present +-- and is the searchable, non-PII form. `statement_raw` is nullable, so a +-- deployment that chooses not to retain it simply leaves it NULL rather than +-- losing the history entirely. +CREATE TABLE IF NOT EXISTS opt_analytics_sql_statement ( + statement_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT, + principal_id TEXT NOT NULL, + statement_normalized TEXT NOT NULL, + statement_raw TEXT, + statement_hash TEXT NOT NULL CHECK (statement_hash ~ '^[0-9a-f]{64}$'), + referenced_objects TEXT[] NOT NULL DEFAULT '{}', + outcome TEXT NOT NULL CHECK (outcome IN ('admitted','refused','succeeded','failed','cancelled')), + error_code TEXT, + refusal_reason TEXT, + submitted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + duration_ms INTEGER, + rows_returned BIGINT, + rows_read BIGINT, + bytes_read BIGINT, + correlation_id TEXT, + -- A refusal must say why. An unreviewable refusal log is not an audit trail, + -- and an analyst probing the boundary is exactly what an operator needs to see. + CHECK (outcome <> 'refused' OR (refusal_reason IS NOT NULL AND length(btrim(refusal_reason)) > 0)) +); +CREATE INDEX IF NOT EXISTS opt_analytics_sql_statement_tenant_idx + ON opt_analytics_sql_statement (tenant_id, submitted_at DESC); +CREATE INDEX IF NOT EXISTS opt_analytics_sql_statement_refused_idx + ON opt_analytics_sql_statement (tenant_id, submitted_at DESC) + WHERE outcome = 'refused'; + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P028 / BI-P031 — METRIC DOMAIN MODEL AND GOVERNANCE +-- ═══════════════════════════════════════════════════════════════════════════ +ALTER TABLE opt_analytics_metric_definition + ADD COLUMN IF NOT EXISTS metric_type TEXT NOT NULL DEFAULT 'simple', + ADD COLUMN IF NOT EXISTS time_dimension TEXT, + ADD COLUMN IF NOT EXISTS unit TEXT, + ADD COLUMN IF NOT EXISTS currency TEXT, + ADD COLUMN IF NOT EXISTS display_format TEXT NOT NULL DEFAULT 'decimal', + ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'DRAFT', + ADD COLUMN IF NOT EXISTS filters_json TEXT NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS certification TEXT NOT NULL DEFAULT 'experimental', + ADD COLUMN IF NOT EXISTS business_definition TEXT, + ADD COLUMN IF NOT EXISTS technical_definition TEXT, + ADD COLUMN IF NOT EXISTS meaning_hash TEXT, + ADD COLUMN IF NOT EXISTS certified_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS certified_by TEXT; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_status' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_status + CHECK (status IN ('DRAFT','REVIEW','ACTIVE','DEPRECATED','RETIRED')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_type' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_type + CHECK (metric_type IN ('simple','ratio','derived','cumulative','conversion')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_format' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_format + CHECK (display_format IN ('integer','decimal','percent','currency','duration_ms','bytes')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_certification' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_certification + CHECK (certification IN ('experimental','certified','deprecated')); + END IF; + -- A currency-formatted metric with no currency is how SEK gets rendered as + -- USD on a dashboard nobody double-checks. + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_currency_pairing' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_currency_pairing + CHECK ( + (display_format = 'currency' AND currency IS NOT NULL AND currency ~ '^[A-Z]{3}$') + OR (display_format <> 'currency' AND currency IS NULL) + ); + END IF; + -- Certification is an ACT: it has an actor and a time, or it has not happened. + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'opt_analytics_metric_certification_act' + AND conrelid = 'opt_analytics_metric_definition'::regclass) THEN + ALTER TABLE opt_analytics_metric_definition ADD CONSTRAINT opt_analytics_metric_certification_act + CHECK ( + certification <> 'certified' + OR (certified_by IS NOT NULL AND certified_at IS NOT NULL + AND business_definition IS NOT NULL AND technical_definition IS NOT NULL) + ); + END IF; +END $$; + +-- ── BI-P031's DoD, in the DATABASE ──────────────────────────────────────── +-- "Certified metric cannot be modified in-place; new version required." +-- +-- The TypeScript check (assertMetricEditAllowed) is layer 1. This trigger is +-- layer 2, and it is the one that holds when something writes to the table +-- without going through the application — a migration script, a psql session, +-- an admin tool. A compiler opinion does not survive `psql`; a trigger does. +-- +-- The rule is on the MEANING HASH, so a change this trigger has never heard of +-- still counts as a change: new fields default to "changes the meaning" rather +-- than to "allowed", which is the fail-closed direction. +CREATE OR REPLACE FUNCTION opt_analytics_metric_certified_immutable() RETURNS trigger AS $$ +BEGIN + IF OLD.certification = 'certified' + AND NEW.current_version = OLD.current_version + AND NEW.meaning_hash IS DISTINCT FROM OLD.meaning_hash THEN + RAISE EXCEPTION + 'BI-P031: metric % is CERTIFIED; its definition cannot be modified in place - mint version %', + OLD.name, OLD.current_version + 1 + USING ERRCODE = 'restrict_violation'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS opt_analytics_metric_certified_guard ON opt_analytics_metric_definition; +CREATE TRIGGER opt_analytics_metric_certified_guard + BEFORE UPDATE ON opt_analytics_metric_definition + FOR EACH ROW EXECUTE FUNCTION opt_analytics_metric_certified_immutable(); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P030 — SEMANTIC MODEL REGISTRY +-- ═══════════════════════════════════════════════════════════════════════════ +-- Persisted SEPARATELY from any implementation engine, which is the whole +-- point of the prompt. There is no column here for a provider manifest, and +-- `assertOnlySemanticModelFields` refuses one in the application; the schema +-- refuses it by simply not having anywhere to put it. +-- +-- sql/008's opt_analytics_semantic_model holds the current model; this table +-- holds the VERSION HISTORY BI-P030 enumerates, and its rows are immutable for +-- the same reason a MetricVersion's are: a dashboard pinned to version 7 must +-- mean the same thing tomorrow. +CREATE TABLE IF NOT EXISTS opt_analytics_semantic_model_version ( + semantic_model_version_id TEXT PRIMARY KEY, + model_id TEXT NOT NULL, + tenant_id TEXT NOT NULL, + project_id TEXT NOT NULL, + version INTEGER NOT NULL CHECK (version >= 1), + datasets TEXT[] NOT NULL DEFAULT '{}', + relationships_json TEXT NOT NULL DEFAULT '[]', + metrics_json TEXT NOT NULL DEFAULT '[]', + dimensions_json TEXT NOT NULL DEFAULT '[]', + valid_from TIMESTAMPTZ NOT NULL DEFAULT now(), + status TEXT NOT NULL CHECK (status IN ('draft','active','superseded','retired')), + hash TEXT NOT NULL CHECK (hash ~ '^[0-9a-f]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT, + UNIQUE (tenant_id, model_id, version) +); +CREATE INDEX IF NOT EXISTS opt_analytics_semantic_model_version_tenant_idx + ON opt_analytics_semantic_model_version (tenant_id, project_id, model_id, version DESC); + +-- At most ONE active version per model per tenant. A partial unique index +-- rather than application discipline: two active versions is a silent +-- correctness failure in which half the dashboards read a different model. +CREATE UNIQUE INDEX IF NOT EXISTS opt_analytics_semantic_model_one_active + ON opt_analytics_semantic_model_version (tenant_id, model_id) + WHERE status = 'active'; + +-- A published version's CONTENT is immutable; only its status and valid_from +-- may move (draft -> active -> superseded -> retired). +CREATE OR REPLACE FUNCTION opt_analytics_semantic_version_content_immutable() RETURNS trigger AS $$ +BEGIN + IF NEW.hash IS DISTINCT FROM OLD.hash + OR NEW.metrics_json IS DISTINCT FROM OLD.metrics_json + OR NEW.dimensions_json IS DISTINCT FROM OLD.dimensions_json + OR NEW.relationships_json IS DISTINCT FROM OLD.relationships_json + OR NEW.datasets IS DISTINCT FROM OLD.datasets + OR NEW.version IS DISTINCT FROM OLD.version THEN + RAISE EXCEPTION + 'BI-P030: semantic model % v% is immutable; publish a new version instead', + OLD.model_id, OLD.version + USING ERRCODE = 'restrict_violation'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS opt_analytics_semantic_version_guard ON opt_analytics_semantic_model_version; +CREATE TRIGGER opt_analytics_semantic_version_guard + BEFORE UPDATE ON opt_analytics_semantic_model_version + FOR EACH ROW EXECUTE FUNCTION opt_analytics_semantic_version_content_immutable(); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P025 / BI-P026 — QUARANTINE AND FRESHNESS +-- ═══════════════════════════════════════════════════════════════════════════ +-- The quarantine record BI-P025's DoD acts on. The READ-side gate lives in the +-- application (`assertProjectionReadable`); this is its durable state, so a +-- process restart does not release a quarantine. +CREATE TABLE IF NOT EXISTS opt_analytics_projection_quarantine ( + tenant_id TEXT NOT NULL, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + quarantined BOOLEAN NOT NULL, + caused_by TEXT[] NOT NULL DEFAULT '{}', + reason TEXT, + since TIMESTAMPTZ, + released_at TIMESTAMPTZ, + released_by TEXT, + PRIMARY KEY (tenant_id, dataset_id), + -- Release is an ACT with an operator behind it, never an automatic + -- consequence of a later passing run: a flapping rule would otherwise + -- quarantine and un-quarantine a dataset between two reads of one dashboard. + CHECK (quarantined OR released_at IS NULL OR (released_by IS NOT NULL AND reason IS NOT NULL)) +); + +-- BI-P026 freshness state, per dataset. `UNKNOWN` is the DEFAULT, deliberately: +-- a dataset nobody has computed freshness for must not read as FRESH. +CREATE TABLE IF NOT EXISTS opt_analytics_dataset_freshness ( + tenant_id TEXT NOT NULL, + dataset_id TEXT NOT NULL REFERENCES opt_analytics_dataset (dataset_id), + state TEXT NOT NULL DEFAULT 'UNKNOWN' + CHECK (state IN ('FRESH','DELAYED','STALE','FAILED','UNKNOWN')), + age_ms BIGINT, + last_successful_load_at TIMESTAMPTZ, + last_run_at TIMESTAMPTZ, + last_run_status TEXT CHECK (last_run_status IN ('succeeded','failed','running','never_run')), + expected_interval_ms BIGINT NOT NULL, + disclosure TEXT NOT NULL, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id, dataset_id), + -- Every state must carry a disclosure a UI can render verbatim. A state with + -- no words attached is a state a dashboard will render as a silent green dot. + CHECK (length(btrim(disclosure)) > 0) +); + +-- ═══════════════════════════════════════════════════════════════════════════ +-- BI-P021 — USAGE LEDGER +-- ═══════════════════════════════════════════════════════════════════════════ +-- What CL0UD quotas, AUD0 usage and B00K accounting read. Append-only. +-- +-- `terminated_by_budget` is recorded because a query the budget killed still +-- COST what it burned before it died; not charging for it is how a tenant +-- learns that the cheapest query is one that gets cancelled. +CREATE TABLE IF NOT EXISTS opt_analytics_usage_ledger ( + usage_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + project_id TEXT, + principal_id TEXT NOT NULL, + query_id TEXT NOT NULL, + engine TEXT NOT NULL, + rows_scanned BIGINT NOT NULL DEFAULT 0, + bytes_scanned BIGINT NOT NULL DEFAULT 0, + storage_read_bytes BIGINT NOT NULL DEFAULT 0, + execution_ms BIGINT NOT NULL DEFAULT 0, + cost_units NUMERIC(20, 6) NOT NULL, + terminated_by_budget BOOLEAN NOT NULL DEFAULT false, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS opt_analytics_usage_ledger_window_idx + ON opt_analytics_usage_ledger (tenant_id, recorded_at DESC); + +DROP TRIGGER IF EXISTS opt_analytics_usage_ledger_immutable ON opt_analytics_usage_ledger; +CREATE TRIGGER opt_analytics_usage_ledger_immutable + BEFORE UPDATE OR DELETE ON opt_analytics_usage_ledger + FOR EACH ROW EXECUTE FUNCTION opt_analytics_forbid_mutation(); diff --git a/src/analytics/cache/query-cache.ts b/src/analytics/cache/query-cache.ts new file mode 100644 index 0000000..f5793cf --- /dev/null +++ b/src/analytics/cache/query-cache.ts @@ -0,0 +1,329 @@ +/** + * src/analytics/cache/query-cache.ts + * =================================== + * BI-P020 — QUERY CACHE. + * + * "Cache key includes: tenant, semantic model version, metric definitions, + * filters, query, data snapshot/version. Never share tenant result cache." + * + * All six are REQUIRED, and the key builder refuses to produce a key when one + * is missing rather than defaulting it. A defaulted component is how a cache + * serves a result computed under a metric definition that no longer exists — + * the answer is not stale, it is WRONG, and it looks current. + * + * "Never share tenant result cache" is enforced structurally, not by + * discipline: + * + * * the key is PREFIXED with the tenant id, and the prefix is derived from + * the verified identity inside this module — a caller cannot pass one; + * * every read re-derives the prefix from the identity and REFUSES a stored + * entry whose recorded tenant does not match, so a prefix collision or a + * shared backing store still cannot serve a foreign result; + * * the stored entry carries its own tenant id, so the check is against the + * DATA, not against the key that found it. + * + * The second and third points exist because the first is a naming convention, + * and naming conventions are what ST0RE was relying on when it turned out to + * have four cross-tenant paths rather than the one reported. + * + * Provider-neutral: `AnalyticsResultCache` is the port. `InMemoryResultCache` + * is the reference implementation and the test double; `ValkeyResultCache` + * adapts any Valkey/Redis client that can GET/SET/DEL with a TTL, taking it as + * an injected interface so this module adds no dependency. + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; + +/** + * Everything that must be true for a cached result to still be the right + * answer. Missing any one of these makes the cache unsound, so the type has no + * optional fields. + */ +export interface CacheKeyComponents { + /** From the verified identity. Not a caller-supplied value. */ + tenant_id: string; + project_id?: string; + dataset_id: string; + /** BI-P030's semantic model version. Bumping it invalidates everything under it. */ + semantic_model_version: number; + /** + * metric_id -> version, for EVERY metric in the query. A metric whose + * definition changed must not serve a result computed under the old one. + */ + metric_versions: Record; + /** The compiled statement hash (BI-P018). Covers filters, grain and limit. */ + compiled_query_hash: string; + /** + * The data snapshot the result was computed from. Whatever identifies "the + * data as of now" for this dataset: a DatasetVersion number, an Iceberg + * snapshot id, a max ingested watermark. Advancing it invalidates. + */ + data_snapshot: string; + /** Result encoding, since a JSON and an Arrow result are different bytes. */ + format: string; +} + +export interface CachedResult { + /** Recorded so the read check is against the DATA, not against the key. */ + tenant_id: string; + key: string; + value: T; + stored_at: string; + expires_at: string; + row_count: number; + /** Kept for targeted invalidation. */ + dataset_id: string; + metric_ids: string[]; + semantic_model_version: number; + data_snapshot: string; +} + +export interface AnalyticsResultCache { + readonly provider: string; + get(components: CacheKeyComponents): Promise | undefined>; + set( + components: CacheKeyComponents, + value: T, + opts: { ttlSeconds: number; rowCount: number }, + ): Promise; + /** Drop every entry for a dataset — the blunt, always-correct invalidation. */ + invalidateDataset(tenantId: string, datasetId: string): Promise; + /** Drop every entry that used a given metric. */ + invalidateMetric(tenantId: string, metricId: string): Promise; + /** Drop everything for a tenant (offboarding, incident response). */ + invalidateTenant(tenantId: string): Promise; +} + +function required(value: unknown, name: string): void { + if ( + value === undefined || + value === null || + value === '' || + (typeof value === 'number' && !Number.isFinite(value)) + ) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.cache', + `cache key component "${name}" is required — a defaulted component serves results computed under a definition that no longer exists`, + ); + } +} + +/** + * Build the key. Deterministic, tenant-prefixed, and total over the six + * components BI-P020 lists. + * + * The tenant prefix is in PLAINTEXT at the front rather than folded into the + * hash, so `invalidateTenant` is a prefix scan and an operator reading the + * store can see which tenant an entry belongs to without a lookup table. + */ +export function buildCacheKey(c: CacheKeyComponents): string { + required(c.tenant_id, 'tenant_id'); + required(c.dataset_id, 'dataset_id'); + required(c.semantic_model_version, 'semantic_model_version'); + required(c.compiled_query_hash, 'compiled_query_hash'); + required(c.data_snapshot, 'data_snapshot'); + required(c.format, 'format'); + if (!c.metric_versions || Object.keys(c.metric_versions).length === 0) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.cache', + 'cache key component "metric_versions" is required and must name every metric in the query', + ); + } + for (const [id, version] of Object.entries(c.metric_versions)) { + required(version, `metric_versions[${id}]`); + } + if (!/^[A-Za-z0-9_-]{1,64}$/.test(c.tenant_id)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'tenant id is not key-safe'); + } + const material = JSON.stringify([ + c.tenant_id, + c.project_id ?? '', + c.dataset_id, + c.semantic_model_version, + Object.keys(c.metric_versions) + .sort() + .map((k) => [k, c.metric_versions[k]]), + c.compiled_query_hash, + c.data_snapshot, + c.format, + ]); + const digest = crypto.createHash('sha256').update(material).digest('hex'); + return `optq:${c.tenant_id}:${digest}`; +} + +/** The prefix `invalidateTenant` scans. Derived, never supplied. */ +export function tenantKeyPrefix(tenantId: string): string { + return `optq:${tenantId}:`; +} + +abstract class BaseResultCache implements AnalyticsResultCache { + abstract readonly provider: string; + protected abstract read(key: string): Promise; + protected abstract write(entry: CachedResult, ttlSeconds: number): Promise; + protected abstract entries(tenantId: string): Promise; + protected abstract remove(keys: string[]): Promise; + + protected readonly now: () => number; + constructor(now: () => number = Date.now) { + this.now = now; + } + + async get(components: CacheKeyComponents): Promise | undefined> { + const key = buildCacheKey(components); + const entry = await this.read(key); + if (!entry) return undefined; + + // The check that survives a prefix collision or a shared backing store: + // the ENTRY says which tenant it belongs to, and it must be this one. + if (entry.tenant_id !== components.tenant_id) { + // Not a miss — a cross-tenant hit is a security event, and returning + // `undefined` would hide it. Drop the poisoned entry and refuse loudly. + await this.remove([key]); + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.cache', + 'cache entry belongs to a different tenant; entry evicted', + ); + } + if (Date.parse(entry.expires_at) <= this.now()) { + await this.remove([key]); + return undefined; + } + return entry as CachedResult; + } + + async set( + components: CacheKeyComponents, + value: T, + opts: { ttlSeconds: number; rowCount: number }, + ): Promise { + if (!Number.isFinite(opts.ttlSeconds) || opts.ttlSeconds <= 0) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'ttlSeconds must be positive'); + } + const key = buildCacheKey(components); + const nowMs = this.now(); + await this.write( + { + tenant_id: components.tenant_id, + key, + value, + stored_at: new Date(nowMs).toISOString(), + expires_at: new Date(nowMs + opts.ttlSeconds * 1000).toISOString(), + row_count: opts.rowCount, + dataset_id: components.dataset_id, + metric_ids: Object.keys(components.metric_versions), + semantic_model_version: components.semantic_model_version, + data_snapshot: components.data_snapshot, + }, + opts.ttlSeconds, + ); + } + + async invalidateDataset(tenantId: string, datasetId: string): Promise { + const victims = (await this.entries(tenantId)).filter((e) => e.dataset_id === datasetId); + await this.remove(victims.map((v) => v.key)); + return victims.length; + } + + async invalidateMetric(tenantId: string, metricId: string): Promise { + const victims = (await this.entries(tenantId)).filter((e) => e.metric_ids.includes(metricId)); + await this.remove(victims.map((v) => v.key)); + return victims.length; + } + + async invalidateTenant(tenantId: string): Promise { + const victims = await this.entries(tenantId); + await this.remove(victims.map((v) => v.key)); + return victims.length; + } +} + +export class InMemoryResultCache extends BaseResultCache { + readonly provider = 'memory'; + private readonly store = new Map(); + + protected async read(key: string): Promise { + return this.store.get(key); + } + protected async write(entry: CachedResult): Promise { + this.store.set(entry.key, entry); + } + protected async entries(tenantId: string): Promise { + const prefix = tenantKeyPrefix(tenantId); + return [...this.store.values()].filter((e) => e.key.startsWith(prefix)); + } + protected async remove(keys: string[]): Promise { + for (const k of keys) this.store.delete(k); + } + /** Test/observability affordance. Never used to bypass the tenant check. */ + get size(): number { + return this.store.size; + } +} + +/** + * The minimum a Valkey/Redis client must offer. Taken as an interface so this + * module adds no dependency and no CVE surface — the same reason BI-P005's + * Arrow writer is hand-written. + */ +export interface ValkeyLikeClient { + get(key: string): Promise; + set(key: string, value: string, opts: { EX: number }): Promise; + del(keys: string[]): Promise; + /** Cursor-based scan. A blocking KEYS on a shared Valkey is an outage. */ + scan(cursor: string, opts: { MATCH: string; COUNT: number }): Promise<{ cursor: string; keys: string[] }>; +} + +export class ValkeyResultCache extends BaseResultCache { + readonly provider = 'valkey'; + constructor( + private readonly client: ValkeyLikeClient, + now: () => number = Date.now, + ) { + super(now); + } + + protected async read(key: string): Promise { + const raw = await this.client.get(key); + if (raw === null) return undefined; + try { + return JSON.parse(raw) as CachedResult; + } catch { + // A corrupt entry is a miss, not a crash — but it is also removed, so a + // permanently unparseable key cannot become a permanent cache miss that + // nobody notices. + await this.client.del([key]); + return undefined; + } + } + + protected async write(entry: CachedResult, ttlSeconds: number): Promise { + // The TTL is set on the STORE as well as recorded in the entry. Relying on + // the recorded timestamp alone leaves expired results occupying memory + // forever on a busy tenant. + await this.client.set(entry.key, JSON.stringify(entry), { EX: Math.ceil(ttlSeconds) }); + } + + protected async entries(tenantId: string): Promise { + const prefix = tenantKeyPrefix(tenantId); + const out: CachedResult[] = []; + let cursor = '0'; + do { + const page = await this.client.scan(cursor, { MATCH: `${prefix}*`, COUNT: 500 }); + cursor = page.cursor; + for (const key of page.keys) { + const entry = await this.read(key); + // Defence in depth: SCAN MATCH is a glob, and a glob is a naming + // convention. Confirm against the entry's own tenant id. + if (entry && entry.tenant_id === tenantId) out.push(entry); + } + } while (cursor !== '0'); + return out; + } + + protected async remove(keys: string[]): Promise { + if (keys.length) await this.client.del(keys); + } +} diff --git a/src/analytics/contract/compile-clickhouse.ts b/src/analytics/contract/compile-clickhouse.ts new file mode 100644 index 0000000..5ff7071 --- /dev/null +++ b/src/analytics/contract/compile-clickhouse.ts @@ -0,0 +1,252 @@ +/** + * src/analytics/contract/compile-clickhouse.ts + * ============================================= + * BI-P016, compile target 1 of 2: the canonical `AnalyticsQueryRequest` + * becomes tenant-forced ClickHouse SQL. + * + * The BI-P011 guarantee is repeated here VERBATIM and by construction: + * + * * `FORCED_TENANT_PREDICATE` is condition[0] with no branch that can skip + * it, and the tenant value comes from the verified identity handed in by + * the engine — this function has no way to read one from a request, + * because `AnalyticsQueryRequest` has no tenant field. + * * every caller VALUE is a `param_*` binding; `sql` never contains one. + * * every caller NAME has already been resolved through the catalog + * (`resolveAnalyticsQuery`), and the resolved physical name is validated + * as an identifier a second time before it reaches the string. + * + * tests/analytics/query-request-tenant-forcing.test.ts asserts the predicate + * survives every request shape this compiler accepts, by ENUMERATING the + * shapes rather than spot-checking two of them. + */ +import { SiaxError } from '../../platform/errors'; +import { assertSafeIdentifier, FORCED_TENANT_PREDICATE, type CompiledQuery } from '../engine/query-compiler'; +import type { AnalyticsTable } from '../engine/schema'; +import type { ResolvedQuery, TimeGrain } from './query-request'; +import type { FilterOperator } from './query-request'; + +/** Grain -> ClickHouse truncation function. A closed map, never caller text. */ +const GRAIN_FN: Record = { + hour: 'toStartOfHour', + day: 'toStartOfDay', + week: 'toStartOfWeek', + month: 'toStartOfMonth', + quarter: 'toStartOfQuarter', + year: 'toStartOfYear', +}; + +/** + * Aggregation -> conditional-aggregate template. `count_distinct` uses + * `uniqExactIf` so a multi-metric request stays one scan instead of N. + */ +const AGG_IF: Record string> = { + sum: (p) => `sumIf(value, metric = {${p}:String})`, + avg: (p) => `avgIf(value, metric = {${p}:String})`, + min: (p) => `minIf(value, metric = {${p}:String})`, + max: (p) => `maxIf(value, metric = {${p}:String})`, + count: (p) => `countIf(metric = {${p}:String})`, + count_distinct: (p) => `uniqExactIf(event_id, metric = {${p}:String})`, +}; + +export interface ClickHouseCompileOptions { + /** From the VERIFIED CL0UD identity. Never from a request body. */ + tenant_id: string; + project_id?: string; + database: string; + table?: AnalyticsTable; + /** BI-P012 ceiling; `limit` is clamped to it, never raised by it. */ + maxResultRows: number; +} + +export interface CompiledAnalyticsQuery extends CompiledQuery { + /** Column alias -> caller-facing metric name, so the caller never sees `m0`. */ + metricAliases: Record; + /** Column alias -> caller-facing dimension name. */ + dimensionAliases: Record; + effectiveLimit: number; +} + +export function compileRequestToClickHouse( + resolved: ResolvedQuery, + opts: ClickHouseCompileOptions, +): CompiledAnalyticsQuery { + if (!opts.tenant_id || !opts.tenant_id.trim()) { + // Fail closed. A blank tenant must never compile to "no predicate". + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query'); + } + assertSafeIdentifier(opts.database, 'database'); + const table: AnalyticsTable = opts.table ?? 'opt_analytics_event'; + assertSafeIdentifier(table, 'table'); + + const params: Record = { param_tenant: opts.tenant_id }; + // ── the forced predicate. First condition, always, no branch. ───────── + const conditions: string[] = [FORCED_TENANT_PREDICATE]; + + if (opts.project_id) { + conditions.push('project_id = {project:String}'); + params.param_project = opts.project_id; + } + conditions.push('dataset_id = {dataset:String}'); + params.param_dataset = resolved.dataset_id; + + // ── metrics: bound keys, compiler-generated aliases ─────────────────── + const selectParts: string[] = []; + const groupParts: string[] = []; + const dimensionAliases: Record = {}; + const metricAliases: Record = {}; + const metricKeyParams: string[] = []; + + resolved.dimensions.forEach((d, i) => { + assertSafeIdentifier(d.field_name, 'dimension'); + const key = `d${i}`; + const alias = `dim_${i}`; + selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS ${alias}`); + groupParts.push(alias); + params[`param_${key}`] = d.field_name; + dimensionAliases[alias] = d.name; + }); + + // ── time grain: a GROUP BY bucket, from the closed GRAIN_FN map ─────── + const grain = resolved.request.timeRange?.grain; + if (grain) { + const timeCol = resolved.timeDimension ?? 'event_time'; + assertSafeIdentifier(timeCol, 'time dimension'); + const alias = 'time_bucket'; + selectParts.push(`${GRAIN_FN[grain]}(${timeCol}) AS ${alias}`); + groupParts.push(alias); + dimensionAliases[alias] = resolved.request.timeRange?.dimension ?? '__time'; + } + + resolved.metrics.forEach((m, i) => { + const key = `m${i}`; + const alias = `metric_${i}`; + const template = AGG_IF[m.aggregation]; + if (!template) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported aggregation "${m.aggregation}"`); + } + selectParts.push(`${template(key)} AS ${alias}`); + params[`param_${key}`] = m.event_metric_key; + metricKeyParams.push(`{${key}:String}`); + metricAliases[alias] = m.name; + }); + + // Restrict the SCAN to the requested metric keys — a correctness AND a cost + // measure: without it every metric row in the dataset is read to be thrown + // away by the `If` conditions. + conditions.push(`metric IN (${metricKeyParams.join(', ')})`); + + // ── time range: bound instants, half-open ───────────────────────────── + const tr = resolved.request.timeRange; + if (tr?.from || tr?.to) { + const timeCol = resolved.timeDimension ?? 'event_time'; + assertSafeIdentifier(timeCol, 'time dimension'); + if (tr.from) { + conditions.push(`${timeCol} >= parseDateTime64BestEffort({fromTime:String})`); + params.param_fromTime = tr.from; + } + if (tr.to) { + conditions.push(`${timeCol} < parseDateTime64BestEffort({toTime:String})`); + params.param_toTime = tr.to; + } + } + + // ── filters: closed operator set, values always bound ───────────────── + resolved.filters.forEach((entry, i) => { + assertSafeIdentifier(entry.dimension.field_name, 'dimension'); + const nameKey = `fn${i}`; + params[`param_${nameKey}`] = entry.dimension.field_name; + const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`; + conditions.push(renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], i, params)); + }); + + // ── order by: resolved back onto compiler-generated aliases only ────── + const aliasForName = new Map(); + for (const [alias, name] of Object.entries(dimensionAliases)) aliasForName.set(name, alias); + for (const [alias, name] of Object.entries(metricAliases)) aliasForName.set(name, alias); + const orderParts: string[] = []; + for (const o of resolved.request.orderBy ?? []) { + const alias = aliasForName.get(o.ref); + if (!alias) { + // Cannot happen after parseAnalyticsQueryRequest, which already refuses + // an orderBy that is not selected. Kept as a hard stop rather than a + // silent skip: a guard that quietly does nothing is how a bypass hides. + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `orderBy "${o.ref}" is not selected`); + } + orderParts.push(`${alias} ${o.direction === 'asc' ? 'ASC' : 'DESC'}`); + } + + const requested = resolved.request.limit ?? 1000; + const effectiveLimit = Math.max(1, Math.min(Math.floor(requested), opts.maxResultRows)); + + const sql = + `SELECT ${selectParts.join(', ')} ` + + `FROM ${opts.database}.${table} ` + + `WHERE ${conditions.join(' AND ')}` + + (groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') + + (orderParts.length ? ` ORDER BY ${orderParts.join(', ')}` : '') + + ` LIMIT ${effectiveLimit} FORMAT JSON`; + + return { + sql, + params, + forcedTenantPredicate: FORCED_TENANT_PREDICATE, + table, + metricAliases, + dimensionAliases, + effectiveLimit, + }; +} + +function renderFilter( + lhs: string, + nameKey: string, + operator: FilterOperator, + values: string[], + index: number, + params: Record, +): string { + const bind = (v: string, suffix: string): string => { + const key = `fv${index}${suffix}`; + params[`param_${key}`] = v; + return `{${key}:String}`; + }; + switch (operator) { + case 'eq': + return `${lhs} = ${bind(values[0], '')}`; + case 'ne': + return `${lhs} != ${bind(values[0], '')}`; + case 'lt': + return `${lhs} < ${bind(values[0], '')}`; + case 'lte': + return `${lhs} <= ${bind(values[0], '')}`; + case 'gt': + return `${lhs} > ${bind(values[0], '')}`; + case 'gte': + return `${lhs} >= ${bind(values[0], '')}`; + case 'contains': + // `position(haystack, needle) > 0` keeps the needle a BOUND VALUE. + // A LIKE with a concatenated pattern would make it SQL text. + return `position(${lhs}, ${bind(values[0], '')}) > 0`; + case 'in': + case 'not_in': { + const bound = values.map((v, i) => bind(v, `_${i}`)); + return `${lhs} ${operator === 'in' ? 'IN' : 'NOT IN'} (${bound.join(', ')})`; + } + case 'is_null': + // Dimension values live in a paired-array map; an absent key yields '' + // rather than NULL, so "is null" means "the key is not present". The + // key itself is the already-bound `param_fn` — no new binding, and + // certainly no spliced identifier. + return `NOT has(dim_names, {${nameKey}:String})`; + case 'is_not_null': + return `has(dim_names, {${nameKey}:String})`; + default: { + const _never: never = operator; + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics', + `unsupported filter operator ${String(_never)}`, + ); + } + } +} diff --git a/src/analytics/contract/compile-semantic.ts b/src/analytics/contract/compile-semantic.ts new file mode 100644 index 0000000..8cb1ac8 --- /dev/null +++ b/src/analytics/contract/compile-semantic.ts @@ -0,0 +1,145 @@ +/** + * src/analytics/contract/compile-semantic.ts + * =========================================== + * BI-P016, compile target 2 of 2 — and the thing that makes BI-P016's DoD + * ("same request can compile to ClickHouse/semantic engine without consumer + * rewrite") a testable claim rather than a design intention. + * + * A `SemanticQuerySpec` is the engine-neutral form a semantic layer consumes. + * It is deliberately shaped like MetricFlow's query interface (metrics / + * group-by / where / order / limit / time-constraint) so BI-P029's + * `MetricFlowProvider` can execute it without an adapter, but it names + * nothing MetricFlow-specific: swapping MetricFlow for Cube or a hand-rolled + * compiler changes this file's CONSUMER, not this file, and never the + * product API. + * + * The `where` clauses here are STRUCTURED, not SQL strings. A semantic layer + * that accepts SQL predicate text (MetricFlow's `--where` does) would put a + * raw-SQL hole behind the closed DSL — so the spec carries operator+values + * and the provider renders them itself under its own escaping rules. + */ +import type { FilterOperator, ResolvedQuery, TimeGrain } from './query-request'; + +export interface SemanticMetricRef { + /** Canonical SIAX metric id — stable across semantic-engine replacement. */ + metric_id: string; + name: string; +} + +export interface SemanticGroupBy { + dimension_id: string; + name: string; + /** Set when this group-by is a time bucket rather than a categorical one. */ + grain?: TimeGrain; +} + +export interface SemanticWhere { + dimension_id: string; + name: string; + operator: FilterOperator; + values: string[]; +} + +export interface SemanticOrderBy { + ref: string; + direction: 'asc' | 'desc'; +} + +export interface SemanticTimeConstraint { + /** Inclusive lower bound. */ + start?: string; + /** EXCLUSIVE upper bound — matched to the ClickHouse target exactly. */ + end?: string; + dimension?: string; +} + +export interface SemanticQuerySpec { + dataset_id: string; + semantic_model_version?: number; + metrics: SemanticMetricRef[]; + groupBy: SemanticGroupBy[]; + where: SemanticWhere[]; + orderBy: SemanticOrderBy[]; + limit: number; + timeConstraint?: SemanticTimeConstraint; + format: 'json' | 'arrow_ipc'; +} + +export function compileRequestToSemantic(resolved: ResolvedQuery): SemanticQuerySpec { + const groupBy: SemanticGroupBy[] = resolved.dimensions.map((d) => ({ + dimension_id: d.dimension_id, + name: d.name, + })); + const grain = resolved.request.timeRange?.grain; + if (grain) { + groupBy.push({ + dimension_id: resolved.request.timeRange?.dimension ?? '__time', + name: resolved.request.timeRange?.dimension ?? '__time', + grain, + }); + } + return { + dataset_id: resolved.dataset_id, + semantic_model_version: resolved.semantic_model_version, + metrics: resolved.metrics.map((m) => ({ metric_id: m.metric_id, name: m.name })), + groupBy, + where: resolved.filters.map((f) => ({ + dimension_id: f.dimension.dimension_id, + name: f.dimension.name, + operator: f.filter.operator, + values: f.filter.values ?? [], + })), + orderBy: (resolved.request.orderBy ?? []).map((o) => ({ ref: o.ref, direction: o.direction })), + limit: resolved.request.limit ?? 1000, + timeConstraint: + resolved.request.timeRange && (resolved.request.timeRange.from || resolved.request.timeRange.to) + ? { + start: resolved.request.timeRange.from, + end: resolved.request.timeRange.to, + dimension: resolved.request.timeRange.dimension, + } + : undefined, + format: resolved.request.format ?? 'json', + }; +} + +/** + * The comparable SHAPE of a compiled query, target-independent. + * + * BI-P016's DoD is about semantic equivalence, and the honest way to test + * that is to project both targets onto the same tuple and demand equality — + * not to eyeball two strings. `tests/analytics/query-request-contract.test.ts` + * builds this from the ClickHouse SQL by PARSING the compiled artefact, so a + * compiler that quietly drops a filter fails the comparison. + */ +export interface QueryShape { + dataset_id: string; + metrics: string[]; + dimensions: string[]; + filters: Array<{ dimension: string; operator: FilterOperator; values: string[] }>; + order: string[]; + limit: number; + timeFrom?: string; + timeTo?: string; + grain?: TimeGrain; +} + +export function shapeOfSemanticSpec(spec: SemanticQuerySpec): QueryShape { + const timeBucket = spec.groupBy.find((g) => g.grain); + return { + dataset_id: spec.dataset_id, + metrics: spec.metrics.map((m) => m.name).sort(), + dimensions: spec.groupBy + .filter((g) => !g.grain) + .map((g) => g.name) + .sort(), + filters: spec.where + .map((w) => ({ dimension: w.name, operator: w.operator, values: [...w.values] })) + .sort((a, b) => (a.dimension + a.operator).localeCompare(b.dimension + b.operator)), + order: spec.orderBy.map((o) => `${o.ref}:${o.direction}`), + limit: spec.limit, + timeFrom: spec.timeConstraint?.start, + timeTo: spec.timeConstraint?.end, + grain: timeBucket?.grain, + }; +} diff --git a/src/analytics/contract/query-request.ts b/src/analytics/contract/query-request.ts new file mode 100644 index 0000000..4f0e9f4 --- /dev/null +++ b/src/analytics/contract/query-request.ts @@ -0,0 +1,489 @@ +/** + * src/analytics/contract/query-request.ts + * ======================================== + * BI-P016 — QUERY REQUEST CONTRACT. + * + * The canonical shape every SIAX product/application API speaks when it wants + * analytics: + * + * AnalyticsQueryRequest { dataset metrics[] dimensions[] filters[] + * timeRange orderBy limit format } + * + * BI-P016's hard rule: "The default product/application API must NOT accept + * arbitrary SQL." That is enforced structurally, not by a review convention — + * there is no field in this type that can carry SQL, and + * `parseAnalyticsQueryRequest` REJECTS unknown keys rather than ignoring them. + * A caller who adds `sql`, `where`, `having` or `expression` gets + * INVALID_REQUEST, not silent truncation; silent truncation is how an + * "unsupported" field becomes a supported one after the next refactor. + * + * Everything in this request is a NAME, resolved through a declared catalog + * (`QueryResolution`) before it can reach a compiler. Caller text therefore + * never becomes a SQL identifier and never becomes a SQL literal — the two + * ways injection happens. + * + * WHY THIS IS SEPARATE FROM `MetricQueryRequest` (engine/query-compiler.ts): + * that type is the ENGINE's internal, already-resolved, single-metric input, + * and it carries `tenant_id` because the engine sets it. This one is the + * CONSUMER-facing contract: it has no tenant field at all, because a consumer + * must never be able to name a tenant. INF0's Wave-2 hole was exactly a tenant + * read out of a request body; the fix that sticks is a request type in which + * the field does not exist. + */ +import { SiaxError } from '../../platform/errors'; + +// ── vocabulary ───────────────────────────────────────────────────────────── + +/** + * Filter operators. A CLOSED list: every one compiles to a fixed SQL shape + * with bound parameters. There is no "raw" or "expression" operator, and + * adding one would have to survive `filter-operators-are-closed` in + * tests/analytics/query-request-contract.test.ts. + */ +export const FILTER_OPERATORS = [ + 'eq', + 'ne', + 'in', + 'not_in', + 'lt', + 'lte', + 'gt', + 'gte', + 'contains', + 'is_null', + 'is_not_null', +] as const; +export type FilterOperator = (typeof FILTER_OPERATORS)[number]; + +/** Operators that take exactly one value. */ +const SINGLE_VALUE_OPERATORS: readonly FilterOperator[] = ['eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'contains']; +/** Operators that take one-or-more values. */ +const MULTI_VALUE_OPERATORS: readonly FilterOperator[] = ['in', 'not_in']; +/** Operators that take NO value. Supplying one is a rejection, not a shrug. */ +const NO_VALUE_OPERATORS: readonly FilterOperator[] = ['is_null', 'is_not_null']; + +export const TIME_GRAINS = ['hour', 'day', 'week', 'month', 'quarter', 'year'] as const; +export type TimeGrain = (typeof TIME_GRAINS)[number]; + +export const ORDER_DIRECTIONS = ['asc', 'desc'] as const; +export type OrderDirection = (typeof ORDER_DIRECTIONS)[number]; + +/** Result encodings, matching `QueryResult.encoding` (BI-P002/BI-P005). */ +export const REQUEST_FORMATS = ['json', 'arrow_ipc'] as const; +export type RequestFormat = (typeof REQUEST_FORMATS)[number]; + +// ── the request ──────────────────────────────────────────────────────────── + +export interface AnalyticsFilter { + /** Caller-facing dimension NAME. Resolved through the catalog, never spliced. */ + dimension: string; + operator: FilterOperator; + /** Literal values. They travel as bound parameters; never as SQL text. */ + values?: string[]; +} + +export interface AnalyticsTimeRange { + /** + * Caller-facing time dimension name. Omitted means "the dataset's declared + * primary time dimension", which the resolution supplies — a consumer never + * has to know the physical column. + */ + dimension?: string; + /** ISO-8601 inclusive lower bound. */ + from?: string; + /** ISO-8601 EXCLUSIVE upper bound (half-open, so adjacent ranges never double count). */ + to?: string; + grain?: TimeGrain; +} + +export interface AnalyticsOrderBy { + /** + * A metric name or a dimension name that ALREADY APPEARS in this request. + * Ordering by something not selected is refused: it is both a correctness + * trap and the obvious place to smuggle an expression. + */ + ref: string; + direction: OrderDirection; +} + +/** BI-P016's canonical request, field-for-field. */ +export interface AnalyticsQueryRequest { + dataset: string; + metrics: string[]; + dimensions: string[]; + filters: AnalyticsFilter[]; + timeRange?: AnalyticsTimeRange; + orderBy?: AnalyticsOrderBy[]; + limit?: number; + format?: RequestFormat; +} + +/** The exact keys the contract accepts at the top level. */ +export const ANALYTICS_QUERY_REQUEST_KEYS = [ + 'dataset', + 'metrics', + 'dimensions', + 'filters', + 'timeRange', + 'orderBy', + 'limit', + 'format', +] as const; + +/** + * Keys a caller might reach for when they want to smuggle SQL. Listed + * explicitly so the rejection MESSAGE names the rule rather than saying + * "unknown key" — an integrator who tries `sql` should learn why, not guess. + * The unknown-key rejection below covers everything, listed or not. + */ +const SQL_SMELLING_KEYS = new Set([ + 'sql', + 'query', + 'rawSql', + 'raw_sql', + 'raw', + 'where', + 'having', + 'expression', + 'expr', + 'select', + 'from', + 'groupBy', + 'group_by', + 'sqlFragment', + 'unsafe', + 'tenant', + 'tenant_id', + 'tenantId', + 'project_id', + 'database', + 'catalog', + 'table', +]); + +const MAX_METRICS = 16; +const MAX_DIMENSIONS = 8; +const MAX_FILTERS = 32; +const MAX_FILTER_VALUES = 500; +const MAX_ORDER_BY = 8; +/** A name is a catalog key, not free text; keep it short and boring. */ +const MAX_NAME_LENGTH = 128; +/** A filter VALUE is data (it is bound, never spliced) but still bounded. */ +const MAX_VALUE_LENGTH = 4096; +export const DEFAULT_REQUEST_LIMIT = 1000; + +function invalid(detail: string): never { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', detail); +} + +function asObject(raw: unknown, what: string): Record { + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`); + return raw as Record; +} + +function asName(raw: unknown, what: string): string { + if (typeof raw !== 'string') invalid(`${what} must be a string`); + const v = raw.trim(); + if (!v) invalid(`${what} must not be empty`); + if (v.length > MAX_NAME_LENGTH) invalid(`${what} exceeds ${MAX_NAME_LENGTH} characters`); + return v; +} + +function rejectUnknownKeys(obj: Record, allowed: readonly string[], what: string): void { + for (const key of Object.keys(obj)) { + if (allowed.includes(key)) continue; + if (SQL_SMELLING_KEYS.has(key)) { + invalid( + `"${key}" is not part of the analytics query contract — the product API does not accept SQL, ` + + `catalogs, or tenant selection (BI-P016)`, + ); + } + invalid(`unknown field "${key}" in ${what}`); + } +} + +/** + * Parse and VALIDATE an untrusted request body into the canonical contract. + * + * Fail-closed at every branch: an unparseable field is a rejection, never a + * default. The only defaulting this function does is `filters: []`, + * `dimensions: []` and `limit`, and each of those defaults is strictly + * narrower than any value a caller could have supplied. + */ +export function parseAnalyticsQueryRequest(raw: unknown): AnalyticsQueryRequest { + const obj = asObject(raw, 'analytics query request'); + rejectUnknownKeys(obj, ANALYTICS_QUERY_REQUEST_KEYS, 'analytics query request'); + + const dataset = asName(obj.dataset, 'dataset'); + + if (!Array.isArray(obj.metrics)) invalid('metrics must be an array'); + if (obj.metrics.length === 0) invalid('at least one metric is required'); + if (obj.metrics.length > MAX_METRICS) invalid(`at most ${MAX_METRICS} metrics per request`); + const metrics = obj.metrics.map((m, i) => asName(m, `metrics[${i}]`)); + if (new Set(metrics).size !== metrics.length) invalid('metrics contains duplicates'); + + const rawDimensions = obj.dimensions ?? []; + if (!Array.isArray(rawDimensions)) invalid('dimensions must be an array'); + if (rawDimensions.length > MAX_DIMENSIONS) invalid(`at most ${MAX_DIMENSIONS} dimensions per request`); + const dimensions = rawDimensions.map((d, i) => asName(d, `dimensions[${i}]`)); + if (new Set(dimensions).size !== dimensions.length) invalid('dimensions contains duplicates'); + + const rawFilters = obj.filters ?? []; + if (!Array.isArray(rawFilters)) invalid('filters must be an array'); + if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters per request`); + const filters = rawFilters.map((f, i) => parseFilter(f, i)); + + const timeRange = obj.timeRange === undefined ? undefined : parseTimeRange(obj.timeRange); + + const rawOrderBy = obj.orderBy ?? []; + if (!Array.isArray(rawOrderBy)) invalid('orderBy must be an array'); + if (rawOrderBy.length > MAX_ORDER_BY) invalid(`at most ${MAX_ORDER_BY} orderBy entries`); + const orderBy = rawOrderBy.map((o, i) => parseOrderBy(o, i)); + const selectable = new Set([...metrics, ...dimensions]); + for (const o of orderBy) { + if (!selectable.has(o.ref)) { + invalid(`orderBy "${o.ref}" is neither a requested metric nor a requested dimension`); + } + } + + let limit = DEFAULT_REQUEST_LIMIT; + if (obj.limit !== undefined) { + if (typeof obj.limit !== 'number' || !Number.isFinite(obj.limit)) + invalid('limit must be a finite number'); + if (!Number.isInteger(obj.limit)) invalid('limit must be an integer'); + if (obj.limit < 1) invalid('limit must be >= 1'); + limit = obj.limit; + } + + let format: RequestFormat = 'json'; + if (obj.format !== undefined) { + if (typeof obj.format !== 'string' || !(REQUEST_FORMATS as readonly string[]).includes(obj.format)) { + invalid(`format must be one of ${REQUEST_FORMATS.join(', ')}`); + } + format = obj.format as RequestFormat; + } + + return { dataset, metrics, dimensions, filters, timeRange, orderBy, limit, format }; +} + +function parseFilter(raw: unknown, index: number): AnalyticsFilter { + const obj = asObject(raw, `filters[${index}]`); + rejectUnknownKeys(obj, ['dimension', 'operator', 'values'], `filters[${index}]`); + const dimension = asName(obj.dimension, `filters[${index}].dimension`); + if (typeof obj.operator !== 'string' || !(FILTER_OPERATORS as readonly string[]).includes(obj.operator)) { + invalid(`filters[${index}].operator must be one of ${FILTER_OPERATORS.join(', ')}`); + } + const operator = obj.operator as FilterOperator; + + if (NO_VALUE_OPERATORS.includes(operator)) { + if (obj.values !== undefined) invalid(`filters[${index}].operator "${operator}" takes no values`); + return { dimension, operator }; + } + + if (!Array.isArray(obj.values)) invalid(`filters[${index}].values must be an array`); + if (obj.values.length === 0) invalid(`filters[${index}].values must not be empty`); + if (obj.values.length > MAX_FILTER_VALUES) { + invalid(`filters[${index}].values exceeds ${MAX_FILTER_VALUES} entries`); + } + if (SINGLE_VALUE_OPERATORS.includes(operator) && obj.values.length !== 1) { + invalid(`filters[${index}].operator "${operator}" takes exactly one value`); + } + if (!MULTI_VALUE_OPERATORS.includes(operator) && !SINGLE_VALUE_OPERATORS.includes(operator)) { + invalid(`filters[${index}].operator "${operator}" is not a value operator`); + } + const values = obj.values.map((v, i) => { + if (typeof v !== 'string') invalid(`filters[${index}].values[${i}] must be a string`); + if (v.length > MAX_VALUE_LENGTH) + invalid(`filters[${index}].values[${i}] exceeds ${MAX_VALUE_LENGTH} characters`); + return v; + }); + return { dimension, operator, values }; +} + +function parseTimeRange(raw: unknown): AnalyticsTimeRange { + const obj = asObject(raw, 'timeRange'); + rejectUnknownKeys(obj, ['dimension', 'from', 'to', 'grain'], 'timeRange'); + const out: AnalyticsTimeRange = {}; + if (obj.dimension !== undefined) out.dimension = asName(obj.dimension, 'timeRange.dimension'); + if (obj.from !== undefined) out.from = asIsoInstant(obj.from, 'timeRange.from'); + if (obj.to !== undefined) out.to = asIsoInstant(obj.to, 'timeRange.to'); + if (obj.grain !== undefined) { + if (typeof obj.grain !== 'string' || !(TIME_GRAINS as readonly string[]).includes(obj.grain)) { + invalid(`timeRange.grain must be one of ${TIME_GRAINS.join(', ')}`); + } + out.grain = obj.grain as TimeGrain; + } + if (out.from && out.to && Date.parse(out.from) >= Date.parse(out.to)) { + invalid('timeRange.from must be strictly before timeRange.to'); + } + return out; +} + +/** + * A timestamp is validated as a real instant, not merely as "a string that + * gets bound". It is bound as a parameter either way, so this is not the + * injection defence — it is the defence against a nonsense range silently + * producing an empty answer that a dashboard renders as "zero sales". + */ +function asIsoInstant(raw: unknown, what: string): string { + if (typeof raw !== 'string') invalid(`${what} must be an ISO-8601 string`); + const v = raw.trim(); + if (!/^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})?)?$/.test(v)) { + invalid(`${what} is not an ISO-8601 timestamp`); + } + if (Number.isNaN(Date.parse(v))) invalid(`${what} is not a valid timestamp`); + return v; +} + +function parseOrderBy(raw: unknown, index: number): AnalyticsOrderBy { + const obj = asObject(raw, `orderBy[${index}]`); + rejectUnknownKeys(obj, ['ref', 'direction'], `orderBy[${index}]`); + const ref = asName(obj.ref, `orderBy[${index}].ref`); + if (typeof obj.direction !== 'string' || !(ORDER_DIRECTIONS as readonly string[]).includes(obj.direction)) { + invalid(`orderBy[${index}].direction must be one of ${ORDER_DIRECTIONS.join(', ')}`); + } + return { ref, direction: obj.direction as OrderDirection }; +} + +// ── resolution: names -> declared catalog entries ────────────────────────── + +/** + * What a metric NAME resolves to. Supplied from `MetricDefinition` rows + * (BI-P002/BI-P028); the compilers never invent one. + */ +export interface ResolvedMetric { + name: string; + metric_id: string; + /** The `metric` key stored in the analytics event table. */ + event_metric_key: string; + aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count' | 'count_distinct'; + /** Dimension names this metric may legitimately be sliced by. */ + allowed_dimensions: readonly string[]; + unit?: string; +} + +/** What a dimension NAME resolves to. Supplied from `DimensionDefinition` rows. */ +export interface ResolvedDimension { + name: string; + dimension_id: string; + /** Physical key inside `dim_names`. Validated as an identifier downstream. */ + field_name: string; +} + +/** + * The declared surface for ONE dataset, for ONE tenant. A compiler is handed + * this; it cannot reach past it. An unresolvable name is a rejection. + */ +export interface QueryResolution { + dataset_id: string; + /** Semantic model version this resolution came from — part of the cache key (BI-P020). */ + semantic_model_version?: number; + metric(name: string): ResolvedMetric | undefined; + dimension(name: string): ResolvedDimension | undefined; + /** Physical time dimension used when `timeRange.dimension` is omitted. */ + defaultTimeDimension?: string; +} + +export function queryResolutionFrom(args: { + dataset_id: string; + metrics: readonly ResolvedMetric[]; + dimensions: readonly ResolvedDimension[]; + semantic_model_version?: number; + defaultTimeDimension?: string; +}): QueryResolution { + const metricMap = new Map(args.metrics.map((m) => [m.name, m])); + const dimMap = new Map(args.dimensions.map((d) => [d.name, d])); + return { + dataset_id: args.dataset_id, + semantic_model_version: args.semantic_model_version, + defaultTimeDimension: args.defaultTimeDimension, + // Map lookups: "__proto__" / "constructor" resolve to undefined, as they must. + metric: (name) => metricMap.get(name), + dimension: (name) => dimMap.get(name), + }; +} + +/** + * Resolve every name in a request, or refuse the request. + * + * This is the single place where "a caller said X" becomes "the catalog + * declares Y". Both compile targets (ClickHouse and semantic) consume the + * RESOLVED form, which is why BI-P016's DoD holds: the consumer writes one + * request and neither target sees consumer text. + */ +export interface ResolvedQuery { + request: AnalyticsQueryRequest; + dataset_id: string; + metrics: ResolvedMetric[]; + dimensions: ResolvedDimension[]; + filters: Array<{ filter: AnalyticsFilter; dimension: ResolvedDimension }>; + timeDimension?: string; + semantic_model_version?: number; +} + +export function resolveAnalyticsQuery( + request: AnalyticsQueryRequest, + resolution: QueryResolution, +): ResolvedQuery { + if (request.dataset !== resolution.dataset_id) { + // The resolution is built FOR a dataset the caller was authorised to read + // (BI-P006 grants). A mismatch means the caller is asking about something + // else entirely, and answering from this resolution would be a leak. + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics', + 'requested dataset does not match the authorised resolution', + ); + } + const metrics = request.metrics.map((name) => { + const m = resolution.metric(name); + if (!m) invalid(`unknown metric "${name}" for this dataset`); + return m; + }); + const dimensions = request.dimensions.map((name) => { + const d = resolution.dimension(name); + if (!d) invalid(`unknown dimension "${name}" for this dataset`); + return d; + }); + // Every requested dimension must be legitimate for EVERY requested metric. + // Slicing a metric by a dimension its definition does not allow is how a + // "harmless" grouping turns into an unintended disclosure. + for (const m of metrics) { + for (const d of dimensions) { + if (!m.allowed_dimensions.includes(d.name)) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics', + `metric "${m.name}" may not be sliced by dimension "${d.name}"`, + ); + } + } + } + const filters = request.filters.map((f) => { + const d = resolution.dimension(f.dimension); + if (!d) invalid(`unknown filter dimension "${f.dimension}"`); + return { filter: f, dimension: d }; + }); + let timeDimension: string | undefined; + if (request.timeRange) { + if (request.timeRange.dimension) { + const d = resolution.dimension(request.timeRange.dimension); + if (!d) invalid(`unknown time dimension "${request.timeRange.dimension}"`); + timeDimension = d.field_name; + } else { + timeDimension = resolution.defaultTimeDimension; + if ((request.timeRange.from || request.timeRange.to) && !timeDimension) { + invalid('dataset declares no default time dimension; timeRange.dimension is required'); + } + } + } + return { + request, + dataset_id: resolution.dataset_id, + metrics, + dimensions, + filters, + timeDimension, + semantic_model_version: resolution.semantic_model_version, + }; +} diff --git a/src/analytics/cost/query-budget.ts b/src/analytics/cost/query-budget.ts new file mode 100644 index 0000000..c4e9734 --- /dev/null +++ b/src/analytics/cost/query-budget.ts @@ -0,0 +1,415 @@ +/** + * src/analytics/cost/query-budget.ts + * =================================== + * BI-P021 — QUERY COST / BUDGET. + * + * ── SAY THE HONEST THING FIRST ──────────────────────────────────────────── + * A budget checked once before execution and never again bounds NOTHING. The + * estimate is a planner's guess; a query that scans ten times its estimate + * runs to completion regardless, and the budget discovers this after the money + * is spent. Every "cost control" that consists only of a pre-flight check is + * an accounting feature wearing a safety feature's name. + * + * ClickHouse's own `max_rows_to_read` (BI-P012) DOES stop a query mid-flight — + * but it bounds ONE query. It cannot express "this tenant may scan 10 billion + * rows per hour across all its queries", because it has no idea what the + * tenant's other queries did. That window is what a budget is for, and it is + * exactly the thing a per-query setting cannot enforce. + * + * So this module has three parts, and only the second one is a control: + * + * 1. ADMISSION — estimate vs. remaining window. Rejects the obviously + * unaffordable before it starts. Cheap, and insufficient. + * 2. IN-FLIGHT — polls the engine's live `read_rows`/`read_bytes` counters + * for the running query and CANCELS it (via BI-P019, with + * verification) the moment window spend would be exceeded. + * THIS is what makes the budget real. + * 3. ACCOUNTING — settles the actual cost into a ledger the control plane + * reads: CL0UD quotas, AUD0 usage events, B00K accounting. + * + * If part 2 is disabled (`enforceInFlight: false`), `describeEnforcement()` + * says so in plain words, and the admission-only posture is reported as + * "advisory". A control that does not control should say which one it is. + * + * ── THE CAVEAT THAT THE LIVE RUN MEASURED ──────────────────────────────── + * In-flight enforcement is granular to the poll interval, so a query can + * OVERSHOOT its budget between polls — and on a columnar engine the overshoot + * is large. Measured against ClickHouse 24.8.14.39 with a 1,000,000-row window + * and a 100ms poll: the query was cancelled at 153,318,696 rows read, 55ms in. + * A 153x overshoot. See evidence/bi-p019-p021/cancellation-and-budget.json, + * which is regenerated by the live suite — the figure is machine-dependent and + * moves between runs (an earlier run on the same box recorded 131M / 130x). + * Do not quote a specific multiple; quote the SHAPE. + * + * That shape is still a bound — the same query, unwatched, read 4,227,318,261 + * rows before it was stopped in the BI-P019 case — but it is a bound of the + * form "within roughly one poll interval of engine throughput", not "at + * exactly the budget". Anyone sizing a budget must size it for that, and + * anyone quoting this as a hard cap is wrong. Shortening the interval narrows + * the overshoot and costs a `system.processes` read per poll per running + * query. + */ +import { SiaxError } from '../../platform/errors'; +import { + cancelAndVerify, + type CancellationOutcome, + type EngineProcessControl, +} from '../execution/cancellation'; + +/** The cost axes BI-P021 names. */ +export interface QueryCost { + rows_scanned: number; + bytes_scanned: number; + execution_ms: number; + engine: string; + /** Bytes read from object storage, when the engine distinguishes it. */ + storage_read_bytes?: number; +} + +/** + * Weights turning raw counters into the single `cost_units` figure BI-P018 + * records and B00K bills. + * + * The defaults are deliberately dominated by BYTES, not rows: on a columnar + * engine a billion-row scan of one narrow column is cheap and a + * hundred-million-row scan of a wide one is not, so a rows-weighted unit would + * bill the wrong query. + */ +export interface CostModel { + unitsPerBillionRows: number; + unitsPerGigabyteScanned: number; + unitsPerSecond: number; + unitsPerGigabyteStorageRead: number; +} + +export const DEFAULT_COST_MODEL: CostModel = { + unitsPerBillionRows: 1, + unitsPerGigabyteScanned: 10, + unitsPerSecond: 2, + unitsPerGigabyteStorageRead: 15, +}; + +export function costUnits(cost: QueryCost, model: CostModel = DEFAULT_COST_MODEL): number { + const units = + (cost.rows_scanned / 1e9) * model.unitsPerBillionRows + + (cost.bytes_scanned / 2 ** 30) * model.unitsPerGigabyteScanned + + (cost.execution_ms / 1000) * model.unitsPerSecond + + ((cost.storage_read_bytes ?? 0) / 2 ** 30) * model.unitsPerGigabyteStorageRead; + // Rounded to six places so the same cost always serialises identically; + // float noise in a billing figure produces support tickets. + return Math.round(units * 1e6) / 1e6; +} + +/** A tenant's allowance over a rolling window. Issued by CL0UD entitlements. */ +export interface QueryBudget { + windowMs: number; + maxRowsScanned: number; + maxBytesScanned: number; + maxCostUnits: number; + /** What to do when the window is exhausted. */ + onExhausted: 'reject' | 'throttle'; + /** For `throttle`: queries are delayed by this much instead of refused. */ + throttleDelayMs?: number; +} + +export const DEFAULT_QUERY_BUDGET: QueryBudget = { + windowMs: 60 * 60 * 1000, + maxRowsScanned: 50_000_000_000, + maxBytesScanned: 2 * 2 ** 40, + maxCostUnits: 10_000, + onExhausted: 'reject', +}; + +interface LedgerEntry { + at: number; + rows: number; + bytes: number; + units: number; +} + +/** Where settled cost goes. AUD0/B00K/CL0UD sit behind this. */ +export interface UsageLedgerSink { + record(entry: { + tenant_id: string; + project_id?: string; + principal_id: string; + query_id: string; + cost: QueryCost; + cost_units: number; + /** True when the query was stopped by the budget rather than by the user. */ + terminated_by_budget: boolean; + at: string; + }): Promise | void; +} + +export interface WindowSpend { + rows_scanned: number; + bytes_scanned: number; + cost_units: number; + queries: number; + window_started_at: string; +} + +/** + * In-memory rolling window per tenant. + * + * Explicitly NOT the durable ledger — this is one process's view, and a + * multi-replica deployment needs the shared counter (Valkey or the CL0UD quota + * service) behind `SharedWindowStore` for the budget to bind across replicas. + * Stated rather than implied: with N replicas and no shared store, a tenant + * can spend N times its budget. `describeEnforcement()` reports this. + */ +export interface SharedWindowStore { + /** Atomically add to the tenant's window and return the new totals. */ + add( + tenantId: string, + entry: { rows: number; bytes: number; units: number; windowMs: number }, + ): Promise; + get(tenantId: string): Promise; +} + +export class InMemoryWindowStore implements SharedWindowStore { + private readonly entries = new Map(); + constructor(private readonly now: () => number = Date.now) {} + + private prune(tenantId: string, windowMs: number): LedgerEntry[] { + const cutoff = this.now() - windowMs; + const kept = (this.entries.get(tenantId) ?? []).filter((e) => e.at >= cutoff); + this.entries.set(tenantId, kept); + return kept; + } + + async add( + tenantId: string, + entry: { rows: number; bytes: number; units: number; windowMs: number }, + ): Promise { + const kept = this.prune(tenantId, entry.windowMs); + kept.push({ at: this.now(), rows: entry.rows, bytes: entry.bytes, units: entry.units }); + this.entries.set(tenantId, kept); + return this.totals(kept, entry.windowMs); + } + + async get(tenantId: string, windowMs = DEFAULT_QUERY_BUDGET.windowMs): Promise { + return this.totals(this.prune(tenantId, windowMs), windowMs); + } + + private totals(entries: LedgerEntry[], windowMs: number): WindowSpend { + return { + rows_scanned: entries.reduce((a, e) => a + e.rows, 0), + bytes_scanned: entries.reduce((a, e) => a + e.bytes, 0), + cost_units: entries.reduce((a, e) => a + e.units, 0), + queries: entries.length, + window_started_at: new Date(this.now() - windowMs).toISOString(), + }; + } +} + +export interface BudgetEnforcerOptions { + budget: QueryBudget; + store: SharedWindowStore; + ledger: UsageLedgerSink; + control?: EngineProcessControl; + costModel?: CostModel; + /** The control. Off means admission-only, and `describeEnforcement` says so. */ + enforceInFlight?: boolean; + pollIntervalMs?: number; + /** True when `store` is shared across replicas. Affects the honest description. */ + storeIsShared?: boolean; + now?: () => number; + sleep?: (ms: number) => Promise; +} + +export interface AdmissionDecision { + admitted: boolean; + reason?: string; + /** For `throttle`: how long the caller must wait first. */ + delay_ms?: number; + remaining: { rows: number; bytes: number; units: number }; +} + +export interface InFlightOutcome { + /** True when the enforcer stopped the query. */ + terminated: boolean; + reason?: 'rows' | 'bytes' | 'units'; + /** Present when it terminated: the verified cancellation. */ + cancellation?: CancellationOutcome; + observed: { rows_read: number; bytes_read: number; elapsed_ms: number }; + polls: number; +} + +export class QueryBudgetEnforcer { + private readonly opts: Required< + Pick + > & + BudgetEnforcerOptions; + private readonly now: () => number; + private readonly sleep: (ms: number) => Promise; + + constructor(options: BudgetEnforcerOptions) { + this.opts = { + enforceInFlight: options.enforceInFlight ?? true, + pollIntervalMs: options.pollIntervalMs ?? 200, + storeIsShared: options.storeIsShared ?? false, + ...options, + }; + this.now = options.now ?? Date.now; + this.sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))); + if (this.opts.enforceInFlight && !this.opts.control) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.cost', + 'in-flight budget enforcement requires an EngineProcessControl; ' + + 'construct with enforceInFlight:false to run admission-only and be honest about it', + ); + } + } + + /** + * Exactly what this enforcer does and does not bound, in words an operator + * can paste into an incident review. Returned by the analytics health + * endpoint so the posture is visible without reading this file. + */ + describeEnforcement(): { + admission: string; + in_flight: string; + across_replicas: string; + binding: boolean; + } { + return { + admission: 'estimate is compared to the remaining window before the query starts', + in_flight: this.opts.enforceInFlight + ? `live engine counters are polled every ${this.opts.pollIntervalMs}ms and the query is cancelled ` + + '(with verification) when window spend would be exceeded' + : 'NOT ENFORCED — this budget is advisory. A query that exceeds its estimate runs to completion.', + across_replicas: this.opts.storeIsShared + ? 'window spend is held in a shared store, so the budget binds across replicas' + : 'window spend is per-process — with N replicas a tenant can spend up to N times this budget', + binding: this.opts.enforceInFlight && this.opts.storeIsShared, + }; + } + + async admit( + tenantId: string, + estimate: { rows_to_read: number; bytes_to_read: number }, + ): Promise { + const spend = await this.opts.store.get(tenantId); + const b = this.opts.budget; + const remaining = { + rows: Math.max(0, b.maxRowsScanned - spend.rows_scanned), + bytes: Math.max(0, b.maxBytesScanned - spend.bytes_scanned), + units: Math.max(0, b.maxCostUnits - spend.cost_units), + }; + const over = + estimate.rows_to_read > remaining.rows + ? 'rows' + : estimate.bytes_to_read > remaining.bytes + ? 'bytes' + : undefined; + if (!over) return { admitted: true, remaining }; + if (b.onExhausted === 'throttle') { + return { + admitted: true, + delay_ms: b.throttleDelayMs ?? 1000, + reason: `tenant window ${over} budget exhausted; throttled`, + remaining, + }; + } + return { admitted: false, reason: `tenant window ${over} budget exhausted`, remaining }; + } + + /** + * Watch a RUNNING query and stop it if it breaches the window. + * + * Runs concurrently with the query itself; the caller awaits both. Returns + * as soon as the query leaves the engine's process list, so a well-behaved + * query costs one extra poll. + */ + async enforceInFlight(tenantId: string, queryId: string, signal?: AbortSignal): Promise { + const observed = { rows_read: 0, bytes_read: 0, elapsed_ms: 0 }; + if (!this.opts.enforceInFlight || !this.opts.control?.progress) { + return { terminated: false, observed, polls: 0 }; + } + const control = this.opts.control; + const spendBefore = await this.opts.store.get(tenantId); + const b = this.opts.budget; + const remainingRows = Math.max(0, b.maxRowsScanned - spendBefore.rows_scanned); + const remainingBytes = Math.max(0, b.maxBytesScanned - spendBefore.bytes_scanned); + const remainingUnits = Math.max(0, b.maxCostUnits - spendBefore.cost_units); + + let polls = 0; + for (;;) { + if (signal?.aborted) return { terminated: false, observed, polls }; + let progress; + try { + progress = await control.progress?.(queryId); + } catch { + progress = undefined; + } + polls += 1; + if (!progress) { + // Gone from the process list: finished, or never started. Either way + // there is nothing left to bound. + return { terminated: false, observed, polls }; + } + observed.rows_read = progress.rows_read; + observed.bytes_read = progress.bytes_read; + observed.elapsed_ms = progress.elapsed_ms; + + const liveUnits = costUnits( + { + rows_scanned: progress.rows_read, + bytes_scanned: progress.bytes_read, + execution_ms: progress.elapsed_ms, + engine: control.engine, + }, + this.opts.costModel, + ); + const breach = + progress.rows_read > remainingRows + ? 'rows' + : progress.bytes_read > remainingBytes + ? 'bytes' + : liveUnits > remainingUnits + ? 'units' + : undefined; + + if (breach) { + const cancellation = await cancelAndVerify(control, queryId, { timeoutMs: 5_000 }); + return { terminated: true, reason: breach, cancellation, observed, polls }; + } + await this.sleep(this.opts.pollIntervalMs); + } + } + + /** Settle the actual cost. Called for EVERY outcome, including failures. */ + async settle(args: { + tenant_id: string; + project_id?: string; + principal_id: string; + query_id: string; + cost: QueryCost; + terminated_by_budget?: boolean; + }): Promise<{ cost_units: number; window: WindowSpend }> { + const units = costUnits(args.cost, this.opts.costModel); + const window = await this.opts.store.add(args.tenant_id, { + rows: args.cost.rows_scanned, + bytes: args.cost.bytes_scanned, + units, + windowMs: this.opts.budget.windowMs, + }); + await this.opts.ledger.record({ + tenant_id: args.tenant_id, + project_id: args.project_id, + principal_id: args.principal_id, + query_id: args.query_id, + cost: args.cost, + cost_units: units, + // A query the budget killed still COST what it burned before it died. + // Not charging for it is how a tenant discovers that the cheapest query + // is one that gets cancelled. + terminated_by_budget: args.terminated_by_budget ?? false, + at: new Date(this.now()).toISOString(), + }); + return { cost_units: units, window }; + } +} diff --git a/src/analytics/execution/cancellation.ts b/src/analytics/execution/cancellation.ts new file mode 100644 index 0000000..3c0d61f --- /dev/null +++ b/src/analytics/execution/cancellation.ts @@ -0,0 +1,183 @@ +/** + * src/analytics/execution/cancellation.ts + * ======================================== + * BI-P019 — QUERY CANCELLATION. + * + * "Integrate engine cancellation. DoD: User cancellation stops expensive + * query within defined timeout." + * + * THE THING THAT MAKES THIS REAL: "stops" is VERIFIED, not requested. + * + * The obvious implementation aborts the HTTP request and issues a KILL. Both + * of those are requests. The client abort stops US waiting; it does not stop + * the server, which keeps burning CPU on a query nobody will read. `KILL QUERY` + * without `SYNC` returns immediately and tells you nothing. Even `KILL QUERY + * SYNC` can return while the query is still winding down. + * + * So `cancelAndVerify` does three things and reports on all three: + * 1. abort the client side; + * 2. issue the engine's kill; + * 3. POLL the engine's process list until the query is GONE, or the deadline + * passes — and if the deadline passes, it says so with + * `verified_stopped: false` rather than returning a cheerful `true`. + * + * A caller that treats `cancelQuery()` returning `true` as "the query stopped" + * is the failure this module exists to prevent, which is why the return type + * is an outcome object and not a boolean. + */ +import { SiaxError } from '../../platform/errors'; + +/** Whatever can list and kill running queries. ClickHouse and Trino both can. */ +export interface EngineProcessControl { + readonly engine: string; + /** True when a query with this id is still running on the engine. */ + isRunning(queryId: string): Promise; + /** Ask the engine to stop it. Returning true means "accepted", not "stopped". */ + requestKill(queryId: string): Promise; + /** Live progress for the running query, when the engine exposes it (BI-P021). */ + progress?(queryId: string): Promise; +} + +export interface QueryProgress { + rows_read: number; + bytes_read: number; + elapsed_ms: number; +} + +export interface CancellationOutcome { + query_id: string; + requested_at: string; + /** The engine accepted the kill request. */ + kill_accepted: boolean; + /** + * The query was OBSERVED to be gone from the engine's process list. + * `false` means it may still be running — the honest answer. + */ + verified_stopped: boolean; + /** Milliseconds from the cancel request to the observation. */ + elapsed_ms: number; + /** The deadline this cancellation was held to. */ + timeout_ms: number; + /** Present when the query had already finished before the request landed. */ + already_finished: boolean; +} + +export interface CancellationOptions { + /** The defined timeout BI-P019's DoD refers to. */ + timeoutMs?: number; + pollIntervalMs?: number; + /** Aborts the client-side wait. */ + abort?: () => void; + now?: () => number; + sleep?: (ms: number) => Promise; +} + +export const DEFAULT_CANCELLATION_TIMEOUT_MS = 5_000; +const DEFAULT_POLL_INTERVAL_MS = 100; + +const defaultSleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +export async function cancelAndVerify( + control: EngineProcessControl, + queryId: string, + opts: CancellationOptions = {}, +): Promise { + if (!queryId?.trim()) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'cancellation requires a query id'); + } + const timeoutMs = opts.timeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS; + const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const now = opts.now ?? Date.now; + const sleep = opts.sleep ?? defaultSleep; + const startedMs = now(); + const requestedAt = new Date(startedMs).toISOString(); + + // Was it even running? Cancelling a finished query is a no-op, and reporting + // it as a successful stop would overstate what happened. + // + // FAIL CLOSED when the engine cannot be asked: an unreachable engine means + // "unknown", and the safe reading of unknown is "still running" — so the + // kill is still issued and the verification loop still has to observe the + // query gone. Letting this throw would abandon the cancellation entirely at + // the exact moment the operator most needs it to be attempted. + let runningBefore: boolean; + try { + runningBefore = await control.isRunning(queryId); + } catch { + runningBefore = true; + } + + // 1. client side: stop waiting. + opts.abort?.(); + + // 2. engine side: ask it to stop. + let killAccepted = false; + try { + killAccepted = await control.requestKill(queryId); + } catch { + killAccepted = false; + } + + if (!runningBefore) { + return { + query_id: queryId, + requested_at: requestedAt, + kill_accepted: killAccepted, + verified_stopped: true, + elapsed_ms: now() - startedMs, + timeout_ms: timeoutMs, + already_finished: true, + }; + } + + // 3. VERIFY. This loop is the whole point of the module. + let verified = false; + for (;;) { + let stillRunning: boolean; + try { + stillRunning = await control.isRunning(queryId); + } catch { + // An engine we cannot ask is an engine we cannot vouch for. Keep trying + // until the deadline, then report unverified. + stillRunning = true; + } + if (!stillRunning) { + verified = true; + break; + } + if (now() - startedMs >= timeoutMs) break; + await sleep(pollIntervalMs); + } + + return { + query_id: queryId, + requested_at: requestedAt, + kill_accepted: killAccepted, + verified_stopped: verified, + elapsed_ms: now() - startedMs, + timeout_ms: timeoutMs, + already_finished: false, + }; +} + +/** + * The same call, for a caller that wants a failure rather than a report. + * Throws `RETRYABLE_FAILURE` when the query outlived the deadline — because + * "we asked it to stop and it did not" is an operational condition someone + * has to see, not a value to ignore. + */ +export async function cancelOrThrow( + control: EngineProcessControl, + queryId: string, + opts: CancellationOptions = {}, +): Promise { + const outcome = await cancelAndVerify(control, queryId, opts); + if (!outcome.verified_stopped) { + throw new SiaxError( + 'RETRYABLE_FAILURE', + `0pt.analytics.${control.engine}`, + `query was not observed to stop within ${outcome.timeout_ms}ms of cancellation`, + ); + } + return outcome; +} diff --git a/src/analytics/execution/clickhouse-process-control.ts b/src/analytics/execution/clickhouse-process-control.ts new file mode 100644 index 0000000..5476497 --- /dev/null +++ b/src/analytics/execution/clickhouse-process-control.ts @@ -0,0 +1,138 @@ +/** + * src/analytics/execution/clickhouse-process-control.ts + * ====================================================== + * BI-P019/BI-P021 — the ClickHouse implementation of `EngineProcessControl`. + * + * Three statements, each chosen for a measured reason: + * + * isRunning `SELECT count() FROM system.processes WHERE query_id = …` + * — the authority on whether the server is still working. Note + * that `KILL QUERY SYNC` returning is NOT this: a killed query + * can still appear here briefly while it unwinds, which is + * exactly why cancellation polls instead of trusting the kill. + * + * requestKill `KILL QUERY WHERE query_id = … ASYNC` — ASYNC, not SYNC, + * because the verification loop is what establishes the stop and + * a SYNC kill would block this call for up to the query's + * remaining lifetime while providing no stronger guarantee. + * + * progress `read_rows`, `read_bytes` and `elapsed` from the same table — + * live counters, which is what makes BI-P021's in-flight budget + * enforcement possible at all. + * + * KILL and the system-table reads are ADMINISTRATIVE. They run on the engine's + * own credential, never on the tenant's restricted user (which has no access + * to system.processes — measured: ACCESS_DENIED, code 497). The tenant-scoping + * check therefore happens HERE, on the query id, before any statement is sent. + */ +import { SiaxError } from '../../platform/errors'; +import type { EngineProcessControl, QueryProgress } from './cancellation'; + +export interface ClickHouseProcessControlOptions { + baseUrl: string; + /** "user:password" for the ADMIN credential. From a secret ref. */ + basicAuth?: string; + timeoutMs?: number; + /** + * Restricts every operation to query ids carrying this prefix, so a caller + * cannot kill or inspect another tenant's work through this object. + */ + queryIdPrefix?: string; +} + +export class ClickHouseProcessControl implements EngineProcessControl { + readonly engine = 'clickhouse'; + private readonly baseUrl: string; + private readonly authHeader?: string; + private readonly timeoutMs: number; + private readonly prefix?: string; + + constructor(opts: ClickHouseProcessControlOptions) { + this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined; + this.timeoutMs = opts.timeoutMs ?? 10_000; + this.prefix = opts.queryIdPrefix; + } + + private assertScoped(queryId: string): void { + if (this.prefix && !queryId.startsWith(this.prefix)) { + throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this scope'); + } + if (!/^[A-Za-z0-9_-]{1,128}$/.test(queryId)) { + // The id reaches ClickHouse as a bound parameter, but a malformed id is + // a bug worth surfacing rather than a string worth binding. + throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'malformed query id'); + } + } + + async isRunning(queryId: string): Promise { + this.assertScoped(queryId); + const body = await this.send( + 'SELECT count() AS n FROM system.processes WHERE query_id = {qid:String} FORMAT JSON', + { param_qid: queryId }, + ); + const n = Number((JSON.parse(body) as { data: Array<{ n: string | number }> }).data[0]?.n ?? 0); + return n > 0; + } + + async requestKill(queryId: string): Promise { + this.assertScoped(queryId); + await this.send( + 'KILL QUERY WHERE query_id = {qid:String} ASYNC FORMAT JSON', + { param_qid: queryId }, + false, + ); + return true; + } + + async progress(queryId: string): Promise { + this.assertScoped(queryId); + const body = await this.send( + 'SELECT read_rows, read_bytes, toUInt64(elapsed * 1000) AS elapsed_ms FROM system.processes ' + + 'WHERE query_id = {qid:String} FORMAT JSON', + { param_qid: queryId }, + ); + const row = (JSON.parse(body) as { data: Array> }).data[0]; + if (!row) return undefined; + return { + rows_read: Number(row.read_rows ?? 0), + bytes_read: Number(row.read_bytes ?? 0), + elapsed_ms: Number(row.elapsed_ms ?? 0), + }; + } + + private async send(sql: string, params: Record, expectJson = true): Promise { + const url = new URL(this.baseUrl); + url.searchParams.set('query', sql); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + // NOT readonly: KILL mutates server state. The other two would tolerate + // readonly, but sharing one path keeps the credential story simple. + url.searchParams.set('max_execution_time', '10'); + const headers: Record = {}; + if (this.authHeader) headers.Authorization = this.authHeader; + let res: Response; + try { + res = await fetch(url.toString(), { + method: 'POST', + headers, + body: '', + signal: AbortSignal.timeout(this.timeoutMs), + }); + } catch (cause) { + throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + } + const text = await res.text(); + if (!res.ok) { + const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status); + throw new SiaxError( + 'PERMANENT_FAILURE', + 'clickhouse', + `process-control statement failed (CH code ${code})`, + ); + } + if (expectJson && !text.trim()) { + throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'empty response from process-control statement'); + } + return text; + } +} diff --git a/src/analytics/execution/query-execution-record.ts b/src/analytics/execution/query-execution-record.ts new file mode 100644 index 0000000..f847cc6 --- /dev/null +++ b/src/analytics/execution/query-execution-record.ts @@ -0,0 +1,274 @@ +/** + * src/analytics/execution/query-execution-record.ts + * ================================================== + * BI-P018 — QUERY EXECUTION MODEL. + * + * Every field BI-P018 lists, verbatim, and one it lists by its absence: + * + * "Every production query has traceable execution metadata WITHOUT + * unnecessarily storing sensitive raw SQL." + * + * So the record carries `compiled_query_hash` and never the statement. That is + * enforced, not documented: `assertNoRawSql` scans a finished record for the + * statement it describes, and + * tests/analytics/query-execution-model.test.ts fails if any field carries it. + * + * The ONE place raw SQL is retained is the analyst statement history + * (BI-P017), where a privileged human ran unreviewed SQL against tenant data + * and "who ran what" has to survive. That history is a separate record with a + * separate retention posture; this one is the per-execution telemetry that + * every query in the estate produces, and it must be safe to keep for a long + * time and to ship to AUD0. + * + * BI-P002's `QueryExecution` entity is the persisted row; this module is the + * lifecycle that produces it, plus the four fields BI-P018 adds to it + * (compiled_query_hash, datasets, metrics, cache_status, cost_units). + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; +import type { ExecutionState } from '../domain/entities'; + +/** BI-P020's outcome, recorded per execution so cache behaviour is measurable. */ +export const CACHE_STATUSES = ['hit', 'miss', 'bypass', 'stale_refused', 'not_cacheable'] as const; +export type CacheStatus = (typeof CACHE_STATUSES)[number]; + +/** Which surface produced the query — the "user/workload" axis BI-P018 names. */ +export const WORKLOAD_KINDS = [ + 'dashboard', + 'report', + 'api', + 'analyst_sql', + 'ai_analyst', + 'scheduled', + 'internal', +] as const; +export type WorkloadKind = (typeof WORKLOAD_KINDS)[number]; + +export interface QueryExecutionRecord { + query_id: string; + execution_id: string; + tenant_id: string; + project_id?: string; + /** The CL0UD principal. A human, a service, or a job — never absent. */ + principal_id: string; + workload: WorkloadKind; + submitted_at: string; + started_at?: string; + completed_at?: string; + engine: string; + /** + * sha256 of the COMPILED statement plus its bound parameter values. + * + * Parameters are included deliberately: two executions of the same SQL with + * different filters are different work with different cost, and a hash that + * collapsed them would make the cost record meaningless. They are HASHED, + * not stored, so the record still carries no tenant data. + */ + compiled_query_hash: string; + datasets: string[]; + metrics: string[]; + status: ExecutionState; + /** Rows RETURNED to the caller. */ + rows?: number; + /** Rows and bytes SCANNED — the cost axis, not the result size. */ + rows_scanned?: number; + bytes_scanned?: number; + duration_ms?: number; + cache_status: CacheStatus; + /** BI-P021 units. Present once the query has settled, including on failure. */ + cost_units?: number; + /** A SiaxErrorCode. Never an engine body, never a fragment of the statement. */ + error?: string; + correlation_id?: string; +} + +/** + * Hash a compiled statement and its bound parameters into the one value the + * record keeps. + * + * Parameters are sorted so two logically identical executions hash equal + * regardless of key iteration order — otherwise the cache-key and cost + * aggregations built on this hash would fragment for no reason. + */ +export function compiledQueryHash(sql: string, params: Record = {}): string { + const canonical = JSON.stringify([ + sql.trim(), + Object.keys(params) + .sort() + .map((k) => [k, params[k]]), + ]); + return crypto.createHash('sha256').update(canonical).digest('hex'); +} + +/** + * The legal state machine. Written down because "status" fields drift into + * meaninglessness when any value can follow any other — a `succeeded` row that + * later becomes `running` makes every downstream aggregation a lie. + */ +const LEGAL_TRANSITIONS: Record = { + queued: ['running', 'cancelled', 'rejected', 'failed'], + running: ['succeeded', 'failed', 'cancelled'], + succeeded: [], + failed: [], + cancelled: [], + rejected: [], +}; + +export function assertLegalTransition(from: ExecutionState, to: ExecutionState): void { + if (!LEGAL_TRANSITIONS[from].includes(to)) { + throw new SiaxError('CONFLICT', '0pt.analytics', `illegal execution transition ${from} -> ${to}`); + } +} + +/** Terminal states: a record in one of these will never change again. */ +export const TERMINAL_STATES: readonly ExecutionState[] = ['succeeded', 'failed', 'cancelled', 'rejected']; + +export interface ExecutionSink { + write(record: QueryExecutionRecord): Promise | void; +} + +export interface ExecutionRecorderOptions { + tenant_id: string; + project_id?: string; + principal_id: string; + workload: WorkloadKind; + engine: string; + correlation_id?: string; + sink: ExecutionSink; + now?: () => number; +} + +/** + * Owns one execution's record from submission to a terminal state. + * + * Every transition WRITES. A recorder that only writes at the end loses + * exactly the executions that matter: the ones that never finished. + */ +export class QueryExecutionRecorder { + private record: QueryExecutionRecord; + private readonly sink: ExecutionSink; + private readonly now: () => number; + + constructor( + opts: ExecutionRecorderOptions, + queryId: string, + compiled: { hash: string; datasets: string[]; metrics: string[] }, + ) { + if (!opts.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics', + 'an execution record requires a verified tenant', + ); + } + if (!opts.principal_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics', + 'an execution record requires a verified principal', + ); + } + this.now = opts.now ?? Date.now; + this.sink = opts.sink; + this.record = { + query_id: queryId, + execution_id: crypto.randomUUID(), + tenant_id: opts.tenant_id, + project_id: opts.project_id, + principal_id: opts.principal_id, + workload: opts.workload, + submitted_at: new Date(this.now()).toISOString(), + engine: opts.engine, + compiled_query_hash: compiled.hash, + datasets: [...compiled.datasets], + metrics: [...compiled.metrics], + status: 'queued', + cache_status: 'miss', + correlation_id: opts.correlation_id, + }; + } + + get current(): QueryExecutionRecord { + return { ...this.record, datasets: [...this.record.datasets], metrics: [...this.record.metrics] }; + } + + async submitted(): Promise { + await this.sink.write(this.current); + } + + async started(): Promise { + assertLegalTransition(this.record.status, 'running'); + this.record.status = 'running'; + this.record.started_at = new Date(this.now()).toISOString(); + await this.sink.write(this.current); + } + + /** A cache hit never reached the engine; record it as such, not as a fast query. */ + async servedFromCache(rows: number, costUnits = 0): Promise { + assertLegalTransition(this.record.status, 'running'); + this.record.status = 'running'; + this.record.started_at = this.record.started_at ?? new Date(this.now()).toISOString(); + this.record.cache_status = 'hit'; + await this.finish('succeeded', { rows, rows_scanned: 0, bytes_scanned: 0, cost_units: costUnits }); + } + + setCacheStatus(status: CacheStatus): void { + this.record.cache_status = status; + } + + async finish( + status: ExecutionState, + stats: { + rows?: number; + rows_scanned?: number; + bytes_scanned?: number; + cost_units?: number; + error?: string; + }, + ): Promise { + assertLegalTransition(this.record.status, status); + const completedMs = this.now(); + this.record.status = status; + this.record.completed_at = new Date(completedMs).toISOString(); + const startedMs = this.record.started_at + ? Date.parse(this.record.started_at) + : Date.parse(this.record.submitted_at); + this.record.duration_ms = Math.max(0, completedMs - startedMs); + if (stats.rows !== undefined) this.record.rows = stats.rows; + if (stats.rows_scanned !== undefined) this.record.rows_scanned = stats.rows_scanned; + if (stats.bytes_scanned !== undefined) this.record.bytes_scanned = stats.bytes_scanned; + if (stats.cost_units !== undefined) this.record.cost_units = stats.cost_units; + if (stats.error !== undefined) this.record.error = stats.error; + assertNoRawSql(this.record); + await this.sink.write(this.current); + return this.current; + } +} + +/** + * Refuse to emit a record that carries statement text. + * + * Checked at runtime rather than trusted to the type, because the fields that + * leak are the string ones a future change widens — `error` becoming an engine + * body is the obvious one, and ClickHouse's error text echoes the query. + */ +export function assertNoRawSql(record: QueryExecutionRecord): void { + const suspicious = /\b(select|insert|update|delete|drop|create|alter|from|where|join|union)\b/i; + const stringFields: Array<[string, string | undefined]> = [ + ['error', record.error], + ['query_id', record.query_id], + ['compiled_query_hash', record.compiled_query_hash], + ['correlation_id', record.correlation_id], + ...record.datasets.map((d, i) => [`datasets[${i}]`, d] as [string, string]), + ...record.metrics.map((m, i) => [`metrics[${i}]`, m] as [string, string]), + ]; + for (const [name, value] of stringFields) { + if (value && suspicious.test(value)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics', + `execution record field "${name}" looks like SQL; BI-P018 stores a hash, never the statement`, + ); + } + } +} diff --git a/src/analytics/metrics/metric-model.ts b/src/analytics/metrics/metric-model.ts new file mode 100644 index 0000000..5f3fd0d --- /dev/null +++ b/src/analytics/metrics/metric-model.ts @@ -0,0 +1,419 @@ +/** + * src/analytics/metrics/metric-model.ts + * ====================================== + * BI-P028 — METRIC DOMAIN MODEL, and BI-P031 — METRIC GOVERNANCE. + * + * BI-P028 names a canonical metric with fifteen fields. BI-P002 (merged in + * 0pt#34) already persists a `MetricDefinition` row with nine of them. This + * module does NOT define a second, parallel metric: `MetricSpec` is the + * BI-P028 shape, `metricSpecFromDefinition`/`metricDefinitionFromSpec` bridge + * it to the persisted entity, and sql/009 adds the six columns BI-P028 needs + * that BI-P002 did not have. A second metric type would be the exact failure + * BI-P028's DoD is about. + * + * ── BI-P028's DoD, made enforceable ─────────────────────────────────────── + * "No production metric exists only inside a dashboard tool." + * + * A statement of intent unless something refuses. `assertPanelUsesRegisteredMetric` + * is that something: a dashboard panel must reference a `metric_id` that the + * registry knows and that is ACTIVE. A panel carrying an inline aggregation, a + * SQL expression, or a metric id nobody registered is refused — so a metric + * cannot come into existence inside Superset and quietly become production. + * + * ── BI-P031's DoD, made enforceable ─────────────────────────────────────── + * "Certified metric cannot be modified in-place; new version required." + * + * `assertMetricEditAllowed` refuses any change to a CERTIFIED metric's MEANING + * — measure, aggregation, filters, time dimension, unit, currency — and points + * the caller at `nextMetricVersion`. Editorial fields (description, owner, + * business definition) stay editable, because forcing a version bump to fix a + * typo is how teams learn to route around governance. + * + * The TypeScript check is layer 1. sql/009 carries the same rule as a Postgres + * trigger, which is layer 2 and the one that holds when something writes to + * the table without going through this code. + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; +import type { MetricAggregation, MetricDefinition } from '../domain/entities'; + +// ── BI-P028 vocabulary ───────────────────────────────────────────────────── + +/** + * `type` in BI-P028's field list. What KIND of metric this is, which is + * separate from how it aggregates: a `ratio` is two measures, a `cumulative` + * is a window over one, a `derived` is an expression over other metrics. + */ +export const METRIC_TYPES = ['simple', 'ratio', 'derived', 'cumulative', 'conversion'] as const; +export type MetricType = (typeof METRIC_TYPES)[number]; + +/** BI-P028's lifecycle, verbatim. */ +export const METRIC_STATUSES = ['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED'] as const; +export type MetricStatus = (typeof METRIC_STATUSES)[number]; + +/** BI-P031's certification axis. Orthogonal to lifecycle status, on purpose. */ +export const METRIC_CERTIFICATIONS = ['experimental', 'certified', 'deprecated'] as const; +export type MetricCertification = (typeof METRIC_CERTIFICATIONS)[number]; + +/** Display format. A number without one is a number a dashboard guesses at. */ +export const METRIC_FORMATS = ['integer', 'decimal', 'percent', 'currency', 'duration_ms', 'bytes'] as const; +export type MetricFormat = (typeof METRIC_FORMATS)[number]; + +/** + * A metric-level filter. STRUCTURED, never a SQL string — same rule as + * BI-P016's request filters, and for the same reason: a metric definition that + * carries SQL is a raw-SQL surface with a governance label on it. + */ +export interface MetricFilter { + dimension: string; + operator: 'eq' | 'ne' | 'in' | 'not_in' | 'lt' | 'lte' | 'gt' | 'gte' | 'is_null' | 'is_not_null'; + values?: string[]; +} + +/** BI-P028's canonical metric, field for field. */ +export interface MetricSpec { + id: string; + name: string; + description: string; + type: MetricType; + /** The measure field aggregated (numerator, for a ratio). */ + measure: string; + aggregation: MetricAggregation; + /** Denominator for a ratio metric. */ + denominator?: string; + /** Dimension names this metric may legitimately be sliced by. */ + dimensions: string[]; + /** Filters baked into the metric's definition. */ + filters: MetricFilter[]; + /** The dimension time ranges and grains apply to. */ + timeDimension: string; + unit?: string; + /** ISO-4217, required when `format` is "currency". */ + currency?: string; + format: MetricFormat; + owner: string; + version: number; + status: MetricStatus; +} + +/** BI-P031's governance envelope around a spec. */ +export interface MetricGovernance { + certification: MetricCertification; + /** What the business means by this number, in prose, for a non-engineer. */ + business_definition: string; + /** How it is actually computed, for an engineer. */ + technical_definition: string; + owner: string; + /** Upstream datasets/models, from BI-P027 lineage. */ + lineage_refs: string[]; + /** Latest quality verdict for the metric's dataset (BI-P025). */ + quality_status?: 'passing' | 'warning' | 'failing' | 'unknown'; + /** Usage, from AUD0's BI-P076 events. Drives deprecation decisions. */ + usage_30d?: { queries: number; distinct_principals: number; dashboards: number }; + certified_at?: string; + certified_by?: string; +} + +export interface GovernedMetric { + spec: MetricSpec; + governance: MetricGovernance; +} + +// ── validation ───────────────────────────────────────────────────────────── + +const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/; + +function invalid(detail: string): never { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.metrics', detail); +} + +/** + * Validate a spec. Every rule here is one that, unenforced, produces a metric + * that renders as a plausible number and means nothing. + */ +export function assertValidMetricSpec(spec: MetricSpec): void { + if (!NAME_RE.test(spec.name)) { + invalid(`metric name "${spec.name}" must be a lowercase snake_case identifier`); + } + if (!spec.description?.trim()) { + // Not pedantry: an undescribed metric is the one two teams compute + // differently and argue about in a board meeting. + invalid(`metric "${spec.name}" has no description`); + } + if (!spec.owner?.trim()) invalid(`metric "${spec.name}" has no owner`); + if (!spec.timeDimension?.trim()) { + invalid( + `metric "${spec.name}" has no time dimension; a metric without one cannot be trended or compared`, + ); + } + if (!(METRIC_TYPES as readonly string[]).includes(spec.type)) invalid(`unknown metric type "${spec.type}"`); + if (!(METRIC_STATUSES as readonly string[]).includes(spec.status)) + invalid(`unknown metric status "${spec.status}"`); + if (!(METRIC_FORMATS as readonly string[]).includes(spec.format)) + invalid(`unknown metric format "${spec.format}"`); + if (spec.version < 1 || !Number.isInteger(spec.version)) + invalid('metric version must be a positive integer'); + + if (spec.type === 'ratio' && !spec.denominator) { + invalid(`ratio metric "${spec.name}" has no denominator`); + } + if (spec.type !== 'ratio' && spec.denominator) { + invalid(`metric "${spec.name}" is not a ratio but carries a denominator`); + } + if (spec.format === 'currency' && !spec.currency) { + // A currency-formatted number with no currency is how SEK gets rendered + // as USD on a dashboard nobody double-checks. + invalid(`metric "${spec.name}" is currency-formatted but names no currency`); + } + if (spec.currency && !/^[A-Z]{3}$/.test(spec.currency)) { + invalid(`metric "${spec.name}" currency must be an ISO-4217 code`); + } + if (spec.format !== 'currency' && spec.currency) { + invalid(`metric "${spec.name}" names a currency but is not currency-formatted`); + } + if (new Set(spec.dimensions).size !== spec.dimensions.length) { + invalid(`metric "${spec.name}" lists a dimension twice`); + } + for (const f of spec.filters) { + if (!spec.dimensions.includes(f.dimension) && f.dimension !== spec.timeDimension) { + invalid(`metric "${spec.name}" filters on "${f.dimension}", which is not one of its dimensions`); + } + const needsValues = !['is_null', 'is_not_null'].includes(f.operator); + if (needsValues && (!f.values || f.values.length === 0)) { + invalid(`metric "${spec.name}" filter on "${f.dimension}" has no values`); + } + if (!needsValues && f.values?.length) { + invalid(`metric "${spec.name}" filter operator "${f.operator}" takes no values`); + } + } +} + +/** + * The hash that makes "did the meaning change?" answerable. + * + * Deliberately covers ONLY the semantic fields. Description, owner and version + * are excluded: a metric whose hash changed when someone fixed a typo would + * make every governance check cry wolf, and a check that cries wolf gets + * disabled. + */ +export function metricMeaningHash(spec: MetricSpec): string { + const meaning = JSON.stringify([ + spec.type, + spec.measure, + spec.aggregation, + spec.denominator ?? '', + [...spec.dimensions].sort(), + [...spec.filters] + .map((f) => [f.dimension, f.operator, [...(f.values ?? [])].sort()]) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))), + spec.timeDimension, + spec.unit ?? '', + spec.currency ?? '', + ]); + return crypto.createHash('sha256').update(meaning).digest('hex'); +} + +/** Fields whose change is EDITORIAL — allowed in place, even when certified. */ +export const EDITORIAL_FIELDS: readonly (keyof MetricSpec)[] = [ + 'description', + 'owner', + 'status', + 'format', + 'name', +]; + +/** + * BI-P031's DoD. Refuses an in-place change to a CERTIFIED metric's meaning. + * + * The check is on the MEANING HASH, not on a field-by-field diff, so a change + * this code has never heard of is still caught — new fields default to + * "changes the meaning" rather than to "allowed", which is the fail-closed + * direction. + */ +export function assertMetricEditAllowed(current: GovernedMetric, proposed: MetricSpec): void { + if (current.spec.id !== proposed.id) { + invalid('a metric edit must not change the metric id'); + } + if (current.governance.certification !== 'certified') return; + if (proposed.version !== current.spec.version) { + // A version bump is the sanctioned path; it is not an "edit". + return; + } + if (metricMeaningHash(current.spec) !== metricMeaningHash(proposed)) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.metrics', + `metric "${current.spec.name}" is CERTIFIED; its definition cannot be modified in place — ` + + `mint version ${current.spec.version + 1} instead`, + ); + } +} + +/** + * Mint the next version. The new version starts as DRAFT and EXPERIMENTAL: + * inheriting `certified` would let a certified metric's meaning change while + * still wearing the badge, which is the thing BI-P031 forbids. + */ +export function nextMetricVersion( + current: GovernedMetric, + changes: Partial, + reason: string, +): { spec: MetricSpec; governance: MetricGovernance; change_reason: string } { + if (!reason?.trim()) { + invalid('a new metric version requires a change reason'); + } + const spec: MetricSpec = { + ...current.spec, + ...changes, + id: current.spec.id, + version: current.spec.version + 1, + status: 'DRAFT', + }; + assertValidMetricSpec(spec); + return { + spec, + governance: { + ...current.governance, + certification: 'experimental', + certified_at: undefined, + certified_by: undefined, + }, + change_reason: reason, + }; +} + +/** Certification is an act with an actor and a time, not a boolean someone set. */ +export function certifyMetric(metric: GovernedMetric, by: string, at: string): GovernedMetric { + if (metric.spec.status !== 'ACTIVE') { + invalid(`only an ACTIVE metric can be certified; "${metric.spec.name}" is ${metric.spec.status}`); + } + if (!metric.governance.business_definition?.trim() || !metric.governance.technical_definition?.trim()) { + invalid( + `metric "${metric.spec.name}" cannot be certified without both a business and a technical definition`, + ); + } + if (!by?.trim()) invalid('certification requires the certifying principal'); + return { + ...metric, + governance: { ...metric.governance, certification: 'certified', certified_by: by, certified_at: at }, + }; +} + +// ── BI-P028 DoD: no metric lives only in a dashboard ─────────────────────── + +/** + * A dashboard panel, as far as governance is concerned. It may name a metric. + * It may not BE one. + */ +export interface DashboardPanelRef { + panel_id: string; + metric_id?: string; + /** Anything a BI tool would let a user type instead of picking a metric. */ + inline_expression?: string; + inline_aggregation?: string; + inline_sql?: string; +} + +export interface MetricLookup { + (metricId: string): GovernedMetric | undefined; +} + +export function assertPanelUsesRegisteredMetric( + panel: DashboardPanelRef, + lookup: MetricLookup, +): GovernedMetric { + for (const field of ['inline_expression', 'inline_aggregation', 'inline_sql'] as const) { + if (panel[field]) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.metrics', + `panel "${panel.panel_id}" defines a metric inline via "${field}" — BI-P028: no production metric ` + + `may exist only inside a dashboard tool. Register it and reference its metric_id.`, + ); + } + } + if (!panel.metric_id) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.metrics', + `panel "${panel.panel_id}" references no metric_id`, + ); + } + const metric = lookup(panel.metric_id); + if (!metric) { + throw new SiaxError( + 'NOT_FOUND', + '0pt.analytics.metrics', + `panel "${panel.panel_id}" references unregistered metric "${panel.metric_id}"`, + ); + } + if (metric.spec.status === 'RETIRED') { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.metrics', + `panel "${panel.panel_id}" references RETIRED metric "${metric.spec.name}"`, + ); + } + return metric; +} + +// ── bridge to the BI-P002 persisted entity ───────────────────────────────── + +/** + * The BI-P028 spec, read out of the BI-P002 row plus sql/009's added columns. + * Kept as an explicit pair of functions rather than merging the types, so it + * is obvious which fields are new and where they are stored. + */ +export interface MetricDefinitionExtension { + metric_type: MetricType; + time_dimension: string; + unit?: string; + currency?: string; + display_format: MetricFormat; + status: MetricStatus; + filters_json: string; + certification: MetricCertification; + business_definition: string; + technical_definition: string; + meaning_hash: string; +} + +export function metricSpecFromDefinition(def: MetricDefinition, ext: MetricDefinitionExtension): MetricSpec { + return { + id: def.metric_id, + name: def.name, + description: def.description ?? '', + type: ext.metric_type, + measure: def.measure_field, + aggregation: def.aggregation, + denominator: def.denominator_field, + dimensions: [...def.allowed_dimension_ids], + filters: JSON.parse(ext.filters_json) as MetricFilter[], + timeDimension: ext.time_dimension, + unit: ext.unit, + currency: ext.currency, + format: ext.display_format, + owner: def.owner_id, + version: def.current_version, + status: ext.status, + }; +} + +export function metricDefinitionExtensionFromSpec( + spec: MetricSpec, + governance: MetricGovernance, +): MetricDefinitionExtension { + return { + metric_type: spec.type, + time_dimension: spec.timeDimension, + unit: spec.unit, + currency: spec.currency, + display_format: spec.format, + status: spec.status, + filters_json: JSON.stringify(spec.filters), + certification: governance.certification, + business_definition: governance.business_definition, + technical_definition: governance.technical_definition, + meaning_hash: metricMeaningHash(spec), + }; +} diff --git a/src/analytics/metrics/semantic-provider.ts b/src/analytics/metrics/semantic-provider.ts new file mode 100644 index 0000000..30d4ea0 --- /dev/null +++ b/src/analytics/metrics/semantic-provider.ts @@ -0,0 +1,249 @@ +/** + * src/analytics/metrics/semantic-provider.ts + * =========================================== + * BI-P029 — METRICFLOW PROVIDER. + * + * ── WHAT IS AND IS NOT PROVEN HERE. READ THIS BEFORE CITING BI-P029. ────── + * + * BI-P029's DoD asks that a golden metric return the same result via the 0pt + * Metric API, the MetricFlow CLI/provider, a Superset integration path, and + * the AI Analyst, within a numerical tolerance. + * + * MetricFlow is a Python package. It was NOT executed for this commit: there + * is no Python environment provisioned for this repo's CI, and adding one is a + * separate decision with its own supply-chain review (BI-P029 says "pin a + * currently verified OSI-compatible version", which requires that review to + * happen first — `METRICFLOW_PINNED_VERSION` below records the intended pin + * and is explicitly marked unverified). + * + * So what IS proven, in tests/integration/semantic-parity.integration.test.ts, + * is the part that does not need MetricFlow: the SAME golden metric, taken + * from the SAME registry version, executed against the SAME live ClickHouse + * data by TWO INDEPENDENT compilers — 0pt's metric compiler, and a renderer + * driven only by the engine-neutral `SemanticQuerySpec` — returns the same + * number. That is a real parity test between two code paths that share no + * SQL-generation code. It is NOT a test of MetricFlow, and this file does not + * claim to be one. + * + * The remaining three DoD surfaces (MetricFlow CLI, Superset, AI Analyst) are + * therefore OPEN. Saying so is the point; a green suite that quietly covered + * one of four surfaces would be the estate's recurring failure. + */ +import { SiaxError } from '../../platform/errors'; +import type { SemanticQuerySpec } from '../contract/compile-semantic'; +import type { MetricSpec } from './metric-model'; +import type { SemanticModelVersion } from './semantic-registry'; + +/** + * The version this provider is written against. + * + * UNVERIFIED: no supply-chain or licence review has been run for it in this + * repo, and it has not been executed. It is a declared intent, not a + * certification, and `providerReadiness()` reports it as such so nothing + * downstream can mistake the constant for a green light. + */ +export const METRICFLOW_PINNED_VERSION = '0.208.1'; + +/** The provider-neutral port. BI-P029's `SiaxSemanticProvider`. */ +export interface SiaxSemanticProvider { + readonly provider: string; + /** + * Emit this provider's own model document from the canonical registry + * version. The registry never stores it (BI-P030); it is derived. + */ + buildManifest(model: SemanticModelVersion): unknown; + /** Compile the engine-neutral spec into whatever this provider executes. */ + compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery; + /** Canonical SIAX metric id -> this provider's own name for it. */ + providerBindings(model: SemanticModelVersion): Record; +} + +export interface CompiledSemanticQuery { + provider: string; + /** The provider's own query representation. Never SQL text from a caller. */ + query: unknown; + /** Canonical metric ids, unchanged — the BI-P030 contract. */ + canonical_metric_ids: string[]; +} + +export interface ProviderReadiness { + provider: string; + pinned_version: string; + /** True only when the pin has been reviewed AND the provider executed. */ + verified: boolean; + /** What has actually been demonstrated. Plain words for an operator. */ + proven: string[]; + /** What has NOT. */ + open: string[]; +} + +/** + * MetricFlow-shaped provider. + * + * The manifest it emits follows MetricFlow's semantic-manifest structure + * (semantic_models with entities/dimensions/measures, and metrics referencing + * them), so a future run of the real MetricFlow has something to consume. It + * is derived from the registry on demand and never persisted — persisting it + * would be exactly the BI-P030 violation the registry's `assertEngineNeutral` + * exists to prevent. + */ +export class MetricFlowProvider implements SiaxSemanticProvider { + readonly provider = 'metricflow'; + + providerReadiness(): ProviderReadiness { + return { + provider: this.provider, + pinned_version: METRICFLOW_PINNED_VERSION, + verified: false, + proven: [ + 'the engine-neutral SemanticQuerySpec compiles to a MetricFlow-shaped query', + 'a MetricFlow semantic manifest is derived from the canonical registry without persisting it', + 'canonical SIAX metric ids survive the compilation unchanged', + "0pt's metric compiler and an independent spec-driven renderer return the same number " + + 'for the golden metric against live ClickHouse data', + ], + open: [ + `MetricFlow ${METRICFLOW_PINNED_VERSION} has NOT been executed — no Python runtime is provisioned`, + 'the pinned version has NOT had a licence or supply-chain review', + 'the Superset integration path is not implemented', + 'the AI Analyst path (inf0#29) has not been compared against this provider', + ], + }; + } + + buildManifest(model: SemanticModelVersion): { + /** Stamped so a document that leaked into a registry row is detectable. */ + _generated_by: string; + _generated_from: { model_id: string; version: number; hash: string }; + semantic_models: Array>; + metrics: Array>; + } { + const semantic_models = model.datasets.map((dataset) => { + const dims = model.dimensions.filter((d) => d.dataset_id === dataset); + const measures = model.metrics + .filter((m) => m.dimensions.some((dim) => dims.some((d) => d.name === dim))) + .map((m) => ({ name: m.measure, agg: mapAggregation(m.aggregation), expr: m.measure })); + return { + name: dataset, + node_relation: { alias: dataset, schema_name: '__resolved_at_execution__' }, + entities: model.relationships + .filter((r) => r.from_dataset === dataset) + .map((r) => ({ name: r.from_field, type: r.kind === 'many_to_one' ? 'foreign' : 'primary' })), + dimensions: dims.map((d) => ({ + name: d.name, + type: d.kind === 'time' ? 'time' : 'categorical', + ...(d.kind === 'time' ? { type_params: { time_granularity: 'day' } } : {}), + })), + measures: dedupeBy(measures, (m) => m.name), + }; + }); + + const metrics = model.metrics.map((m) => ({ + // The CANONICAL id is what a consumer holds. The provider's own name is + // derived from it, never the other way round. + name: m.id, + description: m.description, + type: mapMetricType(m.type), + type_params: metricTypeParams(m), + filter: m.filters.length ? { where_filters: m.filters.map(structuredFilter) } : undefined, + meta: { siax_metric_id: m.id, siax_metric_version: m.version, unit: m.unit, currency: m.currency }, + })); + + return { + _generated_by: `${this.provider}@${METRICFLOW_PINNED_VERSION}`, + _generated_from: { model_id: model.model_id, version: model.version, hash: model.hash }, + semantic_models, + metrics, + }; + } + + compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery { + const known = new Map(model.metrics.map((m) => [m.id, m])); + for (const ref of spec.metrics) { + if (!known.has(ref.metric_id)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `metric "${ref.metric_id}" is not in semantic model ${model.model_id} v${model.version}`, + ); + } + } + const timeBucket = spec.groupBy.find((g) => g.grain); + return { + provider: this.provider, + query: { + metrics: spec.metrics.map((m) => m.metric_id), + group_by: spec.groupBy.filter((g) => !g.grain).map((g) => g.name), + ...(timeBucket ? { time_granularity: timeBucket.grain } : {}), + // STRUCTURED, not a `--where` SQL string. MetricFlow accepts predicate + // text; handing it caller-derived text would put a raw-SQL surface + // behind the closed DSL. + where: spec.where.map((w) => ({ dimension: w.name, operator: w.operator, values: w.values })), + order_by: spec.orderBy.map((o) => ({ name: o.ref, descending: o.direction === 'desc' })), + limit: spec.limit, + ...(spec.timeConstraint + ? { start_time: spec.timeConstraint.start, end_time: spec.timeConstraint.end } + : {}), + }, + canonical_metric_ids: spec.metrics.map((m) => m.metric_id).sort(), + }; + } + + providerBindings(model: SemanticModelVersion): Record { + // Identity for MetricFlow, because its metric names can be the canonical + // ids. A provider whose naming rules differ (Cube's cube.measure form) + // populates this differently — and THAT is the whole replaceability story: + // the bindings change, the canonical ids do not. + return Object.fromEntries(model.metrics.map((m) => [m.id, m.id])); + } +} + +function mapAggregation(agg: MetricSpec['aggregation']): string { + switch (agg) { + case 'count_distinct': + return 'count_distinct'; + case 'ratio': + case 'derived': + // Not a measure aggregation at all — these are metric-level types, and + // the manifest expresses them through type_params instead. + return 'sum'; + default: + return agg; + } +} + +function mapMetricType(type: MetricSpec['type']): string { + switch (type) { + case 'ratio': + return 'ratio'; + case 'derived': + return 'derived'; + case 'cumulative': + return 'cumulative'; + case 'conversion': + return 'conversion'; + default: + return 'simple'; + } +} + +function metricTypeParams(m: MetricSpec): Record { + if (m.type === 'ratio') { + return { numerator: m.measure, denominator: m.denominator }; + } + return { measure: { name: m.measure } }; +} + +function structuredFilter(f: MetricSpec['filters'][number]): Record { + return { dimension: f.dimension, operator: f.operator, values: f.values ?? [] }; +} + +function dedupeBy(items: T[], key: (t: T) => string): T[] { + const seen = new Set(); + return items.filter((i) => { + const k = key(i); + if (seen.has(k)) return false; + seen.add(k); + return true; + }); +} diff --git a/src/analytics/metrics/semantic-registry.ts b/src/analytics/metrics/semantic-registry.ts new file mode 100644 index 0000000..1e6ac08 --- /dev/null +++ b/src/analytics/metrics/semantic-registry.ts @@ -0,0 +1,316 @@ +/** + * src/analytics/metrics/semantic-registry.ts + * =========================================== + * BI-P030 — SEMANTIC MODEL REGISTRY. + * + * "Persist semantic model version SEPARATELY FROM IMPLEMENTATION ENGINE. + * DoD: MetricFlow can be replaced without breaking canonical SIAX metric IDs." + * + * Which means exactly one thing in code, and it is a negative: NOTHING in this + * file may name MetricFlow, Cube, dbt, ClickHouse, Trino or DuckDB. A registry + * that stores a MetricFlow manifest is a MetricFlow registry with a neutral + * name on it. `assertEngineNeutral` enforces the negative at runtime by + * scanning a serialized version for engine vocabulary, and the test asserts it + * catches a manifest that smuggles one in — because the rule is exactly the + * kind that decays silently. + * + * The canonical SIAX metric id is the contract. A provider maps it to whatever + * its own model calls that metric (BI-P029's `providerBindings`), and swapping + * providers changes the BINDINGS, never the ids — so a dashboard, an API + * consumer or a saved query keeps working. + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; +import type { MetricSpec } from './metric-model'; + +export const SEMANTIC_MODEL_STATUSES = ['draft', 'active', 'superseded', 'retired'] as const; +export type SemanticModelStatus = (typeof SEMANTIC_MODEL_STATUSES)[number]; + +export const RELATIONSHIP_KINDS = ['one_to_many', 'many_to_one', 'one_to_one'] as const; +export type RelationshipKind = (typeof RELATIONSHIP_KINDS)[number]; + +export interface SemanticRelationship { + from_dataset: string; + from_field: string; + to_dataset: string; + to_field: string; + kind: RelationshipKind; +} + +export interface SemanticDimension { + id: string; + name: string; + dataset_id: string; + /** "categorical" | "time". A time dimension is what grains apply to. */ + kind: 'categorical' | 'time'; +} + +/** BI-P030's fields, verbatim. */ +export interface SemanticModelVersion { + model_id: string; + version: number; + tenant_id: string; + datasets: string[]; + relationships: SemanticRelationship[]; + /** Canonical SIAX metric ids and their specs at this version. */ + metrics: MetricSpec[]; + dimensions: SemanticDimension[]; + valid_from: string; + status: SemanticModelStatus; + hash: string; +} + +/** + * Engine vocabulary that must never appear in a persisted semantic model. + * + * Checked against the SERIALIZED version, so it catches an engine name hiding + * in a nested field somebody added later — the failure mode a type cannot + * prevent. + */ +const ENGINE_VOCABULARY = [ + 'metricflow', + 'clickhouse', + 'trino', + 'duckdb', + 'cube.js', + 'cubejs', + 'dbt_metric', + 'superset', + 'presto', + 'bigquery', + 'snowflake', +]; + +/** The only top-level keys a persisted semantic model may carry. */ +export const SEMANTIC_MODEL_KEYS = [ + 'model_id', + 'version', + 'tenant_id', + 'datasets', + 'relationships', + 'metrics', + 'dimensions', + 'valid_from', + 'status', + 'hash', +] as const; + +/** + * The STRUCTURAL half of engine neutrality, and the load-bearing one. + * + * `assertEngineNeutral` below is a VOCABULARY scan: it catches a model that + * says "metricflow" or "clickhouse". It cannot catch a document that is + * structurally MetricFlow-shaped while avoiding the word — a limitation found + * by a test that expected it to and did not. So publishing also rejects any + * key outside the BI-P030 field list, which means a provider manifest cannot + * be stored in a registry row at all, whatever it calls itself. + */ +export function assertOnlySemanticModelFields(model: object): void { + for (const key of Object.keys(model)) { + if (!(SEMANTIC_MODEL_KEYS as readonly string[]).includes(key)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `semantic model carries field "${key}", which is not part of BI-P030's model — ` + + `provider manifests are DERIVED on demand, never persisted`, + ); + } + } +} + +export function assertEngineNeutral(model: Omit): void { + const serialized = JSON.stringify(model).toLowerCase(); + for (const term of ENGINE_VOCABULARY) { + if (serialized.includes(term)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `semantic model "${model.model_id}" mentions "${term}" — BI-P030 requires the model to be ` + + `persisted separately from any implementation engine, so it can be replaced without ` + + `breaking canonical SIAX metric ids`, + ); + } + } +} + +/** + * The version hash. Covers everything that defines the model's MEANING; + * `valid_from` and `status` are excluded because activating a model does not + * change what it says. + */ +export function semanticModelHash( + model: Omit, +): string { + const canonical = JSON.stringify([ + model.model_id, + model.version, + model.tenant_id, + [...model.datasets].sort(), + [...model.relationships] + .map((r) => [r.from_dataset, r.from_field, r.to_dataset, r.to_field, r.kind]) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))), + [...model.metrics].map((m) => [m.id, m.version]).sort((a, b) => String(a[0]).localeCompare(String(b[0]))), + [...model.dimensions].map((d) => [d.id, d.name, d.dataset_id, d.kind]).sort(), + ]); + return crypto.createHash('sha256').update(canonical).digest('hex'); +} + +export interface SemanticModelStore { + put(model: SemanticModelVersion): Promise; + get(tenantId: string, modelId: string, version: number): Promise; + active(tenantId: string, modelId: string): Promise; + versions(tenantId: string, modelId: string): Promise; +} + +export class InMemorySemanticModelStore implements SemanticModelStore { + private readonly rows = new Map(); + private key(t: string, m: string, v: number): string { + return `${t}::${m}::${v}`; + } + async put(model: SemanticModelVersion): Promise { + this.rows.set(this.key(model.tenant_id, model.model_id, model.version), model); + } + async get(t: string, m: string, v: number): Promise { + return this.rows.get(this.key(t, m, v)); + } + async versions(t: string, m: string): Promise { + return [...this.rows.values()] + .filter((r) => r.tenant_id === t && r.model_id === m) + .sort((a, b) => a.version - b.version); + } + async active(t: string, m: string): Promise { + return (await this.versions(t, m)).find((r) => r.status === 'active'); + } +} + +export class SemanticModelRegistry { + constructor(private readonly store: SemanticModelStore) {} + + /** + * Publish a new version. Versions are IMMUTABLE — re-publishing an existing + * (model, version) is a CONFLICT, not an update, because a dashboard pinned + * to version 7 must mean the same thing tomorrow. + */ + async publish(input: Omit): Promise { + if (!input.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.semantic', + 'a semantic model requires a verified tenant', + ); + } + if (input.version < 1 || !Number.isInteger(input.version)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.semantic', 'version must be a positive integer'); + } + assertOnlySemanticModelFields(input); + assertEngineNeutral(input); + this.assertReferentialIntegrity(input); + + const existing = await this.store.get(input.tenant_id, input.model_id, input.version); + if (existing) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.semantic', + `semantic model ${input.model_id} v${input.version} already exists and is immutable`, + ); + } + const model: SemanticModelVersion = { ...input, hash: semanticModelHash(input) }; + await this.store.put(model); + return model; + } + + /** Activate a version, superseding the previous active one. */ + async activate( + tenantId: string, + modelId: string, + version: number, + at: string, + ): Promise { + const target = await this.store.get(tenantId, modelId, version); + if (!target) + throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found'); + if (target.status === 'retired') { + throw new SiaxError('CONFLICT', '0pt.analytics.semantic', 'a retired version cannot be re-activated'); + } + const current = await this.store.active(tenantId, modelId); + if (current && current.version !== version) { + await this.store.put({ ...current, status: 'superseded' }); + } + const activated = { ...target, status: 'active' as const, valid_from: at }; + await this.store.put(activated); + return activated; + } + + /** + * BI-P030's DoD, as a query: the canonical metric ids a model exposes. + * The engine-parity test asserts this set is identical before and after a + * provider swap. + */ + async canonicalMetricIds(tenantId: string, modelId: string, version: number): Promise { + const model = await this.store.get(tenantId, modelId, version); + if (!model) + throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found'); + return model.metrics.map((m) => m.id).sort(); + } + + /** + * Every dimension and dataset a metric names must exist in the model. + * + * A model that passes validation while referencing a dimension nobody + * defined produces a metric that fails at query time, per tenant, in + * production — the worst place to find out. + */ + private assertReferentialIntegrity(model: Omit): void { + const datasets = new Set(model.datasets); + const dimensionNames = new Set(model.dimensions.map((d) => d.name)); + const timeDimensions = new Set(model.dimensions.filter((d) => d.kind === 'time').map((d) => d.name)); + + for (const d of model.dimensions) { + if (!datasets.has(d.dataset_id)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `dimension "${d.name}" belongs to dataset "${d.dataset_id}", which the model does not include`, + ); + } + } + for (const r of model.relationships) { + for (const ds of [r.from_dataset, r.to_dataset]) { + if (!datasets.has(ds)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `relationship references dataset "${ds}", which the model does not include`, + ); + } + } + } + const seenIds = new Set(); + for (const m of model.metrics) { + if (seenIds.has(m.id)) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.semantic', + `metric id "${m.id}" appears twice in the model`, + ); + } + seenIds.add(m.id); + for (const dim of m.dimensions) { + if (!dimensionNames.has(dim)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `metric "${m.name}" names dimension "${dim}", which the model does not define`, + ); + } + } + if (!timeDimensions.has(m.timeDimension)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.semantic', + `metric "${m.name}" names time dimension "${m.timeDimension}", which the model does not define as a time dimension`, + ); + } + } + } +} diff --git a/src/analytics/providers/duckdb-provider.ts b/src/analytics/providers/duckdb-provider.ts new file mode 100644 index 0000000..1ad1300 --- /dev/null +++ b/src/analytics/providers/duckdb-provider.ts @@ -0,0 +1,434 @@ +/** + * src/analytics/providers/duckdb-provider.ts + * ========================================== + * BI-P014 — DUCKDB PROVIDER. + * + * Use cases BI-P014 names: Parquet, Iceberg preview, local query, test, CSV, + * ad-hoc files, report build, notebook. Heavy jobs go through N0D. + * + * ── THE SECURITY REQUIREMENT, AND THE MEASURED SURPRISE ─────────────────── + * "No uncontrolled filesystem access. Explicit ST0RE-mounted/input artifacts + * only." + * + * DuckDB is a local engine with `read_csv('/etc/passwd')` built in, so this is + * the whole prompt. Two layers, and the second is not what it looks like: + * + * LAYER 1 — MOUNT REGISTRY. A caller names ARTIFACT IDS, never paths. The + * registry resolves an id to a path, checks the artifact belongs to + * the caller's tenant, and `realpath`s the result to confirm it is + * still inside the mount root once symlinks are followed. The query + * builder is the only thing that ever writes a path into SQL. + * + * LAYER 2 — DUCKDB'S OWN SANDBOX. And here is the finding: + * + * `SET allowed_directories=[...]` ALONE IS A NO-OP. + * + * Measured on DuckDB 1.5.5: with `allowed_directories` set to one + * mount directory and nothing else, `read_csv('/etc/passwd')` + * RETURNED THE FILE, and an https:// URL was fetched over the + * network. Only when `enable_external_access = false` is ALSO set + * does the allow-list bind — the same reads then fail with + * "Permission Error: Cannot access file". `lock_configuration=true` + * afterwards stops a query re-opening either. + * + * So the obvious spelling of this control — set the allow-list and + * move on — yields a setting that appears in `duckdb_settings()`, + * looks like a sandbox, and permits everything. Same shape as + * `trivy --exit-code 0`. Hence `duckDbSessionBootstrap()` emits the + * three statements as one unit, and `assertSessionSandboxed()` + * refuses a session missing any of them. + * + * Evidence: evidence/bi-p014/duckdb-sandbox.json, written by + * tests/duckdb/duckdb-provider.live.test.ts, which runs the bypass + * corpus with LAYER 1 REMOVED and requires layer 2 to refuse it anyway. + */ +import path from 'node:path'; +import fs from 'node:fs'; +import { SiaxError } from '../../platform/errors'; +import type { AnalyticsIdentity } from '../engine/siax-analytics-engine'; + +export const ARTIFACT_FORMATS = ['parquet', 'csv', 'json', 'iceberg', 'arrow_ipc'] as const; +export type ArtifactFormat = (typeof ARTIFACT_FORMATS)[number]; + +/** + * A file the provider is allowed to read. Registered by the ST0RE mount + * process, never by a query. + */ +export interface ArtifactMount { + artifact_id: string; + tenant_id: string; + project_id?: string; + /** Canonical ST0RE reference, for lineage. */ + st0re_ref: string; + /** Path RELATIVE to the mount root. Never absolute, never traversing. */ + relative_path: string; + format: ArtifactFormat; + /** Bytes, from ST0RE metadata. Drives the N0D hand-off policy. */ + size_bytes: number; +} + +const ARTIFACT_ID_RE = /^[A-Za-z0-9_-]{1,128}$/; + +/** + * Resolves artifact ids to real paths, and refuses everything else. + * + * `realpathSync` is the load-bearing call: a relative path that passes a string + * check can still be a symlink into `/etc`. The check is on the RESOLVED path, + * after the filesystem has had its say. + */ +export class ArtifactMountRegistry { + private readonly mounts = new Map(); + private readonly rootReal: string; + + constructor( + /** The single directory every mounted artifact lives under. */ + readonly root: string, + private readonly statPath: (p: string) => string = (p) => fs.realpathSync(p), + ) { + if (!path.isAbsolute(root)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + 'the mount root must be an absolute path', + ); + } + this.rootReal = this.statPath(root); + } + + register(mount: ArtifactMount): void { + if (!ARTIFACT_ID_RE.test(mount.artifact_id)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe'); + } + if (!mount.tenant_id?.trim()) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'a mount must name its owning tenant'); + } + if (path.isAbsolute(mount.relative_path)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + 'a mount path must be relative to the mount root', + ); + } + // Resolve BEFORE storing, so a bad mount is refused at registration rather + // than at query time when the caller is a tenant. + this.resolveWithin(mount.relative_path); + this.mounts.set(mount.artifact_id, mount); + } + + /** + * Resolve an artifact for a VERIFIED identity. + * + * Two refusals that are easy to conflate but must stay distinct: + * * an unknown artifact is NOT_FOUND; + * * an artifact owned by another tenant is AUTHORIZATION_DENIED, and + * deliberately NOT reported as NOT_FOUND — an operator reading the audit + * trail needs to see an attempted cross-tenant read as one. + */ + resolve(identity: AnalyticsIdentity, artifactId: string): { mount: ArtifactMount; absolutePath: string } { + if (!identity?.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.duckdb', + 'artifact access requires a verified tenant', + ); + } + if (!ARTIFACT_ID_RE.test(artifactId)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe'); + } + const mount = this.mounts.get(artifactId); + if (!mount) { + throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', `artifact "${artifactId}" is not mounted`); + } + if (mount.tenant_id !== identity.tenant_id) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.duckdb', + `artifact "${artifactId}" belongs to another tenant`, + ); + } + if (mount.project_id && identity.project_id && mount.project_id !== identity.project_id) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.duckdb', + `artifact "${artifactId}" belongs to another project`, + ); + } + return { mount, absolutePath: this.resolveWithin(mount.relative_path) }; + } + + /** Join, resolve symlinks, and confirm the result is still under the root. */ + private resolveWithin(relative: string): string { + const joined = path.resolve(this.rootReal, relative); + let real: string; + try { + real = this.statPath(joined); + } catch { + throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', 'mounted artifact is not present on disk'); + } + // `startsWith(root)` alone would accept `/mnt-evil` for root `/mnt`. + if (real !== this.rootReal && !real.startsWith(`${this.rootReal}${path.sep}`)) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.duckdb', + 'artifact resolves outside the mount root (traversal or symlink)', + ); + } + return real; + } + + /** Every directory DuckDB is told about. One entry: the root itself. */ + allowedDirectories(): string[] { + return [this.rootReal]; + } + + list(tenantId: string): ArtifactMount[] { + return [...this.mounts.values()].filter((m) => m.tenant_id === tenantId); + } +} + +// ── layer 2: the DuckDB session sandbox ──────────────────────────────────── + +/** + * The three statements, as ONE unit. + * + * Order matters and is not cosmetic: `allowed_directories` must be set while + * the configuration is still open, `enable_external_access = false` is what + * makes the list bind at all, and `lock_configuration = true` must come last + * or a query could undo the other two. + */ +export function duckDbSessionBootstrap(allowedDirectories: readonly string[]): string[] { + if (allowedDirectories.length === 0) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + 'refusing to open a DuckDB session with no allowed directory — an empty list is not a sandbox', + ); + } + for (const dir of allowedDirectories) { + if (!path.isAbsolute(dir)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'an allowed directory must be absolute'); + } + if (dir.includes("'")) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + 'an allowed directory must not contain a quote', + ); + } + } + return [ + `SET allowed_directories=[${allowedDirectories.map((d) => `'${d}'`).join(', ')}]`, + // MEASURED: without this, the line above permits everything, including + // /etc/passwd and outbound HTTP. See this file's header. + 'SET enable_external_access=false', + 'SET lock_configuration=true', + ]; +} + +/** The settings that must all be present for a session to be sandboxed. */ +export const REQUIRED_SANDBOX_SETTINGS = [ + 'allowed_directories', + 'enable_external_access', + 'lock_configuration', +] as const; + +export function assertSessionSandboxed(statements: readonly string[]): void { + const joined = statements.join('\n').toLowerCase(); + for (const setting of REQUIRED_SANDBOX_SETTINGS) { + if (!joined.includes(setting)) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.duckdb', + `DuckDB session is missing "${setting}" — allowed_directories WITHOUT ` + + `enable_external_access=false is a no-op, measured on 1.5.5`, + ); + } + } + if (!/enable_external_access\s*=\s*false/i.test(joined)) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.duckdb', + 'DuckDB session leaves external access ENABLED; the directory allow-list is then not enforced', + ); + } +} + +// ── the provider ─────────────────────────────────────────────────────────── + +export interface DuckDbRunner { + readonly engine: string; + /** Open a session with the bootstrap applied, run `sql`, return rows. */ + run(args: { bootstrap: readonly string[]; sql: string }): Promise>>; +} + +export const DUCKDB_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const; +export type DuckDbAggregate = (typeof DUCKDB_AGGREGATES)[number]; + +/** + * A closed request. There is NO caller SQL on this provider, for the same + * reason `executeMetric` has none: an analyst SQL surface is a separate, + * role-gated thing (BI-P017), and letting the ad-hoc FILE engine grow one would + * put raw SQL next to a filesystem. + */ +export interface DuckDbArtifactQuery { + artifact_id: string; + /** Column names to select. Validated as identifiers. */ + select?: string[]; + aggregate?: { fn: DuckDbAggregate; column: string; alias?: string }; + group_by?: string[]; + /** Equality filters. Names validated, values escaped. */ + filters?: Record; + limit?: number; +} + +const COLUMN_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; + +function column(name: string): string { + if (!COLUMN_RE.test(name)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + `"${name.slice(0, 32)}" is not a column name`, + ); + } + return name; +} + +const READER: Record string> = { + parquet: (p) => `read_parquet('${p}')`, + csv: (p) => `read_csv('${p}', header=true, auto_detect=true)`, + json: (p) => `read_json_auto('${p}')`, + // The iceberg extension reads a table DIRECTORY, not a single file. + iceberg: (p) => `iceberg_scan('${p}')`, + arrow_ipc: (p) => `read_arrow('${p}')`, +}; + +export interface N0dHandoffPolicy { + /** Above this, the query is refused here and must go to N0D. */ + maxLocalBytes: number; +} + +export const DEFAULT_N0D_HANDOFF: N0dHandoffPolicy = { maxLocalBytes: 2 * 1024 * 1024 * 1024 }; + +export class DuckDbAnalyticsProvider { + readonly provider = 'duckdb'; + + constructor( + private readonly registry: ArtifactMountRegistry, + private readonly runner: DuckDbRunner, + private readonly handoff: N0dHandoffPolicy = DEFAULT_N0D_HANDOFF, + ) {} + + /** Formats this build can read. See `readiness()` for what has been PROVEN. */ + supports(format: ArtifactFormat): boolean { + return ['parquet', 'csv', 'json', 'iceberg'].includes(format); + } + + /** + * What has been demonstrated against a real DuckDB, and what has not. + * Same posture as BI-P029's `providerReadiness`: a capability list nobody + * can check is a capability list that overstates. + */ + readiness(): { proven: ArtifactFormat[]; unproven: Array<{ format: ArtifactFormat; reason: string }> } { + return { + proven: ['parquet', 'csv', 'iceberg'], + unproven: [ + { + format: 'arrow_ipc', + reason: + "DuckDB's `arrow` extension is a network install and the offline verification " + + 'environment could not download it, so read_arrow() has NOT been exercised', + }, + { format: 'json', reason: 'no fixture was built for it in the verification run' }, + ], + }; + } + + async query( + identity: AnalyticsIdentity, + request: DuckDbArtifactQuery, + ): Promise>> { + const { mount, absolutePath } = this.registry.resolve(identity, request.artifact_id); + + if (!this.supports(mount.format)) { + throw new SiaxError( + 'CAPABILITY_UNAVAILABLE', + '0pt.analytics.duckdb', + `format "${mount.format}" is not readable by this provider build`, + ); + } + if (mount.size_bytes > this.handoff.maxLocalBytes) { + // BI-P014: "Run heavy jobs through N0D." Refusing is the honest form — + // silently running a 40 GB scan in the API process is how a preview + // engine takes down a control plane. + throw new SiaxError( + 'CAPABILITY_UNAVAILABLE', + '0pt.analytics.duckdb', + `artifact is ${mount.size_bytes} bytes, above the ${this.handoff.maxLocalBytes}-byte local ceiling — ` + + `submit it as an N0D job`, + ); + } + + const source = READER[mount.format](absolutePath); + const selectParts: string[] = []; + const groupParts: string[] = []; + + for (const g of request.group_by ?? []) { + selectParts.push(column(g)); + groupParts.push(column(g)); + } + if (request.aggregate) { + const col = column(request.aggregate.column); + const alias = column(request.aggregate.alias ?? 'value'); + const fn = request.aggregate.fn; + if (!DUCKDB_AGGREGATES.includes(fn)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', `unsupported aggregate "${fn}"`); + } + const expr = + fn === 'count' ? 'count(*)' : fn === 'count_distinct' ? `count(DISTINCT ${col})` : `${fn}(${col})`; + selectParts.push(`${expr} AS ${alias}`); + } else { + for (const s of request.select ?? []) selectParts.push(column(s)); + } + if (selectParts.length === 0) selectParts.push('*'); + + const conditions: string[] = []; + for (const [name, value] of Object.entries(request.filters ?? {})) { + // Not every DuckDB binding exposes a bound-parameter transport, so values + // are ESCAPED here — and the escaping is asserted against a hostile + // corpus in the test rather than assumed. + conditions.push(`${column(name)} = '${escapeLiteral(value)}'`); + } + + const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), 1_000_000)); + const sql = + `SELECT ${selectParts.join(', ')} FROM ${source}` + + (conditions.length ? ` WHERE ${conditions.join(' AND ')}` : '') + + (groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') + + ` LIMIT ${limit}`; + + const bootstrap = duckDbSessionBootstrap(this.registry.allowedDirectories()); + assertSessionSandboxed(bootstrap); + return this.runner.run({ bootstrap, sql }); + } +} + +/** + * Escape a value for a DuckDB string literal. + * + * Doubling `'` is the whole grammar — DuckDB does NOT treat backslash as an + * escape inside a standard single-quoted string, so `\'` does not close one. + * A NUL is REFUSED rather than escaped: a truncating byte inside a literal is + * a parser disagreement waiting to happen, and there is no legitimate filter + * value that contains one. + */ +export function escapeLiteral(value: string): string { + if (value.includes('\u0000')) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.duckdb', + 'a filter value must not contain a NUL byte', + ); + } + return value.replace(/'/g, "''"); +} diff --git a/src/analytics/quality/clickhouse-quality-runner.ts b/src/analytics/quality/clickhouse-quality-runner.ts new file mode 100644 index 0000000..697ea64 --- /dev/null +++ b/src/analytics/quality/clickhouse-quality-runner.ts @@ -0,0 +1,127 @@ +/** + * src/analytics/quality/clickhouse-quality-runner.ts + * =================================================== + * BI-P025 — the ClickHouse `QualityQueryRunner`. + * + * One statement per rule, and it counts BOTH numbers in a single pass: + * + * SELECT countIf() AS violations, count() AS evaluated + * FROM .
WHERE tenant_id = {tenant:String} AND dataset_id = … + * + * `evaluated` is not decoration. It is what makes an EMPTY dataset + * distinguishable from a CLEAN one — the false green + * `NativeQualityProvider` refuses. Counting only violations would make the two + * cases identical at this layer, and no amount of care upstream could recover + * the difference. + * + * The BI-P011 forced tenant predicate applies here exactly as it does to a + * metric query: `tenant_id = {tenant:String}` is condition[0], the value is + * bound, and a quality run therefore cannot read another tenant's rows even + * though it runs on an administrative credential. + * + * `unique` gets its own statement shape, because a duplicate is a property of + * a GROUP, not of a row, and pretending otherwise would make the rule count + * the wrong thing. + */ +import { SiaxError } from '../../platform/errors'; +import { assertSafeIdentifier, FORCED_TENANT_PREDICATE } from '../engine/query-compiler'; +import type { QualityQueryRunner } from './quality-provider'; + +export interface ClickHouseQualityRunnerOptions { + baseUrl: string; + basicAuth?: string; + database: string; + /** dataset_id -> physical table. An unlisted dataset is refused, not guessed. */ + tables: Record; + timeoutMs?: number; + /** Ceiling for a quality scan; a check must not become the expensive query. */ + maxRowsToRead?: number; +} + +/** Marker the provider emits for the `unique` rule. */ +const UNIQUE_MARKER = /^__unique__\(([A-Za-z_][A-Za-z0-9_]{0,62})\)$/; + +export class ClickHouseQualityRunner implements QualityQueryRunner { + readonly engine = 'clickhouse'; + + constructor(private readonly opts: ClickHouseQualityRunnerOptions) { + assertSafeIdentifier(opts.database, 'database'); + for (const table of Object.values(opts.tables)) assertSafeIdentifier(table, 'table'); + } + + async count(args: { + tenant_id: string; + dataset_id: string; + violationExpression: string; + params: Record; + }): Promise<{ violations: number; evaluated: number }> { + if (!args.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.quality', + 'a quality run requires a verified tenant', + ); + } + const table = this.opts.tables[args.dataset_id]; + if (!table) { + throw new SiaxError( + 'NOT_FOUND', + '0pt.analytics.quality', + `dataset "${args.dataset_id}" has no physical table registered for quality runs`, + ); + } + const params: Record = { + ...args.params, + param_tenant: args.tenant_id, + param_dataset: args.dataset_id, + }; + const where = `${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String}`; + + const unique = UNIQUE_MARKER.exec(args.violationExpression.trim()); + const sql = unique + ? // A duplicate is a property of a GROUP. Count the rows that belong to a + // key appearing more than once, and the total rows, in one pass. + `SELECT sum(multiIf(c > 1, c, 0)) AS violations, sum(c) AS evaluated FROM ` + + `(SELECT ${assertSafeIdentifier(unique[1], 'field')} AS k, count() AS c ` + + `FROM ${this.opts.database}.${table} WHERE ${where} GROUP BY k) FORMAT JSON` + : `SELECT countIf(${args.violationExpression}) AS violations, count() AS evaluated ` + + `FROM ${this.opts.database}.${table} WHERE ${where} FORMAT JSON`; + + const url = new URL(this.opts.baseUrl.replace(/\/+$/, '')); + url.searchParams.set('query', sql); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + url.searchParams.set('max_rows_to_read', String(this.opts.maxRowsToRead ?? 5_000_000_000)); + url.searchParams.set('read_overflow_mode', 'throw'); + url.searchParams.set('max_execution_time', '60'); + url.searchParams.set('timeout_overflow_mode', 'throw'); + + const headers: Record = {}; + if (this.opts.basicAuth) { + headers.Authorization = `Basic ${Buffer.from(this.opts.basicAuth).toString('base64')}`; + } + + let res: Response; + try { + res = await fetch(url.toString(), { + method: 'POST', + headers, + body: '', + signal: AbortSignal.timeout(this.opts.timeoutMs ?? 60_000), + }); + } catch (cause) { + throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + } + const text = await res.text(); + if (!res.ok) { + const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status); + throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `quality query failed (CH code ${code})`); + } + const row = ( + JSON.parse(text) as { data: Array<{ violations: string | number; evaluated: string | number }> } + ).data[0]; + if (!row) { + throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'quality query returned no counts'); + } + return { violations: Number(row.violations ?? 0), evaluated: Number(row.evaluated ?? 0) }; + } +} diff --git a/src/analytics/quality/freshness.ts b/src/analytics/quality/freshness.ts new file mode 100644 index 0000000..aaa587b --- /dev/null +++ b/src/analytics/quality/freshness.ts @@ -0,0 +1,196 @@ +/** + * src/analytics/quality/freshness.ts + * =================================== + * BI-P026 — DATA FRESHNESS. + * + * Five states, exactly as BI-P026 names them, and the DoD is a NEGATIVE: + * + * "Stale data is never SILENTLY presented as current." + * + * Two words carry it. "Silently" means the failure mode is not "we served old + * data" — it is "we served old data and the reader could not tell". And + * "never" means the enforcement has to sit at the boundary every result + * crosses, not in each dashboard's rendering code. + * + * So `assertFreshnessDisclosed` refuses to release a result that carries no + * freshness envelope at all. A result with no envelope is not FRESH and not + * UNKNOWN — it is unreleasable, because "we forgot to attach it" and "the data + * is current" are indistinguishable to a reader, and a default of FRESH would + * make forgetting the silent path. + * + * UNKNOWN is likewise never treated as FRESH. It is its own state precisely + * because "we could not determine the age" is the honest thing to display, and + * collapsing it into FRESH is the classic version of this bug. + */ +import { SiaxError } from '../../platform/errors'; + +/** BI-P026's states, verbatim and in severity order. */ +export const FRESHNESS_STATES = ['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN'] as const; +export type FreshnessState = (typeof FRESHNESS_STATES)[number]; + +/** States a reader must be shown explicitly rather than served as current. */ +export const NON_CURRENT_STATES: readonly FreshnessState[] = ['DELAYED', 'STALE', 'FAILED', 'UNKNOWN']; + +export interface FreshnessPolicy { + /** How often the upstream is expected to land data. */ + expectedIntervalMs: number; + /** + * Multiples of the expected interval. Past `delayedAfter` the data is + * DELAYED; past `staleAfter` it is STALE. Defaults are deliberately tight: + * a dashboard that is two cycles behind is already misleading someone. + */ + delayedAfterIntervals?: number; + staleAfterIntervals?: number; +} + +export interface FreshnessInput { + /** When the newest successfully loaded data is timestamped. */ + lastSuccessfulLoadAt?: string; + /** Outcome of the most recent load attempt. */ + lastRunStatus?: 'succeeded' | 'failed' | 'running' | 'never_run'; + lastRunAt?: string; + policy: FreshnessPolicy; + now: number; +} + +export interface FreshnessEnvelope { + state: FreshnessState; + /** Age of the newest data, in ms. Absent when unknown. */ + age_ms?: number; + last_successful_load_at?: string; + last_run_at?: string; + expected_interval_ms: number; + /** Plain-language line a UI can render verbatim. */ + disclosure: string; + computed_at: string; +} + +export function computeFreshness(input: FreshnessInput): FreshnessEnvelope { + const { policy, now } = input; + const delayedAfter = policy.expectedIntervalMs * (policy.delayedAfterIntervals ?? 1.5); + const staleAfter = policy.expectedIntervalMs * (policy.staleAfterIntervals ?? 3); + const computed_at = new Date(now).toISOString(); + const base = { + last_successful_load_at: input.lastSuccessfulLoadAt, + last_run_at: input.lastRunAt, + expected_interval_ms: policy.expectedIntervalMs, + computed_at, + }; + + // FAILED outranks age: a pipeline that just failed may still have + // recently-timestamped data from the run before, and reporting that as FRESH + // is precisely how a broken pipeline stays invisible for a day. + if (input.lastRunStatus === 'failed') { + const age = ageOf(input.lastSuccessfulLoadAt, now); + return { + ...base, + state: 'FAILED', + age_ms: age, + disclosure: `The most recent load FAILED. Showing data from ${input.lastSuccessfulLoadAt ?? 'an unknown time'}.`, + }; + } + + if (!input.lastSuccessfulLoadAt || input.lastRunStatus === 'never_run') { + return { + ...base, + state: 'UNKNOWN', + disclosure: 'The age of this data could not be determined. Do not read it as current.', + }; + } + + const age = ageOf(input.lastSuccessfulLoadAt, now); + if (age === undefined) { + return { ...base, state: 'UNKNOWN', disclosure: 'The age of this data could not be determined.' }; + } + if (age > staleAfter) { + return { + ...base, + state: 'STALE', + age_ms: age, + disclosure: `This data is ${describeAge(age)} old, past the ${describeAge(staleAfter)} staleness threshold.`, + }; + } + if (age > delayedAfter) { + return { + ...base, + state: 'DELAYED', + age_ms: age, + disclosure: `This data is ${describeAge(age)} old and behind its ${describeAge(policy.expectedIntervalMs)} schedule.`, + }; + } + return { + ...base, + state: 'FRESH', + age_ms: age, + disclosure: `Current as of ${input.lastSuccessfulLoadAt}.`, + }; +} + +function ageOf(iso: string | undefined, now: number): number | undefined { + if (!iso) return undefined; + const t = Date.parse(iso); + if (Number.isNaN(t)) return undefined; + return Math.max(0, now - t); +} + +function describeAge(ms: number): string { + const s = Math.round(ms / 1000); + if (s < 90) return `${s}s`; + const m = Math.round(s / 60); + if (m < 90) return `${m}m`; + const h = Math.round(m / 60); + if (h < 48) return `${h}h`; + return `${Math.round(h / 24)}d`; +} + +/** Any result leaving the analytics boundary. */ +export interface DisclosableResult { + freshness?: FreshnessEnvelope; + /** + * Set by a caller that has SEEN a non-current state and chosen to proceed. + * Its presence is what makes serving non-current data not-silent. + */ + acknowledged_non_current?: boolean; +} + +/** + * BI-P026's DoD, as a gate. + * + * Three refusals, in order of how easy they are to get wrong: + * 1. no envelope at all — unreleasable. Not defaulted to FRESH. + * 2. a non-current state with no acknowledgement — refused, so a consumer + * that ignores the envelope cannot render stale numbers as current. + * 3. an acknowledgement with no envelope — refused, because acknowledging + * something you were never shown is not disclosure. + */ +export function assertFreshnessDisclosed(result: DisclosableResult, context: string): FreshnessEnvelope { + if (!result.freshness) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.freshness', + `${context} carries no freshness envelope — BI-P026: data with unknown age must never be released ` + + `as though it were current`, + ); + } + const state = result.freshness.state; + if (!(FRESHNESS_STATES as readonly string[]).includes(state)) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.freshness', + `${context} has an unknown freshness state`, + ); + } + if (NON_CURRENT_STATES.includes(state) && !result.acknowledged_non_current) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.freshness', + `${context} is ${state} and the consumer has not acknowledged it: ${result.freshness.disclosure}`, + ); + } + return result.freshness; +} + +/** True when the state may be presented without a caveat. Only one state may. */ +export function isPresentableAsCurrent(state: FreshnessState): boolean { + return state === 'FRESH'; +} diff --git a/src/analytics/quality/quality-provider.ts b/src/analytics/quality/quality-provider.ts new file mode 100644 index 0000000..1080e62 --- /dev/null +++ b/src/analytics/quality/quality-provider.ts @@ -0,0 +1,418 @@ +/** + * src/analytics/quality/quality-provider.ts + * ========================================== + * BI-P025 — DATA QUALITY PROVIDER. + * + * One interface, three provider kinds, and an honest split between them: + * + * NATIVE — this module compiles the rule into a counting query and runs it + * through an injected runner. It EXECUTES. + * DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against + * the BI-P022 contract). This module INGESTS the results. It does + * not run dbt, and `providerKind` says so rather than implying + * otherwise. + * GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here + * executes it. The adapter maps a GE validation result onto the + * canonical verdict so a GE suite counts toward the same gate. + * + * Calling all three "providers" while only one executes would be the kind of + * overstatement that turns into "we have Great Expectations coverage" in a + * status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or + * `"ingesting"`, and the gate records which produced each verdict. + * + * ── BI-P025's DoD ───────────────────────────────────────────────────────── + * "Failed critical quality gate can QUARANTINE serving projection." + * + * `quarantine.ts` is that, and the important half is what happens on the READ + * side: a quarantined projection is REFUSED, not annotated. An annotation on a + * dashboard nobody reads is not a quarantine. + */ +import { SiaxError } from '../../platform/errors'; +import type { QualityRuleKind, QualitySeverity } from '../domain/entities'; + +/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */ +export const QUALITY_RULES = [ + 'not_null', + 'unique', + 'range', + 'referential', + 'freshness', + 'schema', + 'distribution', + 'custom', +] as const; +export type QualityRule = (typeof QUALITY_RULES)[number]; + +/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */ +export const RULE_KIND_ALIASES: Record = { + not_null: 'not_null', + unique: 'unique', + range: 'range', + referential: 'referential', + freshness: 'freshness', + schema: 'custom', + distribution: 'accepted_values', + custom: 'custom', +}; + +export interface QualityRuleSpec { + rule_id: string; + tenant_id: string; + dataset_id: string; + rule: QualityRule; + /** Column the rule applies to; absent for dataset-level rules. */ + field?: string; + severity: QualitySeverity; + params: QualityRuleParams; + description?: string; +} + +export interface QualityRuleParams { + /** range */ + min?: number; + max?: number; + /** distribution / accepted values */ + accepted_values?: string[]; + /** distribution: the share of rows that must fall in accepted_values */ + min_share?: number; + /** referential */ + references_dataset?: string; + references_field?: string; + /** freshness: the maximum acceptable age of the newest row */ + max_age_seconds?: number; + timestamp_field?: string; + /** schema: the columns and types that must exist */ + expected_columns?: Array<{ name: string; type: string }>; + /** unique / not_null take no parameters */ + /** custom: an operator-authored counting query, reviewed like code */ + custom_violation_sql?: string; +} + +export interface QualityVerdict { + rule_id: string; + dataset_id: string; + tenant_id: string; + rule: QualityRule; + severity: QualitySeverity; + passed: boolean; + violation_count: number; + evaluated_rows: number; + /** Which provider produced this, and whether it EXECUTED or merely ingested. */ + produced_by: string; + provider_kind: 'executing' | 'ingesting'; + evaluated_at: string; + /** Short operator-facing note. Never raw engine output. */ + detail?: string; +} + +export interface SiaxDataQualityProvider { + readonly provider: string; + readonly providerKind: 'executing' | 'ingesting'; + /** Rules this provider can actually evaluate. */ + supports(rule: QualityRule): boolean; + evaluate(spec: QualityRuleSpec): Promise; +} + +/** + * Runs a counting query and returns two numbers. The only engine coupling in + * this module, so a DuckDB or Trino runner drops in unchanged. + */ +export interface QualityQueryRunner { + readonly engine: string; + /** + * Must apply the caller's tenant scoping. The runner is handed the tenant + * explicitly rather than trusting the SQL to carry it. + */ + count(args: { + tenant_id: string; + dataset_id: string; + violationExpression: string; + /** Bound parameters. Values NEVER appear in the expression text. */ + params: Record; + }): Promise<{ violations: number; evaluated: number }>; +} + +const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; + +function assertField(field: string | undefined, rule: QualityRule): string { + if (!field || !IDENT_RE.test(field)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + `rule "${rule}" requires a valid field name`, + ); + } + return field; +} + +/** + * The provider that actually runs checks. + * + * Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and + * the runner counts rows where it is true. That shape is chosen so a rule can + * never return "how many rows are fine", which is the number that looks + * healthy when a table is empty. + */ +export class NativeQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'native'; + readonly providerKind = 'executing' as const; + + constructor( + private readonly runner: QualityQueryRunner, + private readonly now: () => number = Date.now, + ) {} + + supports(rule: QualityRule): boolean { + // `schema` and `referential` need catalog/second-table access the counting + // runner does not have; they are honestly unsupported here rather than + // silently passing. + return !['schema', 'referential'].includes(rule); + } + + async evaluate(spec: QualityRuleSpec): Promise { + if (!this.supports(spec.rule)) { + throw new SiaxError( + 'CAPABILITY_UNAVAILABLE', + '0pt.analytics.quality', + `the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`, + ); + } + const { expression, params } = this.compile(spec); + const { violations, evaluated } = await this.runner.count({ + tenant_id: spec.tenant_id, + dataset_id: spec.dataset_id, + violationExpression: expression, + params, + }); + + // An EMPTY dataset is not a passing dataset. A `not_null` check over zero + // rows finds zero violations and would report success while the pipeline + // is silently broken — the single most common false green in data quality. + const emptyIsFailure = spec.rule !== 'custom'; + const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0; + + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + passed, + violation_count: violations, + evaluated_rows: evaluated, + produced_by: `${this.provider}:${this.runner.engine}`, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: + evaluated === 0 && emptyIsFailure + ? 'dataset is EMPTY — reported as a failure, not as zero violations' + : undefined, + }; + } + + private compile(spec: QualityRuleSpec): { expression: string; params: Record } { + const p = spec.params; + switch (spec.rule) { + case 'not_null': { + const f = assertField(spec.field, spec.rule); + return { expression: `${f} IS NULL OR ${f} = ''`, params: {} }; + } + case 'unique': { + const f = assertField(spec.field, spec.rule); + // Counted over a grouped subquery by the runner; the expression names + // the key so the runner can build it. + return { expression: `__unique__(${f})`, params: {} }; + } + case 'range': { + const f = assertField(spec.field, spec.rule); + if (p.min === undefined && p.max === undefined) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a range rule needs a min or a max', + ); + } + const parts: string[] = []; + const params: Record = {}; + if (p.min !== undefined) { + parts.push(`${f} < {qmin:Float64}`); + params.param_qmin = String(p.min); + } + if (p.max !== undefined) { + parts.push(`${f} > {qmax:Float64}`); + params.param_qmax = String(p.max); + } + return { expression: parts.join(' OR '), params }; + } + case 'freshness': { + const f = assertField(p.timestamp_field ?? spec.field, spec.rule); + if (!p.max_age_seconds || p.max_age_seconds <= 0) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a freshness rule needs max_age_seconds', + ); + } + return { + expression: `${f} < now() - toIntervalSecond({qage:UInt64})`, + params: { param_qage: String(Math.floor(p.max_age_seconds)) }, + }; + } + case 'distribution': { + const f = assertField(spec.field, spec.rule); + if (!p.accepted_values?.length) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a distribution rule needs accepted_values', + ); + } + const params: Record = {}; + const bound = p.accepted_values.map((v, i) => { + params[`param_qv${i}`] = v; + return `{qv${i}:String}`; + }); + return { expression: `${f} NOT IN (${bound.join(', ')})`, params }; + } + case 'custom': { + if (!p.custom_violation_sql?.trim()) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a custom rule needs custom_violation_sql', + ); + } + // Deliberately NOT parsed or guarded here. A custom rule is + // operator-authored, reviewed like code, and stored in the rule table + // — not caller input. It never reaches this function from a request + // body; `QualityRuleSpec` rows are written by an admin path. + return { expression: p.custom_violation_sql, params: {} }; + } + default: + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`); + } + } +} + +/** + * Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against + * the BI-P022 contract, and pretending otherwise here would misreport where + * the work happens. + */ +export interface DbtTestResult { + unique_id: string; + status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped'; + failures: number; + /** dbt does not report rows evaluated; absent, not zero. */ + rows_evaluated?: number; + message?: string; +} + +export class DbtQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'dbt'; + readonly providerKind = 'ingesting' as const; + constructor( + private readonly results: Map, + private readonly now: () => number = Date.now, + ) {} + + supports(): boolean { + return true; + } + + async evaluate(spec: QualityRuleSpec): Promise { + const result = this.results.get(spec.rule_id); + if (!result) { + // A missing result is UNKNOWN, and unknown is a FAILURE for a blocking + // rule. Treating "no result" as "passed" is how a quality gate stops + // gating the moment the test stops running. + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + passed: false, + violation_count: 0, + evaluated_rows: 0, + produced_by: this.provider, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: 'no dbt result for this rule — reported as a failure, not as a pass', + }; + } + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + // `skipped` and `error` are NOT passes. dbt's `skipped` in particular is + // the shape of l0re's silently-skipped Postgres tests. + passed: result.status === 'pass', + violation_count: result.failures, + evaluated_rows: result.rows_evaluated ?? 0, + produced_by: `dbt:${result.unique_id}`, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined, + }; + } +} + +/** Ingests a Great Expectations validation result. Does NOT run GE. */ +export interface GreatExpectationsResult { + expectation_suite_name: string; + success: boolean; + results: Array<{ + expectation_config: { expectation_type: string }; + success: boolean; + result?: { unexpected_count?: number; element_count?: number }; + }>; +} + +export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'great_expectations'; + readonly providerKind = 'ingesting' as const; + constructor( + private readonly suites: Map, + private readonly now: () => number = Date.now, + ) {} + + supports(): boolean { + return true; + } + + async evaluate(spec: QualityRuleSpec): Promise { + const suite = this.suites.get(spec.rule_id); + const base = { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + produced_by: this.provider, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + }; + if (!suite) { + return { + ...base, + passed: false, + violation_count: 0, + evaluated_rows: 0, + detail: 'no GE result — reported as a failure', + }; + } + const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0); + const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0); + return { + ...base, + passed: suite.success, + violation_count: unexpected, + evaluated_rows: elements, + produced_by: `great_expectations:${suite.expectation_suite_name}`, + detail: suite.success ? undefined : 'GE suite failed', + }; + } +} diff --git a/src/analytics/quality/quarantine.ts b/src/analytics/quality/quarantine.ts new file mode 100644 index 0000000..518d232 --- /dev/null +++ b/src/analytics/quality/quarantine.ts @@ -0,0 +1,188 @@ +/** + * src/analytics/quality/quarantine.ts + * ==================================== + * BI-P025's DoD: "Failed critical quality gate can QUARANTINE serving + * projection." + * + * The word that matters is on the READ side. A quarantine that annotates a + * result is a label; a quarantine that REFUSES the read is a quarantine. This + * module does the second, and `assertProjectionReadable` is the single place + * every serving read passes through. + * + * Severity semantics, stated because they are the whole gate: + * `block` — a failure quarantines the projection. Reads are refused. + * `warn` — a failure is recorded and surfaced, and reads continue. + * A rule with no severity does not exist: `QualityRuleSpec.severity` is + * required, so "we forgot to mark it critical" cannot silently become `warn`. + */ +import { SiaxError } from '../../platform/errors'; +import type { QualityVerdict } from './quality-provider'; + +export interface QuarantineRecord { + tenant_id: string; + dataset_id: string; + quarantined: boolean; + /** Rule ids whose blocking failure caused it. */ + caused_by: string[]; + since?: string; + released_at?: string; + released_by?: string; + /** Operator-facing summary. */ + reason?: string; +} + +export interface QuarantineStore { + get(tenantId: string, datasetId: string): Promise; + put(record: QuarantineRecord): Promise; +} + +export class InMemoryQuarantineStore implements QuarantineStore { + private readonly rows = new Map(); + private key(t: string, d: string): string { + return `${t}::${d}`; + } + async get(t: string, d: string): Promise { + return this.rows.get(this.key(t, d)); + } + async put(r: QuarantineRecord): Promise { + this.rows.set(this.key(r.tenant_id, r.dataset_id), r); + } +} + +export interface GateOutcome { + dataset_id: string; + tenant_id: string; + passed: boolean; + quarantined: boolean; + blocking_failures: QualityVerdict[]; + warnings: QualityVerdict[]; + evaluated: number; +} + +export class QualityGate { + constructor( + private readonly store: QuarantineStore, + private readonly now: () => number = Date.now, + ) {} + + /** + * Apply a run's verdicts. + * + * A gate over ZERO verdicts does not pass. "No rules ran" and "every rule + * passed" produce the same green in most implementations, and that is how a + * quality gate stops gating without anyone noticing — the same shape as + * l0re's 28 silently-skipped tests and api0's `trivy --exit-code 0`. + */ + async apply(tenantId: string, datasetId: string, verdicts: QualityVerdict[]): Promise { + if (verdicts.length === 0) { + const record: QuarantineRecord = { + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: true, + caused_by: [], + since: new Date(this.now()).toISOString(), + reason: 'no quality rules were evaluated — an ungated dataset is quarantined, not trusted', + }; + await this.store.put(record); + return { + dataset_id: datasetId, + tenant_id: tenantId, + passed: false, + quarantined: true, + blocking_failures: [], + warnings: [], + evaluated: 0, + }; + } + + for (const v of verdicts) { + if (v.tenant_id !== tenantId || v.dataset_id !== datasetId) { + // A verdict from another tenant's run must never influence this gate. + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.quality', + 'a quality verdict from a different tenant or dataset was supplied to this gate', + ); + } + } + + const blocking = verdicts.filter((v) => !v.passed && v.severity === 'block'); + const warnings = verdicts.filter((v) => !v.passed && v.severity === 'warn'); + + if (blocking.length > 0) { + await this.store.put({ + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: true, + caused_by: blocking.map((v) => v.rule_id), + since: new Date(this.now()).toISOString(), + reason: `${blocking.length} blocking quality rule(s) failed`, + }); + } else { + // Passing does NOT auto-release: releasing is an act with an operator + // behind it (`release()`), because a flapping rule would otherwise let a + // dataset quarantine and un-quarantine itself between reads. + const existing = await this.store.get(tenantId, datasetId); + if (!existing || !existing.quarantined) { + await this.store.put({ + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: false, + caused_by: [], + }); + } + } + + return { + dataset_id: datasetId, + tenant_id: tenantId, + passed: blocking.length === 0, + quarantined: blocking.length > 0 || (await this.isQuarantined(tenantId, datasetId)), + blocking_failures: blocking, + warnings, + evaluated: verdicts.length, + }; + } + + async isQuarantined(tenantId: string, datasetId: string): Promise { + return (await this.store.get(tenantId, datasetId))?.quarantined ?? false; + } + + /** Release requires an operator and a reason. It is not automatic. */ + async release(tenantId: string, datasetId: string, by: string, reason: string): Promise { + if (!by?.trim() || !reason?.trim()) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'releasing a quarantine requires a principal and a reason', + ); + } + const record: QuarantineRecord = { + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: false, + caused_by: [], + released_at: new Date(this.now()).toISOString(), + released_by: by, + reason, + }; + await this.store.put(record); + return record; + } + + /** + * The read-side gate. Every serving projection read passes through here. + * Refuses, rather than annotating — an annotation on a dashboard nobody + * reads is not a quarantine. + */ + async assertProjectionReadable(tenantId: string, datasetId: string): Promise { + const record = await this.store.get(tenantId, datasetId); + if (record?.quarantined) { + throw new SiaxError( + 'DEPENDENCY_UNAVAILABLE', + '0pt.analytics.quality', + `serving projection for "${datasetId}" is QUARANTINED: ${record.reason ?? 'failed quality gate'}`, + ); + } + } +} diff --git a/src/analytics/sql/analyst-sql-session.ts b/src/analytics/sql/analyst-sql-session.ts new file mode 100644 index 0000000..6489837 --- /dev/null +++ b/src/analytics/sql/analyst-sql-session.ts @@ -0,0 +1,376 @@ +/** + * src/analytics/sql/analyst-sql-session.ts + * ========================================= + * BI-P017 — ANALYST SQL CONTRACT. + * + * "For privileged analyst role only. Implement separate AnalyticsSQLSession + * using Trino/read-only endpoints. Security: role-gated; query timeout; + * row/byte limits; read only; tenant/project catalogs; audit; statement + * history." + * + * SEPARATE is the operative word, and it is why this is its own class rather + * than a method on `SiaxAnalyticsEngine`. `executeMetric` has no raw-SQL + * surface at all; if analyst SQL lived on the same object, the closed DSL + * would be one refactor away from growing one. `ClickHouseAnalyticsEngine + * .executeQuery` stays a documented refusal. + * + * The four layers are described in statement-guard.ts. This class owns layers + * 1 (role) and the enforcement of every named limit; the guard is layer 2; + * tenant-catalog.ts renders layers 3 and 4. + * + * AUDIT AND STATEMENT HISTORY — the deliberate split, because BI-P017 requires + * statement history and BI-P018 requires NOT storing sensitive raw SQL: + * + * * the BI-P018 execution record stores a HASH of the compiled statement + * and nothing else; + * * the analyst statement history stores the NORMALIZED statement (every + * literal replaced by `?`) always — that is the searchable, non-PII form; + * * the RAW statement is retained only when `retainRawStatements` is on, and + * the entry is classified so a retention job can find it. It is on by + * default for analyst sessions, because a privileged human executing + * unreviewed SQL against tenant data is exactly the case where "who ran + * what" must survive, and a masked statement cannot answer it. + * + * A caller who needs the non-PII form gets `normalized`; nothing forces raw + * SQL into a downstream sink. + */ +import crypto from 'node:crypto'; +import { SiaxError } from '../../platform/errors'; +import type { AnalyticsIdentity } from '../engine/siax-analytics-engine'; +import type { ResourceBudget } from '../engine/resource-governance'; +import { applyQuota, BUDGET_TIERS, DEFAULT_BUDGETS } from '../engine/resource-governance'; +import { guardAnalystStatement, type GuardedStatement } from './statement-guard'; + +/** The CL0UD role BI-P017 gates on. Verified upstream; never self-asserted. */ +export const ANALYST_SQL_ROLE = 'analytics.sql.analyst'; + +/** + * An identity carrying CL0UD-verified roles. Roles are absent by default and + * absence is a denial — a session must never open because a field was missing. + */ +export interface AnalystIdentity extends AnalyticsIdentity { + roles?: readonly string[]; +} + +export interface AnalystStatementRecord { + statement_id: string; + tenant_id: string; + project_id?: string; + principal_id: string; + /** Literals masked. Always present. */ + normalized: string; + /** Present only when the session retains raw statements. */ + raw?: string; + statement_hash: string; + referenced_objects: string[]; + submitted_at: string; + finished_at?: string; + duration_ms?: number; + rows_returned?: number; + bytes_read?: number; + rows_read?: number; + outcome: 'admitted' | 'refused' | 'succeeded' | 'failed' | 'cancelled'; + /** SiaxErrorCode on refusal/failure. Never an engine body. */ + error_code?: string; + /** Short operator-facing reason. Never echoes the statement. */ + refusal_reason?: string; + correlation_id?: string; +} + +/** Where the history goes. AUD0 is the system of record; this is the port. */ +export interface AnalystAuditSink { + record(entry: AnalystStatementRecord): Promise | void; +} + +/** The read-only endpoint the session executes against. */ +export interface ReadOnlySqlEndpoint { + /** "clickhouse" | "trino" — recorded on every execution. */ + readonly engine: string; + /** + * Execute an ALREADY-GUARDED statement as the tenant's restricted user. + * The implementation must not add privileges, and must apply `settings`. + */ + execute(args: { + sql: string; + queryId: string; + settings: Record; + signal: AbortSignal; + }): Promise; + /** Stop a running statement server-side. Returns true when it is gone. */ + cancel(queryId: string): Promise; +} + +export interface SqlEndpointResult { + columns: string[]; + rows: unknown[][]; + rows_read?: number; + bytes_read?: number; +} + +export interface AnalystSqlSessionOptions { + identity: AnalystIdentity; + endpoint: ReadOnlySqlEndpoint; + /** The tenant's own catalog of pre-filtered views (BI-P017 layer 4). */ + catalog: string; + /** Objects inside it the analyst may read. */ + objects: readonly string[]; + audit: AnalystAuditSink; + /** + * Budget tier. ANALYTICAL is the default for a human analyst; the tier is + * still only ever TIGHTENED by a CL0UD quota, never loosened (BI-P012). + */ + tier?: (typeof BUDGET_TIERS)[number]; + retainRawStatements?: boolean; + /** Injected for deterministic tests. */ + now?: () => number; +} + +export interface AnalystQueryOutcome { + statement_id: string; + query_id: string; + columns: string[]; + rows: unknown[][]; + duration_ms: number; + rows_read?: number; + bytes_read?: number; + /** Objects the statement was permitted to read, as resolved by the guard. */ + referenced_objects: string[]; + truncated: boolean; +} + +export class AnalyticsSQLSession { + private readonly identity: AnalystIdentity; + private readonly endpoint: ReadOnlySqlEndpoint; + private readonly catalog: string; + private readonly objects: readonly string[]; + private readonly audit: AnalystAuditSink; + private readonly budget: ResourceBudget; + private readonly retainRaw: boolean; + private readonly now: () => number; + private readonly running = new Map(); + private closed = false; + + /** + * Opening a session is itself the role gate. A caller without the analyst + * role never obtains an object that can run SQL — as opposed to obtaining + * one whose methods happen to check a flag. + */ + constructor(opts: AnalystSqlSessionOptions) { + const id = opts.identity; + if (!id?.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.sql', + 'analyst SQL requires a verified tenant identity', + ); + } + if (!id.principal_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.sql', + 'analyst SQL requires a verified principal', + ); + } + const roles = id.roles ?? []; + if (!roles.includes(ANALYST_SQL_ROLE)) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.sql', + `analyst SQL requires the ${ANALYST_SQL_ROLE} role`, + ); + } + if (opts.objects.length === 0) { + throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', 'session exposes no readable objects'); + } + this.identity = id; + this.endpoint = opts.endpoint; + this.catalog = opts.catalog; + this.objects = opts.objects; + this.audit = opts.audit; + this.budget = applyQuota(DEFAULT_BUDGETS[opts.tier ?? 'ANALYTICAL'], id.quota); + this.retainRaw = opts.retainRawStatements ?? true; + this.now = opts.now ?? Date.now; + } + + get resourceBudget(): ResourceBudget { + return { ...this.budget }; + } + + /** + * Guard a statement WITHOUT running it. Exposed so a UI can validate as the + * analyst types, and so the adversarial test corpus has a pure entry point. + */ + guard(sql: string): GuardedStatement { + return guardAnalystStatement(sql, { allowedCatalog: this.catalog, allowedObjects: this.objects }); + } + + async query(sql: string): Promise { + if (this.closed) { + throw new SiaxError('CONFLICT', '0pt.analytics.sql', 'session is closed'); + } + const statementId = crypto.randomUUID(); + const submittedAt = new Date(this.now()).toISOString(); + const base = { + statement_id: statementId, + tenant_id: this.identity.tenant_id, + project_id: this.identity.project_id, + principal_id: this.identity.principal_id, + correlation_id: this.identity.correlation_id, + submitted_at: submittedAt, + }; + + let guarded: GuardedStatement; + try { + guarded = this.guard(sql); + } catch (e) { + const err = e as SiaxError; + // A REFUSAL is audited as loudly as an execution. A guard whose + // rejections are invisible cannot be reviewed, and an analyst probing + // the boundary is exactly what an operator needs to see. + await this.audit.record({ + ...base, + normalized: safeNormalize(sql), + statement_hash: hashStatement(sql), + referenced_objects: [], + outcome: 'refused', + error_code: err.code ?? 'PERMANENT_FAILURE', + // `message` is already the sanitised "[CODE] source: detail" form; it + // never contains the caller's statement, only the guard's reason. + refusal_reason: err.message ?? 'refused', + }); + throw e; + } + + const queryId = this.newQueryId(); + const controller = new AbortController(); + this.running.set(queryId, controller); + const startedMs = this.now(); + + const record: AnalystStatementRecord = { + ...base, + normalized: guarded.normalized, + ...(this.retainRaw ? { raw: guarded.raw } : {}), + statement_hash: hashStatement(guarded.raw), + referenced_objects: guarded.referencedObjects, + outcome: 'admitted', + }; + + try { + const result = await this.endpoint.execute({ + // The statement is passed THROUGH unchanged. This class does not + // rewrite SQL: a rewriter must be a correct parser of the dialect to + // be safe, and layers 3+4 do not need it to be one. + sql: guarded.raw, + queryId, + settings: this.executionSettings(), + signal: controller.signal, + }); + const duration = this.now() - startedMs; + const truncated = result.rows.length >= this.budget.maxResultRows; + await this.audit.record({ + ...record, + outcome: 'succeeded', + finished_at: new Date(this.now()).toISOString(), + duration_ms: duration, + rows_returned: result.rows.length, + rows_read: result.rows_read, + bytes_read: result.bytes_read, + }); + return { + statement_id: statementId, + query_id: queryId, + columns: result.columns, + rows: result.rows, + duration_ms: duration, + rows_read: result.rows_read, + bytes_read: result.bytes_read, + referenced_objects: guarded.referencedObjects, + truncated, + }; + } catch (e) { + const err = e as SiaxError; + await this.audit.record({ + ...record, + outcome: controller.signal.aborted ? 'cancelled' : 'failed', + finished_at: new Date(this.now()).toISOString(), + duration_ms: this.now() - startedMs, + error_code: err.code ?? 'PERMANENT_FAILURE', + }); + throw e; + } finally { + this.running.delete(queryId); + } + } + + /** Cancel a statement this session started. Never another session's. */ + async cancel(queryId: string): Promise { + if (!queryId.startsWith(this.queryIdPrefix())) { + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.sql', + 'statement does not belong to this session', + ); + } + this.running.get(queryId)?.abort(); + return this.endpoint.cancel(queryId); + } + + /** Abort everything in flight. Idempotent. */ + async close(): Promise { + this.closed = true; + const ids = [...this.running.keys()]; + for (const id of ids) { + this.running.get(id)?.abort(); + try { + await this.endpoint.cancel(id); + } catch { + /* best effort; the abort already stopped us waiting */ + } + this.running.delete(id); + } + } + + /** + * Every limit BI-P017 names, as engine settings. `readonly` is `1`, not `2`: + * `readonly = 2` permits SETTINGS changes, which would let a statement raise + * its own ceilings. + */ + private executionSettings(): Record { + return { + readonly: '1', + max_execution_time: String(this.budget.maxExecutionTimeSec), + timeout_overflow_mode: 'throw', + max_result_rows: String(this.budget.maxResultRows), + result_overflow_mode: 'throw', + max_rows_to_read: String(this.budget.maxRowsToRead), + max_bytes_to_read: String(this.budget.maxBytesToRead), + read_overflow_mode: 'throw', + max_memory_usage: String(this.budget.maxMemoryUsageBytes), + }; + } + + private queryIdPrefix(): string { + return `sql_${crypto.createHash('sha256').update(this.identity.tenant_id).digest('hex').slice(0, 12)}_`; + } + + private newQueryId(): string { + return `${this.queryIdPrefix()}${crypto.randomUUID().replace(/-/g, '')}`; + } +} + +export function hashStatement(sql: string): string { + return crypto.createHash('sha256').update(sql.trim()).digest('hex'); +} + +/** + * Mask literals in a statement that FAILED the guard, so a refusal can be + * audited without copying whatever the caller pasted. Best-effort by design: + * the statement did not parse cleanly, so neither will this. + */ +function safeNormalize(sql: string): string { + return sql + .replace(/'(?:[^'\\]|\\.|'')*'/g, '?') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 2000); +} diff --git a/src/analytics/sql/statement-guard.ts b/src/analytics/sql/statement-guard.ts new file mode 100644 index 0000000..c9ba31c --- /dev/null +++ b/src/analytics/sql/statement-guard.ts @@ -0,0 +1,516 @@ +/** + * src/analytics/sql/statement-guard.ts + * ===================================== + * BI-P017 — ANALYST SQL CONTRACT, layer 2 (statement admission). + * + * READ THIS BEFORE TRUSTING IT. + * + * This module is a GUARD, not a proof. It is the second of four layers, and + * it is the WEAKEST of the four, because it reasons about SQL text without + * being ClickHouse's parser. A guard that reads a dialect it does not + * implement can always be wrong about some string. The layers below it are + * the ones that hold when it is: + * + * 1. ROLE — only a CL0UD-verified analyst role reaches a session + * at all (analyst-sql-session.ts). + * 2. THIS GUARD — statement admission: one read-only statement, no + * banned construct, every table reference resolved + * into the tenant's own catalog. + * 3. GRANTS — the session authenticates as a per-tenant ClickHouse + * user with `readonly = 1` and SELECT granted ONLY on + * that tenant's catalog. A statement this guard wrongly + * admits still cannot read a table the user cannot see. + * 4. VIEW PREDICATE — the tenant catalog contains VIEWS, never base tables, + * and each view bakes `WHERE tenant_id = ''` + * into its own definition. A row policy on the base + * tables repeats it server-side. + * + * tests/integration/analyst-sql-clickhouse.integration.test.ts runs the + * adversarial corpus against a live ClickHouse with layer 2 REMOVED, and + * requires layers 3 and 4 to refuse it anyway. A defence-in-depth claim that + * has never been tested with a layer removed is not evidence — this estate + * has shipped several. + * + * DESIGN NOTES that are load-bearing: + * + * * Identifiers must resolve to an ALLOW-LIST. The deny-lists below are a + * second opinion and a better error message, never the mechanism. Every + * table reference is checked positively against the tenant's catalog. + * * Outside string literals the statement must be ASCII. Homoglyph and + * zero-width tricks against keyword matching stop being expressible. + * * The FROM/JOIN scan is CONSERVATIVE where it is unsure, and that costs + * something worth naming: ClickHouse's `ARRAY JOIN arr` is a legitimate + * construct whose target is a COLUMN, not a table, and this guard refuses + * it because the token after JOIN is neither a CTE nor a qualified object. + * A false refusal an analyst can see and report is the right side of that + * trade; a false ADMISSION is not. Supporting it means teaching the scan + * which JOIN flavours take a non-table target, which is the direction that + * grows holes, so it stays refused until someone needs it and can add it + * WITH a corpus entry proving the narrowing did not open anything else. + * * Comments and string literals are removed BEFORE any keyword scan, using + * ClickHouse's own lexical rules (`--` and `#` to end of line, `/* *\/` + * non-nesting, `'` with backslash and doubled-quote escapes, `"` and + * backtick as identifier quotes). + */ +import { SiaxError } from '../../platform/errors'; + +export interface StatementGuardOptions { + /** + * The ONLY database name a table reference may carry. This is the tenant's + * own catalog of pre-filtered views, not the shared analytics database. + */ + allowedCatalog: string; + /** Object names inside that catalog the analyst may read. */ + allowedObjects: readonly string[]; + /** Hard ceiling on statement length; a megabyte of SQL is an attack, not a query. */ + maxLength?: number; +} + +export interface GuardedStatement { + /** The statement as submitted, unchanged. */ + raw: string; + /** Comments and literals removed — what the keyword scan actually saw. */ + stripped: string; + /** Literals replaced by `?`, for statement history that is not a PII dump. */ + normalized: string; + /** Fully-qualified objects the statement reads, resolved against the catalog. */ + referencedObjects: string[]; + /** Common-table-expression names defined by the statement itself. */ + cteNames: string[]; +} + +const DEFAULT_MAX_LENGTH = 20_000; + +/** + * Statement-level constructs that make a statement not-a-read. Matched as + * whole words against the stripped text. + * + * `SETTINGS` is here because a trailing `SETTINGS max_rows_to_read = …` is the + * obvious way to try to lift the governance ceiling from inside the query. + * `FORMAT` is here because the session appends its own; a caller-supplied one + * is either a duplicate or an attempt at `FORMAT ...` side effects. + */ +const BANNED_KEYWORDS = [ + 'insert', + 'update', + 'delete', + 'drop', + 'alter', + 'create', + 'truncate', + 'rename', + 'attach', + 'detach', + 'replace', + 'optimize', + 'grant', + 'revoke', + 'kill', + 'system', + 'use', + 'set', + 'settings', + 'exchange', + 'move', + 'undrop', + 'backup', + 'restore', + 'outfile', + 'infile', + 'format', + 'watch', + 'check', + 'explain', + 'show', + 'exists', + 'call', + 'execute', + 'declare', + 'begin', + 'commit', + 'rollback', +] as const; + +/** + * Functions that read something other than a table the grants control: + * the filesystem, the network, another server, a dictionary, an executable. + * ClickHouse table functions are the whole reason an analyst SQL surface is + * dangerous — `SELECT * FROM url('http://…')` needs no table privilege at all. + */ +const BANNED_FUNCTIONS = [ + 'file', + 'url', + 'urlcluster', + 's3', + 's3cluster', + 'gcs', + 'azureblobstorage', + 'remote', + 'remotesecure', + 'cluster', + 'clusterallreplicas', + 'hdfs', + 'hdfscluster', + 'mysql', + 'postgresql', + 'mongodb', + 'jdbc', + 'odbc', + 'sqlite', + 'redis', + 'deltalake', + 'hudi', + 'iceberg', + 'executable', + 'input', + 'generaterandom', + 'numbers_mt', + 'merge', + 'view', + 'viewifpermitted', + 'loop', + 'fuzzjson', + 'fuzzquery', + 'dictget', + 'dictgetordefault', + 'dictgetornull', + 'dictgetstring', + 'dictgetuint64', + 'dictgethierarchy', + 'dictisin', + 'dicthas', + 'dictgetchildren', + 'dictgetdescendants', + 'filesystemavailable', + 'filesystemfree', + 'filesystemcapacity', + 'currentdatabase', + 'currentuser', + 'hostname', + 'fqdn', + 'uptime', + 'version', + 'getsetting', + 'getmacro', + 'shardnum', + 'shardcount', + 'buildid', + 'servertimezone', + 'readwkt', + 'structurefromjson', + 's3queue', + 'timeseries', + 'mergetreeindex', +] as const; + +/** + * Constructs that a whole-word keyword ban gets WRONG, so they get a pattern. + * + * `DESC` is the case that proves the point: banning the word breaks every + * `ORDER BY x DESC` an analyst will ever write, and the test corpus caught it + * immediately. What is actually dangerous is `DESCRIBE`/`DESC` used as a + * STATEMENT — including inside a subquery, where ClickHouse accepts + * `SELECT * FROM (DESCRIBE TABLE x)` and the FROM-scan sees only a `(`. + */ +const BANNED_PATTERNS: Array<{ re: RegExp; reason: string }> = [ + { re: /\bdesc(ribe)?\s+(table|\()/i, reason: 'DESCRIBE is not permitted in a read-only analyst session' }, + { re: /\bcurrent_user\b/i, reason: 'CURRENT_USER is not permitted in an analyst session' }, + { re: /\binto\s+outfile\b/i, reason: 'INTO OUTFILE is not permitted in an analyst session' }, +]; + +/** Names that are catalogs an analyst may never reach, whatever the grants say. */ +const BANNED_CATALOGS = [ + 'system', + 'information_schema', + 'INFORMATION_SCHEMA', + 'default', + 'opt_analytics', + 'opt_scraped', +]; + +const IDENT = '[A-Za-z_][A-Za-z0-9_]*'; +const QUALIFIED_RE = new RegExp(`^${IDENT}\\.${IDENT}$`); + +function deny(detail: string): never { + throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', detail); +} + +function invalid(detail: string): never { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', detail); +} + +interface StripResult { + stripped: string; + normalized: string; + /** Identifier tokens that were quoted, kept so they can still be resolved. */ + hadQuotedIdentifier: boolean; +} + +/** + * Remove comments and string literals using ClickHouse's lexical rules. + * + * Two outputs, deliberately: `stripped` keeps the literal DELIMITERS so the + * token stream still parses (`'x'` becomes `''`), and `normalized` replaces + * each literal with `?` so a statement history is searchable without being a + * copy of the tenant's data. + * + * An unterminated literal or comment is a REJECTION. Silently treating the + * rest of the statement as a string is exactly how a guard and an engine come + * to disagree about where the query ends. + */ +export function stripSqlText(sql: string): StripResult { + let stripped = ''; + let normalized = ''; + let hadQuotedIdentifier = false; + let i = 0; + const n = sql.length; + + while (i < n) { + const c = sql[i]; + const next = sql[i + 1]; + + // -- line comment + if (c === '-' && next === '-') { + while (i < n && sql[i] !== '\n') i += 1; + stripped += ' '; + normalized += ' '; + continue; + } + // # line comment (ClickHouse accepts `#` and `#!`) + if (c === '#') { + while (i < n && sql[i] !== '\n') i += 1; + stripped += ' '; + normalized += ' '; + continue; + } + // /* block comment */ — ClickHouse does NOT nest these + if (c === '/' && next === '*') { + const end = sql.indexOf('*/', i + 2); + if (end === -1) invalid('unterminated block comment'); + i = end + 2; + stripped += ' '; + normalized += ' '; + continue; + } + // 'string literal' + if (c === "'") { + i += 1; + while (i < n) { + if (sql[i] === '\\') { + i += 2; + continue; + } + if (sql[i] === "'") { + if (sql[i + 1] === "'") { + i += 2; + continue; + } + break; + } + i += 1; + } + if (i >= n) invalid('unterminated string literal'); + i += 1; // closing quote + stripped += "''"; + normalized += '?'; + continue; + } + // "quoted identifier" / `quoted identifier` + if (c === '"' || c === '`') { + const quote = c; + let ident = ''; + i += 1; + while (i < n) { + if (sql[i] === '\\') { + ident += sql[i + 1] ?? ''; + i += 2; + continue; + } + if (sql[i] === quote) { + if (sql[i + 1] === quote) { + ident += quote; + i += 2; + continue; + } + break; + } + ident += sql[i]; + i += 1; + } + if (i >= n) invalid('unterminated quoted identifier'); + i += 1; + hadQuotedIdentifier = true; + // The identifier is put back UNQUOTED so the table-reference scan sees + // it exactly as it would see a bare one. A quoted identifier that does + // not survive that scan is refused like any other. + if (!new RegExp(`^${IDENT}$`).test(ident)) { + deny(`quoted identifier "${ident.slice(0, 32)}" is not a plain name`); + } + stripped += ident; + normalized += ident; + continue; + } + stripped += c; + normalized += c; + i += 1; + } + return { stripped, normalized, hadQuotedIdentifier }; +} + +/** + * Admit a statement, or refuse it. + * + * Returns the parsed view of what was admitted; the caller (the session) is + * responsible for actually running it under the restricted user. + */ +export function guardAnalystStatement(sql: string, opts: StatementGuardOptions): GuardedStatement { + const maxLength = opts.maxLength ?? DEFAULT_MAX_LENGTH; + if (typeof sql !== 'string') invalid('statement must be a string'); + const raw = sql.trim(); + if (!raw) invalid('statement is empty'); + if (raw.length > maxLength) invalid(`statement exceeds ${maxLength} characters`); + if (!opts.allowedCatalog || !new RegExp(`^${IDENT}$`).test(opts.allowedCatalog)) { + // A misconfigured session must not degrade to "any catalog". + deny('session has no valid tenant catalog'); + } + if (BANNED_CATALOGS.includes(opts.allowedCatalog)) { + deny(`session catalog "${opts.allowedCatalog}" is a reserved catalog`); + } + if (opts.allowedObjects.length === 0) { + deny('session exposes no readable objects'); + } + + const { stripped, normalized } = stripSqlText(raw); + + // ── ASCII-only outside literals ──────────────────────────────────────── + // Homoglyphs, zero-width joiners and RTL overrides stop being expressible + // against the keyword scan. Values are unaffected: they live in literals, + // which have already been removed at this point. + // Written as a code-point scan rather than a character class: a regex of + // control characters is unreadable, trips `no-control-regex`, and is the + // kind of expression a reviewer nods at without checking. Permitted: tab, + // LF, CR and printable ASCII. Everything else, NUL included, is refused. + for (let i = 0; i < stripped.length; i += 1) { + const code = stripped.charCodeAt(i); + const printable = code >= 0x20 && code <= 0x7e; + const allowedWhitespace = code === 0x09 || code === 0x0a || code === 0x0d; + if (!printable && !allowedWhitespace) { + deny( + `statement contains a disallowed character (U+${code.toString(16).toUpperCase().padStart(4, '0')}) ` + + `outside a string literal`, + ); + } + } + + // ── exactly one statement ────────────────────────────────────────────── + const withoutTrailingSemi = stripped.replace(/;\s*$/, ''); + if (withoutTrailingSemi.includes(';')) { + deny('multiple statements are not allowed in an analyst SQL session'); + } + + const lower = ` ${withoutTrailingSemi.toLowerCase().replace(/\s+/g, ' ').trim()} `; + + // ── must be a read ───────────────────────────────────────────────────── + if (!/^\s*(select|with)\b/i.test(withoutTrailingSemi.trim())) { + deny('analyst SQL sessions accept SELECT and WITH statements only'); + } + + for (const kw of BANNED_KEYWORDS) { + if (new RegExp(`(^|[^A-Za-z0-9_])${kw}([^A-Za-z0-9_]|$)`).test(lower)) { + deny(`"${kw.toUpperCase()}" is not permitted in a read-only analyst session`); + } + } + + for (const { re, reason } of BANNED_PATTERNS) { + if (re.test(lower)) deny(reason); + } + + // ── banned functions, matched as `name(` ─────────────────────────────── + const calls = [...lower.matchAll(new RegExp(`([a-z_][a-z0-9_]*)\\s*\\(`, 'g'))].map((m) => m[1]); + for (const call of calls) { + if ((BANNED_FUNCTIONS as readonly string[]).includes(call)) { + deny(`function "${call}" is not permitted in an analyst session`); + } + } + + // ── table references: positively resolved against the catalog ────────── + const cteNames = collectCteNames(withoutTrailingSemi); + const referencedObjects = collectTableReferences(withoutTrailingSemi, cteNames, opts); + + if (referencedObjects.length === 0) { + // A SELECT with no FROM (`SELECT 1`) is harmless, but so is refusing it, + // and "no table reference" is also what a successful parser evasion looks + // like. Refuse: an analyst session exists to read tables. + deny('statement references no readable object in the tenant catalog'); + } + + return { raw, stripped, normalized, referencedObjects, cteNames }; +} + +/** `WITH x AS (…), y AS (…)` — names the statement defines for itself. */ +function collectCteNames(stripped: string): string[] { + const names: string[] = []; + const re = new RegExp(`(?:^|[,\\s(])(${IDENT})\\s+as\\s*\\(`, 'gi'); + for (const m of stripped.matchAll(re)) names.push(m[1].toLowerCase()); + return names; +} + +/** + * Every `FROM` / `JOIN` target, resolved. + * + * Rules, all fail-closed: + * * `(` after FROM/JOIN is a subquery — fine, its own FROM is scanned too. + * * `name(` after FROM/JOIN is a TABLE FUNCTION — refused outright, even if + * the name is not on the deny-list, because an unknown table function is + * precisely the case the deny-list cannot cover. + * * a bare `name` must be a CTE this statement defined. + * * a `db.name` must have `db === allowedCatalog` and `name` in the + * allow-list. Three-part `a.b.c` names are refused. + */ +function collectTableReferences(stripped: string, cteNames: string[], opts: StatementGuardOptions): string[] { + const allowed = new Set(opts.allowedObjects.map((o) => o.toLowerCase())); + const ctes = new Set(cteNames); + const found = new Set(); + + const re = /(^|[^A-Za-z0-9_])(from|join)([^A-Za-z0-9_]|$)/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(stripped)) !== null) { + const after = stripped.slice(m.index + m[0].length - 1); + const target = /^\s*([A-Za-z_][A-Za-z0-9_.]*|\()/.exec(after); + if (!target) deny('could not resolve a FROM/JOIN target'); + const token = target[1]; + if (token === '(') continue; // subquery; its own FROM is scanned by this loop + + const rest = after.slice(target[0].length); + if (/^\s*\(/.test(rest)) { + deny(`table function "${token}" is not permitted in an analyst session`); + } + + const parts = token.split('.'); + if (parts.length === 1) { + if (!ctes.has(parts[0].toLowerCase())) { + deny( + `unqualified table "${parts[0]}" — an analyst statement must name ` + + `${opts.allowedCatalog}., or a CTE it defined itself`, + ); + } + continue; + } + if (parts.length !== 2 || !QUALIFIED_RE.test(token)) { + deny(`"${token}" is not a valid . reference`); + } + const [catalog, object] = parts; + if (catalog.toLowerCase() !== opts.allowedCatalog.toLowerCase()) { + deny(`catalog "${catalog}" is not readable by this session`); + } + if (!allowed.has(object.toLowerCase())) { + deny(`object "${catalog}.${object}" is not readable by this session`); + } + found.add(`${catalog}.${object}`); + } + return [...found]; +} + +/** Exported for the test corpus and for operator documentation. */ +export const GUARD_BANNED_KEYWORDS = BANNED_KEYWORDS; +export const GUARD_BANNED_FUNCTIONS = BANNED_FUNCTIONS; diff --git a/src/analytics/sql/tenant-catalog.ts b/src/analytics/sql/tenant-catalog.ts new file mode 100644 index 0000000..c10fa6a --- /dev/null +++ b/src/analytics/sql/tenant-catalog.ts @@ -0,0 +1,238 @@ +/** + * src/analytics/sql/tenant-catalog.ts + * ==================================== + * BI-P017 — ANALYST SQL CONTRACT, layers 3 and 4. + * + * The analyst never touches a base table. Each tenant gets its OWN ClickHouse + * database containing nothing but VIEWS, and every view has the BI-P011 forced + * predicate baked into its own definition: + * + * CREATE VIEW t_alpha_sql.opt_analytics_event AS + * SELECT … FROM opt_analytics.opt_analytics_event + * WHERE tenant_id = 't_alpha' + * + * That is the difference between "the guard rewrites your SQL to add a WHERE" + * (fragile — it depends on the guard being a correct SQL parser) and "there is + * no unfiltered object in reach" (structural — it depends only on the grants). + * + * The session's ClickHouse user is created with `readonly = 1` and is granted + * SELECT on that database ALONE. `GRANT SELECT ON opt_analytics.*` is never + * issued, so a statement the guard wrongly admits still cannot read the base + * table: ClickHouse answers "Not enough privileges", not rows. + * + * The `tenant_id` column is deliberately DROPPED from the view projection. + * An analyst cannot filter on, group by, or select a column that does not + * exist in the object they can see — which removes the whole class of + * "predicate on tenant_id looks legitimate" confusion from the audit trail. + */ +import { SiaxError } from '../../platform/errors'; +import { assertSafeIdentifier } from '../engine/query-compiler'; +import type { ResourceBudget } from '../engine/resource-governance'; + +export interface TenantSqlCatalogSpec { + /** Shared analytics database holding the base tables. */ + sourceDatabase: string; + /** Verified tenant id. Embedded as a literal — validated strictly first. */ + tenantId: string; + /** The tenant's own catalog name, e.g. `t_alpha_sql`. */ + catalog: string; + /** Base tables to expose, with the columns each view projects. */ + objects: ReadonlyArray<{ table: string; columns: readonly string[] }>; +} + +export interface TenantSqlUserSpec extends TenantSqlCatalogSpec { + userName: string; + /** From a secret ref. Never a literal in code, never logged. */ + password: string; + /** + * BI-P012 ceilings. MEASURED CONSTRAINT, not a style choice: a ClickHouse + * user whose profile carries `readonly = 1` cannot have ANY setting changed + * from the request — the server answers + * + * Code: 164. Cannot modify 'max_rows_to_read' setting in readonly mode. + * + * (measured on 24.8.14.39; see evidence/bi-p017/). So the ceilings cannot be + * client-supplied per request the way `clickhouseSettings()` supplies them + * for the non-readonly engine user. They must be baked into the user's + * SETTINGS PROFILE at provisioning time. + * + * That is strictly BETTER: the ceiling no longer depends on the client + * remembering to send it, and the same `readonly = 1` that forbids raising + * it also forbids lowering it or turning it off. + */ + budget?: ResourceBudget; + /** Explicit profile name; defaults to `_profile`. */ + settingsProfile?: string; +} + +const TENANT_ID_RE = /^[A-Za-z0-9_-]{1,64}$/; + +function assertTenantId(tenantId: string): string { + if (!TENANT_ID_RE.test(tenantId)) { + // ClickHouse view definitions take no bound parameters, so validation is + // the only protection available here and it is applied strictly. + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', 'tenant id is not catalog-safe'); + } + return tenantId; +} + +/** + * DDL creating the tenant's read-only catalog of pre-filtered views. + * Idempotent; safe to re-run on every deploy. + */ +export function tenantSqlCatalogDdl(spec: TenantSqlCatalogSpec): string[] { + assertSafeIdentifier(spec.sourceDatabase, 'database'); + assertSafeIdentifier(spec.catalog, 'catalog'); + assertTenantId(spec.tenantId); + if (spec.catalog === spec.sourceDatabase) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.sql', + 'tenant catalog must not be the source database', + ); + } + const stmts: string[] = [`CREATE DATABASE IF NOT EXISTS ${spec.catalog}`]; + for (const obj of spec.objects) { + assertSafeIdentifier(obj.table, 'table'); + if (obj.columns.length === 0) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', `view ${obj.table} projects no columns`); + } + for (const col of obj.columns) { + assertSafeIdentifier(col, 'column'); + if (col === 'tenant_id') { + // Not an oversight to guard against — a deliberate refusal. Exposing + // tenant_id inside a single-tenant view invites a reader to believe + // filtering on it means something. + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.sql', + 'tenant_id must not be projected into a tenant SQL view', + ); + } + } + stmts.push( + `CREATE OR REPLACE VIEW ${spec.catalog}.${obj.table} AS ` + + `SELECT ${obj.columns.join(', ')} FROM ${spec.sourceDatabase}.${obj.table} ` + + `WHERE tenant_id = '${spec.tenantId}'`, + ); + } + return stmts; +} + +/** + * DDL for the session's ClickHouse user. + * + * Every line here is a layer-3 control: + * * `readonly = 1` — no writes AND no settings changes from inside a query, + * which is what stops `SETTINGS max_rows_to_read = …` lifting the ceiling. + * * `allow_ddl = 0` — belt to readonly's braces. + * * SELECT granted on the tenant catalog ONLY. The base database is never + * granted, so a guard bypass hits a privilege error. + * * A row policy on the BASE tables as well, so even a mistaken grant, or a + * view accidentally created without its predicate, still filters. + */ +export function tenantSqlUserDdl(spec: TenantSqlUserSpec): string[] { + assertSafeIdentifier(spec.userName, 'user'); + assertSafeIdentifier(spec.catalog, 'catalog'); + assertSafeIdentifier(spec.sourceDatabase, 'database'); + assertTenantId(spec.tenantId); + if (spec.password.includes("'") || spec.password.includes('\\')) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.sql', + 'password contains characters that cannot be embedded', + ); + } + if (spec.settingsProfile) assertSafeIdentifier(spec.settingsProfile, 'settings profile'); + + const profileName = spec.settingsProfile ?? `${spec.userName}_profile`; + assertSafeIdentifier(profileName, 'settings profile'); + + const stmts: string[] = []; + if (spec.budget) { + // Every BI-P012 ceiling, plus the overflow modes. `throw` is explicit + // because ClickHouse's default for some of these is `break`, which returns + // a PARTIAL result that looks complete — the analytics equivalent of a + // false green. + // `OR REPLACE`, for the same measured reason as the row policy below: + // `CREATE SETTINGS PROFILE IF NOT EXISTS` is a silent no-op when the + // profile already exists, so a CHANGED budget never lands. Found the hard + // way — a re-provisioning run kept a stale 60-row ceiling and starved + // every legitimate analyst statement in the integration suite, while the + // provisioning DDL reported success on every statement. + stmts.push( + `CREATE SETTINGS PROFILE OR REPLACE ${profileName} SETTINGS ` + + `max_execution_time = ${spec.budget.maxExecutionTimeSec}, ` + + `timeout_overflow_mode = 'throw', ` + + `max_result_rows = ${spec.budget.maxResultRows}, ` + + `result_overflow_mode = 'throw', ` + + `max_rows_to_read = ${spec.budget.maxRowsToRead}, ` + + `max_bytes_to_read = ${spec.budget.maxBytesToRead}, ` + + `read_overflow_mode = 'throw', ` + + `max_memory_usage = ${spec.budget.maxMemoryUsageBytes}, ` + + `readonly = 1`, + ); + } + const settings = spec.budget ? `SETTINGS PROFILE ${profileName}` : `SETTINGS readonly = 1, allow_ddl = 0`; + + stmts.push( + `CREATE USER IF NOT EXISTS ${spec.userName} IDENTIFIED WITH sha256_password BY '${spec.password}' ${settings}`, + ); + // Re-applied on every provisioning run so a rotated budget actually lands on + // an already-existing user; CREATE USER IF NOT EXISTS would silently skip it. + stmts.push(`ALTER USER ${spec.userName} ${settings}`); + stmts.push(`GRANT SELECT ON ${spec.catalog}.* TO ${spec.userName}`); + for (const obj of spec.objects) { + assertSafeIdentifier(obj.table, 'table'); + // A view reads its source with the DEFINER's rights in ClickHouse only for + // parameterised/materialised cases; for a plain view the reader still needs + // SELECT on the source. Grant it on the SOURCE TABLE and pin a row policy + // to it, so the grant can never be wider than the tenant. + stmts.push(`GRANT SELECT ON ${spec.sourceDatabase}.${obj.table} TO ${spec.userName}`); + // `OR REPLACE`, never `IF NOT EXISTS`. MEASURED FAILURE, found by a test + // that attempted the bypass: + // + // Dropping and re-creating the user empties the policy's apply-to list + // (ClickHouse removes the dropped user from it). On the next + // provisioning run `CREATE ROW POLICY IF NOT EXISTS` sees the policy + // already exists and does NOTHING — so the policy survives with + // `apply_to_list = []`, applying to nobody, while `system.row_policies` + // still shows the right `select_filter`. An operator reading that table + // sees a policy protecting `opt_analytics.sales` and is wrong: the + // restricted user read 100 rows of 100, both tenants. + // + // This is the "green check that does not gate" shape. `OR REPLACE` makes + // provisioning idempotent in the way that was intended, and + // `rowPolicyVerificationSql` below turns "the policy is applied" from an + // assumption into a query. + stmts.push( + `CREATE ROW POLICY OR REPLACE ${spec.userName}_${obj.table}_rp ON ${spec.sourceDatabase}.${obj.table} ` + + `AS restrictive FOR SELECT USING tenant_id = '${spec.tenantId}' TO ${spec.userName}`, + ); + } + return stmts; +} + +/** + * The query that PROVES the row policies landed, rather than assuming it. + * + * Returns one row per (table, policy) with `applied_to_user` — which must be + * 1 for every object. Run it after provisioning and fail the deploy if any + * row reads 0; a policy with an empty apply-to list is indistinguishable from + * a working one in `SHOW ROW POLICIES`. + */ +export function rowPolicyVerificationSql(spec: TenantSqlUserSpec): string { + assertSafeIdentifier(spec.userName, 'user'); + assertSafeIdentifier(spec.sourceDatabase, 'database'); + assertTenantId(spec.tenantId); + return ( + `SELECT table, short_name, has(apply_to_list, '${spec.userName}') AS applied_to_user, select_filter ` + + `FROM system.row_policies WHERE database = '${spec.sourceDatabase}' ` + + `AND short_name LIKE '${spec.userName}%' FORMAT JSON` + ); +} + +/** Object names the guard's allow-list is built from. */ +export function catalogObjectNames(spec: TenantSqlCatalogSpec): string[] { + return spec.objects.map((o) => o.table); +} diff --git a/src/analytics/transform/dbt-contract.ts b/src/analytics/transform/dbt-contract.ts new file mode 100644 index 0000000..76e5c5d --- /dev/null +++ b/src/analytics/transform/dbt-contract.ts @@ -0,0 +1,374 @@ +/** + * src/analytics/transform/dbt-contract.ts + * ======================================== + * BI-P022 — DBT CORE CANONICAL TRANSFORMATION. + * + * ── SCOPE, STATED FIRST ─────────────────────────────────────────────────── + * This module defines the CONTRACT. It does not run dbt, and nothing in this + * repo does. Execution belongs to N0D (n0d#38), which consumes this contract + * from the other side. There is deliberately no `DbtExecutor` implementation + * here — only the interface — so that "0pt can run dbt" cannot become true by + * accident, and so the two repos have one shared, versioned shape rather than + * two drifting ones. + * + * ── THE VERSION RULE BI-P022 STATES ─────────────────────────────────────── + * "Use stable dbt Core production line AFTER current license/security + * verification. Do NOT use alpha development engine in production unless + * separately certified." + * + * So the pin is recorded together with its certification STATE, and the state + * is `unverified` until someone does the review. `assertDbtRuntimeCertified` + * refuses a production run against an uncertified or alpha runtime — which + * makes the sentence above a gate rather than a paragraph. A version constant + * on its own would be exactly the "documented but unenforced" pattern. + * + * ── THE TIER MAPPING ────────────────────────────────────────────────────── + * BI-P022's DoD names raw → staging → curated → serving. BI-P003 (merged) + * already owns RAW/NORMALIZED/CURATED/SERVING, so the dbt layers map onto it + * rather than inventing a fifth vocabulary: + * + * sources/ -> RAW (declared, never modelled) + * staging/ -> NORMALIZED (rename, cast, deduplicate; no business logic) + * intermediate/ -> CURATED (joins, business logic, conformed entities) + * marts/ -> SERVING (what a dashboard or a metric reads) + * + * A model in the wrong directory for its tier is a rejection, not a warning: + * the tier is what BI-P003's read rule gates on, and a mart that is really a + * staging model is how RAW data reaches a dashboard. + */ +import { SiaxError } from '../../platform/errors'; +import type { AnalyticsTier } from '../domain/tiers'; + +/** dbt layers, in dependency order. */ +export const DBT_LAYERS = ['sources', 'staging', 'intermediate', 'marts'] as const; +export type DbtLayer = (typeof DBT_LAYERS)[number]; + +export const DBT_LAYER_TIERS: Record = { + sources: 'RAW', + staging: 'NORMALIZED', + intermediate: 'CURATED', + marts: 'SERVING', +}; + +/** Directories BI-P022 requires under `analytics/dbt/`. */ +export const REQUIRED_DBT_DIRECTORIES = [ + 'models/sources', + 'models/staging', + 'models/intermediate', + 'models/marts', + 'tests', + 'snapshots', +] as const; + +// ── runtime certification ────────────────────────────────────────────────── + +export type DbtReleaseLine = 'stable' | 'prerelease' | 'alpha'; + +export interface DbtRuntimePin { + /** e.g. "1.10.13". Exact, never a range. */ + version: string; + line: DbtReleaseLine; + /** Adapter and its pin, e.g. { name: "dbt-clickhouse", version: "1.9.2" }. */ + adapter: { name: string; version: string }; + /** Has a licence + supply-chain review been completed FOR THIS PIN? */ + licenseReviewed: boolean; + securityReviewed: boolean; + reviewedBy?: string; + reviewedAt?: string; + /** Required by BI-P022 to run an alpha line in production at all. */ + separateCertificationRef?: string; +} + +/** + * The pin this contract is written against. + * + * BOTH review flags are FALSE. No licence or supply-chain review of dbt Core + * has been performed in this repo, and none is claimed. The constant records + * an intended pin; `assertDbtRuntimeCertified` is what stops it being treated + * as an approval. Flipping these to true without doing the review is the + * failure this shape exists to make visible. + */ +export const DBT_RUNTIME_PIN: DbtRuntimePin = { + version: '1.10.13', + line: 'stable', + adapter: { name: 'dbt-clickhouse', version: '1.9.2' }, + licenseReviewed: false, + securityReviewed: false, +}; + +export type DbtRunEnvironment = 'development' | 'ci' | 'production'; + +/** BI-P022's version rule, as a gate. */ +export function assertDbtRuntimeCertified(pin: DbtRuntimePin, environment: DbtRunEnvironment): void { + if (!/^\d+\.\d+\.\d+$/.test(pin.version)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + 'the dbt runtime pin must be an exact version', + ); + } + if (environment !== 'production') return; + if (pin.line === 'alpha' && !pin.separateCertificationRef) { + throw new SiaxError( + 'ENTITLEMENT_REQUIRED', + '0pt.analytics.dbt', + 'BI-P022: an alpha dbt engine may not run in production without a separate certification reference', + ); + } + if (!pin.licenseReviewed || !pin.securityReviewed) { + throw new SiaxError( + 'ENTITLEMENT_REQUIRED', + '0pt.analytics.dbt', + `BI-P022: dbt ${pin.version} has not completed licence and security verification; ` + + `it may not run in production`, + ); + } +} + +// ── the job contract N0D executes ────────────────────────────────────────── + +export const DBT_COMMANDS = [ + 'build', + 'run', + 'test', + 'snapshot', + 'seed', + 'compile', + 'source_freshness', +] as const; +export type DbtCommand = (typeof DBT_COMMANDS)[number]; + +/** + * What 0pt hands N0D. Deliberately narrow: there is no free-form `args` field, + * because a passthrough argument list is a shell surface, and a shell surface + * on a job that holds warehouse credentials is the whole problem. + */ +export interface DbtJobRequest { + /** Verified, from CL0UD. Never read from a request body. */ + tenant_id: string; + project_id: string; + /** Which analytics project to run — a registered name, not a path. */ + dbt_project: string; + /** Git ref of the analytics repo the models come from. Exact SHA. */ + revision: string; + command: DbtCommand; + /** dbt node selectors. Validated; never concatenated into a shell string. */ + select?: string[]; + exclude?: string[]; + /** Target layer, which N0D maps to a credential with matching write scope. */ + target_layer: DbtLayer; + environment: DbtRunEnvironment; + runtime: DbtRuntimePin; + /** Idempotency key, per the standard N0D job contract. */ + idempotency_key: string; + correlation_id?: string; + /** Fail the run if it would write outside this tenant's schema. */ + expected_target_schema: string; +} + +/** The artefacts N0D must return. dbt produces all of them; they are not optional. */ +export const REQUIRED_DBT_ARTIFACTS = ['manifest.json', 'run_results.json'] as const; + +export interface DbtJobResult { + tenant_id: string; + dbt_project: string; + revision: string; + command: DbtCommand; + status: 'success' | 'error' | 'partial'; + started_at: string; + finished_at: string; + /** ST0RE artifact refs, one per REQUIRED_DBT_ARTIFACTS entry. */ + artifacts: Record; + /** Per-node outcomes, the input to BI-P025's DbtQualityProvider. */ + nodes: DbtNodeResult[]; + /** dbt's own version, as reported by the runtime that actually ran. */ + observed_dbt_version: string; +} + +export interface DbtNodeResult { + unique_id: string; + resource_type: 'model' | 'test' | 'snapshot' | 'seed' | 'source'; + status: 'success' | 'error' | 'skipped' | 'pass' | 'fail' | 'warn'; + rows_affected?: number; + execution_time_s?: number; + message?: string; +} + +const SELECTOR_RE = /^[A-Za-z0-9_.:*+@ -]{1,200}$/; +const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/; +const SHA_RE = /^[0-9a-f]{40}$/; + +export function assertValidDbtJobRequest(req: DbtJobRequest): void { + if (!req.tenant_id?.trim()) { + throw new SiaxError('UNAUTHENTICATED', '0pt.analytics.dbt', 'a dbt job requires a verified tenant'); + } + if (!NAME_RE.test(req.dbt_project)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + 'dbt_project must be a registered snake_case name', + ); + } + if (!SHA_RE.test(req.revision)) { + // A branch name is not reproducible, and BI-P022's DoD is about + // reproducibility. "main" today and "main" tomorrow are different projects. + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + 'revision must be an exact 40-character commit SHA — a branch name is not reproducible', + ); + } + if (!(DBT_COMMANDS as readonly string[]).includes(req.command)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unsupported dbt command "${req.command}"`); + } + if (!(DBT_LAYERS as readonly string[]).includes(req.target_layer)) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unknown target layer "${req.target_layer}"`); + } + if (req.target_layer === 'sources') { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + 'sources are DECLARED, never built — a job cannot target the sources layer', + ); + } + for (const selector of [...(req.select ?? []), ...(req.exclude ?? [])]) { + if (!SELECTOR_RE.test(selector)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + `dbt selector "${selector.slice(0, 40)}" contains characters that are not selector syntax`, + ); + } + } + if (!req.idempotency_key?.trim()) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + 'the N0D job contract requires an idempotency key', + ); + } + if (!req.expected_target_schema.includes(req.tenant_id)) { + // The target schema must be derivable from the tenant. A job that can name + // an arbitrary schema is a cross-tenant write waiting to happen. + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.dbt', + "expected_target_schema must be scoped to the job's tenant", + ); + } + assertDbtRuntimeCertified(req.runtime, req.environment); +} + +/** + * Validate what came BACK. A result is untrusted input: it arrives from + * another repo's executor, and the estate has been burned by trusting a green + * status that meant nothing. + */ +export function assertValidDbtJobResult(result: DbtJobResult, request: DbtJobRequest): void { + if (result.tenant_id !== request.tenant_id) { + throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.dbt', 'dbt result is for a different tenant'); + } + if (result.revision !== request.revision) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.dbt', + 'dbt result is for a different revision than was requested — the run is not reproducible', + ); + } + for (const artifact of REQUIRED_DBT_ARTIFACTS) { + if (!result.artifacts[artifact]) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.dbt', + `dbt result is missing "${artifact}" — without it the run cannot be audited or its tests ingested`, + ); + } + } + if (result.observed_dbt_version !== request.runtime.version) { + // A runtime that is not the pinned one silently changes what the models + // mean. Catching it here is the difference between a pin and a wish. + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.dbt', + `dbt ran as ${result.observed_dbt_version} but ${request.runtime.version} was pinned`, + ); + } + if (result.status === 'success' && result.nodes.length === 0) { + // A "successful" run that built nothing is the dbt-shaped false green: a + // selector that matched no models exits 0. + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.dbt', + 'dbt reported success but executed zero nodes — a selector that matches nothing exits 0', + ); + } + const failed = result.nodes.filter((n) => n.status === 'error' || n.status === 'fail'); + if (result.status === 'success' && failed.length > 0) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.dbt', + `dbt reported success while ${failed.length} node(s) failed`, + ); + } +} + +/** + * The executor N0D implements. Declared so both repos compile against one + * shape; deliberately NOT implemented here. + */ +export interface DbtExecutor { + readonly runsIn: 'n0d'; + execute(request: DbtJobRequest): Promise; +} + +/** Which tier a model path belongs to, or a rejection. */ +export function tierForModelPath(modelPath: string): AnalyticsTier { + const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(modelPath); + if (!match) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + `model "${modelPath}" is not under one of ${DBT_LAYERS.join('/')} — BI-P003 cannot assign it a tier`, + ); + } + return DBT_LAYER_TIERS[match[1] as DbtLayer]; +} + +/** + * A staging model must not reach past sources, a mart must not read a source + * directly, and nothing may skip a layer. Enforced on the dependency graph + * from `manifest.json`, because "we follow the convention" is not a check. + */ +const LAYER_ORDER: DbtLayer[] = ['sources', 'staging', 'intermediate', 'marts']; + +export function assertLayerDependenciesLegal(edges: Array<{ from: string; to: string }>): void { + for (const edge of edges) { + const fromLayer = layerOf(edge.from); + const toLayer = layerOf(edge.to); + const fromIdx = LAYER_ORDER.indexOf(fromLayer); + const toIdx = LAYER_ORDER.indexOf(toLayer); + if (toIdx > fromIdx) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + `"${edge.from}" (${fromLayer}) depends on "${edge.to}" (${toLayer}) — a model may not read a later layer`, + ); + } + if (fromIdx - toIdx > 1) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.dbt', + `"${edge.from}" (${fromLayer}) skips a layer to read "${edge.to}" (${toLayer}) — ` + + `raw data reaching a mart directly is what BI-P003's tier rule exists to prevent`, + ); + } + } +} + +function layerOf(path: string): DbtLayer { + const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(path); + if (!match) { + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `model "${path}" is not in a known layer`); + } + return match[1] as DbtLayer; +} diff --git a/tests/analytics/analyst-sql-bypass-corpus.ts b/tests/analytics/analyst-sql-bypass-corpus.ts new file mode 100644 index 0000000..4d7d893 --- /dev/null +++ b/tests/analytics/analyst-sql-bypass-corpus.ts @@ -0,0 +1,362 @@ +/** + * tests/analytics/analyst-sql-bypass-corpus.ts + * ============================================= + * BI-P017 — the bypass attempts, in one place, so the SAME corpus can be run + * twice: once against the guard (unit) and once against a LIVE ClickHouse with + * the guard REMOVED (integration), where the grants and the view predicate + * must refuse it anyway. + * + * Running it only against the guard would prove the guard, not the system. + * This estate has repeatedly shipped guards that were never tested against a + * real attempt: INF0's tenant-from-body hole was exploitable against LIVE + * production, and ST0RE turned out to have FOUR cross-tenant paths, not the + * one reported. + * + * The integration test asserts that with the guard removed the engine either + * ERRORS or returns zero foreign rows — never that "the guard said no", which + * is the assertion that would keep passing while the system was open. + * + * Session under test: tenant `t_alpha`, catalog `t_alpha_sql`, objects + * `sales` and `events`. The targets are `t_beta_sql`, the shared base database + * `opt_analytics`, and `system`. + */ + +export interface BypassAttempt { + id: string; + /** What the attacker is trying to do. */ + goal: string; + sql: string; +} + +export const BYPASS_CORPUS: BypassAttempt[] = [ + // ── reach another tenant's catalog ────────────────────────────────────── + { + id: 'cross-catalog-direct', + goal: "read the other tenant's catalog", + sql: 'SELECT * FROM t_beta_sql.sales', + }, + { + id: 'cross-catalog-union', + goal: 'union in the other tenant', + sql: 'SELECT customer FROM t_alpha_sql.sales UNION ALL SELECT customer FROM t_beta_sql.sales', + }, + { + id: 'cross-catalog-subquery', + goal: 'subquery on the other tenant', + sql: 'SELECT (SELECT count() FROM t_beta_sql.sales) AS n FROM t_alpha_sql.sales LIMIT 1', + }, + { + id: 'cross-catalog-join', + goal: 'join the other tenant', + sql: 'SELECT a.customer FROM t_alpha_sql.sales a JOIN t_beta_sql.sales b ON a.customer = b.customer', + }, + { + id: 'cross-catalog-cte', + goal: 'hide the other tenant in a CTE', + sql: 'WITH other AS (SELECT * FROM t_beta_sql.sales) SELECT count() AS n FROM other', + }, + { + id: 'cross-catalog-quoted', + goal: 'quote the identifier to dodge the scan', + sql: 'SELECT * FROM "t_beta_sql"."sales"', + }, + { + id: 'cross-catalog-backtick', + goal: 'backtick the identifier', + sql: 'SELECT * FROM `t_beta_sql`.`sales`', + }, + + // ── reach the unfiltered base table ───────────────────────────────────── + { + id: 'base-table-direct', + goal: 'read the shared base table, which holds every tenant', + sql: 'SELECT * FROM opt_analytics.opt_analytics_event', + }, + { + id: 'base-table-in-cte', + goal: 'base table behind a CTE', + sql: 'WITH raw AS (SELECT * FROM opt_analytics.opt_analytics_event) SELECT count() AS n FROM raw', + }, + { + id: 'base-table-union', + goal: 'union the base table in', + sql: 'SELECT customer FROM t_alpha_sql.sales UNION ALL SELECT tenant_id FROM opt_analytics.opt_analytics_event', + }, + + // ── system / metadata catalogs ────────────────────────────────────────── + { + id: 'system-tables', + goal: 'enumerate every table on the server', + sql: 'SELECT database, name FROM system.tables', + }, + { id: 'system-columns', goal: 'enumerate columns', sql: 'SELECT * FROM system.columns' }, + { id: 'system-users', goal: 'enumerate users', sql: 'SELECT * FROM system.users' }, + { + id: 'system-processes', + goal: "read other tenants' running queries", + sql: 'SELECT query FROM system.processes', + }, + { + id: 'system-query-log', + goal: "read other tenants' statement text out of the log", + sql: 'SELECT query FROM system.query_log', + }, + { + id: 'information-schema', + goal: 'metadata via the ANSI schema', + sql: 'SELECT * FROM information_schema.tables', + }, + { id: 'one-table', goal: 'the always-present one-row table', sql: 'SELECT * FROM system.one' }, + + // ── table functions: these need NO table privilege at all ─────────────── + { + id: 'tf-url', + goal: 'exfiltrate over HTTP', + sql: "SELECT * FROM url('http://127.0.0.1:1/x', 'JSONEachRow', 'a String')", + }, + { + id: 'tf-file', + goal: 'read the server filesystem', + sql: "SELECT * FROM file('/etc/passwd', 'LineAsString', 'l String')", + }, + { + id: 'tf-s3', + goal: 'read object storage directly', + sql: "SELECT * FROM s3('http://minio/b/*', 'CSV', 'a String')", + }, + { + id: 'tf-remote', + goal: 'query another ClickHouse server', + sql: "SELECT * FROM remote('127.0.0.1:9000', 'opt_analytics', 'opt_analytics_event')", + }, + { + id: 'tf-mysql', + goal: 'reach an external database', + sql: "SELECT * FROM mysql('h:3306', 'db', 't', 'u', 'p')", + }, + { id: 'tf-executable', goal: 'run a program', sql: "SELECT * FROM executable('sh', 'TSV', 'a String')" }, + { + id: 'tf-numbers', + goal: 'a CPU bomb that touches no table', + sql: 'SELECT count() FROM numbers_mt(100000000000)', + }, + { id: 'tf-generate', goal: 'unbounded synthetic scan', sql: "SELECT * FROM generateRandom('a String')" }, + { + id: 'tf-merge', + goal: 'merge() glob across a database', + sql: "SELECT * FROM merge('opt_analytics', '.*')", + }, + { + id: 'tf-cluster', + goal: 'fan out across the cluster', + sql: "SELECT * FROM cluster('default', 'opt_analytics', 'opt_analytics_event')", + }, + { + id: 'tf-view', + goal: 'wrap a subquery in view()', + sql: 'SELECT * FROM view(SELECT * FROM opt_analytics.opt_analytics_event)', + }, + { + id: 'fn-dictget', + goal: 'read a dictionary, bypassing table grants', + sql: "SELECT dictGet('d', 'v', toUInt64(1)) AS x FROM t_alpha_sql.sales", + }, + + // ── write / DDL / privilege escalation ────────────────────────────────── + { + id: 'stacked-drop', + goal: 'stacked statement', + sql: 'SELECT 1 FROM t_alpha_sql.sales; DROP TABLE opt_analytics.opt_analytics_event', + }, + { id: 'insert', goal: 'write', sql: "INSERT INTO t_alpha_sql.sales VALUES ('x')" }, + { + id: 'insert-select', + goal: 'write from a read', + sql: 'INSERT INTO t_alpha_sql.sales SELECT * FROM t_alpha_sql.sales', + }, + { id: 'alter-delete', goal: 'mutate', sql: 'ALTER TABLE opt_analytics.opt_analytics_event DELETE WHERE 1' }, + { + id: 'create-view-escape', + goal: 'create an unfiltered view to read later', + sql: 'CREATE VIEW t_alpha_sql.escape AS SELECT * FROM opt_analytics.opt_analytics_event', + }, + { id: 'grant-self', goal: 'grant itself more', sql: 'GRANT SELECT ON opt_analytics.* TO CURRENT_USER' }, + { id: 'system-command', goal: 'operate the server', sql: 'SYSTEM RELOAD DICTIONARIES' }, + { id: 'kill-query', goal: "kill another tenant's query", sql: 'KILL QUERY WHERE 1' }, + { id: 'truncate', goal: 'destroy data', sql: 'TRUNCATE TABLE t_alpha_sql.sales' }, + { + id: 'optimize', + goal: 'trigger heavy background work', + sql: 'OPTIMIZE TABLE opt_analytics.opt_analytics_event FINAL', + }, + { + id: 'outfile', + goal: 'write to the filesystem', + sql: "SELECT * FROM t_alpha_sql.sales INTO OUTFILE '/tmp/x.csv'", + }, + + // ── lift the governance ceiling from inside the statement ─────────────── + { + id: 'settings-raise-rows', + goal: 'raise max_rows_to_read', + sql: 'SELECT count() AS n FROM t_alpha_sql.sales SETTINGS max_rows_to_read = 0', + }, + { + id: 'settings-readonly-off', + goal: 'turn readonly off', + sql: 'SELECT 1 FROM t_alpha_sql.sales SETTINGS readonly = 0', + }, + { id: 'set-statement', goal: 'SET before the query', sql: 'SET max_execution_time = 0' }, + { + id: 'format-override', + goal: 'choose the wire format', + sql: 'SELECT * FROM t_alpha_sql.sales FORMAT Native', + }, + + // ── lexical evasion of the keyword scan ───────────────────────────────── + { + id: 'comment-split-name', + goal: 'hide a catalog boundary behind a comment', + sql: 'SELECT * FROM opt_analytics/**/.opt_analytics_event', + }, + { + id: 'comment-line-hide', + goal: 'hide the tail after a -- comment', + sql: 'SELECT customer FROM t_alpha_sql.sales -- \nUNION ALL SELECT name FROM system.tables', + }, + { + id: 'comment-hash-hide', + goal: 'hide the tail after a # comment', + sql: 'SELECT customer FROM t_alpha_sql.sales # \nUNION ALL SELECT name FROM system.tables', + }, + { + id: 'unterminated-comment', + goal: 'run off the end of a comment', + sql: 'SELECT * FROM t_alpha_sql.sales /* x', + }, + { + id: 'unterminated-string', + goal: 'run off the end of a literal', + sql: "SELECT * FROM t_alpha_sql.sales WHERE customer = 'x", + }, + { + id: 'unicode-homoglyph', + // U+0430 CYRILLIC SMALL LETTER A in place of the ASCII "a". + goal: 'cyrillic homoglyph inside the catalog name', + sql: 'SELECT * FROM t_\u0430lpha_sql.sales', + }, + { + id: 'zero-width', + // U+200B ZERO WIDTH SPACE inside "system". + goal: 'zero-width space inside a reserved catalog name', + sql: 'SELECT * FROM sys\u200btem.tables', + }, + { + id: 'nul-byte', + goal: 'NUL truncation between two halves of a statement', + sql: 'SELECT customer FROM t_alpha_sql.sales \u0000 UNION ALL SELECT name FROM system.tables', + }, + { id: 'case-mixing', goal: 'mixed case keywords and names', sql: 'SeLeCt * FrOm SyStEm.TaBlEs' }, + { + id: 'whitespace-newlines', + goal: 'newlines and tabs between tokens', + sql: 'SELECT\n\t*\nFROM\n\tsystem.tables', + }, + { + id: 'describe-subquery', + goal: 'DESCRIBE a base table from inside a subquery, where the FROM-scan sees only a paren', + sql: 'SELECT * FROM (DESCRIBE TABLE opt_analytics.opt_analytics_event)', + }, + { + id: 'desc-subquery-short', + goal: 'the same, using the DESC abbreviation', + sql: 'SELECT * FROM (DESC TABLE opt_analytics.opt_analytics_event)', + }, + { + id: 'current-user', + goal: "identify the session's ClickHouse principal", + sql: 'SELECT currentUser() AS u FROM t_alpha_sql.sales', + }, + { id: 'three-part-name', goal: 'a three-part name', sql: 'SELECT * FROM a.b.c' }, + { + id: 'unqualified-table', + goal: "unqualified name resolved by the connection's default database", + sql: 'SELECT * FROM opt_analytics_event', + }, + { + id: 'cte-shadow', + goal: 'shadow a legitimate name with a CTE while also reading the real base table', + sql: 'WITH sales AS (SELECT 1 AS x) SELECT count() AS n FROM opt_analytics.opt_analytics_event', + }, + { id: 'no-from', goal: 'a statement that reads no table at all', sql: 'SELECT 1' }, + { + id: 'object-not-granted', + goal: 'an object in the right catalog that the session was not given', + sql: 'SELECT * FROM t_alpha_sql.secret_ledger', + }, +]; + +/** + * Attempts the GUARD ADMITS — and which only layers 3 and 4 refuse. + * + * This list exists because the first run of the corpus found one, and hiding + * it by quietly deleting the case would have been the dishonest fix. The guard + * reasons about statement STRUCTURE; it does not know which columns a view + * projects, and teaching it would mean giving it a schema cache that goes + * stale — a hardcoded list, which is how api0's container job died this week. + * + * `SELECT tenant_id FROM t_alpha_sql.sales` is structurally a legitimate read + * of an allowed object. It fails at the ENGINE, because the view does not + * project `tenant_id` at all (tenant-catalog.ts refuses to build one that + * does). The integration test asserts that engine-side refusal directly, so + * this case is covered by evidence rather than by an assumption. + */ +export const GUARD_ADMITS_ENGINE_REFUSES: BypassAttempt[] = [ + { + id: 'hidden-tenant-column', + goal: 'select the tenant column the view does not project', + sql: 'SELECT tenant_id FROM t_alpha_sql.sales', + }, + { + id: 'hidden-column-star-then-filter', + goal: 'filter on a column the view does not expose', + sql: "SELECT count() AS n FROM t_alpha_sql.sales WHERE tenant_id = 't_beta'", + }, +]; + +/** Statements that MUST be admitted — a guard that refuses everything is useless. */ +export const LEGITIMATE_CORPUS: Array<{ id: string; sql: string }> = [ + { id: 'simple', sql: 'SELECT customer, amount FROM t_alpha_sql.sales LIMIT 10' }, + { + id: 'aggregate', + sql: 'SELECT customer, sum(amount) AS total FROM t_alpha_sql.sales GROUP BY customer ORDER BY total DESC LIMIT 5', + }, + { + id: 'cte', + sql: + 'WITH top AS (SELECT customer, sum(amount) AS total FROM t_alpha_sql.sales GROUP BY customer) ' + + 'SELECT customer, total FROM top ORDER BY total DESC LIMIT 3', + }, + { + id: 'self-join', + sql: 'SELECT a.customer FROM t_alpha_sql.sales a JOIN t_alpha_sql.events b ON a.customer = b.customer LIMIT 10', + }, + { + id: 'literal-with-quotes', + sql: "SELECT count() AS n FROM t_alpha_sql.sales WHERE customer = 'O''Brien'", + }, + { + id: 'literal-containing-sql-text', + sql: "SELECT count() AS n FROM t_alpha_sql.sales WHERE customer = 'DROP TABLE system.tables'", + }, + { id: 'comment-then-query', sql: '-- daily revenue\nSELECT sum(amount) AS total FROM t_alpha_sql.sales' }, + { + id: 'subquery', + sql: 'SELECT customer FROM t_alpha_sql.sales WHERE amount > (SELECT avg(amount) FROM t_alpha_sql.sales)', + }, + { + id: 'union-own-catalog', + sql: 'SELECT customer FROM t_alpha_sql.sales UNION ALL SELECT customer FROM t_alpha_sql.events', + }, + { id: 'trailing-semicolon', sql: 'SELECT count() AS n FROM t_alpha_sql.sales;' }, +]; diff --git a/tests/analytics/analyst-sql-evidence.test.ts b/tests/analytics/analyst-sql-evidence.test.ts new file mode 100644 index 0000000..f614f4a --- /dev/null +++ b/tests/analytics/analyst-sql-evidence.test.ts @@ -0,0 +1,108 @@ +/** + * tests/analytics/analyst-sql-evidence.test.ts + * ============================================= + * BI-P017 — the test that does NOT skip. + * + * The live suite (tests/integration/analyst-sql-clickhouse.integration.test.ts) + * only runs when `CLICKHOUSE_URL` is set. On a runner without ClickHouse it + * contributes nothing, and a suite that contributes nothing while reporting + * green is how l0re's 28 Postgres tests stayed silently skipped for months. + * + * So the live run WRITES its result to + * `evidence/bi-p017/analyst-sql-live-bypass.json`, that file is committed, and + * this test — which always runs — checks it against the corpus: + * + * * every attempt in the corpus must appear in the evidence, so adding an + * attack without re-running the live suite BREAKS THE BUILD rather than + * quietly going untested; + * * zero attempts may have leaked; + * * the legitimate statements must all have succeeded, so the evidence + * cannot be a recording of a database that refuses everything; + * * the counterfactual numbers must be present, so the evidence cannot be a + * recording of an EMPTY database, where "no foreign row came back" is + * true and meaningless. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { BYPASS_CORPUS, GUARD_ADMITS_ENGINE_REFUSES, LEGITIMATE_CORPUS } from './analyst-sql-bypass-corpus'; + +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p017/analyst-sql-live-bypass.json'); + +interface Evidence { + clickhouse_version: string; + recorded_at: string; + base_table_rows_both_tenants: number; + tenant_view_rows_alpha: number; + attempts_total: number; + attempts_that_leaked: number; + legitimate_statements_that_succeeded: number; + legitimate_statements_total: number; + attempts: Array<{ + id: string; + engine_http_status: number; + engine_succeeded: boolean; + leaked_foreign_tenant_row: boolean; + leaked_ungranted_table_row: boolean; + leaked_server_metadata: boolean; + guard_verdict_for_reference: string; + }>; + legitimate: Array<{ id: string; ok: boolean; contains_beta_canary: boolean }>; +} + +describe('BI-P017 — the committed live evidence covers the current corpus', () => { + it('the evidence file exists', () => { + expect( + fs.existsSync(EVIDENCE_PATH), + 'evidence/bi-p017/analyst-sql-live-bypass.json is missing — run the live suite with CLICKHOUSE_URL set', + ).toBe(true); + }); + + const evidence: Evidence = fs.existsSync(EVIDENCE_PATH) + ? (JSON.parse(fs.readFileSync(EVIDENCE_PATH, 'utf8')) as Evidence) + : ({ attempts: [], legitimate: [] } as unknown as Evidence); + + it('was recorded against a real ClickHouse, named by version', () => { + expect(evidence.clickhouse_version).toMatch(/^\d+\.\d+/); + expect(Date.parse(evidence.recorded_at)).toBeGreaterThan(0); + }); + + it("records the counterfactual, so 'nothing leaked' is not a statement about an empty database", () => { + expect(evidence.base_table_rows_both_tenants).toBe(100); + expect(evidence.tenant_view_rows_alpha).toBe(50); + }); + + it('covers EVERY attempt in the corpus — a new attack without a re-run fails here', () => { + const recorded = new Set(evidence.attempts.map((a) => a.id)); + const missing = [...BYPASS_CORPUS, ...GUARD_ADMITS_ENGINE_REFUSES] + .map((a) => a.id) + .filter((id) => !recorded.has(id)); + expect(missing, `attempts never run against a live engine: ${missing.join(', ')}`).toEqual([]); + expect(evidence.attempts_total).toBe(BYPASS_CORPUS.length + GUARD_ADMITS_ENGINE_REFUSES.length); + }); + + it('no attempt leaked a foreign-tenant row, an ungranted row, or a foreign object name', () => { + const leaked = evidence.attempts.filter( + (a) => a.leaked_foreign_tenant_row || a.leaked_ungranted_table_row || a.leaked_server_metadata, + ); + expect(leaked.map((a) => a.id)).toEqual([]); + expect(evidence.attempts_that_leaked).toBe(0); + }); + + it('records which attempts the ENGINE alone stopped, with the guard removed', () => { + // The interesting number: attempts that the guard would have admitted and + // the engine refused anyway. If it were zero, the layered claim would rest + // entirely on the guard. + const engineOnly = evidence.attempts.filter((a) => a.guard_verdict_for_reference === 'admitted'); + expect(engineOnly.length).toBeGreaterThan(0); + for (const a of engineOnly) { + expect(a.engine_succeeded, `${a.id} was admitted by the guard AND succeeded at the engine`).toBe(false); + } + }); + + it('legitimate analyst work succeeded — the evidence is not a recording of a database that refuses everything', () => { + expect(evidence.legitimate_statements_total).toBe(LEGITIMATE_CORPUS.length); + expect(evidence.legitimate_statements_that_succeeded).toBe(LEGITIMATE_CORPUS.length); + expect(evidence.legitimate.every((l) => l.contains_beta_canary === false)).toBe(true); + }); +}); diff --git a/tests/analytics/analyst-sql-guard.test.ts b/tests/analytics/analyst-sql-guard.test.ts new file mode 100644 index 0000000..e84ac7a --- /dev/null +++ b/tests/analytics/analyst-sql-guard.test.ts @@ -0,0 +1,380 @@ +/** + * tests/analytics/analyst-sql-guard.test.ts + * ========================================== + * BI-P017 — layer 2 (statement admission) against the bypass corpus. + * + * BI-P017's DoD: "INSERT, UPDATE, DROP, unsafe functions and unauthorized + * catalogs fail." That is the floor. This file tests the floor and then keeps + * going, because the interesting attacks are not `DROP TABLE` — they are + * `url()`, `merge()`, a quoted identifier, a homoglyph, and a comment placed + * where the guard and the engine might disagree about where the query ends. + * + * What this file does NOT prove: that the SYSTEM is safe. It proves the guard + * refuses these statements. The system claim is + * tests/integration/analyst-sql-clickhouse.integration.test.ts, which runs the + * same corpus against a live ClickHouse with this layer REMOVED. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + guardAnalystStatement, + stripSqlText, + type StatementGuardOptions, +} from '@/src/analytics/sql/statement-guard'; +import { + AnalyticsSQLSession, + ANALYST_SQL_ROLE, + type AnalystStatementRecord, + type ReadOnlySqlEndpoint, +} from '@/src/analytics/sql/analyst-sql-session'; +import { tenantSqlCatalogDdl, tenantSqlUserDdl } from '@/src/analytics/sql/tenant-catalog'; +import { BYPASS_CORPUS, GUARD_ADMITS_ENGINE_REFUSES, LEGITIMATE_CORPUS } from './analyst-sql-bypass-corpus'; + +const OPTS: StatementGuardOptions = { + allowedCatalog: 't_alpha_sql', + allowedObjects: ['sales', 'events'], +}; + +describe('BI-P017 layer 2 — the bypass corpus is refused', () => { + it('covers every attack family the DoD names, and more', () => { + // A corpus that shrank silently would make this suite pass by testing + // nothing. Pin the size and the presence of the named families. + expect(BYPASS_CORPUS.length).toBeGreaterThanOrEqual(55); + const ids = new Set(BYPASS_CORPUS.map((a) => a.id)); + for (const required of [ + 'insert', + 'alter-delete', + 'create-view-escape', + 'tf-url', + 'system-tables', + 'cross-catalog-direct', + ]) { + expect(ids.has(required)).toBe(true); + } + expect(new Set(BYPASS_CORPUS.map((a) => a.sql)).size).toBe(BYPASS_CORPUS.length); + }); + + it.each(BYPASS_CORPUS.map((a) => [a.id, a.goal, a.sql] as const))('refuses %s (%s)', (_id, _goal, sql) => { + let thrown: unknown; + try { + guardAnalystStatement(sql, OPTS); + } catch (e) { + thrown = e; + } + expect(thrown, 'statement was ADMITTED — this is a bypass').toBeInstanceOf(SiaxError); + const err = thrown as SiaxError; + expect(['AUTHORIZATION_DENIED', 'INVALID_REQUEST']).toContain(err.code); + // A refusal with an empty reason is unreviewable. + expect(err.message.length).toBeGreaterThan(20); + }); +}); + +describe('BI-P017 layer 2 — what the guard honestly does NOT catch', () => { + // Recorded, not hidden. The guard admits these; the engine refuses them + // because the view does not project the column. The claim is proven in + // tests/integration/analyst-sql-clickhouse.integration.test.ts, not here. + it.each(GUARD_ADMITS_ENGINE_REFUSES.map((a) => [a.id, a.sql] as const))( + 'the guard admits %s — layers 3/4 are what refuse it', + (_id, sql) => { + expect(() => guardAnalystStatement(sql, OPTS)).not.toThrow(); + }, + ); +}); + +describe('BI-P017 layer 2 — legitimate analyst work is admitted', () => { + it.each(LEGITIMATE_CORPUS.map((a) => [a.id, a.sql] as const))('admits %s', (_id, sql) => { + const guarded = guardAnalystStatement(sql, OPTS); + expect(guarded.referencedObjects.length).toBeGreaterThan(0); + for (const obj of guarded.referencedObjects) { + expect(obj.startsWith('t_alpha_sql.')).toBe(true); + } + }); + + it('a guard that refused everything would fail this file', () => { + // Stated explicitly because "all tests pass" is compatible with a guard + // that throws unconditionally, and that guard would be useless. + expect(LEGITIMATE_CORPUS.length).toBeGreaterThanOrEqual(10); + }); +}); + +describe("BI-P017 — literal and comment handling matches ClickHouse's lexer", () => { + it('does not see keywords that are inside a string literal', () => { + const g = guardAnalystStatement( + "SELECT count() AS n FROM t_alpha_sql.sales WHERE customer = 'DROP TABLE system.tables'", + OPTS, + ); + expect(g.stripped).not.toContain('DROP'); + expect(g.stripped).not.toContain('system'); + }); + + it('masks literals in the normalized form so statement history is not a data dump', () => { + const g = guardAnalystStatement( + "SELECT customer FROM t_alpha_sql.sales WHERE customer = 'Anna Andersson, 19850101-1234'", + OPTS, + ); + expect(g.normalized).not.toContain('Andersson'); + expect(g.normalized).toContain('?'); + expect(g.raw).toContain('Andersson'); // the raw form is retained deliberately + }); + + it('handles doubled and backslash-escaped quotes without losing the statement end', () => { + expect(stripSqlText("SELECT 'a''b'").stripped).toBe("SELECT ''"); + expect(stripSqlText("SELECT 'a\\'b'").stripped).toBe("SELECT ''"); + // The classic: an escaped quote that a naive stripper would treat as the + // close, leaving `; DROP …` outside a literal in the guard's view but + // inside one in the engine's. Both must agree it stays inside. + expect(stripSqlText("SELECT 'a\\'; DROP TABLE x; --'").stripped).toBe("SELECT ''"); + }); + + it('refuses rather than guesses when a literal or comment never ends', () => { + expect(() => guardAnalystStatement("SELECT * FROM t_alpha_sql.sales WHERE c = 'x", OPTS)).toThrowError( + SiaxError, + ); + expect(() => guardAnalystStatement('SELECT * FROM t_alpha_sql.sales /* x', OPTS)).toThrowError(SiaxError); + }); + + it('treats a comment as whitespace, so it cannot glue two identifiers together', () => { + // `a/**/b` must not become `ab`. + expect(stripSqlText('a/**/b').stripped).toBe('a b'); + // A line comment leaves its terminating newline in place, so the result is + // "a \nb" — separated, which is the property that matters. + expect(stripSqlText('a--x\nb').stripped).toMatch(/^a\s+b$/); + expect(stripSqlText('a--x\nb').stripped).not.toContain('ab'); + }); +}); + +describe('BI-P017 — session role gate', () => { + const endpoint: ReadOnlySqlEndpoint = { + engine: 'clickhouse', + async execute() { + return { columns: ['n'], rows: [[1]] }; + }, + async cancel() { + return true; + }, + }; + const audit = { + records: [] as AnalystStatementRecord[], + record(e: AnalystStatementRecord) { + this.records.push(e); + }, + }; + + const open = (roles: string[] | undefined) => + new AnalyticsSQLSession({ + identity: { tenant_id: 't_alpha', principal_id: 'u_1', roles }, + endpoint, + catalog: 't_alpha_sql', + objects: ['sales', 'events'], + audit, + }); + + it('refuses to open without the analyst role', () => { + for (const roles of [undefined, [], ['analytics.reader'], ['admin'], ['analytics.sql.analystX']]) { + try { + open(roles as string[] | undefined); + throw new Error(`session opened with roles=${JSON.stringify(roles)} — role gate is not closed`); + } catch (e) { + expect(e).toBeInstanceOf(SiaxError); + expect((e as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + } + } + }); + + it('opens with the analyst role', () => { + expect(open([ANALYST_SQL_ROLE])).toBeInstanceOf(AnalyticsSQLSession); + }); + + it('refuses to open without a verified tenant or principal', () => { + for (const identity of [ + { tenant_id: '', principal_id: 'u_1', roles: [ANALYST_SQL_ROLE] }, + { tenant_id: ' ', principal_id: 'u_1', roles: [ANALYST_SQL_ROLE] }, + { tenant_id: 't_alpha', principal_id: '', roles: [ANALYST_SQL_ROLE] }, + ]) { + expect( + () => + new AnalyticsSQLSession({ + identity, + endpoint, + catalog: 't_alpha_sql', + objects: ['sales'], + audit, + }), + ).toThrowError(SiaxError); + } + }); +}); + +describe('BI-P017 — audit and statement history', () => { + function makeSession(opts?: { retainRaw?: boolean }) { + const records: AnalystStatementRecord[] = []; + const session = new AnalyticsSQLSession({ + identity: { + tenant_id: 't_alpha', + principal_id: 'u_1', + roles: [ANALYST_SQL_ROLE], + correlation_id: 'c_1', + }, + endpoint: { + engine: 'clickhouse', + async execute() { + return { columns: ['n'], rows: [[7]], rows_read: 42, bytes_read: 4242 }; + }, + async cancel() { + return true; + }, + }, + catalog: 't_alpha_sql', + objects: ['sales', 'events'], + audit: { record: (e) => void records.push(e) }, + retainRawStatements: opts?.retainRaw, + }); + return { session, records }; + } + + it('records a REFUSAL, not only a success — a guard whose rejections are invisible cannot be reviewed', async () => { + const { session, records } = makeSession(); + await expect(session.query('SELECT * FROM system.tables')).rejects.toBeInstanceOf(SiaxError); + expect(records).toHaveLength(1); + expect(records[0].outcome).toBe('refused'); + expect(records[0].error_code).toBe('AUTHORIZATION_DENIED'); + expect(records[0].refusal_reason).toBeTruthy(); + // Even a refused statement is masked before it is stored. + expect(records[0].normalized).not.toContain("'"); + }); + + it('records the executed statement with objects, timings and byte counts', async () => { + const { session, records } = makeSession(); + const out = await session.query('SELECT count() AS n FROM t_alpha_sql.sales'); + expect(out.rows).toEqual([[7]]); + expect(records).toHaveLength(1); + expect(records[0].outcome).toBe('succeeded'); + expect(records[0].referenced_objects).toEqual(['t_alpha_sql.sales']); + expect(records[0].rows_read).toBe(42); + expect(records[0].bytes_read).toBe(4242); + expect(records[0].statement_hash).toMatch(/^[0-9a-f]{64}$/); + expect(records[0].correlation_id).toBe('c_1'); + }); + + it('omits the raw statement when retention is off, but never the normalized one', async () => { + const { session, records } = makeSession({ retainRaw: false }); + await session.query("SELECT count() AS n FROM t_alpha_sql.sales WHERE customer = 'Anna'"); + expect(records[0].raw).toBeUndefined(); + expect(records[0].normalized).toContain('?'); + expect(records[0].normalized).not.toContain('Anna'); + }); + + it('applies every limit BI-P017 names, with readonly=1 (not 2, which permits SETTINGS changes)', async () => { + let seen: Record = {}; + const session = new AnalyticsSQLSession({ + identity: { tenant_id: 't_alpha', principal_id: 'u_1', roles: [ANALYST_SQL_ROLE] }, + endpoint: { + engine: 'clickhouse', + async execute(args) { + seen = args.settings; + return { columns: [], rows: [] }; + }, + async cancel() { + return true; + }, + }, + catalog: 't_alpha_sql', + objects: ['sales'], + audit: { record: () => undefined }, + }); + await session.query('SELECT count() AS n FROM t_alpha_sql.sales'); + expect(seen.readonly).toBe('1'); + for (const key of [ + 'max_execution_time', + 'max_result_rows', + 'max_rows_to_read', + 'max_bytes_to_read', + 'max_memory_usage', + ]) { + expect(Number(seen[key])).toBeGreaterThan(0); + } + // A limit that returns a PARTIAL result silently is a false green. + expect(seen.timeout_overflow_mode).toBe('throw'); + expect(seen.result_overflow_mode).toBe('throw'); + expect(seen.read_overflow_mode).toBe('throw'); + }); + + it('a CL0UD quota may only tighten the budget, never loosen it', () => { + const session = new AnalyticsSQLSession({ + identity: { + tenant_id: 't_alpha', + principal_id: 'u_1', + roles: [ANALYST_SQL_ROLE], + quota: { maxRowsToRead: 10, maxExecutionTimeSec: 999_999 }, + }, + endpoint: { + engine: 'clickhouse', + async execute() { + return { columns: [], rows: [] }; + }, + async cancel() { + return true; + }, + }, + catalog: 't_alpha_sql', + objects: ['sales'], + audit: { record: () => undefined }, + }); + expect(session.resourceBudget.maxRowsToRead).toBe(10); + expect(session.resourceBudget.maxExecutionTimeSec).toBeLessThanOrEqual(60); + }); + + it('refuses to cancel a statement belonging to another session', async () => { + const { session } = makeSession(); + await expect(session.cancel('sql_deadbeefcafe_notmine')).rejects.toBeInstanceOf(SiaxError); + }); +}); + +describe('BI-P017 layers 3 and 4 — the DDL that holds when the guard does not', () => { + const spec = { + sourceDatabase: 'opt_analytics', + tenantId: 't_alpha', + catalog: 't_alpha_sql', + objects: [{ table: 'opt_analytics_event', columns: ['event_id', 'metric', 'value', 'event_time'] }], + }; + + it('bakes the tenant predicate into every view definition', () => { + const ddl = tenantSqlCatalogDdl(spec); + const view = ddl.find((s) => s.startsWith('CREATE OR REPLACE VIEW')); + expect(view).toContain("WHERE tenant_id = 't_alpha'"); + expect(view).toContain('FROM opt_analytics.opt_analytics_event'); + }); + + it('refuses to project tenant_id into a single-tenant view', () => { + expect(() => + tenantSqlCatalogDdl({ ...spec, objects: [{ table: 'x', columns: ['tenant_id', 'v'] }] }), + ).toThrowError(SiaxError); + }); + + it('refuses a tenant id that is not catalog-safe rather than embedding it', () => { + for (const bad of ["t_alpha'; DROP", 't alpha', "'", '', "t_alpha' OR '1'='1"]) { + expect(() => tenantSqlCatalogDdl({ ...spec, tenantId: bad })).toThrowError(SiaxError); + } + }); + + it('refuses to point the tenant catalog at the shared source database', () => { + expect(() => tenantSqlCatalogDdl({ ...spec, catalog: 'opt_analytics' })).toThrowError(SiaxError); + }); + + it('grants SELECT only on the tenant catalog and pins a row policy to the base table', () => { + const ddl = tenantSqlUserDdl({ ...spec, userName: 't_alpha_sql_user', password: 'x' }); + expect(ddl.some((s) => s.includes('GRANT SELECT ON t_alpha_sql.* TO t_alpha_sql_user'))).toBe(true); + // The wide grant that would make the whole design pointless: + expect(ddl.some((s) => s.includes('GRANT SELECT ON opt_analytics.* '))).toBe(false); + expect( + ddl.some((s) => s.includes('CREATE ROW POLICY') && s.includes("USING tenant_id = 't_alpha'")), + ).toBe(true); + expect(ddl[0]).toContain('readonly = 1'); + }); + + it('never embeds a password containing a quote or a backslash', () => { + for (const bad of ["a'b", 'a\\b']) { + expect(() => tenantSqlUserDdl({ ...spec, userName: 'u', password: bad })).toThrowError(SiaxError); + } + }); +}); diff --git a/tests/analytics/analytics-domain-model.test.ts b/tests/analytics/analytics-domain-model.test.ts index e1d9040..ca98f0d 100644 --- a/tests/analytics/analytics-domain-model.test.ts +++ b/tests/analytics/analytics-domain-model.test.ts @@ -10,14 +10,14 @@ * rename, or an entity quietly dropped from the migration, fails HERE rather * than at 3am against a database that turns out to be missing a table. */ -import { describe, it, expect } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; import { ANALYTICS_DOMAIN_ENTITIES, ANALYTICS_ENTITY_TABLES, IMMUTABLE_VERSION_ENTITIES, -} from "@/src/analytics/domain/entities"; +} from '@/src/analytics/domain/entities'; import { ANALYTICS_TIERS, assertReadableByBiSurface, @@ -25,40 +25,37 @@ import { BI_READABLE_TIERS, isAnalyticsTier, TIER_RANK, -} from "@/src/analytics/domain/tiers"; +} from '@/src/analytics/domain/tiers'; -const MIGRATION = fs.readFileSync( - path.join(process.cwd(), "sql", "008_analytics_domain_model.sql"), - "utf8", -); +const MIGRATION = fs.readFileSync(path.join(process.cwd(), 'sql', '008_analytics_domain_model.sql'), 'utf8'); -describe("BI-P002 canonical entity set", () => { - it("lists exactly the nineteen entities the program names, verbatim", () => { +describe('BI-P002 canonical entity set', () => { + it('lists exactly the nineteen entities the program names, verbatim', () => { expect(ANALYTICS_DOMAIN_ENTITIES).toEqual([ - "AnalyticsDataset", - "DatasetVersion", - "DatasetField", - "DatasetGrant", - "MetricDefinition", - "MetricVersion", - "DimensionDefinition", - "SemanticModel", - "AnalyticsQuery", - "QueryExecution", - "QueryResult", - "DashboardDefinition", - "DashboardVersion", - "ReportDefinition", - "ReportRun", - "DataQualityRule", - "DataQualityRun", - "LineageReference", - "BIApplicationBinding", + 'AnalyticsDataset', + 'DatasetVersion', + 'DatasetField', + 'DatasetGrant', + 'MetricDefinition', + 'MetricVersion', + 'DimensionDefinition', + 'SemanticModel', + 'AnalyticsQuery', + 'QueryExecution', + 'QueryResult', + 'DashboardDefinition', + 'DashboardVersion', + 'ReportDefinition', + 'ReportRun', + 'DataQualityRule', + 'DataQualityRun', + 'LineageReference', + 'BIApplicationBinding', ]); expect(ANALYTICS_DOMAIN_ENTITIES).toHaveLength(19); }); - it("sql/008 defines a table for every entity (no entity is a type without storage)", () => { + it('sql/008 defines a table for every entity (no entity is a type without storage)', () => { const missing: string[] = []; for (const entity of ANALYTICS_DOMAIN_ENTITIES) { const table = ANALYTICS_ENTITY_TABLES[entity]; @@ -69,14 +66,14 @@ describe("BI-P002 canonical entity set", () => { expect(missing).toEqual([]); }); - it("every tenant-owned table carries tenant_id AND project_id", () => { + it('every tenant-owned table carries tenant_id AND project_id', () => { // Split on the table definitions and check each block. const blocks = MIGRATION.split(/CREATE TABLE IF NOT EXISTS /).slice(1); const offenders: string[] = []; for (const block of blocks) { const name = block.split(/[\s(]/)[0]; if (!Object.values(ANALYTICS_ENTITY_TABLES).includes(name)) continue; - const body = block.slice(0, block.indexOf(");")); + const body = block.slice(0, block.indexOf(');')); if (!/\btenant_id TEXT NOT NULL\b/.test(body) || !/\bproject_id TEXT NOT NULL\b/.test(body)) { offenders.push(name); } @@ -84,12 +81,12 @@ describe("BI-P002 canonical entity set", () => { expect(offenders).toEqual([]); }); - it("the four immutable version entities have a BEFORE UPDATE OR DELETE trigger in sql/008", () => { + it('the four immutable version entities have a BEFORE UPDATE OR DELETE trigger in sql/008', () => { expect(IMMUTABLE_VERSION_ENTITIES).toEqual([ - "DatasetVersion", - "MetricVersion", - "SemanticModel", - "DashboardVersion", + 'DatasetVersion', + 'MetricVersion', + 'SemanticModel', + 'DashboardVersion', ]); for (const entity of IMMUTABLE_VERSION_ENTITIES) { const table = ANALYTICS_ENTITY_TABLES[entity]; @@ -100,16 +97,17 @@ describe("BI-P002 canonical entity set", () => { } }); - it("BIApplicationBinding storage carries NO credential column — the ownership boundary is structural", () => { - const block = MIGRATION.split("CREATE TABLE IF NOT EXISTS opt_analytics_bi_application_binding")[1] ?? ""; - const body = block.slice(0, block.indexOf(");")); - for (const forbidden of ["password", "dsn", "secret", "connection_string", "clickhouse_url", "token"]) { + it('BIApplicationBinding storage carries NO credential column — the ownership boundary is structural', () => { + const block = MIGRATION.split('CREATE TABLE IF NOT EXISTS opt_analytics_bi_application_binding')[1] ?? ''; + const body = block.slice(0, block.indexOf(');')); + for (const forbidden of ['password', 'dsn', 'secret', 'connection_string', 'clickhouse_url', 'token']) { expect(body.toLowerCase()).not.toContain(forbidden); } }); - it("sql/008 is idempotent-shaped: every CREATE is IF NOT EXISTS / OR REPLACE", () => { - const creates = MIGRATION.match(/^CREATE (TABLE|INDEX|TRIGGER|FUNCTION|OR REPLACE FUNCTION)[^\n]*/gm) ?? []; + it('sql/008 is idempotent-shaped: every CREATE is IF NOT EXISTS / OR REPLACE', () => { + const creates = + MIGRATION.match(/^CREATE (TABLE|INDEX|TRIGGER|FUNCTION|OR REPLACE FUNCTION)[^\n]*/gm) ?? []; const nonIdempotent = creates.filter( (line) => !/IF NOT EXISTS/.test(line) && !/OR REPLACE/.test(line) && !/^CREATE TRIGGER/.test(line), ); @@ -118,70 +116,77 @@ describe("BI-P002 canonical entity set", () => { // preceded by a DROP TRIGGER IF EXISTS to stay replayable. const triggers = MIGRATION.match(/^CREATE TRIGGER (\w+)/gm) ?? []; for (const t of triggers) { - const name = t.replace("CREATE TRIGGER ", ""); + const name = t.replace('CREATE TRIGGER ', ''); expect(MIGRATION).toContain(`DROP TRIGGER IF EXISTS ${name}`); } }); }); -describe("BI-P003 analytics data tiers", () => { - it("canonicalizes exactly RAW / NORMALIZED / CURATED / SERVING", () => { - expect(ANALYTICS_TIERS).toEqual(["RAW", "NORMALIZED", "CURATED", "SERVING"]); - expect(isAnalyticsTier("CURATED")).toBe(true); - expect(isAnalyticsTier("raw")).toBe(false); - expect(isAnalyticsTier("BRONZE")).toBe(false); +describe('BI-P003 analytics data tiers', () => { + it('canonicalizes exactly RAW / NORMALIZED / CURATED / SERVING', () => { + expect(ANALYTICS_TIERS).toEqual(['RAW', 'NORMALIZED', 'CURATED', 'SERVING']); + expect(isAnalyticsTier('CURATED')).toBe(true); + expect(isAnalyticsTier('raw')).toBe(false); + expect(isAnalyticsTier('BRONZE')).toBe(false); }); - it("no business dashboard reads a RAW/NORMALIZED dataset by default", () => { - expect(BI_READABLE_TIERS).toEqual(["SERVING", "CURATED"]); - for (const tier of ["RAW", "NORMALIZED"] as const) { + it('no business dashboard reads a RAW/NORMALIZED dataset by default', () => { + expect(BI_READABLE_TIERS).toEqual(['SERVING', 'CURATED']); + for (const tier of ['RAW', 'NORMALIZED'] as const) { expect(() => - assertReadableByBiSurface({ dataset_id: "ds_1", tier }, "dashboard:exec-overview"), + assertReadableByBiSurface({ dataset_id: 'ds_1', tier }, 'dashboard:exec-overview'), ).toThrowError(/may not read/); } - for (const tier of ["SERVING", "CURATED"] as const) { - expect(() => assertReadableByBiSurface({ dataset_id: "ds_1", tier }, "dashboard:x")).not.toThrow(); + for (const tier of ['SERVING', 'CURATED'] as const) { + expect(() => assertReadableByBiSurface({ dataset_id: 'ds_1', tier }, 'dashboard:x')).not.toThrow(); } }); - it("a recorded exception is the only way a RAW dataset reaches a BI surface", () => { + it('a recorded exception is the only way a RAW dataset reaches a BI surface', () => { expect(() => assertReadableByBiSurface( - { dataset_id: "ds_1", tier: "RAW", raw_access_exception: "INC-2026-09-06 forensic review, expires 2026-10-01" }, - "dashboard:incident", + { + dataset_id: 'ds_1', + tier: 'RAW', + raw_access_exception: 'INC-2026-09-06 forensic review, expires 2026-10-01', + }, + 'dashboard:incident', ), ).not.toThrow(); // An empty/whitespace "exception" is not an exception. expect(() => - assertReadableByBiSurface({ dataset_id: "ds_1", tier: "RAW", raw_access_exception: " " }, "dashboard:x"), + assertReadableByBiSurface( + { dataset_id: 'ds_1', tier: 'RAW', raw_access_exception: ' ' }, + 'dashboard:x', + ), ).toThrowError(/may not read/); }); - it("tier order is enforced: a CURATED dataset cannot be built from SERVING", () => { + it('tier order is enforced: a CURATED dataset cannot be built from SERVING', () => { expect(TIER_RANK.RAW).toBeLessThan(TIER_RANK.NORMALIZED); expect(TIER_RANK.NORMALIZED).toBeLessThan(TIER_RANK.CURATED); expect(TIER_RANK.CURATED).toBeLessThan(TIER_RANK.SERVING); - expect(() => assertTierLineageSane("CURATED", "SERVING")).toThrowError(/tier order/); - expect(() => assertTierLineageSane("SERVING", "CURATED")).not.toThrow(); - expect(() => assertTierLineageSane("CURATED", "CURATED")).not.toThrow(); + expect(() => assertTierLineageSane('CURATED', 'SERVING')).toThrowError(/tier order/); + expect(() => assertTierLineageSane('SERVING', 'CURATED')).not.toThrow(); + expect(() => assertTierLineageSane('CURATED', 'CURATED')).not.toThrow(); }); - it("sql/008 constrains tier at the DATABASE level, not only in TypeScript", () => { + it('sql/008 constrains tier at the DATABASE level, not only in TypeScript', () => { expect(MIGRATION).toContain("CHECK (tier IN ('RAW','NORMALIZED','CURATED','SERVING'))"); }); it("BI-P003's eight required declarations are all NOT NULL in sql/008", () => { - const block = MIGRATION.split("CREATE TABLE IF NOT EXISTS opt_analytics_dataset ")[1] ?? ""; - const body = block.slice(0, block.indexOf(");")); + const block = MIGRATION.split('CREATE TABLE IF NOT EXISTS opt_analytics_dataset ')[1] ?? ''; + const body = block.slice(0, block.indexOf(');')); for (const col of [ - "tier", - "schema_contract_id", - "owner_id", - "source_ref", - "freshness_slo_seconds", - "retention_days", - "classification", - "quality_rule_ids", + 'tier', + 'schema_contract_id', + 'owner_id', + 'source_ref', + 'freshness_slo_seconds', + 'retention_days', + 'classification', + 'quality_rule_ids', ]) { expect(new RegExp(`\\b${col}\\b[^,]*NOT NULL`).test(body), `${col} is nullable`).toBe(true); } diff --git a/tests/analytics/analytics-domain-postgres.realinfra.test.ts b/tests/analytics/analytics-domain-postgres.realinfra.test.ts index 81c7f92..09216d3 100644 --- a/tests/analytics/analytics-domain-postgres.realinfra.test.ts +++ b/tests/analytics/analytics-domain-postgres.realinfra.test.ts @@ -19,17 +19,17 @@ * found a table no file ever created; this suite keeps that class of defect * from coming back through sql/008. */ -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; -import postgres from "postgres"; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import postgres from 'postgres'; import { ANALYTICS_DOMAIN_ENTITIES, ANALYTICS_ENTITY_TABLES, IMMUTABLE_VERSION_ENTITIES, -} from "@/src/analytics/domain/entities"; -import { AnalyticsDomainRepository } from "@/src/analytics/repositories/analytics-domain-repository"; -import { contractHash, type DatasetContract } from "@/src/analytics/domain/contract"; +} from '@/src/analytics/domain/entities'; +import { AnalyticsDomainRepository } from '@/src/analytics/repositories/analytics-domain-repository'; +import { contractHash, type DatasetContract } from '@/src/analytics/domain/contract'; const TEST_DATABASE_URL = process.env.ANALYTICS_TEST_DATABASE_URL; const run = TEST_DATABASE_URL ? describe : describe.skip; @@ -39,12 +39,12 @@ const SCHEMA = `bi_p002_${Date.now().toString(36)}`; let sql: ReturnType | undefined; let reachable = false; -let unreachableReason = ""; +let unreachableReason = ''; /** Which sql/*.sql files the restore drill actually applied. */ const applied: string[] = []; -const ALPHA = { tenant_id: "t_alpha", project_id: "p_main" }; -const BETA = { tenant_id: "t_beta", project_id: "p_main" }; +const ALPHA = { tenant_id: 't_alpha', project_id: 'p_main' }; +const BETA = { tenant_id: 't_beta', project_id: 'p_main' }; function repo(): AnalyticsDomainRepository { return new AnalyticsDomainRepository(sql!); @@ -53,38 +53,43 @@ function repo(): AnalyticsDomainRepository { function contract(datasetId: string): DatasetContract { return { dataset_id: datasetId, - version: "1.0.0", - tier: "CURATED", - fields: [{ name: "id", type: "string", nullable: false, role: "identifier", primary_identifier: true }], - primary_identifiers: ["id"], + version: '1.0.0', + tier: 'CURATED', + fields: [{ name: 'id', type: 'string', nullable: false, role: 'identifier', primary_identifier: true }], + primary_identifiers: ['id'], dimensions: [], measures: [], freshness_slo_seconds: 3600, retention_days: 365, - pii_classification: "none", - source_lineage: ["src_x"], + pii_classification: 'none', + source_lineage: ['src_x'], quality_rules: [], - allowed_purposes: ["analytics"], + allowed_purposes: ['analytics'], }; } beforeAll(async () => { if (!TEST_DATABASE_URL) { - console.warn("[realinfra] ANALYTICS_TEST_DATABASE_URL not set — skipping honestly, not fabricating success."); + console.warn( + '[realinfra] ANALYTICS_TEST_DATABASE_URL not set — skipping honestly, not fabricating success.', + ); return; } try { sql = postgres(TEST_DATABASE_URL, { prepare: false, ssl: false, max: 3 }); await Promise.race([ sql`SELECT 1`, - new Promise((_, reject) => setTimeout(() => reject(new Error("connect timeout")), 5000)), + new Promise((_, reject) => setTimeout(() => reject(new Error('connect timeout')), 5000)), ]); // RESTORE DRILL: a brand-new schema, then every migration in order. await sql.unsafe(`CREATE SCHEMA ${SCHEMA}`); await sql.unsafe(`SET search_path TO ${SCHEMA}, public`); - const dir = path.join(process.cwd(), "sql"); - for (const file of fs.readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) { - const body = fs.readFileSync(path.join(dir, file), "utf8"); + const dir = path.join(process.cwd(), 'sql'); + for (const file of fs + .readdirSync(dir) + .filter((f) => f.endsWith('.sql')) + .sort()) { + const body = fs.readFileSync(path.join(dir, file), 'utf8'); try { await sql.unsafe(`SET search_path TO ${SCHEMA}, public; ${body}`); applied.push(file); @@ -109,17 +114,17 @@ afterAll(async () => { await sql?.end({ timeout: 5 }); }); -run("BI-P002 analytics domain model against a LIVE Postgres", () => { - it("restore drill: sql/008 applies cleanly on top of the replayed chain", (ctx) => { +run('BI-P002 analytics domain model against a LIVE Postgres', () => { + it('restore drill: sql/008 applies cleanly on top of the replayed chain', (ctx) => { if (!reachable) ctx.skip(); - expect(unreachableReason).toBe(""); - const p008 = applied.find((f) => f.startsWith("008_")); - expect(p008, `sql/008 was not applied. Applied: ${applied.join(", ")}`).toBe( - "008_analytics_domain_model.sql", + expect(unreachableReason).toBe(''); + const p008 = applied.find((f) => f.startsWith('008_')); + expect(p008, `sql/008 was not applied. Applied: ${applied.join(', ')}`).toBe( + '008_analytics_domain_model.sql', ); }); - it("all nineteen entity tables really exist after the restore", async (ctx) => { + it('all nineteen entity tables really exist after the restore', async (ctx) => { if (!reachable) ctx.skip(); const rows = await sql!<{ table_name: string }[]>` SELECT table_name FROM information_schema.tables WHERE table_schema = ${SCHEMA}`; @@ -128,33 +133,33 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { expect(missing).toEqual([]); }); - it("replaying sql/008 a second time is a no-op, not an error", async (ctx) => { + it('replaying sql/008 a second time is a no-op, not an error', async (ctx) => { if (!reachable) ctx.skip(); - const body = fs.readFileSync(path.join(process.cwd(), "sql", "008_analytics_domain_model.sql"), "utf8"); + const body = fs.readFileSync(path.join(process.cwd(), 'sql', '008_analytics_domain_model.sql'), 'utf8'); await expect(sql!.unsafe(`SET search_path TO ${SCHEMA}, public; ${body}`)).resolves.toBeDefined(); }); - it("persists a dataset and reads it back through the repository", async (ctx) => { + it('persists a dataset and reads it back through the repository', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); const created = await r.createDataset(ALPHA, { - dataset_id: "ds_alpha_1", - name: "Alpha listings", - tier: "CURATED", - schema_contract_id: "dc_alpha", - schema_contract_version: "1.0.0", - owner_id: "team_data", - source_ref: "src_feed", + dataset_id: 'ds_alpha_1', + name: 'Alpha listings', + tier: 'CURATED', + schema_contract_id: 'dc_alpha', + schema_contract_version: '1.0.0', + owner_id: 'team_data', + source_ref: 'src_feed', freshness_slo_seconds: 3600, quality_rule_ids: [], retention_days: 365, - classification: "none", - status: "active", - created_by: "usr_1", + classification: 'none', + status: 'active', + created_by: 'usr_1', }); - expect(created.tenant_id).toBe("t_alpha"); - const read = await r.getDataset(ALPHA, "ds_alpha_1"); - expect(read?.name).toBe("Alpha listings"); + expect(created.tenant_id).toBe('t_alpha'); + const read = await r.getDataset(ALPHA, 'ds_alpha_1'); + expect(read?.name).toBe('Alpha listings'); expect(read?.freshness_slo_seconds).toBe(3600); // Cross-check with direct SQL — proving the row is really in the table, // not an artifact of the repository's own bookkeeping. @@ -166,7 +171,7 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { it("TENANT ISOLATION: t_beta cannot read t_alpha's dataset, even with the exact id", async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); - expect(await r.getDataset(BETA, "ds_alpha_1")).toBeUndefined(); + expect(await r.getDataset(BETA, 'ds_alpha_1')).toBeUndefined(); expect(await r.listDatasets(BETA)).toEqual([]); // And the row IS there — the empty result is scoping, not absence. const direct = await sql!<{ c: string }[]>`SELECT count(*)::text AS c FROM opt_analytics_dataset`; @@ -177,34 +182,34 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { if (!reachable) ctx.skip(); await expect( repo().createDatasetVersion(BETA, { - dataset_id: "ds_alpha_1", - contract_hash: "deadbeef", - schema_contract_version: "1.0.0", + dataset_id: 'ds_alpha_1', + contract_hash: 'deadbeef', + schema_contract_version: '1.0.0', }), - ).rejects.toMatchObject({ code: "NOT_FOUND" }); + ).rejects.toMatchObject({ code: 'NOT_FOUND' }); }); - it("dataset versions are monotonic and carry the contract hash", async (ctx) => { + it('dataset versions are monotonic and carry the contract hash', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); - const c = contract("ds_alpha_1"); + const c = contract('ds_alpha_1'); const v1 = await r.createDatasetVersion(ALPHA, { - dataset_id: "ds_alpha_1", + dataset_id: 'ds_alpha_1', contract_hash: contractHash(c), - schema_contract_version: "1.0.0", + schema_contract_version: '1.0.0', }); const v2 = await r.createDatasetVersion(ALPHA, { - dataset_id: "ds_alpha_1", - contract_hash: contractHash({ ...c, version: "2.0.0" }), - schema_contract_version: "2.0.0", + dataset_id: 'ds_alpha_1', + contract_hash: contractHash({ ...c, version: '2.0.0' }), + schema_contract_version: '2.0.0', }); expect(v1.version).toBe(1); expect(v2.version).toBe(2); expect(v2.contract_hash).not.toBe(v1.contract_hash); - expect((await r.listDatasetVersions(ALPHA, "ds_alpha_1")).map((v) => v.version)).toEqual([1, 2]); + expect((await r.listDatasetVersions(ALPHA, 'ds_alpha_1')).map((v) => v.version)).toEqual([1, 2]); }); - it("IMMUTABILITY is enforced by the DATABASE: UPDATE and DELETE on a version row are refused", async (ctx) => { + it('IMMUTABILITY is enforced by the DATABASE: UPDATE and DELETE on a version row are refused', async (ctx) => { if (!reachable) ctx.skip(); // Direct SQL, bypassing every TypeScript `readonly` — the only test that // proves the guarantee survives a psql session. @@ -220,40 +225,40 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { const rows = await sql!<{ contract_hash: string }[]>` SELECT contract_hash FROM opt_analytics_dataset_version WHERE dataset_id = 'ds_alpha_1'`; expect(rows).toHaveLength(2); - for (const r of rows) expect(r.contract_hash).not.toBe("tampered"); + for (const r of rows) expect(r.contract_hash).not.toBe('tampered'); }); - it("all four immutable entities refuse an UPDATE, not just the one that was tested first", async (ctx) => { + it('all four immutable entities refuse an UPDATE, not just the one that was tested first', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); await r.createDataset(ALPHA, { - dataset_id: "ds_imm", - name: "Immutability fixture", - tier: "SERVING", - schema_contract_id: "dc_i", - schema_contract_version: "1.0.0", - owner_id: "team_data", - source_ref: "src", + dataset_id: 'ds_imm', + name: 'Immutability fixture', + tier: 'SERVING', + schema_contract_id: 'dc_i', + schema_contract_version: '1.0.0', + owner_id: 'team_data', + source_ref: 'src', freshness_slo_seconds: 60, quality_rule_ids: [], retention_days: 30, - classification: "none", - status: "active", + classification: 'none', + status: 'active', }); const metric = await r.createMetric(ALPHA, { - metric_id: "m_imm", - name: "immutable metric", - dataset_id: "ds_imm", - aggregation: "sum", - measure_field: "value", + metric_id: 'm_imm', + name: 'immutable metric', + dataset_id: 'ds_imm', + aggregation: 'sum', + measure_field: 'value', allowed_dimension_ids: [], - owner_id: "team_data", + owner_id: 'team_data', }); await r.createMetricVersion(ALPHA, { metric_id: metric.metric_id, definition_json: '{"v":1}' }); await r.createDatasetVersion(ALPHA, { - dataset_id: "ds_imm", - contract_hash: "h1", - schema_contract_version: "1.0.0", + dataset_id: 'ds_imm', + contract_hash: 'h1', + schema_contract_version: '1.0.0', }); await sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_semantic_model (semantic_model_id, tenant_id, project_id, name, version, model_json, model_hash, owner_id) @@ -275,21 +280,21 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { } }); - it("a metric version after the first must state WHY the number changed meaning", async (ctx) => { + it('a metric version after the first must state WHY the number changed meaning', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); await expect( - r.createMetricVersion(ALPHA, { metric_id: "m_imm", definition_json: '{"v":2}' }), + r.createMetricVersion(ALPHA, { metric_id: 'm_imm', definition_json: '{"v":2}' }), ).rejects.toThrow(); const v2 = await r.createMetricVersion(ALPHA, { - metric_id: "m_imm", + metric_id: 'm_imm', definition_json: '{"v":2}', - change_reason: "excluded internal test accounts from the numerator", + change_reason: 'excluded internal test accounts from the numerator', }); expect(v2.version).toBe(2); }); - it("a DatasetGrant without a purpose is refused by the DATABASE (purpose limitation)", async (ctx) => { + it('a DatasetGrant without a purpose is refused by the DATABASE (purpose limitation)', async (ctx) => { if (!reachable) ctx.skip(); // Regression pin: the first version of this constraint was written // `array_length(allowed_purposes, 1) >= 1`, which is NULL for '{}' — and a @@ -309,7 +314,7 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { ).resolves.toBeDefined(); }); - it("a BIApplicationBinding without a purpose is refused too, and carries no credential column", async (ctx) => { + it('a BIApplicationBinding without a purpose is refused too, and carries no credential column', async (ctx) => { if (!reachable) ctx.skip(); await expect( sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_bi_application_binding @@ -321,28 +326,28 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { SELECT column_name FROM information_schema.columns WHERE table_schema = ${SCHEMA} AND table_name = 'opt_analytics_bi_application_binding'`; const names = cols.map((c) => c.column_name.toLowerCase()); - for (const forbidden of ["password", "secret", "dsn", "token", "connection_string", "clickhouse_url"]) { + for (const forbidden of ['password', 'secret', 'dsn', 'token', 'connection_string', 'clickhouse_url']) { expect(names, `BI binding must never store ${forbidden}`).not.toContain(forbidden); } }); - it("the dimension catalog is tenant-scoped and FAILS CLOSED for a tenant with none", async (ctx) => { + it('the dimension catalog is tenant-scoped and FAILS CLOSED for a tenant with none', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); await r.createDimension(ALPHA, { - dimension_id: "dim_1", - name: "make", - dataset_id: "ds_imm", - field_name: "vehicle_make", - field_type: "string", - classification: "none", + dimension_id: 'dim_1', + name: 'make', + dataset_id: 'ds_imm', + field_name: 'vehicle_make', + field_type: 'string', + classification: 'none', }); - expect(await r.dimensionCatalog(ALPHA)).toEqual({ make: "vehicle_make" }); + expect(await r.dimensionCatalog(ALPHA)).toEqual({ make: 'vehicle_make' }); // t_beta declared nothing, so it may reference nothing. expect(await r.dimensionCatalog(BETA)).toEqual({}); }); - it("a dimension whose physical field is not identifier-safe is refused by the DATABASE", async (ctx) => { + it('a dimension whose physical field is not identifier-safe is refused by the DATABASE', async (ctx) => { if (!reachable) ctx.skip(); await expect( sql!.unsafe(`INSERT INTO ${SCHEMA}.opt_analytics_dimension_definition @@ -351,14 +356,14 @@ run("BI-P002 analytics domain model against a LIVE Postgres", () => { ).rejects.toThrow(); }); - it("the repository refuses to act without a verified tenant scope", async (ctx) => { + it('the repository refuses to act without a verified tenant scope', async (ctx) => { if (!reachable) ctx.skip(); const r = repo(); - await expect(r.getDataset({ tenant_id: "", project_id: "p" }, "ds_imm")).rejects.toMatchObject({ - code: "UNAUTHENTICATED", + await expect(r.getDataset({ tenant_id: '', project_id: 'p' }, 'ds_imm')).rejects.toMatchObject({ + code: 'UNAUTHENTICATED', }); - await expect(r.listDatasets({ tenant_id: "t_alpha", project_id: " " })).rejects.toMatchObject({ - code: "UNAUTHENTICATED", + await expect(r.listDatasets({ tenant_id: 't_alpha', project_id: ' ' })).rejects.toMatchObject({ + code: 'UNAUTHENTICATED', }); }); }); diff --git a/tests/analytics/analytics-engine-contract.test.ts b/tests/analytics/analytics-engine-contract.test.ts index e986474..1a0f75c 100644 --- a/tests/analytics/analytics-engine-contract.test.ts +++ b/tests/analytics/analytics-engine-contract.test.ts @@ -11,114 +11,114 @@ * REQUEST BODY (INF0 Wave 2). So the contract asserts that a request field * named `tenant_id` cannot influence the engine at all. */ -import { describe, it, expect } from "vitest"; -import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { describe, it, expect } from 'vitest'; +import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; import { SIAX_ANALYTICS_ENGINE_CAPABILITIES, type AnalyticsIdentity, type SiaxAnalyticsEngine, -} from "@/src/analytics/engine/siax-analytics-engine"; +} from '@/src/analytics/engine/siax-analytics-engine'; const IDENTITY: AnalyticsIdentity = { - tenant_id: "t_alpha", - project_id: "p_default", - principal_id: "usr_1", + tenant_id: 't_alpha', + project_id: 'p_default', + principal_id: 'usr_1', }; /** Reusable contract suite — future providers call this with their factory. */ export function analyticsEngineContract(name: string, make: () => SiaxAnalyticsEngine): void { describe(`SiaxAnalyticsEngine contract: ${name}`, () => { - it("implements all eight BI-P009 capabilities as callable methods", () => { + it('implements all eight BI-P009 capabilities as callable methods', () => { const engine = make() as unknown as Record; for (const cap of SIAX_ANALYTICS_ENGINE_CAPABILITIES) { - expect(typeof engine[cap], `${cap} is not a method`).toBe("function"); + expect(typeof engine[cap], `${cap} is not a method`).toBe('function'); } expect(SIAX_ANALYTICS_ENGINE_CAPABILITIES).toEqual([ - "health", - "executeMetric", - "executeQuery", - "estimateQuery", - "cancelQuery", - "explain", - "ingestProjection", - "refreshProjection", + 'health', + 'executeMetric', + 'executeQuery', + 'estimateQuery', + 'cancelQuery', + 'explain', + 'ingestProjection', + 'refreshProjection', ]); }); - it("names its provider", () => { + it('names its provider', () => { expect(make().provider.length).toBeGreaterThan(0); }); - it("refuses every tenant-scoped capability without a verified identity", async () => { + it('refuses every tenant-scoped capability without a verified identity', async () => { const engine = make(); - const blank = { tenant_id: "", project_id: "p", principal_id: "usr_1" }; - const noPrincipal = { tenant_id: "t_alpha", project_id: "p", principal_id: "" }; + const blank = { tenant_id: '', project_id: 'p', principal_id: 'usr_1' }; + const noPrincipal = { tenant_id: 't_alpha', project_id: 'p', principal_id: '' }; for (const bad of [blank, noPrincipal]) { - await expect(engine.executeMetric(bad, { metric: "m", aggregate: "sum" })).rejects.toMatchObject({ - code: "UNAUTHENTICATED", + await expect(engine.executeMetric(bad, { metric: 'm', aggregate: 'sum' })).rejects.toMatchObject({ + code: 'UNAUTHENTICATED', }); - await expect(engine.estimateQuery(bad, { metric: "m", aggregate: "sum" })).rejects.toMatchObject({ - code: "UNAUTHENTICATED", + await expect(engine.estimateQuery(bad, { metric: 'm', aggregate: 'sum' })).rejects.toMatchObject({ + code: 'UNAUTHENTICATED', }); - await expect(engine.ingestProjection(bad, { dataset_id: "ds", rows: [] })).rejects.toMatchObject({ - code: "UNAUTHENTICATED", + await expect(engine.ingestProjection(bad, { dataset_id: 'ds', rows: [] })).rejects.toMatchObject({ + code: 'UNAUTHENTICATED', }); - await expect(engine.refreshProjection(bad, "ds")).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); - await expect(engine.cancelQuery(bad, "q")).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); + await expect(engine.refreshProjection(bad, 'ds')).rejects.toMatchObject({ code: 'UNAUTHENTICATED' }); + await expect(engine.cancelQuery(bad, 'q')).rejects.toMatchObject({ code: 'UNAUTHENTICATED' }); } }); - it("a tenant_id smuggled into the REQUEST cannot influence the engine (INF0 Wave-2 bug class)", async () => { + it('a tenant_id smuggled into the REQUEST cannot influence the engine (INF0 Wave-2 bug class)', async () => { const engine = make(); - const smuggled = { metric: "m", aggregate: "sum", tenant_id: "t_victim" } as never; + const smuggled = { metric: 'm', aggregate: 'sum', tenant_id: 't_victim' } as never; // The request type has no tenant field; if a provider ever started // reading one, this call would target t_victim. The engine must still // fail on the IDENTITY, proving the request field is inert. await expect( - engine.executeMetric({ tenant_id: "", project_id: "p", principal_id: "u" }, smuggled), - ).rejects.toMatchObject({ code: "UNAUTHENTICATED" }); + engine.executeMetric({ tenant_id: '', project_id: 'p', principal_id: 'u' }, smuggled), + ).rejects.toMatchObject({ code: 'UNAUTHENTICATED' }); }); - it("cancelQuery refuses a query id belonging to another tenant", async () => { + it('cancelQuery refuses a query id belonging to another tenant', async () => { const engine = make(); - await expect(engine.cancelQuery(IDENTITY, "opt_deadbeefdead_someoneelses")).rejects.toMatchObject({ - code: "AUTHORIZATION_DENIED", + await expect(engine.cancelQuery(IDENTITY, 'opt_deadbeefdead_someoneelses')).rejects.toMatchObject({ + code: 'AUTHORIZATION_DENIED', }); }); - it("health() reports unhealthy rather than throwing when the engine is unreachable", async () => { + it('health() reports unhealthy rather than throwing when the engine is unreachable', async () => { const report = await make().health(); expect(report.healthy).toBe(false); expect(report.provider.length).toBeGreaterThan(0); // Never leak an engine body or a DSN into the operator-facing detail. - expect(report.detail ?? "").not.toMatch(/http|password|@/i); + expect(report.detail ?? '').not.toMatch(/http|password|@/i); }); }); } analyticsEngineContract( - "clickhouse (unreachable host — contract-level assertions only)", + 'clickhouse (unreachable host — contract-level assertions only)', () => new ClickHouseAnalyticsEngine({ - baseUrl: "http://clickhouse.invalid:8123", - database: "opt_contract_test", + baseUrl: 'http://clickhouse.invalid:8123', + database: 'opt_contract_test', timeoutMs: 500, }), ); -describe("BI-P009 refusals that are deliberate, not missing", () => { - it("executeQuery (analyst SQL) is declared and REFUSED until BI-P017/BI-P033", async () => { - const engine = new ClickHouseAnalyticsEngine({ baseUrl: "http://clickhouse.invalid:8123" }); - await expect(engine.executeQuery(IDENTITY, { sql: "SELECT 1", dataset_ids: [] })).rejects.toMatchObject({ - code: "CAPABILITY_UNAVAILABLE", +describe('BI-P009 refusals that are deliberate, not missing', () => { + it('executeQuery (analyst SQL) is declared and REFUSED until BI-P017/BI-P033', async () => { + const engine = new ClickHouseAnalyticsEngine({ baseUrl: 'http://clickhouse.invalid:8123' }); + await expect(engine.executeQuery(IDENTITY, { sql: 'SELECT 1', dataset_ids: [] })).rejects.toMatchObject({ + code: 'CAPABILITY_UNAVAILABLE', }); }); - it("the PRESERVED ClickHouseAnalytics is reachable through the engine, not replaced by it", () => { - const engine = new ClickHouseAnalyticsEngine({ baseUrl: "http://clickhouse.invalid:8123" }); - expect(engine.legacyAnalytics.constructor.name).toBe("ClickHouseAnalytics"); - expect(typeof engine.legacyAnalytics.queryMetric).toBe("function"); - expect(typeof engine.legacyAnalytics.ensureSchema).toBe("function"); - expect(typeof engine.legacyAnalytics.ingest).toBe("function"); + it('the PRESERVED ClickHouseAnalytics is reachable through the engine, not replaced by it', () => { + const engine = new ClickHouseAnalyticsEngine({ baseUrl: 'http://clickhouse.invalid:8123' }); + expect(engine.legacyAnalytics.constructor.name).toBe('ClickHouseAnalytics'); + expect(typeof engine.legacyAnalytics.queryMetric).toBe('function'); + expect(typeof engine.legacyAnalytics.ensureSchema).toBe('function'); + expect(typeof engine.legacyAnalytics.ingest).toBe('function'); }); }); diff --git a/tests/analytics/analytics-governance-postgres.realinfra.test.ts b/tests/analytics/analytics-governance-postgres.realinfra.test.ts new file mode 100644 index 0000000..d92ce10 --- /dev/null +++ b/tests/analytics/analytics-governance-postgres.realinfra.test.ts @@ -0,0 +1,349 @@ +/** + * tests/analytics/analytics-governance-postgres.realinfra.test.ts + * ================================================================ + * sql/009 against a REAL Postgres. + * + * A migration nobody ran is a migration that does not work, and a trigger + * nobody fired is a comment. So this applies sql/008 and sql/009 into a + * throwaway schema and then attempts, against the database, every write the + * constraints exist to refuse: + * + * BI-P031 UPDATE a CERTIFIED metric's meaning in place -> must RAISE. + * This is the layer that holds when something writes without going + * through the application: a migration script, a psql session, an + * admin tool. A TypeScript guard does not survive `psql`. + * BI-P030 UPDATE a published semantic model version's content -> must RAISE. + * Two ACTIVE versions of one model -> must violate a unique index. + * BI-P028 A currency-formatted metric with no currency -> must be refused. + * Certification without an actor or the two definitions -> refused. + * BI-P018 A `compiled_query_hash` that is not a 64-hex digest -> refused. + * BI-P021 UPDATE or DELETE a usage-ledger row -> must RAISE (append-only). + * + * Gated on ANALYTICS_TEST_DATABASE_URL, matching the convention in + * analytics-domain-postgres.realinfra.test.ts. With it set and Postgres down, + * this FAILS rather than skipping. + * + * docker run -d --name pg -p 25433:5432 -e POSTGRES_PASSWORD=test \ + * -e POSTGRES_DB=opt postgres:16-alpine + * ANALYTICS_TEST_DATABASE_URL=postgres://postgres:test@127.0.0.1:25433/opt \ + * pnpm vitest run tests/analytics/analytics-governance-postgres.realinfra.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import postgres from 'postgres'; + +const TEST_DATABASE_URL = process.env.ANALYTICS_TEST_DATABASE_URL; +const run = TEST_DATABASE_URL ? describe : describe.skip; +const SCHEMA = `bi_p009_${Date.now().toString(36)}`; + +let sql: ReturnType; + +/** Did the statement RAISE? Returns the message, or undefined if it succeeded. */ +async function refusalOf(fn: () => Promise): Promise { + try { + await fn(); + return undefined; + } catch (e) { + return String((e as Error).message); + } +} + +beforeAll(async () => { + if (!TEST_DATABASE_URL) return; + // The search_path is pinned on the CONNECTION rather than prefixed to every + // statement. Prefixing makes each call a multi-statement simple query, and + // postgres.js then returns the LAST statement's result — which silently + // turned `SELECT state FROM ...` into the connection object in the first + // version of this file, and made two assertions compare against nonsense. + const admin = postgres(TEST_DATABASE_URL, { max: 1, onnotice: () => undefined }); + await admin.unsafe(`CREATE SCHEMA IF NOT EXISTS ${SCHEMA}`); + await admin.end({ timeout: 5 }); + + sql = postgres(TEST_DATABASE_URL, { + max: 2, + onnotice: () => undefined, + connection: { search_path: SCHEMA }, + }); + for (const file of [ + 'sql/008_analytics_domain_model.sql', + 'sql/009_analytics_execution_metrics_governance.sql', + ]) { + await sql.unsafe(fs.readFileSync(path.join(process.cwd(), file), 'utf8')); + } + + // A dataset and a metric to govern. + await sql.unsafe(` + INSERT INTO opt_analytics_dataset + (dataset_id, tenant_id, project_id, name, tier, schema_contract_id, schema_contract_version, + owner_id, source_ref, freshness_slo_seconds, retention_days, classification, status) + VALUES ('ds_sales','t_alpha','p_main','sales','SERVING','c_sales','1.0.0', + 'u_1','0pt://sourcing/sales', 3600, 90, 'none', 'active') + ON CONFLICT DO NOTHING;`); +}, 120_000); + +afterAll(async () => { + if (!TEST_DATABASE_URL || !sql) return; + await sql.unsafe(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await sql.end({ timeout: 5 }); +}); + +run('sql/009 applies, and replays', () => { + it('is idempotent — a replay onto an already-migrated schema succeeds', async () => { + const text = fs.readFileSync( + path.join(process.cwd(), 'sql/009_analytics_execution_metrics_governance.sql'), + 'utf8', + ); + // The restore drill replays every file in order onto a live database. + await expect(sql.unsafe(text)).resolves.toBeDefined(); + }); + + it('added every BI-P018 column the execution record needs', async () => { + const rows = await sql.unsafe( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = '${SCHEMA}' AND table_name = 'opt_analytics_query_execution'`, + ); + const names = new Set((rows as unknown as Array<{ column_name: string }>).map((r) => r.column_name)); + for (const col of [ + 'principal_id', + 'workload', + 'compiled_query_hash', + 'datasets', + 'metrics', + 'cache_status', + 'cost_units', + ]) { + expect(names.has(col), `missing ${col}`).toBe(true); + } + // And the one it must NOT have. + expect(names.has('query_text')).toBe(false); + expect(names.has('sql')).toBe(false); + }); +}); + +run('BI-P031 — a CERTIFIED metric cannot be modified in place, enforced by the DATABASE', () => { + const base = ''; + + beforeAll(async () => { + await sql.unsafe(`${base} + INSERT INTO opt_analytics_metric_definition + (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, + allowed_dimension_ids, current_version, owner_id, description, + metric_type, time_dimension, unit, currency, display_format, status, filters_json, + certification, business_definition, technical_definition, meaning_hash, + certified_at, certified_by) + VALUES ('m_revenue','t_alpha','p_main','net_revenue','ds_sales','sum','amount_sek', + ARRAY['country'], 3, 'finance@siax.io', 'Invoiced revenue net of credits', + 'simple','invoiced_at','SEK','SEK','currency','ACTIVE','[]', + 'certified','What we invoiced','sum(amount_sek)', repeat('a',64), + now(), 'u_1') + ON CONFLICT (metric_id) DO NOTHING;`); + }); + + it('REFUSES an in-place change to the meaning hash', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_metric_definition + SET meaning_hash = repeat('b',64) WHERE metric_id = 'm_revenue'`), + ); + expect(message, "a certified metric's meaning was changed in place, in the database").toBeDefined(); + expect(message).toContain('BI-P031'); + expect(message).toContain('mint version 4'); + }); + + it('ALLOWS an editorial change that leaves the meaning hash alone', async () => { + await expect( + sql.unsafe(`${base} UPDATE opt_analytics_metric_definition + SET description = 'Clearer wording' WHERE metric_id = 'm_revenue'`), + ).resolves.toBeDefined(); + }); + + it('ALLOWS a meaning change that comes with a version bump', async () => { + await expect( + sql.unsafe(`${base} UPDATE opt_analytics_metric_definition + SET meaning_hash = repeat('c',64), current_version = 4 + WHERE metric_id = 'm_revenue'`), + ).resolves.toBeDefined(); + // Put it back for the remaining assertions. + await sql.unsafe(`${base} UPDATE opt_analytics_metric_definition + SET current_version = 3 WHERE metric_id = 'm_revenue'`); + }); + + it('refuses a currency-formatted metric with no currency', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_metric_definition + (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, + owner_id, display_format, currency, time_dimension) + VALUES ('m_bad','t_alpha','p_main','bad_metric','ds_sales','sum','x','u','currency',NULL,'t')`), + ); + expect(message).toContain('currency_pairing'); + }); + + it('refuses a currency on a non-currency format', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_metric_definition + (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, + owner_id, display_format, currency, time_dimension) + VALUES ('m_bad2','t_alpha','p_main','bad_metric2','ds_sales','sum','x','u','integer','SEK','t')`), + ); + expect(message).toContain('currency_pairing'); + }); + + it('refuses certification without an actor, a time, and both definitions', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_metric_definition + (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, + owner_id, display_format, time_dimension, certification) + VALUES ('m_bad3','t_alpha','p_main','bad_metric3','ds_sales','sum','x','u','decimal','t','certified')`), + ); + expect(message).toContain('certification_act'); + }); + + it('refuses an unknown lifecycle status', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_metric_definition + SET status = 'PROBABLY_FINE' WHERE metric_id = 'm_revenue'`), + ); + expect(message).toContain('metric_status'); + }); +}); + +run('BI-P030 — a published semantic model version is immutable, and only one is active', () => { + const base = ''; + + beforeAll(async () => { + for (const [id, version, status] of [ + ['smv_1', 1, 'active'], + ['smv_2', 2, 'draft'], + ] as const) { + await sql.unsafe(`${base} + INSERT INTO opt_analytics_semantic_model_version + (semantic_model_version_id, model_id, tenant_id, project_id, version, + datasets, relationships_json, metrics_json, dimensions_json, status, hash) + VALUES ('${id}','sm_sales','t_alpha','p_main',${version}, + ARRAY['ds_sales'],'[]','[]','[]','${status}', repeat('${version}',64)) + ON CONFLICT DO NOTHING;`); + } + }); + + it("REFUSES an update to a published version's content", async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_semantic_model_version + SET metrics_json = '[{"id":"m_new"}]' WHERE semantic_model_version_id = 'smv_1'`), + ); + expect(message, 'a pinned semantic model version was rewritten under a dashboard').toBeDefined(); + expect(message).toContain('BI-P030'); + }); + + it('ALLOWS a status transition, which is not a content change', async () => { + await expect( + sql.unsafe(`${base} UPDATE opt_analytics_semantic_model_version + SET status = 'superseded' WHERE semantic_model_version_id = 'smv_1'`), + ).resolves.toBeDefined(); + await sql.unsafe(`${base} UPDATE opt_analytics_semantic_model_version + SET status = 'active' WHERE semantic_model_version_id = 'smv_1'`); + }); + + it('refuses TWO active versions of the same model — half the dashboards would read the other one', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_semantic_model_version + SET status = 'active' WHERE semantic_model_version_id = 'smv_2'`), + ); + expect(message).toContain('opt_analytics_semantic_model_one_active'); + }); + + it('refuses a hash that is not a sha256 digest', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} + INSERT INTO opt_analytics_semantic_model_version + (semantic_model_version_id, model_id, tenant_id, project_id, version, + datasets, relationships_json, metrics_json, dimensions_json, status, hash) + VALUES ('smv_bad','sm_sales','t_alpha','p_main',9,ARRAY['ds_sales'],'[]','[]','[]','draft','not-a-hash')`), + ); + expect(message).toBeDefined(); + }); +}); + +run('BI-P018 / BI-P021 — the execution record and the usage ledger', () => { + const base = ''; + + it('refuses a compiled_query_hash that is not a 64-hex digest', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_query_execution + (execution_id, tenant_id, project_id, engine_query_id, state, provider, compiled_query_hash) + VALUES ('e_bad','t_alpha','p_main','q1','queued','clickhouse','SELECT * FROM sales')`), + ); + expect(message).toContain('hash_shape'); + }); + + it('accepts a proper execution record', async () => { + await expect( + sql.unsafe(`${base} INSERT INTO opt_analytics_query_execution + (execution_id, tenant_id, project_id, engine_query_id, state, provider, + compiled_query_hash, principal_id, workload, cache_status, cost_units) + VALUES ('e_ok','t_alpha','p_main','q1','succeeded','clickhouse', + repeat('a',64),'u_1','dashboard','hit',0.5)`), + ).resolves.toBeDefined(); + }); + + it('refuses an unknown cache status and an unknown workload', async () => { + expect( + await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_query_execution + SET cache_status = 'probably' WHERE execution_id = 'e_ok'`), + ), + ).toContain('cache_status'); + expect( + await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_query_execution + SET workload = 'vibes' WHERE execution_id = 'e_ok'`), + ), + ).toContain('workload'); + }); + + it('the usage ledger is APPEND-ONLY — update and delete both raise', async () => { + await sql.unsafe(`${base} INSERT INTO opt_analytics_usage_ledger + (usage_id, tenant_id, principal_id, query_id, engine, rows_scanned, bytes_scanned, + execution_ms, cost_units, terminated_by_budget) + VALUES ('u_1','t_alpha','p_1','q1','clickhouse',1000,2000,50,0.25,true)`); + + expect( + await refusalOf(() => + sql.unsafe(`${base} UPDATE opt_analytics_usage_ledger SET cost_units = 0 WHERE usage_id = 'u_1'`), + ), + 'a billing row was editable', + ).toBeDefined(); + expect( + await refusalOf(() => + sql.unsafe(`${base} DELETE FROM opt_analytics_usage_ledger WHERE usage_id = 'u_1'`), + ), + 'a billing row was deletable', + ).toBeDefined(); + }); + + it('an analyst statement refusal must carry a reason', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_sql_statement + (statement_id, tenant_id, principal_id, statement_normalized, statement_hash, outcome) + VALUES ('s_bad','t_alpha','u_1','SELECT ?', repeat('a',64), 'refused')`), + ); + expect(message, 'a refusal with no reason was accepted — an unreviewable audit trail').toBeDefined(); + }); + + it('a freshness row must carry a disclosure a UI can render', async () => { + const message = await refusalOf(() => + sql.unsafe(`${base} INSERT INTO opt_analytics_dataset_freshness + (tenant_id, dataset_id, state, expected_interval_ms, disclosure) + VALUES ('t_alpha','ds_sales','STALE',3600000,' ')`), + ); + expect(message).toBeDefined(); + }); + + it('freshness defaults to UNKNOWN, never to FRESH', async () => { + await sql.unsafe(`${base} INSERT INTO opt_analytics_dataset_freshness + (tenant_id, dataset_id, expected_interval_ms, disclosure) + VALUES ('t_beta','ds_sales',3600000,'not yet computed')`); + const rows = await sql.unsafe(`${base} SELECT state FROM opt_analytics_dataset_freshness + WHERE tenant_id = 't_beta'`); + expect((rows as unknown as Array<{ state: string }>)[0].state).toBe('UNKNOWN'); + }); +}); diff --git a/tests/analytics/arrow-fixture.test.ts b/tests/analytics/arrow-fixture.test.ts index 807dac5..bf7a171 100644 --- a/tests/analytics/arrow-fixture.test.ts +++ b/tests/analytics/arrow-fixture.test.ts @@ -10,45 +10,45 @@ * the SHIPPED module produces, not a copy of the encoder living in a * script. */ -import { describe, it, expect } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; -import crypto from "node:crypto"; -import { type ArrowColumn, encodeArrowStream } from "@/src/analytics/arrow/ipc"; +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { type ArrowColumn, encodeArrowStream } from '@/src/analytics/arrow/ipc'; /** The same three rows the ClickHouse side of the DoD produces. */ export const BI_P005_FIXTURE: ArrowColumn[] = [ - { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, - { name: "n", type: "int64", values: [1, 2, 3] }, - { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, - { name: "flag", type: "bool", values: [true, false, true] }, + { name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', 't_gamma'] }, + { name: 'n', type: 'int64', values: [1, 2, 3] }, + { name: 'v', type: 'float64', values: [1.5, 2.5, 3.25] }, + { name: 'flag', type: 'bool', values: [true, false, true] }, { - name: "ts", - type: "timestamp_ms", + name: 'ts', + type: 'timestamp_ms', values: [ - new Date("2026-01-01T00:00:00.000Z"), - new Date("2026-06-15T08:09:10.123Z"), - new Date("2026-09-06T12:34:56.789Z"), + new Date('2026-01-01T00:00:00.000Z'), + new Date('2026-06-15T08:09:10.123Z'), + new Date('2026-09-06T12:34:56.789Z'), ], }, ]; -describe("BI-P005 canonical Arrow fixture", () => { - it("encodes deterministically", () => { +describe('BI-P005 canonical Arrow fixture', () => { + it('encodes deterministically', () => { const a = encodeArrowStream(BI_P005_FIXTURE); const b = encodeArrowStream(BI_P005_FIXTURE); expect(Buffer.from(a).equals(Buffer.from(b))).toBe(true); expect(a.length).toBeGreaterThan(0); }); - it("writes the fixture for the external roundtrip when asked", () => { + it('writes the fixture for the external roundtrip when asked', () => { const dir = process.env.ARROW_FIXTURE_DIR; if (!dir) return; // not an error: the normal test run has nowhere to put it const bytes = encodeArrowStream(BI_P005_FIXTURE); - const out = path.join(dir, "02_0pt_encoded.arrow"); + const out = path.join(dir, '02_0pt_encoded.arrow'); fs.writeFileSync(out, bytes); expect(fs.statSync(out).size).toBe(bytes.length); // Print the digest so the evidence file can be tied to these exact bytes. - console.log(`ARROW_FIXTURE_SHA256=${crypto.createHash("sha256").update(bytes).digest("hex")}`); + console.log(`ARROW_FIXTURE_SHA256=${crypto.createHash('sha256').update(bytes).digest('hex')}`); }); }); diff --git a/tests/analytics/arrow-ipc.test.ts b/tests/analytics/arrow-ipc.test.ts index c734602..1dc8643 100644 --- a/tests/analytics/arrow-ipc.test.ts +++ b/tests/analytics/arrow-ipc.test.ts @@ -15,7 +15,7 @@ * That distinction is the point: a self-roundtrip that agrees with its own * mistakes is exactly the kind of green this estate has been burned by. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect } from 'vitest'; import { ARROW_TYPE_SUPPORT, arrowStreamSettings, @@ -23,11 +23,11 @@ import { clickhouseArrowStreamFormat, decodeArrowStream, encodeArrowStream, -} from "@/src/analytics/arrow/ipc"; +} from '@/src/analytics/arrow/ipc'; -describe("BI-P005 Arrow IPC", () => { - it("emits a well-formed IPC stream: continuation markers, schema, batch, EOS", () => { - const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [1, 2, 3] }]); +describe('BI-P005 Arrow IPC', () => { + it('emits a well-formed IPC stream: continuation markers, schema, batch, EOS', () => { + const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3] }]); const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); expect(dv.getUint32(0, true)).toBe(0xffffffff); // Last 8 bytes are the end-of-stream marker: 0xFFFFFFFF then length 0. @@ -35,66 +35,72 @@ describe("BI-P005 Arrow IPC", () => { expect(dv.getInt32(bytes.length - 4, true)).toBe(0); }); - it("round-trips every supported column type with nulls", () => { + it('round-trips every supported column type with nulls', () => { const bytes = encodeArrowStream([ - { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", null] }, - { name: "n", type: "int64", values: [1, 2, 3] }, - { name: "v", type: "float64", values: [1.5, 2.5, null] }, - { name: "flag", type: "bool", values: [true, false, true] }, - { name: "ts", type: "timestamp_ms", values: [new Date("2026-01-01T00:00:00Z"), null, new Date("2026-09-06T12:34:56.789Z")] }, + { name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', null] }, + { name: 'n', type: 'int64', values: [1, 2, 3] }, + { name: 'v', type: 'float64', values: [1.5, 2.5, null] }, + { name: 'flag', type: 'bool', values: [true, false, true] }, + { + name: 'ts', + type: 'timestamp_ms', + values: [new Date('2026-01-01T00:00:00Z'), null, new Date('2026-09-06T12:34:56.789Z')], + }, ]); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(3); expect(info.batchCount).toBe(1); - expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); + expect(info.fields.map((f) => f.name)).toEqual(['tenant_id', 'n', 'v', 'flag', 'ts']); // Arrow Type union tags: Int=2, FloatingPoint=3, Utf8=5, Bool=6, Timestamp=10. expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); }); - it("handles an empty result set without producing a malformed stream", () => { - const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [] }]); + it('handles an empty result set without producing a malformed stream', () => { + const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [] }]); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(0); - expect(info.fields).toEqual([{ name: "n", typeType: 2 }]); + expect(info.fields).toEqual([{ name: 'n', typeType: 2 }]); }); - it("rejects ragged columns rather than emitting a stream that decodes to garbage", () => { + it('rejects ragged columns rather than emitting a stream that decodes to garbage', () => { expect(() => encodeArrowStream([ - { name: "a", type: "int64", values: [1, 2] }, - { name: "b", type: "int64", values: [1] }, + { name: 'a', type: 'int64', values: [1, 2] }, + { name: 'b', type: 'int64', values: [1] }, ]), ).toThrowError(/expected 2/); expect(() => encodeArrowStream([])).toThrowError(/at least one column/); }); - it("refuses an int64 value outside the representable range instead of wrapping it", () => { + it('refuses an int64 value outside the representable range instead of wrapping it', () => { expect(() => - encodeArrowStream([{ name: "n", type: "int64", values: [BigInt("9223372036854775808")] }]), + encodeArrowStream([{ name: 'n', type: 'int64', values: [BigInt('9223372036854775808')] }]), ).toThrowError(/out of int64 range/); }); - it("rejects bytes that are not an Arrow IPC stream", () => { - expect(() => decodeArrowStream(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrowError(/not an Arrow IPC stream/); + it('rejects bytes that are not an Arrow IPC stream', () => { + expect(() => decodeArrowStream(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrowError( + /not an Arrow IPC stream/, + ); }); - it("BI-P012: Arrow is not a way around max_result_rows", () => { - const bytes = encodeArrowStream([{ name: "n", type: "int64", values: [1, 2, 3, 4, 5] }]); + it('BI-P012: Arrow is not a way around max_result_rows', () => { + const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3, 4, 5] }]); expect(() => assertArrowWithinResultBudget(bytes, 3)).toThrowError(/over the 3-row budget/); expect(assertArrowWithinResultBudget(bytes, 5).rowCount).toBe(5); }); - it("the ClickHouse pass-through asks for text Strings, not Binary", () => { - expect(clickhouseArrowStreamFormat()).toBe("ArrowStream"); + it('the ClickHouse pass-through asks for text Strings, not Binary', () => { + expect(clickhouseArrowStreamFormat()).toBe('ArrowStream'); // Without this setting ClickHouse emits String as Arrow Binary, which // pyarrow and DuckDB read as bytes — silent semantic loss. - expect(arrowStreamSettings().output_format_arrow_string_as_string).toBe("1"); + expect(arrowStreamSettings().output_format_arrow_string_as_string).toBe('1'); }); - it("documents the supported type mapping including where loss WOULD occur", () => { + it('documents the supported type mapping including where loss WOULD occur', () => { expect(ARROW_TYPE_SUPPORT.length).toBeGreaterThanOrEqual(5); const arrowTypes = ARROW_TYPE_SUPPORT.map((r) => r.arrow); - expect(new Set(arrowTypes)).toEqual(new Set(["int64", "float64", "utf8", "bool", "timestamp_ms"])); + expect(new Set(arrowTypes)).toEqual(new Set(['int64', 'float64', 'utf8', 'bool', 'timestamp_ms'])); // Every row states its caveat; an empty note would be a silent promise. for (const row of ARROW_TYPE_SUPPORT) expect(row.note.length).toBeGreaterThan(10); }); diff --git a/tests/analytics/dataset-contract.test.ts b/tests/analytics/dataset-contract.test.ts index 769af32..38a59ed 100644 --- a/tests/analytics/dataset-contract.test.ts +++ b/tests/analytics/dataset-contract.test.ts @@ -7,7 +7,7 @@ * be ACCEPTED). A gate that only ever says yes, or only ever says no, would * pass a one-sided test; this one cannot. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect } from 'vitest'; import { assertContractCompatible, checkContractCompatibility, @@ -16,28 +16,30 @@ import { parseSemVer, validateContract, type DatasetContract, -} from "@/src/analytics/domain/contract"; +} from '@/src/analytics/domain/contract'; function base(): DatasetContract { return { - dataset_id: "ds_vehicle_listings", - version: "1.2.0", - tier: "CURATED", + dataset_id: 'ds_vehicle_listings', + version: '1.2.0', + tier: 'CURATED', fields: [ - { name: "listing_id", type: "string", nullable: false, role: "identifier", primary_identifier: true }, - { name: "make", type: "string", nullable: false, role: "dimension" }, - { name: "asking_price", type: "int64", nullable: true, role: "measure" }, - { name: "listed_at", type: "timestamp", nullable: false, role: "attribute" }, + { name: 'listing_id', type: 'string', nullable: false, role: 'identifier', primary_identifier: true }, + { name: 'make', type: 'string', nullable: false, role: 'dimension' }, + { name: 'asking_price', type: 'int64', nullable: true, role: 'measure' }, + { name: 'listed_at', type: 'timestamp', nullable: false, role: 'attribute' }, ], - primary_identifiers: ["listing_id"], - dimensions: ["make"], - measures: ["asking_price"], + primary_identifiers: ['listing_id'], + dimensions: ['make'], + measures: ['asking_price'], freshness_slo_seconds: 3600, retention_days: 730, - pii_classification: "none", - source_lineage: ["src_dealer_feed"], - quality_rules: [{ rule_id: "qr_listing_id_unique", kind: "unique", field_name: "listing_id", severity: "block" }], - allowed_purposes: ["market_analytics"], + pii_classification: 'none', + source_lineage: ['src_dealer_feed'], + quality_rules: [ + { rule_id: 'qr_listing_id_unique', kind: 'unique', field_name: 'listing_id', severity: 'block' }, + ], + allowed_purposes: ['market_analytics'], }; } @@ -50,93 +52,223 @@ function next(mutate: (c: DatasetContract) => void, version: string): DatasetCon } const BREAKING_CASES: { name: string; mutate: (c: DatasetContract) => void; code: string }[] = [ - { name: "field removed", mutate: (c) => { c.fields = c.fields.filter((f) => f.name !== "make"); c.dimensions = []; }, code: "field_removed" }, - { name: "field type changed", mutate: (c) => { c.fields[2].type = "float64"; }, code: "field_type_changed" }, - { name: "field became nullable", mutate: (c) => { c.fields[1].nullable = true; }, code: "field_became_nullable" }, - { name: "field role changed", mutate: (c) => { c.fields[1].role = "measure"; }, code: "field_role_changed" }, - { name: "required field added", mutate: (c) => { c.fields.push({ name: "vat_rate", type: "float64", nullable: false, role: "attribute" }); }, code: "required_field_added" }, - { name: "primary identifier changed", mutate: (c) => { c.primary_identifiers = ["make"]; }, code: "primary_identifier_changed" }, - { name: "dimension removed", mutate: (c) => { c.dimensions = []; }, code: "dimension_removed" }, - { name: "measure removed", mutate: (c) => { c.measures = []; }, code: "measure_removed" }, - { name: "freshness SLO loosened", mutate: (c) => { c.freshness_slo_seconds = 86400; }, code: "freshness_slo_loosened" }, - { name: "retention shortened", mutate: (c) => { c.retention_days = 30; }, code: "retention_shortened" }, - { name: "allowed purpose removed", mutate: (c) => { c.allowed_purposes = []; }, code: "purpose_removed" }, - { name: "PII classification raised", mutate: (c) => { c.pii_classification = "personal"; }, code: "pii_classification_raised" }, - { name: "blocking quality rule removed", mutate: (c) => { c.quality_rules = []; }, code: "blocking_quality_rule_removed" }, - { name: "blocking quality rule downgraded", mutate: (c) => { c.quality_rules[0].severity = "warn"; }, code: "blocking_quality_rule_downgraded" }, - { name: "tier changed", mutate: (c) => { c.tier = "SERVING"; }, code: "tier_changed" }, + { + name: 'field removed', + mutate: (c) => { + c.fields = c.fields.filter((f) => f.name !== 'make'); + c.dimensions = []; + }, + code: 'field_removed', + }, + { + name: 'field type changed', + mutate: (c) => { + c.fields[2].type = 'float64'; + }, + code: 'field_type_changed', + }, + { + name: 'field became nullable', + mutate: (c) => { + c.fields[1].nullable = true; + }, + code: 'field_became_nullable', + }, + { + name: 'field role changed', + mutate: (c) => { + c.fields[1].role = 'measure'; + }, + code: 'field_role_changed', + }, + { + name: 'required field added', + mutate: (c) => { + c.fields.push({ name: 'vat_rate', type: 'float64', nullable: false, role: 'attribute' }); + }, + code: 'required_field_added', + }, + { + name: 'primary identifier changed', + mutate: (c) => { + c.primary_identifiers = ['make']; + }, + code: 'primary_identifier_changed', + }, + { + name: 'dimension removed', + mutate: (c) => { + c.dimensions = []; + }, + code: 'dimension_removed', + }, + { + name: 'measure removed', + mutate: (c) => { + c.measures = []; + }, + code: 'measure_removed', + }, + { + name: 'freshness SLO loosened', + mutate: (c) => { + c.freshness_slo_seconds = 86400; + }, + code: 'freshness_slo_loosened', + }, + { + name: 'retention shortened', + mutate: (c) => { + c.retention_days = 30; + }, + code: 'retention_shortened', + }, + { + name: 'allowed purpose removed', + mutate: (c) => { + c.allowed_purposes = []; + }, + code: 'purpose_removed', + }, + { + name: 'PII classification raised', + mutate: (c) => { + c.pii_classification = 'personal'; + }, + code: 'pii_classification_raised', + }, + { + name: 'blocking quality rule removed', + mutate: (c) => { + c.quality_rules = []; + }, + code: 'blocking_quality_rule_removed', + }, + { + name: 'blocking quality rule downgraded', + mutate: (c) => { + c.quality_rules[0].severity = 'warn'; + }, + code: 'blocking_quality_rule_downgraded', + }, + { + name: 'tier changed', + mutate: (c) => { + c.tier = 'SERVING'; + }, + code: 'tier_changed', + }, ]; -describe("BI-P004 compatibility gate", () => { - it.each(BREAKING_CASES)("REFUSES $name under a minor bump", ({ mutate, code }) => { - const verdict = checkContractCompatibility(base(), next(mutate, "1.3.0")); +describe('BI-P004 compatibility gate', () => { + it.each(BREAKING_CASES)('REFUSES $name under a minor bump', ({ mutate, code }) => { + const verdict = checkContractCompatibility(base(), next(mutate, '1.3.0')); expect(verdict.compatible).toBe(false); expect(verdict.breaking.map((b) => b.code)).toContain(code); expect(verdict.reason).toMatch(/MAJOR version bump/); }); - it.each(BREAKING_CASES)("ACCEPTS $name once explicitly versioned (major bump)", ({ mutate, code }) => { - const verdict = checkContractCompatibility(base(), next(mutate, "2.0.0")); + it.each(BREAKING_CASES)('ACCEPTS $name once explicitly versioned (major bump)', ({ mutate, code }) => { + const verdict = checkContractCompatibility(base(), next(mutate, '2.0.0')); expect(verdict.compatible).toBe(true); expect(verdict.explicitly_versioned).toBe(true); expect(verdict.breaking.map((b) => b.code)).toContain(code); }); - it("a patch bump is equally refused — the gate is about the MAJOR, not about moving at all", () => { - const verdict = checkContractCompatibility(base(), next((c) => c.fields.pop(), "1.2.1")); + it('a patch bump is equally refused — the gate is about the MAJOR, not about moving at all', () => { + const verdict = checkContractCompatibility( + base(), + next((c) => c.fields.pop(), '1.2.1'), + ); expect(verdict.compatible).toBe(false); }); - it("allows genuinely additive changes without a major bump", () => { + it('allows genuinely additive changes without a major bump', () => { const additive = next((c) => { - c.fields.push({ name: "mileage_km", type: "int64", nullable: true, role: "measure" }); - c.measures.push("mileage_km"); - c.quality_rules.push({ rule_id: "qr_mileage_range", kind: "range", field_name: "mileage_km", severity: "warn" }); - c.allowed_purposes.push("valuation"); + c.fields.push({ name: 'mileage_km', type: 'int64', nullable: true, role: 'measure' }); + c.measures.push('mileage_km'); + c.quality_rules.push({ + rule_id: 'qr_mileage_range', + kind: 'range', + field_name: 'mileage_km', + severity: 'warn', + }); + c.allowed_purposes.push('valuation'); c.retention_days = 1095; c.freshness_slo_seconds = 1800; - }, "1.3.0"); + }, '1.3.0'); const verdict = checkContractCompatibility(base(), additive); expect(verdict.compatible).toBe(true); expect(verdict.breaking).toEqual([]); expect(verdict.explicitly_versioned).toBe(false); }); - it("refuses a version that does not move forward", () => { - expect(checkContractCompatibility(base(), next(() => {}, "1.2.0")).compatible).toBe(false); - expect(checkContractCompatibility(base(), next(() => {}, "1.1.9")).compatible).toBe(false); - expect(checkContractCompatibility(base(), next(() => {}, "0.9.0")).compatible).toBe(false); + it('refuses a version that does not move forward', () => { + expect( + checkContractCompatibility( + base(), + next(() => {}, '1.2.0'), + ).compatible, + ).toBe(false); + expect( + checkContractCompatibility( + base(), + next(() => {}, '1.1.9'), + ).compatible, + ).toBe(false); + expect( + checkContractCompatibility( + base(), + next(() => {}, '0.9.0'), + ).compatible, + ).toBe(false); }); - it("treats date -> timestamp as a safe widening, and int64 -> float64 as NOT one", () => { + it('treats date -> timestamp as a safe widening, and int64 -> float64 as NOT one', () => { const widened = next((c) => { - c.fields.push({ name: "sold_on", type: "date", nullable: true, role: "attribute" }); - }, "1.3.0"); + c.fields.push({ name: 'sold_on', type: 'date', nullable: true, role: 'attribute' }); + }, '1.3.0'); const later = JSON.parse(JSON.stringify(widened)) as DatasetContract; - later.fields[later.fields.length - 1].type = "timestamp"; - later.version = "1.4.0"; + later.fields[later.fields.length - 1].type = 'timestamp'; + later.version = '1.4.0'; expect(checkContractCompatibility(widened, later).compatible).toBe(true); - const lossy = next((c) => { c.fields[2].type = "float64"; }, "1.3.0"); + const lossy = next((c) => { + c.fields[2].type = 'float64'; + }, '1.3.0'); expect(checkContractCompatibility(base(), lossy).compatible).toBe(false); }); - it("assertContractCompatible throws a CONFLICT rather than returning a verdict nobody reads", () => { - expect(() => assertContractCompatible(base(), next((c) => c.fields.pop(), "1.3.0"))).toThrowError(/MAJOR/); - expect(() => assertContractCompatible(base(), next(() => {}, "1.3.0"))).not.toThrow(); + it('assertContractCompatible throws a CONFLICT rather than returning a verdict nobody reads', () => { + expect(() => + assertContractCompatible( + base(), + next((c) => c.fields.pop(), '1.3.0'), + ), + ).toThrowError(/MAJOR/); + expect(() => + assertContractCompatible( + base(), + next(() => {}, '1.3.0'), + ), + ).not.toThrow(); }); - it("refuses to compare contracts for different datasets", () => { - const other = next((c) => { c.dataset_id = "ds_other"; }, "1.3.0"); + it('refuses to compare contracts for different datasets', () => { + const other = next((c) => { + c.dataset_id = 'ds_other'; + }, '1.3.0'); expect(() => diffContracts(base(), other)).toThrowError(/SAME dataset/); }); - it("validateContract catches internally inconsistent contracts", () => { + it('validateContract catches internally inconsistent contracts', () => { expect(() => validateContract(base())).not.toThrow(); - expect(() => validateContract({ ...base(), version: "1.2" })).toThrowError(/major.minor.patch/); + expect(() => validateContract({ ...base(), version: '1.2' })).toThrowError(/major.minor.patch/); expect(() => validateContract({ ...base(), fields: [] })).toThrowError(/no fields/); - expect(() => validateContract({ ...base(), dimensions: ["nope"] })).toThrowError(/unknown field/); - expect(() => validateContract({ ...base(), primary_identifiers: [] })).toThrowError(/no primary identifier/); + expect(() => validateContract({ ...base(), dimensions: ['nope'] })).toThrowError(/unknown field/); + expect(() => validateContract({ ...base(), primary_identifiers: [] })).toThrowError( + /no primary identifier/, + ); expect(() => validateContract({ ...base(), allowed_purposes: [] })).toThrowError(/allowed_purposes/); expect(() => validateContract({ ...base(), freshness_slo_seconds: 0 })).toThrowError(/freshness/); const dup = base(); @@ -144,7 +276,7 @@ describe("BI-P004 compatibility gate", () => { expect(() => validateContract(dup)).toThrowError(/duplicate field/); }); - it("contractHash is order-insensitive over fields but sensitive to meaning", () => { + it('contractHash is order-insensitive over fields but sensitive to meaning', () => { const a = base(); const reordered = JSON.parse(JSON.stringify(a)) as DatasetContract; reordered.fields.reverse(); @@ -155,9 +287,9 @@ describe("BI-P004 compatibility gate", () => { expect(contractHash(changed)).not.toBe(contractHash(a)); }); - it("parseSemVer rejects anything that is not major.minor.patch", () => { - expect(parseSemVer("2.10.3")).toEqual({ major: 2, minor: 10, patch: 3 }); - for (const bad of ["v1.0.0", "1.0", "1.0.0-rc1", "latest", ""]) { + it('parseSemVer rejects anything that is not major.minor.patch', () => { + expect(parseSemVer('2.10.3')).toEqual({ major: 2, minor: 10, patch: 3 }); + for (const bad of ['v1.0.0', '1.0', '1.0.0-rc1', 'latest', '']) { expect(() => parseSemVer(bad)).toThrowError(/major.minor.patch/); } }); diff --git a/tests/analytics/dbt-contract.test.ts b/tests/analytics/dbt-contract.test.ts new file mode 100644 index 0000000..29a7a0d --- /dev/null +++ b/tests/analytics/dbt-contract.test.ts @@ -0,0 +1,321 @@ +/** + * tests/analytics/dbt-contract.test.ts + * ===================================== + * BI-P022 — DBT CORE CANONICAL TRANSFORMATION (the CONTRACT). + * + * This repo does not run dbt; N0D (n0d#38) does, against this contract. So + * what is testable here is exactly what the contract must refuse, and the + * interesting refusals are the ones that look like successes: + * + * * `status: "success"` with ZERO nodes executed — a dbt selector that + * matches nothing exits 0. The dbt-shaped false green. + * * a result whose observed dbt version differs from the pin — the models + * mean something different and nothing says so. + * * a branch name as the revision — "main" today and "main" tomorrow are + * different projects, and BI-P022's DoD is about reproducibility. + * * a production run against an unreviewed runtime, which is BI-P022's + * "after current license/security verification" turned into a gate. + * + * The scaffold itself is asserted against the filesystem, because a directory + * convention that nothing checks is a directory convention that drifts. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + assertDbtRuntimeCertified, + assertLayerDependenciesLegal, + assertValidDbtJobRequest, + assertValidDbtJobResult, + DBT_LAYER_TIERS, + DBT_LAYERS, + DBT_RUNTIME_PIN, + REQUIRED_DBT_ARTIFACTS, + REQUIRED_DBT_DIRECTORIES, + tierForModelPath, + type DbtJobRequest, + type DbtJobResult, +} from '@/src/analytics/transform/dbt-contract'; + +const DBT_ROOT = path.join(process.cwd(), 'analytics/dbt'); +const SHA = 'a'.repeat(40); + +const CERTIFIED_PIN = { + ...DBT_RUNTIME_PIN, + licenseReviewed: true, + securityReviewed: true, + reviewedBy: 'u_1', + reviewedAt: '2026-09-07T00:00:00Z', +}; + +function request(over: Partial = {}): DbtJobRequest { + return { + tenant_id: 't_alpha', + project_id: 'p_main', + dbt_project: 'siax_analytics', + revision: SHA, + command: 'build', + target_layer: 'marts', + environment: 'ci', + runtime: DBT_RUNTIME_PIN, + idempotency_key: 'idem-1', + expected_target_schema: 't_alpha_marts', + ...over, + }; +} + +function result(over: Partial = {}): DbtJobResult { + return { + tenant_id: 't_alpha', + dbt_project: 'siax_analytics', + revision: SHA, + command: 'build', + status: 'success', + started_at: '2026-09-07T00:00:00Z', + finished_at: '2026-09-07T00:05:00Z', + artifacts: { + 'manifest.json': 'st0re://a/manifest.json', + 'run_results.json': 'st0re://a/run_results.json', + }, + nodes: [ + { unique_id: 'model.siax_analytics.mart_sales_daily', resource_type: 'model', status: 'success' }, + ], + observed_dbt_version: DBT_RUNTIME_PIN.version, + ...over, + }; +} + +describe('BI-P022 — the scaffold exists and matches the declared conventions', () => { + it.each([...REQUIRED_DBT_DIRECTORIES])('analytics/dbt/%s exists', (dir) => { + expect(fs.existsSync(path.join(DBT_ROOT, dir)), `${dir} is missing`).toBe(true); + }); + + it('declares a reference project covering raw -> staging -> curated -> serving', () => { + expect(fs.existsSync(path.join(DBT_ROOT, 'models/sources/sources.yml'))).toBe(true); + expect(fs.existsSync(path.join(DBT_ROOT, 'models/staging/stg_sales_events.sql'))).toBe(true); + expect(fs.existsSync(path.join(DBT_ROOT, 'models/intermediate/int_sales_enriched.sql'))).toBe(true); + expect(fs.existsSync(path.join(DBT_ROOT, 'models/marts/mart_sales_daily.sql'))).toBe(true); + expect(fs.existsSync(path.join(DBT_ROOT, 'snapshots/snap_customers.sql'))).toBe(true); + }); + + it('the staging model applies the tenant predicate at the first modelled layer', () => { + const sql = fs.readFileSync(path.join(DBT_ROOT, 'models/staging/stg_sales_events.sql'), 'utf8'); + expect(sql).toContain('where tenant_id = \'{{ var("siax_tenant_id") }}\''); + }); + + it('the mart reads intermediate, never a source directly', () => { + const sql = fs.readFileSync(path.join(DBT_ROOT, 'models/marts/mart_sales_daily.sql'), 'utf8'); + expect(sql).toContain("ref('int_sales_enriched')"); + expect(sql).not.toContain('source('); + }); + + it('ships a test that fails on an EMPTY mart', () => { + // Without it, every column test passes trivially over zero rows. + const sql = fs.readFileSync(path.join(DBT_ROOT, 'tests/assert_mart_is_not_empty.sql'), 'utf8'); + expect(sql).toContain('= 0'); + }); + + it('ships a cross-tenant test even though it should be impossible to fail', () => { + const sql = fs.readFileSync(path.join(DBT_ROOT, 'tests/assert_no_cross_tenant_rows.sql'), 'utf8'); + expect(sql).toContain('tenant_id !='); + }); + + it("maps the four layers onto BI-P003's tiers", () => { + expect(DBT_LAYER_TIERS).toEqual({ + sources: 'RAW', + staging: 'NORMALIZED', + intermediate: 'CURATED', + marts: 'SERVING', + }); + expect([...DBT_LAYERS]).toEqual(['sources', 'staging', 'intermediate', 'marts']); + }); + + it('assigns a tier from a model path, and refuses one it cannot place', () => { + expect(tierForModelPath('models/marts/mart_sales_daily.sql')).toBe('SERVING'); + expect(tierForModelPath('analytics/dbt/models/staging/x.sql')).toBe('NORMALIZED'); + expect(() => tierForModelPath('models/scratch/x.sql')).toThrowError(SiaxError); + }); +}); + +describe('BI-P022 — the version rule is a GATE, not a paragraph', () => { + it('the shipped pin is honestly UNREVIEWED', () => { + // If someone flips these without doing the review, that is a deliberate + // act visible in a diff — which is the point of storing the state. + expect(DBT_RUNTIME_PIN.licenseReviewed).toBe(false); + expect(DBT_RUNTIME_PIN.securityReviewed).toBe(false); + expect(DBT_RUNTIME_PIN.line).toBe('stable'); + }); + + it('refuses a PRODUCTION run against an unreviewed runtime', () => { + expect(() => assertDbtRuntimeCertified(DBT_RUNTIME_PIN, 'production')).toThrowError(SiaxError); + }); + + it('allows development and CI against the same unreviewed runtime', () => { + expect(() => assertDbtRuntimeCertified(DBT_RUNTIME_PIN, 'ci')).not.toThrow(); + expect(() => assertDbtRuntimeCertified(DBT_RUNTIME_PIN, 'development')).not.toThrow(); + }); + + it('allows production once both reviews are recorded', () => { + expect(() => assertDbtRuntimeCertified(CERTIFIED_PIN, 'production')).not.toThrow(); + }); + + it('refuses an ALPHA line in production without a separate certification reference', () => { + const alpha = { ...CERTIFIED_PIN, line: 'alpha' as const }; + expect(() => assertDbtRuntimeCertified(alpha, 'production')).toThrowError(SiaxError); + expect(() => + assertDbtRuntimeCertified({ ...alpha, separateCertificationRef: 'CERT-2026-11' }, 'production'), + ).not.toThrow(); + }); + + it('refuses a version range — a pin is exact or it is not a pin', () => { + expect(() => assertDbtRuntimeCertified({ ...CERTIFIED_PIN, version: '1.10' }, 'ci')).toThrowError( + SiaxError, + ); + expect(() => assertDbtRuntimeCertified({ ...CERTIFIED_PIN, version: '^1.10.0' }, 'ci')).toThrowError( + SiaxError, + ); + }); +}); + +describe('BI-P022 — the job request', () => { + it('accepts a well-formed request', () => { + expect(() => assertValidDbtJobRequest(request())).not.toThrow(); + }); + + it("refuses a branch name as the revision — BI-P022's DoD is reproducibility", () => { + for (const rev of ['main', 'HEAD', 'v1.0.0', 'a'.repeat(39)]) { + expect(() => assertValidDbtJobRequest(request({ revision: rev }))).toThrowError(SiaxError); + } + }); + + it('refuses a target schema that is not scoped to the tenant', () => { + let thrown: unknown; + try { + assertValidDbtJobRequest(request({ expected_target_schema: 't_beta_marts' })); + } catch (e) { + thrown = e; + } + expect((thrown as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + }); + + it('refuses a selector carrying shell metacharacters', () => { + for (const selector of ['stg_sales; rm -rf /', '$(whoami)', 'a`b`', 'x && y', 'a|b']) { + expect(() => assertValidDbtJobRequest(request({ select: [selector] }))).toThrowError(SiaxError); + } + // ...while accepting real dbt selector syntax. + expect(() => + assertValidDbtJobRequest( + request({ select: ['tag:tier:SERVING+', 'state:modified', '+mart_sales_daily'] }), + ), + ).not.toThrow(); + }); + + it('refuses a job targeting the sources layer — sources are declared, not built', () => { + expect(() => assertValidDbtJobRequest(request({ target_layer: 'sources' }))).toThrowError(SiaxError); + }); + + it('refuses a request with no idempotency key', () => { + expect(() => assertValidDbtJobRequest(request({ idempotency_key: '' }))).toThrowError(SiaxError); + }); + + it('refuses a production request on the unreviewed pin', () => { + expect(() => + assertValidDbtJobRequest( + request({ environment: 'production', expected_target_schema: 't_alpha_marts' }), + ), + ).toThrowError(SiaxError); + }); +}); + +describe('BI-P022 — the job RESULT is untrusted input', () => { + it('accepts a well-formed result', () => { + expect(() => assertValidDbtJobResult(result(), request())).not.toThrow(); + }); + + it('REFUSES success with zero nodes — a selector matching nothing exits 0', () => { + let thrown: unknown; + try { + assertValidDbtJobResult(result({ nodes: [] }), request()); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(SiaxError); + expect((thrown as SiaxError).message).toContain('zero nodes'); + }); + + it('refuses success while nodes failed', () => { + expect(() => + assertValidDbtJobResult( + result({ nodes: [{ unique_id: 't1', resource_type: 'test', status: 'fail' }] }), + request(), + ), + ).toThrowError(SiaxError); + }); + + it('refuses a runtime that is not the pinned one', () => { + expect(() => assertValidDbtJobResult(result({ observed_dbt_version: '1.9.0' }), request())).toThrowError( + SiaxError, + ); + }); + + it('refuses a result for a different revision', () => { + expect(() => assertValidDbtJobResult(result({ revision: 'b'.repeat(40) }), request())).toThrowError( + SiaxError, + ); + }); + + it('refuses a result for a different tenant', () => { + expect(() => assertValidDbtJobResult(result({ tenant_id: 't_beta' }), request())).toThrowError(SiaxError); + }); + + it.each([...REQUIRED_DBT_ARTIFACTS])('refuses a result missing %s', (artifact) => { + const artifacts = { ...result().artifacts }; + delete artifacts[artifact]; + expect(() => assertValidDbtJobResult(result({ artifacts }), request())).toThrowError(SiaxError); + }); +}); + +describe('BI-P022 — layer dependencies are checked against the graph, not the convention', () => { + it("accepts the reference project's own edges", () => { + expect(() => + assertLayerDependenciesLegal([ + { from: 'models/staging/stg_sales_events.sql', to: 'models/sources/sources.yml' }, + { from: 'models/intermediate/int_sales_enriched.sql', to: 'models/staging/stg_sales_events.sql' }, + { from: 'models/marts/mart_sales_daily.sql', to: 'models/intermediate/int_sales_enriched.sql' }, + ]), + ).not.toThrow(); + }); + + it('refuses a mart reading a source directly — raw data reaching a dashboard', () => { + expect(() => + assertLayerDependenciesLegal([ + { from: 'models/marts/mart_sales_daily.sql', to: 'models/sources/sources.yml' }, + ]), + ).toThrowError(SiaxError); + }); + + it('refuses a mart reading staging — one layer skipped is still a skip', () => { + expect(() => + assertLayerDependenciesLegal([ + { from: 'models/marts/mart_sales_daily.sql', to: 'models/staging/stg_sales_events.sql' }, + ]), + ).toThrowError(SiaxError); + }); + + it('refuses a staging model reading a mart — an upward edge', () => { + expect(() => + assertLayerDependenciesLegal([ + { from: 'models/staging/stg_sales_events.sql', to: 'models/marts/mart_sales_daily.sql' }, + ]), + ).toThrowError(SiaxError); + }); + + it('allows a model reading another in the same layer', () => { + expect(() => + assertLayerDependenciesLegal([ + { from: 'models/staging/stg_sales_events.sql', to: 'models/staging/stg_customers.sql' }, + ]), + ).not.toThrow(); + }); +}); diff --git a/tests/analytics/duckdb-provider.test.ts b/tests/analytics/duckdb-provider.test.ts new file mode 100644 index 0000000..43fcac7 --- /dev/null +++ b/tests/analytics/duckdb-provider.test.ts @@ -0,0 +1,326 @@ +/** + * tests/analytics/duckdb-provider.test.ts + * ======================================== + * BI-P014 — the parts provable without a DuckDB, plus the pin on the parts + * that are not. + * + * The live suite (tests/duckdb/duckdb-provider.live.test.ts) only + * runs when DUCKDB_PYTHON is set. On a runner without it, it contributes + * nothing — so the recorded evidence is committed and THIS file, which never + * skips, checks it. In particular it pins the finding that makes layer 2 real: + * + * `allowed_directories` WITHOUT `enable_external_access=false` permits + * /etc/passwd. Measured, recorded, and asserted here — because a comment + * saying so would rot the moment someone "simplified" the bootstrap. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + ArtifactMountRegistry, + assertSessionSandboxed, + duckDbSessionBootstrap, + escapeLiteral, + DuckDbAnalyticsProvider, + REQUIRED_SANDBOX_SETTINGS, + type DuckDbRunner, +} from '@/src/analytics/providers/duckdb-provider'; + +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p014/duckdb-sandbox.json'); +const FIXTURES = path.join(process.cwd(), 'tests/fixtures/duckdb'); +const IDENTITY = { tenant_id: 't_alpha', principal_id: 'u_1' }; + +const capturingRunner = (): DuckDbRunner & { lastSql?: string; lastBootstrap?: readonly string[] } => { + const r: DuckDbRunner & { lastSql?: string; lastBootstrap?: readonly string[] } = { + engine: 'duckdb', + async run({ bootstrap, sql }) { + r.lastSql = sql; + r.lastBootstrap = bootstrap; + return []; + }, + }; + return r; +}; + +describe('BI-P014 — the session bootstrap is one unit, not three settings', () => { + it('emits all three statements, external access LAST-but-one and the lock last', () => { + const stmts = duckDbSessionBootstrap(['/mnt/artifacts']); + expect(stmts).toHaveLength(3); + expect(stmts[0]).toContain('allowed_directories'); + expect(stmts[1]).toBe('SET enable_external_access=false'); + expect(stmts[2]).toBe('SET lock_configuration=true'); + }); + + it('refuses an EMPTY allow-list — that is not a sandbox', () => { + expect(() => duckDbSessionBootstrap([])).toThrowError(SiaxError); + }); + + it('refuses a relative or quote-carrying directory', () => { + expect(() => duckDbSessionBootstrap(['relative/path'])).toThrowError(SiaxError); + expect(() => duckDbSessionBootstrap(["/mnt/it's"])).toThrowError(SiaxError); + }); + + it.each([...REQUIRED_SANDBOX_SETTINGS])( + 'assertSessionSandboxed refuses a session missing %s', + (setting) => { + const stmts = duckDbSessionBootstrap(['/mnt/artifacts']).filter((s) => !s.includes(setting)); + expect(() => assertSessionSandboxed(stmts)).toThrowError(SiaxError); + }, + ); + + it('refuses a session that sets the allow-list but leaves external access ON', () => { + // The exact naive spelling. It looks configured and permits everything. + const naive = [ + "SET allowed_directories=['/mnt/artifacts']", + 'SET enable_external_access=true', + 'SET lock_configuration=true', + ]; + expect(() => assertSessionSandboxed(naive)).toThrowError(SiaxError); + }); +}); + +describe('BI-P014 — the mount registry refuses everything that is not a mounted artifact', () => { + function registry(): ArtifactMountRegistry { + // A fake realpath, so path semantics are tested without touching disk. + const fakeReal = (p: string) => (p.includes('symlinked') ? '/etc/passwd' : p); + const reg = new ArtifactMountRegistry('/mnt/artifacts', fakeReal); + reg.register({ + artifact_id: 'a_ok', + tenant_id: 't_alpha', + st0re_ref: 'st0re://x', + relative_path: 'sales.parquet', + format: 'parquet', + size_bytes: 10, + }); + return reg; + } + + it('resolves a mounted artifact for its owner', () => { + expect(registry().resolve(IDENTITY, 'a_ok').absolutePath).toBe('/mnt/artifacts/sales.parquet'); + }); + + it('refuses an absolute mount path', () => { + expect(() => + registry().register({ + artifact_id: 'a_abs', + tenant_id: 't_alpha', + st0re_ref: 'x', + relative_path: '/etc/passwd', + format: 'csv', + size_bytes: 1, + }), + ).toThrowError(SiaxError); + }); + + it('refuses a path that resolves outside the root once symlinks are followed', () => { + expect(() => + registry().register({ + artifact_id: 'a_link', + tenant_id: 't_alpha', + st0re_ref: 'x', + relative_path: 'symlinked.parquet', + format: 'parquet', + size_bytes: 1, + }), + ).toThrowError(SiaxError); + }); + + it("refuses a sibling directory that merely shares the root's prefix", () => { + // `startsWith('/mnt/artifacts')` alone would accept `/mnt/artifacts-evil`. + const reg = new ArtifactMountRegistry('/mnt/artifacts', (p) => + p.includes('sibling') ? '/mnt/artifacts-evil/x.parquet' : p, + ); + expect(() => + reg.register({ + artifact_id: 'a_sib', + tenant_id: 't_alpha', + st0re_ref: 'x', + relative_path: 'sibling.parquet', + format: 'parquet', + size_bytes: 1, + }), + ).toThrowError(SiaxError); + }); + + it('refuses a request with no verified tenant', () => { + expect(() => registry().resolve({ tenant_id: '', principal_id: 'u' }, 'a_ok')).toThrowError(SiaxError); + }); + + it('distinguishes NOT_FOUND from AUTHORIZATION_DENIED', () => { + const reg = registry(); + try { + reg.resolve(IDENTITY, 'a_missing'); + } catch (e) { + expect((e as SiaxError).code).toBe('NOT_FOUND'); + } + try { + reg.resolve({ tenant_id: 't_beta', principal_id: 'u' }, 'a_ok'); + } catch (e) { + // An attempted cross-tenant read must be visible in the audit trail as + // one, not disguised as a missing object. + expect((e as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + } + }); + + it('refuses an artifact id that is not registry-safe', () => { + for (const bad of ['../x', 'a b', "a'b", 'a/b', '']) { + expect(() => registry().resolve(IDENTITY, bad)).toThrowError(SiaxError); + } + }); +}); + +describe('BI-P014 — the query builder has no caller-SQL surface', () => { + function provider(runner: DuckDbRunner, sizeBytes = 10) { + const reg = new ArtifactMountRegistry('/mnt/artifacts', (p) => p); + reg.register({ + artifact_id: 'a_ok', + tenant_id: 't_alpha', + st0re_ref: 'x', + relative_path: 'sales.parquet', + format: 'parquet', + size_bytes: sizeBytes, + }); + return new DuckDbAnalyticsProvider(reg, runner); + } + + it('builds a reader from the RESOLVED path, never from caller text', async () => { + const r = capturingRunner(); + await provider(r).query(IDENTITY, { artifact_id: 'a_ok', select: ['country'] }); + expect(r.lastSql).toContain("read_parquet('/mnt/artifacts/sales.parquet')"); + }); + + it('always applies the sandbox bootstrap before the query', async () => { + const r = capturingRunner(); + await provider(r).query(IDENTITY, { artifact_id: 'a_ok' }); + expect(() => assertSessionSandboxed(r.lastBootstrap ?? [])).not.toThrow(); + }); + + it.each(['a b', 'x; DROP TABLE t', '*', '1=1', "a'", '../x'])( + 'refuses %s as a column name', + async (col) => { + await expect( + provider(capturingRunner()).query(IDENTITY, { artifact_id: 'a_ok', select: [col] }), + ).rejects.toBeInstanceOf(SiaxError); + }, + ); + + it('hands a heavy artifact to N0D instead of running it locally', async () => { + let thrown: unknown; + try { + await provider(capturingRunner(), 40 * 1024 * 1024 * 1024).query(IDENTITY, { artifact_id: 'a_ok' }); + } catch (e) { + thrown = e; + } + expect((thrown as SiaxError).code).toBe('CAPABILITY_UNAVAILABLE'); + expect((thrown as SiaxError).message).toContain('N0D'); + }); + + it("escapes a literal by DOUBLING the quote, which is DuckDB's only escape", () => { + // MEASURED on DuckDB 1.5.5 rather than assumed, because the whole escaping + // strategy rests on it: `SELECT length('a\\b')` returns 4, so a backslash + // is a literal character; and `SELECT 'a\''` is a PARSER ERROR + // ("unterminated quoted string"), which is what a non-escaping backslash + // produces. A dialect where `\'` closed the literal would make doubling + // insufficient. + expect(escapeLiteral("O'Brien")).toBe("O''Brien"); + // A backslash is NOT an escape inside a DuckDB single-quoted string, so it + // must survive untouched rather than being "helpfully" doubled. + expect(escapeLiteral("a\\'b")).toBe("a\\''b"); + expect(escapeLiteral("x' OR '1'='1")).toBe("x'' OR ''1''=''1"); + }); + + it('refuses a NUL byte in a filter value rather than escaping it', () => { + expect(() => escapeLiteral('a\u0000b')).toThrowError(SiaxError); + }); + + it('declares which formats have actually been PROVEN, and why the others have not', () => { + const readiness = provider(capturingRunner()).readiness(); + expect(readiness.proven).toContain('parquet'); + expect(readiness.proven).toContain('iceberg'); + const arrow = readiness.unproven.find((u) => u.format === 'arrow_ipc'); + expect(arrow?.reason).toContain('network install'); + }); +}); + +describe('BI-P014 — the committed evidence, checked by a test that never skips', () => { + it('the evidence file exists', () => { + expect( + fs.existsSync(EVIDENCE_PATH), + 'evidence/bi-p014/duckdb-sandbox.json is missing — run the live suite with DUCKDB_PYTHON set', + ).toBe(true); + }); + + const evidence = fs.existsSync(EVIDENCE_PATH) + ? (JSON.parse(fs.readFileSync(EVIDENCE_PATH, 'utf8')) as Record) + : ({} as Record); + + it('was recorded against a real DuckDB', () => { + expect(String(evidence.duckdb_version ?? '')).toMatch(/^\d+\.\d+/); + }); + + it('PINS the finding: allowed_directories alone read /etc/passwd', () => { + // The load-bearing assertion of this whole file. If someone "simplifies" + // the bootstrap down to the allow-list, the module comment would become a + // lie quietly; this keeps the measurement attached to the build. + const naive = evidence.naive_allowed_directories_is_a_noop as unknown as { + read_etc_passwd_succeeded: boolean; + statements: string[]; + }; + expect(naive).toBeDefined(); + expect(naive.read_etc_passwd_succeeded).toBe(true); + expect(naive.statements.join(' ')).not.toContain('enable_external_access'); + }); + + it('records that NO bypass attempt succeeded with the full bootstrap', () => { + const attempts = evidence.sandbox_bypass_attempts as unknown as Array<{ id: string; succeeded: boolean }>; + expect(Array.isArray(attempts)).toBe(true); + expect(attempts.length).toBeGreaterThanOrEqual(10); + expect(attempts.filter((a) => a.succeeded).map((a) => a.id)).toEqual([]); + }); + + it('records the counterfactual — the mounted file WAS readable in the same session', () => { + // Without this, "no attempt succeeded" is equally consistent with a + // sandbox that blocks everything, which is an outage, not a control. + expect(evidence.counterfactual_mounted_file_readable).toBe(true); + }); + + it('records the Iceberg snapshot read, whose fixture is deliberately not committed', () => { + // pyiceberg embeds ABSOLUTE paths in its manifests, so a committed Iceberg + // table would only work on the machine that built it. The recorded result + // is the artefact instead. + const iceberg = evidence.iceberg as unknown as { totals: Record }; + expect(iceberg?.totals).toEqual({ NO: 20, SE: 40 }); + }); + + it('records that a locked session could not re-open its own sandbox', () => { + const reopened = evidence.reconfiguration_refused as unknown as Array<{ refused: boolean }>; + expect(reopened.length).toBe(3); + expect(reopened.every((r) => r.refused)).toBe(true); + }); +}); + +describe('BI-P014 — the committed fixtures', () => { + it('ships the Parquet and CSV fixtures the live suite reads', () => { + expect(fs.existsSync(path.join(FIXTURES, 'sales.parquet'))).toBe(true); + expect(fs.existsSync(path.join(FIXTURES, 'customers.csv'))).toBe(true); + }); + + it('does NOT ship the Iceberg table, because its manifests carry absolute paths', () => { + // Asserted against GIT, not the filesystem. The claim is about what is + // COMMITTED; the live suite REBUILDS that table locally, so a filesystem + // check fails whenever both suites run in one pass — it was testing the + // wrong thing rather than catching a real problem. + const tracked = execFileSync('git', ['ls-files', 'tests/fixtures/duckdb/'], { + encoding: 'utf8', + cwd: process.cwd(), + }) + .split('\n') + .filter(Boolean); + // The parquet/csv fixtures ARE tracked, so a version that shipped nothing + // would fail here too rather than passing vacuously. + expect(tracked.length).toBeGreaterThan(0); + expect(tracked.filter((f) => f.includes('/iceberg'))).toEqual([]); + }); +}); diff --git a/tests/analytics/metric-governance.test.ts b/tests/analytics/metric-governance.test.ts new file mode 100644 index 0000000..f661e4c --- /dev/null +++ b/tests/analytics/metric-governance.test.ts @@ -0,0 +1,483 @@ +/** + * tests/analytics/metric-governance.test.ts + * ========================================== + * BI-P028 (metric domain model), BI-P030 (semantic model registry), + * BI-P031 (metric governance), BI-P029 (semantic provider — the parts not + * needing a Python runtime). + * + * Each prompt's DoD is turned into something that can FAIL: + * + * BI-P028 "No production metric exists only inside a dashboard tool." + * A dashboard panel carrying an inline aggregation / expression / + * SQL is REFUSED, and so is one naming an unregistered metric. + * + * BI-P030 "MetricFlow can be replaced without breaking canonical SIAX + * metric IDs." Two providers with different naming rules compile + * the same registry version; the canonical id set must be + * identical and the provider BINDINGS must differ — otherwise the + * test would pass for a registry that simply has one provider. + * + * BI-P031 "Certified metric cannot be modified in-place; new version + * required." Every semantic field is mutated one at a time and + * each must be refused; every editorial field is mutated and each + * must be allowed. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + assertMetricEditAllowed, + assertPanelUsesRegisteredMetric, + assertValidMetricSpec, + certifyMetric, + metricMeaningHash, + nextMetricVersion, + METRIC_STATUSES, + type GovernedMetric, + type MetricSpec, +} from '@/src/analytics/metrics/metric-model'; +import { + assertEngineNeutral, + assertOnlySemanticModelFields, + InMemorySemanticModelStore, + SemanticModelRegistry, + semanticModelHash, + type SemanticModelVersion, +} from '@/src/analytics/metrics/semantic-registry'; +import { MetricFlowProvider, type SiaxSemanticProvider } from '@/src/analytics/metrics/semantic-provider'; +import { compileRequestToSemantic } from '@/src/analytics/contract/compile-semantic'; +import { + parseAnalyticsQueryRequest, + queryResolutionFrom, + resolveAnalyticsQuery, +} from '@/src/analytics/contract/query-request'; + +const REVENUE: MetricSpec = { + id: 'm_revenue', + name: 'net_revenue', + description: 'Invoiced revenue net of credit notes, in SEK, recognised on invoice date.', + type: 'simple', + measure: 'amount_sek', + aggregation: 'sum', + dimensions: ['country', 'channel'], + filters: [{ dimension: 'channel', operator: 'ne', values: ['internal'] }], + timeDimension: 'invoiced_at', + unit: 'SEK', + currency: 'SEK', + format: 'currency', + owner: 'finance@siax.io', + version: 3, + status: 'ACTIVE', +}; + +const GOVERNED: GovernedMetric = { + spec: REVENUE, + governance: { + certification: 'certified', + business_definition: 'What we actually invoiced, after credits.', + technical_definition: "sum(amount_sek) on ds_sales where channel != 'internal', by invoiced_at.", + owner: 'finance@siax.io', + lineage_refs: ['ds_sales@v42'], + quality_status: 'passing', + }, +}; + +describe('BI-P028 — a metric that cannot mean nothing', () => { + it('accepts the golden metric', () => { + expect(() => assertValidMetricSpec(REVENUE)).not.toThrow(); + }); + + const rejections: Array<[string, Partial]> = [ + ['no description', { description: '' }], + ['no owner', { owner: '' }], + ['no time dimension', { timeDimension: '' }], + ['a ratio with no denominator', { type: 'ratio' }], + ['a denominator on a non-ratio', { denominator: 'orders' }], + ['currency format with no currency', { currency: undefined }], + ['a currency on a non-currency format', { format: 'integer' }], + ['a non-ISO currency', { currency: 'kronor' as string }], + ['a name that is not snake_case', { name: 'Net Revenue' }], + ['a duplicate dimension', { dimensions: ['country', 'country'] }], + [ + 'a filter on an undeclared dimension', + { filters: [{ dimension: 'segment', operator: 'eq', values: ['x'] }] }, + ], + ['a value operator with no values', { filters: [{ dimension: 'channel', operator: 'eq', values: [] }] }], + [ + 'a presence operator with values', + { filters: [{ dimension: 'channel', operator: 'is_null', values: ['x'] }] }, + ], + ['version zero', { version: 0 }], + ]; + + it.each(rejections)('refuses %s', (_label, override) => { + expect(() => assertValidMetricSpec({ ...REVENUE, ...override })).toThrowError(SiaxError); + }); + + it('declares the five lifecycle statuses BI-P028 names, in order', () => { + expect([...METRIC_STATUSES]).toEqual(['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED']); + }); +}); + +describe('BI-P028 DoD — no production metric exists only inside a dashboard tool', () => { + const lookup = (id: string) => (id === 'm_revenue' ? GOVERNED : undefined); + + it.each(['inline_expression', 'inline_aggregation', 'inline_sql'])( + 'refuses a panel that defines a metric via %s', + (field) => { + expect(() => + assertPanelUsesRegisteredMetric({ panel_id: 'p1', [field]: 'sum(amount)' }, lookup), + ).toThrowError(SiaxError); + }, + ); + + it('refuses a panel that names an unregistered metric', () => { + expect(() => + assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_invented' }, lookup), + ).toThrowError(SiaxError); + }); + + it('refuses a panel with no metric at all', () => { + expect(() => assertPanelUsesRegisteredMetric({ panel_id: 'p1' }, lookup)).toThrowError(SiaxError); + }); + + it('refuses a panel pointing at a RETIRED metric', () => { + const retired = { ...GOVERNED, spec: { ...REVENUE, status: 'RETIRED' as const } }; + expect(() => + assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, () => retired), + ).toThrowError(SiaxError); + }); + + it('accepts a panel that references a registered, live metric', () => { + expect(assertPanelUsesRegisteredMetric({ panel_id: 'p1', metric_id: 'm_revenue' }, lookup).spec.id).toBe( + 'm_revenue', + ); + }); +}); + +describe('BI-P031 DoD — a certified metric cannot be modified in place', () => { + /** Every field that changes what the number MEANS. */ + const semanticEdits: Array<[string, Partial]> = [ + ['the measure', { measure: 'amount_eur' }], + ['the aggregation', { aggregation: 'avg' }], + ['the type', { type: 'ratio', denominator: 'orders' }], + ['the dimensions', { dimensions: ['country'] }], + ['the filters', { filters: [] }], + ['the time dimension', { timeDimension: 'paid_at' }], + ['the unit', { unit: 'EUR' }], + ['the currency', { currency: 'EUR' }], + ]; + + it.each(semanticEdits)('refuses an in-place change to %s', (_label, override) => { + const proposed = { ...REVENUE, ...override }; + let thrown: unknown; + try { + assertMetricEditAllowed(GOVERNED, proposed); + } catch (e) { + thrown = e; + } + expect(thrown, "a certified metric's meaning was changed in place").toBeInstanceOf(SiaxError); + expect((thrown as SiaxError).code).toBe('CONFLICT'); + expect((thrown as SiaxError).message).toContain('version 4'); + }); + + /** Editorial fields stay editable — forcing a version bump for a typo is how teams route around governance. */ + const editorialEdits: Array<[string, Partial]> = [ + ['the description', { description: 'Clearer wording.' }], + ['the owner', { owner: 'controlling@siax.io' }], + ['the lifecycle status', { status: 'DEPRECATED' }], + ]; + + it.each(editorialEdits)('allows an in-place change to %s', (_label, override) => { + expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, ...override })).not.toThrow(); + }); + + it('allows any change to an EXPERIMENTAL metric', () => { + const experimental = { + ...GOVERNED, + governance: { ...GOVERNED.governance, certification: 'experimental' as const }, + }; + expect(() => assertMetricEditAllowed(experimental, { ...REVENUE, measure: 'anything' })).not.toThrow(); + }); + + it('mints a new version that is DRAFT and EXPERIMENTAL, never inheriting the certification', () => { + const next = nextMetricVersion( + GOVERNED, + { measure: 'amount_eur', unit: 'EUR', currency: 'EUR' }, + 'switch to EUR', + ); + expect(next.spec.version).toBe(4); + expect(next.spec.status).toBe('DRAFT'); + expect(next.governance.certification).toBe('experimental'); + expect(next.governance.certified_at).toBeUndefined(); + expect(next.change_reason).toBe('switch to EUR'); + }); + + it('requires a change reason for a new version', () => { + expect(() => nextMetricVersion(GOVERNED, { measure: 'x' }, ' ')).toThrowError(SiaxError); + }); + + it('refuses to change the metric id under the guise of an edit', () => { + expect(() => assertMetricEditAllowed(GOVERNED, { ...REVENUE, id: 'm_other' })).toThrowError(SiaxError); + }); + + it('the meaning hash ignores editorial fields and catches semantic ones', () => { + expect(metricMeaningHash({ ...REVENUE, description: 'different', owner: 'x', status: 'DRAFT' })).toBe( + metricMeaningHash(REVENUE), + ); + expect(metricMeaningHash({ ...REVENUE, measure: 'other' })).not.toBe(metricMeaningHash(REVENUE)); + // Filter ORDER is not meaning; filter CONTENT is. + expect( + metricMeaningHash({ + ...REVENUE, + filters: [ + { dimension: 'country', operator: 'eq', values: ['SE', 'NO'] }, + { dimension: 'channel', operator: 'ne', values: ['internal'] }, + ], + }), + ).not.toBe(metricMeaningHash(REVENUE)); + }); + + it('certification is an act with an actor and a time, and needs both definitions', () => { + const draft: GovernedMetric = { + spec: { ...REVENUE, status: 'DRAFT' }, + governance: { ...GOVERNED.governance, certification: 'experimental' }, + }; + expect(() => certifyMetric(draft, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError); + + const active = { ...draft, spec: { ...draft.spec, status: 'ACTIVE' as const } }; + const noDefs = { ...active, governance: { ...active.governance, technical_definition: '' } }; + expect(() => certifyMetric(noDefs, 'u_1', '2026-09-07T00:00:00Z')).toThrowError(SiaxError); + + const certified = certifyMetric(active, 'u_1', '2026-09-07T00:00:00Z'); + expect(certified.governance.certification).toBe('certified'); + expect(certified.governance.certified_by).toBe('u_1'); + }); +}); + +// ── BI-P030 ──────────────────────────────────────────────────────────────── + +function model(version = 1): Omit { + return { + model_id: 'sm_sales', + version, + tenant_id: 't_alpha', + datasets: ['ds_sales'], + relationships: [], + metrics: [REVENUE], + dimensions: [ + { id: 'd_country', name: 'country', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_channel', name: 'channel', dataset_id: 'ds_sales', kind: 'categorical' }, + { id: 'd_invoiced', name: 'invoiced_at', dataset_id: 'ds_sales', kind: 'time' }, + ], + valid_from: '2026-09-01T00:00:00Z', + status: 'draft', + }; +} + +describe('BI-P030 — the registry is engine-neutral and its versions are immutable', () => { + it('refuses a model that names an implementation engine anywhere in it', async () => { + for (const smuggled of [ + { model_id: 'sm_metricflow_sales' }, + { datasets: ['clickhouse_sales'] }, + { metrics: [{ ...REVENUE, description: 'computed by MetricFlow' }] }, + ]) { + expect(() => + assertEngineNeutral({ ...model(), ...smuggled } as Omit), + ).toThrowError(SiaxError); + } + }); + + it('refuses a metric naming a dimension the model does not define', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + await expect( + registry.publish({ ...model(), metrics: [{ ...REVENUE, dimensions: ['country', 'segment'] }] }), + ).rejects.toBeInstanceOf(SiaxError); + }); + + it('refuses a metric whose time dimension is not declared as a time dimension', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + await expect( + registry.publish({ ...model(), metrics: [{ ...REVENUE, timeDimension: 'country' }] }), + ).rejects.toBeInstanceOf(SiaxError); + }); + + it('refuses to republish an existing version — a pinned dashboard must mean the same thing tomorrow', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + await registry.publish(model(1)); + await expect(registry.publish(model(1))).rejects.toBeInstanceOf(SiaxError); + }); + + it('supersedes the previous active version on activation', async () => { + const store = new InMemorySemanticModelStore(); + const registry = new SemanticModelRegistry(store); + await registry.publish(model(1)); + await registry.publish(model(2)); + await registry.activate('t_alpha', 'sm_sales', 1, '2026-09-01T00:00:00Z'); + await registry.activate('t_alpha', 'sm_sales', 2, '2026-09-05T00:00:00Z'); + expect((await store.get('t_alpha', 'sm_sales', 1))?.status).toBe('superseded'); + expect((await store.active('t_alpha', 'sm_sales'))?.version).toBe(2); + }); + + it('the hash covers meaning and ignores activation', () => { + const a = semanticModelHash(model(1)); + const b = semanticModelHash({ + ...model(1), + valid_from: '2030-01-01T00:00:00Z', + status: 'active', + } as never); + expect(a).toBe(b); + const c = semanticModelHash({ ...model(1), metrics: [{ ...REVENUE, version: 4 }] }); + expect(c).not.toBe(a); + }); + + it('requires a verified tenant', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + await expect(registry.publish({ ...model(), tenant_id: '' })).rejects.toBeInstanceOf(SiaxError); + }); +}); + +describe('BI-P030 DoD — the semantic engine can be replaced without breaking metric IDs', () => { + /** + * A SECOND provider with DIFFERENT naming rules, so the test cannot pass + * trivially for a registry that only ever had one provider. Cube-shaped: + * metric names are `cube.measure`, which is exactly the kind of difference + * that would break a consumer if canonical ids were not the contract. + */ + class CubeShapedProvider implements SiaxSemanticProvider { + readonly provider = 'cube-shaped'; + buildManifest(m: SemanticModelVersion) { + return { cubes: m.datasets.map((d) => ({ name: d, measures: m.metrics.map((x) => x.measure) })) }; + } + compile(spec: ReturnType, m: SemanticModelVersion) { + const bindings = this.providerBindings(m); + return { + provider: this.provider, + query: { measures: spec.metrics.map((x) => bindings[x.metric_id]) }, + canonical_metric_ids: spec.metrics.map((x) => x.metric_id).sort(), + }; + } + providerBindings(m: SemanticModelVersion): Record { + return Object.fromEntries(m.metrics.map((x) => [x.id, `ds_sales.${x.measure}`])); + } + } + + it('both providers preserve the canonical ids while naming them differently', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + const published = await registry.publish(model(1)); + + const resolution = queryResolutionFrom({ + dataset_id: 'ds_sales', + metrics: [ + { + name: 'net_revenue', + metric_id: 'm_revenue', + event_metric_key: 'amount_sek', + aggregation: 'sum', + allowed_dimensions: ['country', 'channel'], + }, + ], + dimensions: [{ name: 'country', dimension_id: 'd_country', field_name: 'country_code' }], + semantic_model_version: 1, + }); + const spec = compileRequestToSemantic( + resolveAnalyticsQuery( + parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['net_revenue'], + dimensions: ['country'], + }), + resolution, + ), + ); + + const mf = new MetricFlowProvider(); + const cube = new CubeShapedProvider(); + + expect(mf.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']); + expect(cube.compile(spec, published).canonical_metric_ids).toEqual(['m_revenue']); + expect(await registry.canonicalMetricIds('t_alpha', 'sm_sales', 1)).toEqual(['m_revenue']); + + // ...and the providers genuinely differ, so the equality above is not + // trivially true. + expect(mf.providerBindings(published)).not.toEqual(cube.providerBindings(published)); + expect(cube.providerBindings(published).m_revenue).toBe('ds_sales.amount_sek'); + }); + + it('the MetricFlow manifest is DERIVED, never persisted into the registry', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + const published = await registry.publish(model(1)); + const manifest = new MetricFlowProvider().buildManifest(published); + expect(manifest.metrics[0].name).toBe('m_revenue'); + expect(manifest._generated_from.hash).toBe(published.hash); + + // Two independent reasons a registry row cannot become a provider manifest. + // The vocabulary scan catches the generator stamp... + expect(() => assertEngineNeutral({ ...published, ...(manifest as object) } as never)).toThrowError( + SiaxError, + ); + // ...and the structural check catches ANY extra field, which is what holds + // for a manifest that avoids the word. The first version of this test + // asserted only the vocabulary scan and passed for the wrong reason: + // buildManifest did not mention "metricflow" anywhere, so nothing fired. + expect(() => assertOnlySemanticModelFields({ ...published, semantic_models: [] })).toThrowError( + SiaxError, + ); + }); + + it("publish refuses a model carrying any field outside BI-P030's list", async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + await expect(registry.publish({ ...model(1), compiled_sql: 'SELECT 1' } as never)).rejects.toBeInstanceOf( + SiaxError, + ); + }); + + it('refuses to compile a metric that is not in the model version', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + const published = await registry.publish(model(1)); + expect(() => + new MetricFlowProvider().compile( + { + dataset_id: 'ds_sales', + metrics: [{ metric_id: 'm_ghost', name: 'ghost' }], + groupBy: [], + where: [], + orderBy: [], + limit: 10, + format: 'json', + }, + published, + ), + ).toThrowError(SiaxError); + }); + + it('puts no SQL predicate text into the compiled provider query', async () => { + const registry = new SemanticModelRegistry(new InMemorySemanticModelStore()); + const published = await registry.publish(model(1)); + const compiled = new MetricFlowProvider().compile( + { + dataset_id: 'ds_sales', + metrics: [{ metric_id: 'm_revenue', name: 'net_revenue' }], + groupBy: [], + where: [{ dimension_id: 'd_country', name: 'country', operator: 'eq', values: ["SE' OR '1'='1"] }], + orderBy: [], + limit: 10, + format: 'json', + }, + published, + ); + const serialized = JSON.stringify(compiled.query); + expect(serialized).toContain("SE' OR '1'='1"); // survives as DATA + expect(serialized).not.toMatch(/\bWHERE\b|\bSELECT\b/); + }); +}); + +describe('BI-P029 — the provider states what it has NOT proven', () => { + it('reports itself unverified, and names the open surfaces', () => { + const readiness = new MetricFlowProvider().providerReadiness(); + expect(readiness.verified).toBe(false); + expect(readiness.open.join(' ')).toContain('NOT been executed'); + expect(readiness.open.join(' ')).toContain('Superset'); + expect(readiness.open.join(' ')).toContain('AI Analyst'); + expect(readiness.proven.length).toBeGreaterThan(0); + }); +}); diff --git a/tests/analytics/quality-and-freshness.test.ts b/tests/analytics/quality-and-freshness.test.ts new file mode 100644 index 0000000..2140c85 --- /dev/null +++ b/tests/analytics/quality-and-freshness.test.ts @@ -0,0 +1,377 @@ +/** + * tests/analytics/quality-and-freshness.test.ts + * ============================================== + * BI-P025 (data quality provider) and BI-P026 (data freshness). + * + * Both DoDs are about a FALSE GREEN, so both are tested by constructing the + * false green and requiring a failure: + * + * BI-P025 A quality gate that passes when nothing ran. A `not_null` check + * over an EMPTY table finds zero violations — the single most + * common false green in data quality. Both are asserted to FAIL. + * + * BI-P026 UNKNOWN age quietly rendering as current, and a result released + * with no freshness envelope at all. Both are refused. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + DbtQualityProvider, + GreatExpectationsQualityProvider, + NativeQualityProvider, + QUALITY_RULES, + type QualityQueryRunner, + type QualityRuleSpec, + type QualityVerdict, +} from '@/src/analytics/quality/quality-provider'; +import { InMemoryQuarantineStore, QualityGate } from '@/src/analytics/quality/quarantine'; +import { + assertFreshnessDisclosed, + computeFreshness, + FRESHNESS_STATES, + isPresentableAsCurrent, + NON_CURRENT_STATES, +} from '@/src/analytics/quality/freshness'; + +function runner( + violations: number, + evaluated: number, +): QualityQueryRunner & { lastExpression?: string; lastParams?: Record } { + const r: QualityQueryRunner & { lastExpression?: string; lastParams?: Record } = { + engine: 'clickhouse', + async count(args) { + r.lastExpression = args.violationExpression; + r.lastParams = args.params; + return { violations, evaluated }; + }, + }; + return r; +} + +const RULE: QualityRuleSpec = { + rule_id: 'q_not_null_customer', + tenant_id: 't_alpha', + dataset_id: 'ds_sales', + rule: 'not_null', + field: 'customer', + severity: 'block', + params: {}, +}; + +describe('BI-P025 — the native provider EXECUTES, and says so', () => { + it('declares every rule BI-P025 names', () => { + expect([...QUALITY_RULES]).toEqual([ + 'not_null', + 'unique', + 'range', + 'referential', + 'freshness', + 'schema', + 'distribution', + 'custom', + ]); + }); + + it('passes a clean dataset', async () => { + const verdict = await new NativeQualityProvider(runner(0, 1000)).evaluate(RULE); + expect(verdict.passed).toBe(true); + expect(verdict.provider_kind).toBe('executing'); + }); + + it('FAILS an empty dataset instead of reporting zero violations', async () => { + // The false green this whole file exists for: a not-null check over zero + // rows finds zero violations and looks healthy while the pipeline is dead. + const verdict = await new NativeQualityProvider(runner(0, 0)).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('EMPTY'); + }); + + it('fails a dataset with violations', async () => { + const verdict = await new NativeQualityProvider(runner(7, 1000)).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.violation_count).toBe(7); + }); + + it('binds rule parameters instead of splicing them into the expression', async () => { + const r = runner(0, 10); + await new NativeQualityProvider(r).evaluate({ + ...RULE, + rule: 'distribution', + field: 'channel', + params: { accepted_values: ['web', "retail'; DROP TABLE x; --"] }, + }); + expect(r.lastExpression).not.toContain('DROP'); + expect(Object.values(r.lastParams ?? {})).toContain("retail'; DROP TABLE x; --"); + }); + + it('refuses a field name that is not an identifier', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + await expect(p.evaluate({ ...RULE, field: 'customer; DROP TABLE x' })).rejects.toBeInstanceOf(SiaxError); + }); + + it('refuses an incomplete rule rather than evaluating a meaningless one', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + await expect(p.evaluate({ ...RULE, rule: 'range', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'freshness', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'distribution', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'custom', params: {} })).rejects.toBeInstanceOf(SiaxError); + }); + + it('says CAPABILITY_UNAVAILABLE for rules it cannot evaluate, rather than passing them', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + for (const rule of ['schema', 'referential'] as const) { + await expect(p.evaluate({ ...RULE, rule })).rejects.toBeInstanceOf(SiaxError); + expect(p.supports(rule)).toBe(false); + } + }); +}); + +describe('BI-P025 — the ingesting providers do not pretend to execute', () => { + it('dbt and GE declare themselves as ingesting', async () => { + expect(new DbtQualityProvider(new Map()).providerKind).toBe('ingesting'); + expect(new GreatExpectationsQualityProvider(new Map()).providerKind).toBe('ingesting'); + }); + + it('a MISSING dbt result is a failure, not a pass', async () => { + // Treating "no result" as "passed" is how a gate stops gating the moment + // the test stops running. + const verdict = await new DbtQualityProvider(new Map()).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('no dbt result'); + }); + + it.each(['error', 'warn', 'skipped'] as const)('dbt status %s is NOT a pass', async (status) => { + const results = new Map([[RULE.rule_id, { unique_id: 't1', status, failures: 0 }]]); + const verdict = await new DbtQualityProvider(results).evaluate(RULE); + expect(verdict.passed).toBe(false); + }); + + it('dbt pass is a pass', async () => { + const results = new Map([[RULE.rule_id, { unique_id: 't1', status: 'pass' as const, failures: 0 }]]); + expect((await new DbtQualityProvider(results).evaluate(RULE)).passed).toBe(true); + }); + + it('a missing GE suite is a failure', async () => { + expect((await new GreatExpectationsQualityProvider(new Map()).evaluate(RULE)).passed).toBe(false); + }); + + it("a GE suite's unexpected counts are carried through", async () => { + const suites = new Map([ + [ + RULE.rule_id, + { + expectation_suite_name: 'sales_suite', + success: false, + results: [ + { + expectation_config: { expectation_type: 'expect_column_values_to_not_be_null' }, + success: false, + result: { unexpected_count: 12, element_count: 1000 }, + }, + ], + }, + ], + ]); + const verdict = await new GreatExpectationsQualityProvider(suites).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.violation_count).toBe(12); + expect(verdict.evaluated_rows).toBe(1000); + }); +}); + +describe('BI-P025 DoD — a failed critical gate QUARANTINES the serving projection', () => { + function verdict(overrides: Partial = {}): QualityVerdict { + return { + rule_id: 'q1', + dataset_id: 'ds_sales', + tenant_id: 't_alpha', + rule: 'not_null', + severity: 'block', + passed: true, + violation_count: 0, + evaluated_rows: 100, + produced_by: 'native:clickhouse', + provider_kind: 'executing', + evaluated_at: new Date().toISOString(), + ...overrides, + }; + } + + it('a blocking failure quarantines, and the READ is then REFUSED', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + // Before: readable. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + + const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'block' })]); + expect(outcome.quarantined).toBe(true); + expect(outcome.passed).toBe(false); + + // After: refused. Not annotated — refused. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('a WARN failure does not quarantine, but is surfaced', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'warn' })]); + expect(outcome.quarantined).toBe(false); + expect(outcome.warnings).toHaveLength(1); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it("a gate over ZERO verdicts QUARANTINES — 'no rules ran' is not 'everything passed'", async () => { + // The same shape as l0re's silently-skipped tests and api0's + // `trivy --exit-code 0`: a check that produced nothing reporting green. + const gate = new QualityGate(new InMemoryQuarantineStore()); + const outcome = await gate.apply('t_alpha', 'ds_sales', []); + expect(outcome.passed).toBe(false); + expect(outcome.quarantined).toBe(true); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('a later passing run does NOT auto-release a quarantine', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: true })]); + // A flapping rule would otherwise quarantine and un-quarantine the dataset + // between two reads of the same dashboard. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('release requires a principal and a reason', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await expect(gate.release('t_alpha', 'ds_sales', '', 'x')).rejects.toBeInstanceOf(SiaxError); + await expect(gate.release('t_alpha', 'ds_sales', 'u_1', ' ')).rejects.toBeInstanceOf(SiaxError); + const released = await gate.release('t_alpha', 'ds_sales', 'u_1', 'upstream backfilled'); + expect(released.released_by).toBe('u_1'); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it("quarantine is tenant-scoped: one tenant's failure does not block another", async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await expect(gate.assertProjectionReadable('t_beta', 'ds_sales')).resolves.toBeUndefined(); + }); + + it('refuses a verdict belonging to another tenant or dataset', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await expect( + gate.apply('t_alpha', 'ds_sales', [verdict({ tenant_id: 't_beta' })]), + ).rejects.toBeInstanceOf(SiaxError); + await expect( + gate.apply('t_alpha', 'ds_sales', [verdict({ dataset_id: 'ds_other' })]), + ).rejects.toBeInstanceOf(SiaxError); + }); +}); + +describe('BI-P026 — five states, and UNKNOWN is not FRESH', () => { + const HOUR = 3_600_000; + const NOW = Date.parse('2026-09-07T12:00:00Z'); + const policy = { expectedIntervalMs: HOUR }; + + it('declares the five states BI-P026 names', () => { + expect([...FRESHNESS_STATES]).toEqual(['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN']); + }); + + it('only FRESH may be presented as current', () => { + expect(isPresentableAsCurrent('FRESH')).toBe(true); + for (const s of NON_CURRENT_STATES) expect(isPresentableAsCurrent(s)).toBe(false); + expect(NON_CURRENT_STATES).toContain('UNKNOWN'); + }); + + const cases: Array<[string, Parameters[0], string]> = [ + [ + 'recent load', + { lastSuccessfulLoadAt: '2026-09-07T11:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'FRESH', + ], + [ + 'behind schedule', + { lastSuccessfulLoadAt: '2026-09-07T09:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'DELAYED', + ], + [ + 'well past the threshold', + { lastSuccessfulLoadAt: '2026-09-06T12:00:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'STALE', + ], + [ + 'the last run FAILED, even though the data looks recent', + { lastSuccessfulLoadAt: '2026-09-07T11:59:00Z', lastRunStatus: 'failed', policy, now: NOW }, + 'FAILED', + ], + ['never run', { lastRunStatus: 'never_run', policy, now: NOW }, 'UNKNOWN'], + ['no load timestamp at all', { policy, now: NOW }, 'UNKNOWN'], + ['an unparseable timestamp', { lastSuccessfulLoadAt: 'not a date', policy, now: NOW }, 'UNKNOWN'], + ]; + + it.each(cases)('%s -> %s', (_label, input, expected) => { + expect(computeFreshness(input).state).toBe(expected); + }); + + it('FAILED outranks a recent timestamp — a broken pipeline must not look FRESH', () => { + const env = computeFreshness({ + lastSuccessfulLoadAt: '2026-09-07T11:59:59Z', + lastRunStatus: 'failed', + policy, + now: NOW, + }); + expect(env.state).toBe('FAILED'); + expect(env.disclosure).toContain('FAILED'); + }); + + it('every state carries a disclosure a UI can render verbatim', () => { + for (const [, input] of cases) { + const env = computeFreshness(input); + expect(env.disclosure.length).toBeGreaterThan(10); + } + }); +}); + +describe('BI-P026 DoD — stale data is never SILENTLY presented as current', () => { + const env = computeFreshness({ + lastSuccessfulLoadAt: '2026-09-01T00:00:00Z', + lastRunStatus: 'succeeded', + policy: { expectedIntervalMs: 3_600_000 }, + now: Date.parse('2026-09-07T12:00:00Z'), + }); + + it('refuses to release a result with NO freshness envelope', () => { + // Not defaulted to FRESH, not defaulted to UNKNOWN: unreleasable. "We + // forgot to attach it" and "the data is current" are indistinguishable to + // a reader, so a default would make forgetting the silent path. + expect(() => assertFreshnessDisclosed({}, 'dashboard panel p1')).toThrowError(SiaxError); + }); + + it('refuses to release STALE data that the consumer has not acknowledged', () => { + expect(env.state).toBe('STALE'); + let thrown: unknown; + try { + assertFreshnessDisclosed({ freshness: env }, 'dashboard panel p1'); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(SiaxError); + expect((thrown as SiaxError).message).toContain('STALE'); + }); + + it('releases STALE data once the consumer has acknowledged it', () => { + expect(assertFreshnessDisclosed({ freshness: env, acknowledged_non_current: true }, 'p1').state).toBe( + 'STALE', + ); + }); + + it.each(NON_CURRENT_STATES)('refuses an unacknowledged %s result', (state) => { + expect(() => assertFreshnessDisclosed({ freshness: { ...env, state } }, 'p1')).toThrowError(SiaxError); + }); + + it('releases FRESH data without ceremony', () => { + expect(assertFreshnessDisclosed({ freshness: { ...env, state: 'FRESH' } }, 'p1').state).toBe('FRESH'); + }); + + it('refuses an envelope carrying a state that is not one of the five', () => { + expect(() => + assertFreshnessDisclosed({ freshness: { ...env, state: 'PROBABLY_FINE' as never } }, 'p1'), + ).toThrowError(SiaxError); + }); +}); diff --git a/tests/analytics/query-cache.test.ts b/tests/analytics/query-cache.test.ts new file mode 100644 index 0000000..cc5a185 --- /dev/null +++ b/tests/analytics/query-cache.test.ts @@ -0,0 +1,291 @@ +/** + * tests/analytics/query-cache.test.ts + * ==================================== + * BI-P020 — QUERY CACHE. + * + * DoD: "Invalidation tests prove changed dataset/metric does not serve stale + * unauthorized result." + * + * Two words in that sentence carry the weight — STALE and UNAUTHORIZED — and + * they are different failures, so they are tested separately: + * + * STALE a component of the key changed (metric version, semantic + * model version, data snapshot) and the cache still answered. + * Tested by changing each of the six components ONE AT A TIME + * and requiring a miss for every one; a table-driven loop, so + * adding a component to the key without adding it to the loop + * is caught by the count assertion. + * + * UNAUTHORIZED another tenant's result came back. Tested three ways: a + * different tenant misses; a poisoned entry stored under this + * tenant's key but carrying another tenant's id is REFUSED and + * evicted rather than returned; and the same components under + * two tenants produce different keys. + * + * The poisoning test is the one that matters. Key namespacing is a naming + * convention, and a naming convention is what ST0RE was relying on when it + * turned out to have four cross-tenant paths rather than the one reported. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + buildCacheKey, + InMemoryResultCache, + tenantKeyPrefix, + ValkeyResultCache, + type CacheKeyComponents, + type CachedResult, + type ValkeyLikeClient, +} from '@/src/analytics/cache/query-cache'; + +const BASE: CacheKeyComponents = { + tenant_id: 't_alpha', + project_id: 'p_main', + dataset_id: 'ds_sales', + semantic_model_version: 7, + metric_versions: { m_revenue: 3, m_orders: 1 }, + compiled_query_hash: 'a'.repeat(64), + data_snapshot: 'dsv_42', + format: 'json', +}; + +describe('BI-P020 — the key requires all six components', () => { + const omissions: Array<[string, Partial]> = [ + ['tenant_id', { tenant_id: '' }], + ['dataset_id', { dataset_id: '' }], + ['semantic_model_version', { semantic_model_version: undefined as unknown as number }], + ['metric_versions', { metric_versions: {} }], + ['compiled_query_hash', { compiled_query_hash: '' }], + ['data_snapshot', { data_snapshot: '' }], + ['format', { format: '' }], + ]; + + it.each(omissions)('refuses to build a key without %s', (_name, override) => { + expect(() => buildCacheKey({ ...BASE, ...override })).toThrowError(SiaxError); + }); + + it('refuses a metric whose version is missing', () => { + expect(() => + buildCacheKey({ ...BASE, metric_versions: { m_revenue: undefined as unknown as number } }), + ).toThrowError(SiaxError); + }); + + it('is stable under key ordering — the same query does not fragment the cache', () => { + const a = buildCacheKey({ ...BASE, metric_versions: { m_orders: 1, m_revenue: 3 } }); + const b = buildCacheKey({ ...BASE, metric_versions: { m_revenue: 3, m_orders: 1 } }); + expect(a).toBe(b); + }); + + it('carries the tenant in plaintext so invalidation is a prefix scan', () => { + expect(buildCacheKey(BASE).startsWith(tenantKeyPrefix('t_alpha'))).toBe(true); + }); + + it('refuses a tenant id that is not key-safe', () => { + for (const bad of ['t alpha', 't:alpha', 't*', '../t_beta']) { + expect(() => buildCacheKey({ ...BASE, tenant_id: bad })).toThrowError(SiaxError); + } + }); +}); + +describe('BI-P020 — a changed component never serves a STALE result', () => { + /** + * Each entry changes ONE component. Every one must miss. The count is + * asserted so that adding a component to `CacheKeyComponents` without adding + * it here is caught rather than silently untested. + */ + const mutations: Array<[string, Partial]> = [ + ['semantic model version bumped', { semantic_model_version: 8 }], + ['a metric definition versioned', { metric_versions: { m_revenue: 4, m_orders: 1 } }], + ['a metric removed from the query', { metric_versions: { m_revenue: 3 } }], + ['the data snapshot advanced', { data_snapshot: 'dsv_43' }], + ['the compiled query changed (different filters)', { compiled_query_hash: 'b'.repeat(64) }], + ['a different dataset', { dataset_id: 'ds_other' }], + ['a different result format', { format: 'arrow_ipc' }], + ['a different project', { project_id: 'p_other' }], + ['a different tenant', { tenant_id: 't_beta' }], + ]; + + it('covers every component of the key', () => { + const covered = new Set(mutations.flatMap(([, m]) => Object.keys(m))); + const componentKeys = Object.keys(BASE); + expect([...covered].sort()).toEqual([...componentKeys].sort()); + }); + + it.each(mutations)('misses after %s', async (_label, override) => { + const cache = new InMemoryResultCache(); + await cache.set(BASE, { rows: [{ value: 100 }] }, { ttlSeconds: 300, rowCount: 1 }); + expect(await cache.get(BASE)).toBeDefined(); + const changed = { ...BASE, ...override }; + expect(await cache.get(changed), 'a changed component served the old result').toBeUndefined(); + }); + + it('expires an entry rather than serving it past its TTL', async () => { + let clock = 1_000_000; + const cache = new InMemoryResultCache(() => clock); + await cache.set(BASE, { rows: [] }, { ttlSeconds: 10, rowCount: 0 }); + expect(await cache.get(BASE)).toBeDefined(); + clock += 11_000; + expect(await cache.get(BASE)).toBeUndefined(); + }); + + it('refuses a non-positive TTL rather than caching forever', async () => { + const cache = new InMemoryResultCache(); + for (const ttl of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + await expect(cache.set(BASE, {}, { ttlSeconds: ttl, rowCount: 0 })).rejects.toBeInstanceOf(SiaxError); + } + }); +}); + +describe('BI-P020 — invalidation', () => { + async function seed() { + const cache = new InMemoryResultCache(); + await cache.set(BASE, { v: 1 }, { ttlSeconds: 300, rowCount: 1 }); + await cache.set( + { ...BASE, compiled_query_hash: 'c'.repeat(64) }, + { v: 2 }, + { ttlSeconds: 300, rowCount: 1 }, + ); + await cache.set( + { ...BASE, dataset_id: 'ds_other', metric_versions: { m_other: 1 } }, + { v: 3 }, + { ttlSeconds: 300, rowCount: 1 }, + ); + await cache.set({ ...BASE, tenant_id: 't_beta' }, { v: 4 }, { ttlSeconds: 300, rowCount: 1 }); + return cache; + } + + it("invalidateDataset drops that dataset's entries and nothing else", async () => { + const cache = await seed(); + const dropped = await cache.invalidateDataset('t_alpha', 'ds_sales'); + expect(dropped).toBe(2); + expect(await cache.get(BASE)).toBeUndefined(); + expect( + await cache.get({ ...BASE, dataset_id: 'ds_other', metric_versions: { m_other: 1 } }), + ).toBeDefined(); + // The other tenant is untouched — invalidation is tenant-scoped too. + expect(await cache.get({ ...BASE, tenant_id: 't_beta' })).toBeDefined(); + }); + + it('invalidateMetric drops every entry that used the metric', async () => { + const cache = await seed(); + const dropped = await cache.invalidateMetric('t_alpha', 'm_revenue'); + expect(dropped).toBe(2); + expect(await cache.get(BASE)).toBeUndefined(); + expect( + await cache.get({ ...BASE, dataset_id: 'ds_other', metric_versions: { m_other: 1 } }), + ).toBeDefined(); + }); + + it('invalidateTenant drops only that tenant', async () => { + const cache = await seed(); + const dropped = await cache.invalidateTenant('t_alpha'); + expect(dropped).toBe(3); + expect(await cache.get({ ...BASE, tenant_id: 't_beta' })).toBeDefined(); + }); +}); + +describe('BI-P020 — never share a tenant result cache', () => { + it('two tenants asking the identical question get different keys', () => { + expect(buildCacheKey(BASE)).not.toBe(buildCacheKey({ ...BASE, tenant_id: 't_beta' })); + }); + + it("REFUSES and evicts an entry whose recorded tenant is not the caller's", async () => { + // The scenario key namespacing alone does not cover: a poisoned or + // mis-migrated entry sitting under the right key with the wrong owner. + // Returning `undefined` here would hide a security event; this refuses. + const client = new FakeValkey(); + const cache = new ValkeyResultCache(client); + await cache.set(BASE, { secret: 'alpha data' }, { ttlSeconds: 300, rowCount: 1 }); + const key = buildCacheKey(BASE); + const poisoned = JSON.parse(client.data.get(key) as string) as CachedResult; + poisoned.tenant_id = 't_beta'; + poisoned.value = { secret: 'BETA DATA' }; + client.data.set(key, JSON.stringify(poisoned)); + + await expect(cache.get(BASE)).rejects.toBeInstanceOf(SiaxError); + // ...and the poisoned entry is gone, so the next read is a clean miss + // rather than a repeated alarm nobody can clear. + expect(client.data.has(key)).toBe(false); + expect(await cache.get(BASE)).toBeUndefined(); + }); + + it('a tenant-scoped scan confirms ownership against the entry, not the glob', async () => { + const client = new FakeValkey(); + const cache = new ValkeyResultCache(client); + await cache.set(BASE, { v: 1 }, { ttlSeconds: 300, rowCount: 1 }); + // An entry whose KEY matches t_alpha's prefix but whose owner is t_beta. + // SCAN MATCH is a glob; the glob would include it, the ownership check + // must not. + client.data.set( + `${tenantKeyPrefix('t_alpha')}forged`, + JSON.stringify({ ...JSON.parse(client.data.get(buildCacheKey(BASE)) as string), tenant_id: 't_beta' }), + ); + const dropped = await cache.invalidateTenant('t_alpha'); + expect(dropped).toBe(1); + }); +}); + +describe('BI-P020 — the Valkey adapter', () => { + it('sets a store-side TTL, not only a recorded timestamp', async () => { + const client = new FakeValkey(); + const cache = new ValkeyResultCache(client); + await cache.set(BASE, { v: 1 }, { ttlSeconds: 42, rowCount: 1 }); + expect(client.ttls.get(buildCacheKey(BASE))).toBe(42); + }); + + it('treats a corrupt entry as a miss AND removes it', async () => { + const client = new FakeValkey(); + const cache = new ValkeyResultCache(client); + client.data.set(buildCacheKey(BASE), '{not json'); + expect(await cache.get(BASE)).toBeUndefined(); + expect(client.data.has(buildCacheKey(BASE))).toBe(false); + }); + + it('pages through SCAN rather than blocking on KEYS', async () => { + const client = new FakeValkey(); + const cache = new ValkeyResultCache(client); + for (let i = 0; i < 1200; i += 1) { + await cache.set( + { ...BASE, compiled_query_hash: String(i).padStart(64, '0') }, + { i }, + { ttlSeconds: 300, rowCount: 1 }, + ); + } + expect(await cache.invalidateTenant('t_alpha')).toBe(1200); + expect(client.scanCalls).toBeGreaterThan(1); + }); +}); + +class FakeValkey implements ValkeyLikeClient { + readonly data = new Map(); + readonly ttls = new Map(); + scanCalls = 0; + + async get(key: string): Promise { + return this.data.get(key) ?? null; + } + async set(key: string, value: string, opts: { EX: number }): Promise { + this.data.set(key, value); + this.ttls.set(key, opts.EX); + return 'OK'; + } + async del(keys: string[]): Promise { + for (const k of keys) { + this.data.delete(k); + this.ttls.delete(k); + } + return keys.length; + } + async scan( + cursor: string, + opts: { MATCH: string; COUNT: number }, + ): Promise<{ cursor: string; keys: string[] }> { + this.scanCalls += 1; + const prefix = opts.MATCH.replace(/\*$/, ''); + const all = [...this.data.keys()].filter((k) => k.startsWith(prefix)); + const start = Number(cursor); + const page = all.slice(start, start + opts.COUNT); + const next = start + opts.COUNT >= all.length ? '0' : String(start + opts.COUNT); + return { cursor: next, keys: page }; + } +} diff --git a/tests/analytics/query-compiler-injection.test.ts b/tests/analytics/query-compiler-injection.test.ts index d001722..a1df7d0 100644 --- a/tests/analytics/query-compiler-injection.test.ts +++ b/tests/analytics/query-compiler-injection.test.ts @@ -8,217 +8,219 @@ * real server — and the with-fix/without-fix proof BI-P011 actually demands — * are tests/integration/analytics-tenant-isolation.integration.test.ts. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect } from 'vitest'; import { assertSafeIdentifier, compileMetricQuery, compileProjectionQuery, dimensionCatalogFrom, restrictedTenantUserDdl, -} from "@/src/analytics/engine/query-compiler"; +} from '@/src/analytics/engine/query-compiler'; -const catalog = dimensionCatalogFrom({ severity: "severity", make: "vehicle_make" }); +const catalog = dimensionCatalogFrom({ severity: 'severity', make: 'vehicle_make' }); const HOSTILE = [ "x'; DROP TABLE opt_analytics_event; --", - "1 OR 1=1", - "tenant_id", - "*/ OR 1=1 /*", - "severity UNION ALL SELECT tenant_id FROM opt_analytics_event", - "dim_values[1]", - "__proto__", - "constructor", - "sever ity", - "sévérité", - "a".repeat(200), - "", + '1 OR 1=1', + 'tenant_id', + '*/ OR 1=1 /*', + 'severity UNION ALL SELECT tenant_id FROM opt_analytics_event', + 'dim_values[1]', + '__proto__', + 'constructor', + 'sever ity', + 'sévérité', + 'a'.repeat(200), + '', ]; -describe("BI-P011 injection surfaces at the compiler", () => { - it("dimension identifier injection: hostile group_by names are refused, not escaped", () => { +describe('BI-P011 injection surfaces at the compiler', () => { + it('dimension identifier injection: hostile group_by names are refused, not escaped', () => { for (const bad of HOSTILE) { - expect(() => - compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", group_by: [bad] }, - "opt_analytics", - catalog, - 1000, - ), + expect( + () => + compileMetricQuery( + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', group_by: [bad] }, + 'opt_analytics', + catalog, + 1000, + ), `group_by "${bad}" was not refused`, ).toThrowError(/unknown group_by dimension/); } }); - it("filter injection: hostile filter KEYS are refused by catalog resolution", () => { + it('filter injection: hostile filter KEYS are refused by catalog resolution', () => { for (const bad of HOSTILE) { - expect(() => - compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", filters: { [bad]: "v" } }, - "opt_analytics", - catalog, - 1000, - ), + expect( + () => + compileMetricQuery( + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', filters: { [bad]: 'v' } }, + 'opt_analytics', + catalog, + 1000, + ), `filter key "${bad}" was not refused`, ).toThrowError(/unknown filter dimension/); } }); - it("prototype-chain keys resolve to undefined rather than to Object.prototype members", () => { + it('prototype-chain keys resolve to undefined rather than to Object.prototype members', () => { // A naive `catalog[name]` lookup would resolve "constructor" to a // function and "toString" to a method; the Map-based catalog must not. - for (const key of ["constructor", "toString", "hasOwnProperty", "__proto__", "valueOf"]) { + for (const key of ['constructor', 'toString', 'hasOwnProperty', '__proto__', 'valueOf']) { expect(catalog.resolve(key)).toBeUndefined(); } }); - it("filter VALUES are never spliced, however hostile", () => { + it('filter VALUES are never spliced, however hostile', () => { const compiled = compileMetricQuery( { - tenant_id: "t_alpha", - metric: "m", - aggregate: "sum", + tenant_id: 't_alpha', + metric: 'm', + aggregate: 'sum', filters: { make: "volvo'; DROP TABLE opt_analytics_event; --" }, }, - "opt_analytics", + 'opt_analytics', catalog, 1000, ); - expect(compiled.sql).not.toContain("DROP TABLE"); - expect(compiled.sql).not.toContain("volvo"); + expect(compiled.sql).not.toContain('DROP TABLE'); + expect(compiled.sql).not.toContain('volvo'); expect(compiled.params.param_f0v).toBe("volvo'; DROP TABLE opt_analytics_event; --"); // The physical column name came from the catalog, not the caller. - expect(compiled.params.param_f0n).toBe("vehicle_make"); + expect(compiled.params.param_f0n).toBe('vehicle_make'); }); - it("metric and dataset VALUES are never spliced either", () => { + it('metric and dataset VALUES are never spliced either', () => { const compiled = compileMetricQuery( { - tenant_id: "t_alpha", + tenant_id: 't_alpha', metric: "m'; SELECT 1; --", dataset_id: "ds'; SELECT 1; --", - aggregate: "sum", + aggregate: 'sum', }, - "opt_analytics", + 'opt_analytics', catalog, 1000, ); - expect(compiled.sql).not.toContain("SELECT 1"); + expect(compiled.sql).not.toContain('SELECT 1'); expect(compiled.sql.match(/SELECT/g)).toHaveLength(1); }); - it("raw SQL attempt: there is no request shape that reaches a second table", () => { + it('raw SQL attempt: there is no request shape that reaches a second table', () => { const compiled = compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum" }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum' }, + 'opt_analytics', catalog, 1000, ); expect(compiled.sql).not.toMatch(/\bUNION\b/i); expect(compiled.sql).not.toMatch(/\bJOIN\b/i); expect((compiled.sql.match(/\bFROM\b/gi) ?? []).length).toBe(1); - expect(compiled.table).toBe("opt_analytics_event"); + expect(compiled.table).toBe('opt_analytics_event'); }); - it("a forged table name is refused rather than reaching the FROM clause", () => { + it('a forged table name is refused rather than reaching the FROM clause', () => { expect(() => compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", table: "system.tables" as never }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', table: 'system.tables' as never }, + 'opt_analytics', catalog, 1000, ), ).toThrowError(/invalid table identifier/); }); - it("an unsupported aggregate is refused rather than being interpolated", () => { + it('an unsupported aggregate is refused rather than being interpolated', () => { expect(() => compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum(1),tenant_id" as never }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum(1),tenant_id' as never }, + 'opt_analytics', catalog, 1000, ), ).toThrowError(/unsupported aggregate/); }); - it("group-by cardinality is capped so a caller cannot explode the GROUP BY", () => { - const many = Array.from({ length: 9 }, () => "severity"); + it('group-by cardinality is capped so a caller cannot explode the GROUP BY', () => { + const many = Array.from({ length: 9 }, () => 'severity'); expect(() => compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", group_by: many }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', group_by: many }, + 'opt_analytics', catalog, 1000, ), ).toThrowError(/at most 8 group-by/); }); - it("LIMIT is clamped to the governance ceiling, and never below 1", () => { + it('LIMIT is clamped to the governance ceiling, and never below 1', () => { const big = compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", limit: 10_000_000 }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', limit: 10_000_000 }, + 'opt_analytics', catalog, 500, ); - expect(big.sql).toContain("LIMIT 500"); + expect(big.sql).toContain('LIMIT 500'); const zero = compileMetricQuery( - { tenant_id: "t_alpha", metric: "m", aggregate: "sum", limit: 0 }, - "opt_analytics", + { tenant_id: 't_alpha', metric: 'm', aggregate: 'sum', limit: 0 }, + 'opt_analytics', catalog, 500, ); - expect(zero.sql).toContain("LIMIT 1"); + expect(zero.sql).toContain('LIMIT 1'); }); - it("the projection compiler forces the tenant and parameterizes entity keys", () => { + it('the projection compiler forces the tenant and parameterizes entity keys', () => { const compiled = compileProjectionQuery( - { tenant_id: "t_alpha", dataset_id: "ds_x", entity_keys: ["a'; DROP", "b"] }, - "opt_analytics", + { tenant_id: 't_alpha', dataset_id: 'ds_x', entity_keys: ["a'; DROP", 'b'] }, + 'opt_analytics', 100, ); expect(compiled.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); - expect(compiled.sql).not.toContain("DROP"); - expect(compiled.params.param_entityKeys).toBe(JSON.stringify(["a'; DROP", "b"])); + expect(compiled.sql).not.toContain('DROP'); + expect(compiled.params.param_entityKeys).toBe(JSON.stringify(["a'; DROP", 'b'])); }); - it("assertSafeIdentifier accepts real identifiers and rejects everything else", () => { - for (const good of ["severity", "vehicle_make", "_x", "a1_b2"]) { - expect(assertSafeIdentifier(good, "test")).toBe(good); + it('assertSafeIdentifier accepts real identifiers and rejects everything else', () => { + for (const good of ['severity', 'vehicle_make', '_x', 'a1_b2']) { + expect(assertSafeIdentifier(good, 'test')).toBe(good); } - for (const bad of ["1abc", "a-b", "a b", "a;b", "", "a".repeat(64)]) { - expect(() => assertSafeIdentifier(bad, "test")).toThrowError(/invalid test identifier/); + for (const bad of ['1abc', 'a-b', 'a b', 'a;b', '', 'a'.repeat(64)]) { + expect(() => assertSafeIdentifier(bad, 'test')).toThrowError(/invalid test identifier/); } }); - it("restrictedTenantUserDdl refuses a tenant id or password it cannot safely embed", () => { + it('restrictedTenantUserDdl refuses a tenant id or password it cannot safely embed', () => { expect(() => restrictedTenantUserDdl({ - db: "opt_analytics", + db: 'opt_analytics', tenantId: "t_alpha' OR '1'='1", - userName: "opt_t_alpha", - password: "pw", - tables: ["opt_analytics_event"], + userName: 'opt_t_alpha', + password: 'pw', + tables: ['opt_analytics_event'], }), ).toThrowError(/not policy-safe/); expect(() => restrictedTenantUserDdl({ - db: "opt_analytics", - tenantId: "t_alpha", - userName: "opt_t_alpha", + db: 'opt_analytics', + tenantId: 't_alpha', + userName: 'opt_t_alpha', password: "pw' OR '1'='1", - tables: ["opt_analytics_event"], + tables: ['opt_analytics_event'], }), ).toThrowError(/cannot be embedded/); const stmts = restrictedTenantUserDdl({ - db: "opt_analytics", - tenantId: "t_alpha", - userName: "opt_t_alpha", - password: "safe_pw_123", - tables: ["opt_analytics_event"], + db: 'opt_analytics', + tenantId: 't_alpha', + userName: 'opt_t_alpha', + password: 'safe_pw_123', + tables: ['opt_analytics_event'], }); - expect(stmts.join(" ")).toContain("CREATE ROW POLICY IF NOT EXISTS"); - expect(stmts.join(" ")).toContain("USING tenant_id = 't_alpha'"); + expect(stmts.join(' ')).toContain('CREATE ROW POLICY IF NOT EXISTS'); + expect(stmts.join(' ')).toContain("USING tenant_id = 't_alpha'"); }); }); diff --git a/tests/analytics/query-execution-model.test.ts b/tests/analytics/query-execution-model.test.ts new file mode 100644 index 0000000..3f1a8ab --- /dev/null +++ b/tests/analytics/query-execution-model.test.ts @@ -0,0 +1,404 @@ +/** + * tests/analytics/query-execution-model.test.ts + * ============================================== + * BI-P018 (execution model), BI-P019 (cancellation logic) and BI-P021 + * (cost/budget) — the parts provable without an engine. + * + * The engine-dependent claims — "cancellation actually stops a running query", + * "the budget actually stops a running query" — are NOT asserted here. They + * live in tests/integration/query-cancellation-budget.integration.test.ts + * against a live ClickHouse, because a fake process-control that returns + * `isRunning: false` on the second call would make this file pass while + * proving nothing about the server. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + assertLegalTransition, + assertNoRawSql, + compiledQueryHash, + QueryExecutionRecorder, + TERMINAL_STATES, + type QueryExecutionRecord, +} from '@/src/analytics/execution/query-execution-record'; +import { cancelAndVerify, type EngineProcessControl } from '@/src/analytics/execution/cancellation'; +import { + costUnits, + DEFAULT_QUERY_BUDGET, + InMemoryWindowStore, + QueryBudgetEnforcer, +} from '@/src/analytics/cost/query-budget'; + +// ── BI-P018 ──────────────────────────────────────────────────────────────── + +function recorder(sink: { write: (r: QueryExecutionRecord) => void }, now = () => 1_000_000) { + return new QueryExecutionRecorder( + { + tenant_id: 't_alpha', + project_id: 'p_main', + principal_id: 'u_1', + workload: 'dashboard', + engine: 'clickhouse', + correlation_id: 'c_1', + sink, + now, + }, + 'opt_q1', + { + hash: compiledQueryHash('SELECT 1', { param_tenant: 't_alpha' }), + datasets: ['ds_sales'], + metrics: ['revenue'], + }, + ); +} + +describe('BI-P018 — every field the prompt names, and none it forbids', () => { + it('carries all eighteen required fields once settled', async () => { + const written: QueryExecutionRecord[] = []; + let clock = 1_000_000; + const r = recorder({ write: (x) => void written.push(x) }, () => clock); + await r.submitted(); + clock += 50; + await r.started(); + clock += 700; + const final = await r.finish('succeeded', { + rows: 12, + rows_scanned: 40_960, + bytes_scanned: 1_234_567, + cost_units: 0.5, + }); + + for (const field of [ + 'query_id', + 'tenant_id', + 'project_id', + 'principal_id', + 'workload', + 'submitted_at', + 'started_at', + 'completed_at', + 'engine', + 'compiled_query_hash', + 'datasets', + 'metrics', + 'status', + 'rows', + 'rows_scanned', + 'bytes_scanned', + 'duration_ms', + 'cache_status', + 'cost_units', + ]) { + expect(final[field as keyof QueryExecutionRecord], `missing ${field}`).toBeDefined(); + } + expect(final.duration_ms).toBe(700); + expect(final.status).toBe('succeeded'); + }); + + it('writes at EVERY transition, so an execution that never finishes is still visible', async () => { + const written: QueryExecutionRecord[] = []; + const r = recorder({ write: (x) => void written.push(x) }); + await r.submitted(); + await r.started(); + expect(written.map((w) => w.status)).toEqual(['queued', 'running']); + // A recorder that only wrote at the end would have zero rows here — and + // would lose precisely the executions worth investigating. + }); + + it('stores a hash, never the statement', async () => { + const written: QueryExecutionRecord[] = []; + const r = recorder({ write: (x) => void written.push(x) }); + await r.submitted(); + await r.started(); + const final = await r.finish('succeeded', { rows: 1 }); + const serialized = JSON.stringify(final); + expect(serialized).not.toMatch(/SELECT/i); + expect(final.compiled_query_hash).toMatch(/^[0-9a-f]{64}$/); + }); + + it('REFUSES to emit a record whose error field carries statement text', () => { + const record: QueryExecutionRecord = { + query_id: 'q', + execution_id: 'e', + tenant_id: 't_alpha', + principal_id: 'u', + workload: 'api', + submitted_at: new Date().toISOString(), + engine: 'clickhouse', + compiled_query_hash: 'a'.repeat(64), + datasets: [], + metrics: [], + status: 'failed', + cache_status: 'miss', + // The realistic leak: ClickHouse's error text echoes the query. + error: 'Code: 47. Unknown identifier in scope SELECT tenant_id FROM opt_analytics.sales', + }; + expect(() => assertNoRawSql(record)).toThrowError(SiaxError); + }); + + it('hashes the same statement with different parameters differently', () => { + const a = compiledQueryHash('SELECT 1 WHERE t = {t:String}', { param_t: 't_alpha' }); + const b = compiledQueryHash('SELECT 1 WHERE t = {t:String}', { param_t: 't_beta' }); + expect(a).not.toBe(b); + // ...and identically regardless of key order. + expect(compiledQueryHash('q', { b: '2', a: '1' })).toBe(compiledQueryHash('q', { a: '1', b: '2' })); + }); + + it('refuses an illegal state transition rather than overwriting a terminal row', () => { + for (const terminal of TERMINAL_STATES) { + expect(() => assertLegalTransition(terminal, 'running')).toThrowError(SiaxError); + } + expect(() => assertLegalTransition('queued', 'succeeded')).toThrowError(SiaxError); + expect(() => assertLegalTransition('running', 'succeeded')).not.toThrow(); + }); + + it('records a cache hit as a cache hit, not as a very fast query', async () => { + const written: QueryExecutionRecord[] = []; + const r = recorder({ write: (x) => void written.push(x) }); + await r.submitted(); + await r.servedFromCache(42); + const final = written.at(-1) as QueryExecutionRecord; + expect(final.cache_status).toBe('hit'); + expect(final.rows_scanned).toBe(0); + expect(final.bytes_scanned).toBe(0); + }); + + it('requires a verified tenant and principal', () => { + for (const bad of [{ tenant_id: '' }, { principal_id: '' }]) { + expect( + () => + new QueryExecutionRecorder( + { + tenant_id: 't_alpha', + principal_id: 'u_1', + workload: 'api', + engine: 'clickhouse', + sink: { write: () => undefined }, + ...bad, + }, + 'q', + { hash: 'h', datasets: [], metrics: [] }, + ), + ).toThrowError(SiaxError); + } + }); +}); + +// ── BI-P019 (logic only; the engine claim is in the integration suite) ───── + +describe('BI-P019 — cancellation reports what it VERIFIED, not what it requested', () => { + function control(script: boolean[]): EngineProcessControl & { kills: number } { + let i = 0; + return { + engine: 'test', + kills: 0, + async isRunning() { + const v = script[Math.min(i, script.length - 1)]; + i += 1; + return v; + }, + async requestKill() { + (this as { kills: number }).kills += 1; + return true; + }, + }; + } + + it('reports verified_stopped only after OBSERVING the query gone', async () => { + // running, running, running, gone + const c = control([true, true, true, false]); + const out = await cancelAndVerify(c, 'q1', { timeoutMs: 5_000, pollIntervalMs: 1 }); + expect(out.verified_stopped).toBe(true); + expect(out.kill_accepted).toBe(true); + expect(c.kills).toBe(1); + }); + + it('reports verified_stopped=false when the query outlives the deadline — it does not claim success', async () => { + const c = control([true]); // never stops + let clock = 0; + const out = await cancelAndVerify(c, 'q1', { + timeoutMs: 100, + pollIntervalMs: 10, + now: () => (clock += 30), + sleep: async () => undefined, + }); + expect(out.verified_stopped).toBe(false); + expect(out.already_finished).toBe(false); + }); + + it('does not claim to have stopped a query that had already finished', async () => { + const c = control([false]); + const out = await cancelAndVerify(c, 'q1', { timeoutMs: 100 }); + expect(out.already_finished).toBe(true); + expect(out.verified_stopped).toBe(true); + }); + + it('keeps polling when the engine is unreachable, then reports unverified', async () => { + let clock = 0; + const flaky: EngineProcessControl = { + engine: 'test', + isRunning: async () => { + throw new Error('engine unreachable'); + }, + requestKill: async () => true, + }; + // isRunning throws on the FIRST call too, which cancelAndVerify treats as + // "assume it was running" — the fail-closed reading. + const out = await cancelAndVerify(flaky, 'q1', { + timeoutMs: 50, + pollIntervalMs: 10, + now: () => (clock += 20), + sleep: async () => undefined, + }); + expect(out.verified_stopped).toBe(false); + }); + + it('aborts the client side as well as the server side', async () => { + let aborted = false; + await cancelAndVerify(control([true, false]), 'q1', { + timeoutMs: 1_000, + pollIntervalMs: 1, + abort: () => { + aborted = true; + }, + }); + expect(aborted).toBe(true); + }); +}); + +// ── BI-P021 ──────────────────────────────────────────────────────────────── + +describe('BI-P021 — the budget says plainly what it does and does not bound', () => { + const ledger = { + entries: [] as unknown[], + record(e: unknown) { + this.entries.push(e); + }, + }; + + it('refuses to be constructed as an in-flight enforcer without a process control', () => { + expect( + () => + new QueryBudgetEnforcer({ + budget: DEFAULT_QUERY_BUDGET, + store: new InMemoryWindowStore(), + ledger, + enforceInFlight: true, + }), + ).toThrowError(SiaxError); + }); + + it('declares admission-only mode as advisory, in words', () => { + const e = new QueryBudgetEnforcer({ + budget: DEFAULT_QUERY_BUDGET, + store: new InMemoryWindowStore(), + ledger, + enforceInFlight: false, + }); + expect(e.describeEnforcement().binding).toBe(false); + expect(e.describeEnforcement().in_flight).toContain('NOT ENFORCED'); + }); + + it('declares the per-process window as non-binding across replicas', () => { + const e = new QueryBudgetEnforcer({ + budget: DEFAULT_QUERY_BUDGET, + store: new InMemoryWindowStore(), + ledger, + control: { + engine: 't', + isRunning: async () => false, + requestKill: async () => true, + progress: async () => undefined, + }, + enforceInFlight: true, + storeIsShared: false, + }); + expect(e.describeEnforcement().across_replicas).toContain('N times this budget'); + expect(e.describeEnforcement().binding).toBe(false); + }); + + it('rejects an estimate that does not fit the remaining window', async () => { + const store = new InMemoryWindowStore(); + const e = new QueryBudgetEnforcer({ + budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 1_000 }, + store, + ledger, + enforceInFlight: false, + }); + expect((await e.admit('t_alpha', { rows_to_read: 100, bytes_to_read: 1 })).admitted).toBe(true); + await store.add('t_alpha', { rows: 999, bytes: 0, units: 0, windowMs: DEFAULT_QUERY_BUDGET.windowMs }); + const decision = await e.admit('t_alpha', { rows_to_read: 100, bytes_to_read: 1 }); + expect(decision.admitted).toBe(false); + expect(decision.reason).toContain('rows'); + }); + + it('throttles instead of rejecting when the budget says so', async () => { + const store = new InMemoryWindowStore(); + const e = new QueryBudgetEnforcer({ + budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 10, onExhausted: 'throttle', throttleDelayMs: 250 }, + store, + ledger, + enforceInFlight: false, + }); + await store.add('t_alpha', { rows: 100, bytes: 0, units: 0, windowMs: DEFAULT_QUERY_BUDGET.windowMs }); + const decision = await e.admit('t_alpha', { rows_to_read: 1, bytes_to_read: 1 }); + expect(decision.admitted).toBe(true); + expect(decision.delay_ms).toBe(250); + }); + + it('charges for the work a cancelled query already did', async () => { + const store = new InMemoryWindowStore(); + const entries: Array<{ terminated_by_budget: boolean; cost_units: number }> = []; + const e = new QueryBudgetEnforcer({ + budget: DEFAULT_QUERY_BUDGET, + store, + ledger: { record: (x) => void entries.push(x as never) }, + enforceInFlight: false, + }); + const settled = await e.settle({ + tenant_id: 't_alpha', + principal_id: 'u', + query_id: 'q', + cost: { rows_scanned: 1e9, bytes_scanned: 2 ** 30, execution_ms: 1000, engine: 'clickhouse' }, + terminated_by_budget: true, + }); + expect(settled.cost_units).toBeGreaterThan(0); + expect(entries[0].terminated_by_budget).toBe(true); + // Not charging for it is how a tenant learns the cheapest query is one + // that gets cancelled. + expect(entries[0].cost_units).toBe(settled.cost_units); + }); + + it('weights bytes above rows, so a wide scan is not billed like a narrow one', () => { + const narrow = costUnits({ rows_scanned: 1e9, bytes_scanned: 1e8, execution_ms: 0, engine: 'e' }); + const wide = costUnits({ rows_scanned: 1e8, bytes_scanned: 1e10, execution_ms: 0, engine: 'e' }); + expect(wide).toBeGreaterThan(narrow); + }); + + it('expires spend out of the rolling window', async () => { + let clock = 0; + const store = new InMemoryWindowStore(() => clock); + await store.add('t_alpha', { rows: 100, bytes: 0, units: 1, windowMs: 1000 }); + expect((await store.get('t_alpha', 1000)).rows_scanned).toBe(100); + clock += 1500; + expect((await store.get('t_alpha', 1000)).rows_scanned).toBe(0); + }); + + it('in-flight enforcement returns immediately when the query is already gone', async () => { + const e = new QueryBudgetEnforcer({ + budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 1 }, + store: new InMemoryWindowStore(), + ledger, + control: { + engine: 't', + isRunning: async () => false, + requestKill: async () => true, + progress: async () => undefined, + }, + enforceInFlight: true, + }); + const out = await e.enforceInFlight('t_alpha', 'q'); + expect(out.terminated).toBe(false); + expect(out.polls).toBe(1); + }); +}); diff --git a/tests/analytics/query-request-contract.test.ts b/tests/analytics/query-request-contract.test.ts new file mode 100644 index 0000000..bd7cc6f --- /dev/null +++ b/tests/analytics/query-request-contract.test.ts @@ -0,0 +1,554 @@ +/** + * tests/analytics/query-request-contract.test.ts + * ============================================== + * BI-P016 — QUERY REQUEST CONTRACT. + * + * Two claims are under test, and both are stated so they can FAIL: + * + * 1. "The default product/application API must NOT accept arbitrary SQL." + * Tested by feeding the parser every field name a caller would reach for + * to smuggle SQL and asserting a rejection — not by asserting the type + * lacks the field, which a compiler already guarantees and a JSON body + * ignores. + * + * 2. "Same request can compile to ClickHouse/semantic engine without + * consumer rewrite." Tested by projecting BOTH compiled artefacts onto a + * comparable shape and demanding equality. The ClickHouse side of that + * projection is obtained by PARSING the emitted SQL and its bound + * parameters here in the test — not by trusting a field the compiler + * helpfully returned. A compiler that silently drops a filter therefore + * fails this test instead of passing it. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + ANALYTICS_QUERY_REQUEST_KEYS, + FILTER_OPERATORS, + parseAnalyticsQueryRequest, + queryResolutionFrom, + resolveAnalyticsQuery, + type AnalyticsQueryRequest, + type FilterOperator, + type ResolvedDimension, + type ResolvedMetric, +} from '@/src/analytics/contract/query-request'; +import { + compileRequestToClickHouse, + type CompiledAnalyticsQuery, +} from '@/src/analytics/contract/compile-clickhouse'; +import { + compileRequestToSemantic, + shapeOfSemanticSpec, + type QueryShape, +} from '@/src/analytics/contract/compile-semantic'; +import { FORCED_TENANT_PREDICATE } from '@/src/analytics/engine/query-compiler'; + +// ── the declared catalog under test ──────────────────────────────────────── + +const METRICS: ResolvedMetric[] = [ + { + name: 'orders', + metric_id: 'm_orders', + event_metric_key: 'order_count', + aggregation: 'count', + allowed_dimensions: ['country', 'channel', 'segment'], + }, + { + name: 'revenue', + metric_id: 'm_revenue', + event_metric_key: 'revenue_sek', + aggregation: 'sum', + allowed_dimensions: ['country', 'channel', 'segment'], + unit: 'SEK', + }, + { + name: 'restricted', + metric_id: 'm_restricted', + event_metric_key: 'restricted_measure', + aggregation: 'sum', + // Deliberately NOT sliceable by `segment`. + allowed_dimensions: ['country'], + }, +]; + +const DIMENSIONS: ResolvedDimension[] = [ + { name: 'country', dimension_id: 'd_country', field_name: 'country_code' }, + { name: 'channel', dimension_id: 'd_channel', field_name: 'sales_channel' }, + { name: 'segment', dimension_id: 'd_segment', field_name: 'customer_segment' }, +]; + +const resolution = queryResolutionFrom({ + dataset_id: 'ds_sales', + metrics: METRICS, + dimensions: DIMENSIONS, + semantic_model_version: 7, + defaultTimeDimension: 'event_time', +}); + +const FIELD_TO_NAME = new Map(DIMENSIONS.map((d) => [d.field_name, d.name])); +const KEY_TO_METRIC = new Map(METRICS.map((m) => [m.event_metric_key, m.name])); + +function compile(req: AnalyticsQueryRequest): CompiledAnalyticsQuery { + return compileRequestToClickHouse(resolveAnalyticsQuery(req, resolution), { + tenant_id: 't_alpha', + project_id: 'p_main', + database: 'opt_analytics', + maxResultRows: 10_000, + }); +} + +// ── independent parser: SQL text + bound params -> comparable shape ──────── + +/** Split a WHERE clause on top-level " AND " (parenthesised lists survive). */ +function splitConditions(where: string): string[] { + const out: string[] = []; + let depth = 0; + let current = ''; + for (let i = 0; i < where.length; i += 1) { + const c = where[i]; + if (c === '(') depth += 1; + if (c === ')') depth -= 1; + if (depth === 0 && where.startsWith(' AND ', i)) { + out.push(current); + current = ''; + i += 4; + continue; + } + current += c; + } + if (current) out.push(current); + return out; +} + +const OP_SYMBOL: Record = { + '=': 'eq', + '!=': 'ne', + '<': 'lt', + '<=': 'lte', + '>': 'gt', + '>=': 'gte', +}; + +/** + * Reconstruct the query's meaning from the artefact the engine would actually + * send. Anything the compiler failed to emit is invisible here, which is the + * point. + */ +function shapeOfCompiledSql(c: CompiledAnalyticsQuery): QueryShape { + const p = (key: string): string => { + const v = c.params[`param_${key}`]; + if (v === undefined) throw new Error(`compiled SQL references unbound parameter ${key}`); + return v; + }; + const whereMatch = /\sWHERE\s(.*?)(?:\sGROUP BY\s|\sORDER BY\s|\sLIMIT\s)/.exec(c.sql); + if (!whereMatch) throw new Error('compiled SQL has no WHERE clause'); + const conditions = splitConditions(whereMatch[1]); + + const filters: QueryShape['filters'] = []; + let timeFrom: string | undefined; + let timeTo: string | undefined; + + for (const cond of conditions) { + if (cond === FORCED_TENANT_PREDICATE) continue; + if (/^project_id = \{project:String\}$/.test(cond)) continue; + if (/^dataset_id = \{dataset:String\}$/.test(cond)) continue; + if (/^metric IN \(/.test(cond)) continue; + + let m = /^\w+ >= parseDateTime64BestEffort\(\{fromTime:String\}\)$/.exec(cond); + if (m) { + timeFrom = p('fromTime'); + continue; + } + m = /^\w+ < parseDateTime64BestEffort\(\{toTime:String\}\)$/.exec(cond); + if (m) { + timeTo = p('toTime'); + continue; + } + m = /^position\(dim_values\[indexOf\(dim_names, \{(fn\d+):String\}\)\], \{(fv\d+):String\}\) > 0$/.exec( + cond, + ); + if (m) { + filters.push({ dimension: dimNameFor(p(m[1])), operator: 'contains', values: [p(m[2])] }); + continue; + } + m = /^(NOT )?has\(dim_names, \{(fn\d+):String\}\)$/.exec(cond); + if (m) { + filters.push({ + dimension: dimNameFor(p(m[2])), + operator: m[1] ? 'is_null' : 'is_not_null', + values: [], + }); + continue; + } + m = /^dim_values\[indexOf\(dim_names, \{(fn\d+):String\}\)\] (NOT IN|IN) \((.+)\)$/.exec(cond); + if (m) { + const keys = m[3].split(', ').map((t) => /^\{(fv\d+_\d+):String\}$/.exec(t)?.[1]); + if (keys.some((k) => !k)) throw new Error(`IN list contains a non-parameter token: ${m[3]}`); + filters.push({ + dimension: dimNameFor(p(m[1])), + operator: m[2] === 'NOT IN' ? 'not_in' : 'in', + values: keys.map((k) => p(k as string)), + }); + continue; + } + m = /^dim_values\[indexOf\(dim_names, \{(fn\d+):String\}\)\] (=|!=|<=|>=|<|>) \{(fv\d+):String\}$/.exec( + cond, + ); + if (m) { + filters.push({ dimension: dimNameFor(p(m[1])), operator: OP_SYMBOL[m[2]], values: [p(m[3])] }); + continue; + } + throw new Error(`unrecognised compiled condition (possible unbound SQL text): ${cond}`); + } + + const selectMatch = /^SELECT (.*?) FROM /.exec(c.sql); + if (!selectMatch) throw new Error('compiled SQL has no SELECT list'); + const metricKeys = [...selectMatch[1].matchAll(/\{(m\d+):String\}/g)].map((mm) => p(mm[1])); + const dimKeys = [...selectMatch[1].matchAll(/\{(d\d+):String\}/g)].map((mm) => p(mm[1])); + const grainMatch = /toStartOf(\w+)\(/.exec(selectMatch[1]); + + const orderMatch = /\sORDER BY\s(.*?)\sLIMIT\s/.exec(c.sql); + const order = orderMatch + ? orderMatch[1].split(', ').map((tok) => { + const [alias, dir] = tok.split(' '); + const name = c.metricAliases[alias] ?? c.dimensionAliases[alias]; + if (!name) throw new Error(`ORDER BY references unknown alias ${alias}`); + return `${name}:${dir.toLowerCase()}`; + }) + : []; + + const limitMatch = /\sLIMIT (\d+) FORMAT JSON$/.exec(c.sql); + if (!limitMatch) throw new Error('compiled SQL has no LIMIT'); + + return { + dataset_id: p('dataset'), + metrics: metricKeys.map((k) => KEY_TO_METRIC.get(k) ?? k).sort(), + dimensions: dimKeys.map((k) => FIELD_TO_NAME.get(k) ?? k).sort(), + filters: filters.sort((a, b) => (a.dimension + a.operator).localeCompare(b.dimension + b.operator)), + order, + limit: Number(limitMatch[1]), + timeFrom, + timeTo, + grain: grainMatch ? (grainMatch[1].toLowerCase() as QueryShape['grain']) : undefined, + }; +} + +function dimNameFor(field: string): string { + const name = FIELD_TO_NAME.get(field); + if (!name) throw new Error(`compiled SQL bound an unknown physical dimension "${field}"`); + return name; +} + +// ── 1. the API does not accept SQL ───────────────────────────────────────── + +describe('BI-P016 — the product API accepts no SQL', () => { + const base = { dataset: 'ds_sales', metrics: ['orders'] }; + + const smuggling = [ + 'sql', + 'query', + 'rawSql', + 'raw_sql', + 'raw', + 'where', + 'having', + 'expression', + 'expr', + 'select', + 'from', + 'groupBy', + 'group_by', + 'unsafe', + ]; + + it.each(smuggling)('rejects a request carrying a %s field', (key) => { + expect(() => parseAnalyticsQueryRequest({ ...base, [key]: 'SELECT 1' })).toThrowError(SiaxError); + try { + parseAnalyticsQueryRequest({ ...base, [key]: 'SELECT 1' }); + } catch (e) { + expect((e as SiaxError).code).toBe('INVALID_REQUEST'); + } + }); + + // The INF0 Wave-2 lesson: a tenant read out of a request body. The contract + // must not merely IGNORE such a field; ignoring it is what lets a later + // refactor start honouring it. + it.each(['tenant', 'tenant_id', 'tenantId', 'project_id', 'database', 'catalog', 'table'])( + 'rejects caller-supplied %s (tenant/catalog selection is never a request field)', + (key) => { + expect(() => parseAnalyticsQueryRequest({ ...base, [key]: 't_beta' })).toThrowError(SiaxError); + }, + ); + + it('rejects ANY unknown key, not just the ones on the smell list', () => { + expect(() => parseAnalyticsQueryRequest({ ...base, somethingNewNextQuarter: 1 })).toThrowError(SiaxError); + }); + + it('accepts exactly the eight BI-P016 fields and no more', () => { + expect([...ANALYTICS_QUERY_REQUEST_KEYS]).toEqual([ + 'dataset', + 'metrics', + 'dimensions', + 'filters', + 'timeRange', + 'orderBy', + 'limit', + 'format', + ]); + }); + + it('rejects SQL smuggled into a nested filter object', () => { + expect(() => + parseAnalyticsQueryRequest({ + ...base, + filters: [{ dimension: 'country', operator: 'eq', values: ['SE'], sql: 'OR 1=1' }], + }), + ).toThrowError(SiaxError); + }); + + it('keeps the filter operator set closed', () => { + expect([...FILTER_OPERATORS]).toEqual([ + 'eq', + 'ne', + 'in', + 'not_in', + 'lt', + 'lte', + 'gt', + 'gte', + 'contains', + 'is_null', + 'is_not_null', + ]); + expect(() => + parseAnalyticsQueryRequest({ + ...base, + filters: [{ dimension: 'country', operator: 'raw', values: ['x'] }], + }), + ).toThrowError(SiaxError); + }); + + it('refuses an orderBy that is not selected (the obvious expression smuggling slot)', () => { + expect(() => + parseAnalyticsQueryRequest({ ...base, orderBy: [{ ref: '(SELECT 1)', direction: 'asc' }] }), + ).toThrowError(SiaxError); + }); + + it('refuses a metric the catalog does not declare', () => { + const req = parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['orders OR 1=1'] }); + expect(() => resolveAnalyticsQuery(req, resolution)).toThrowError(SiaxError); + }); + + it('refuses slicing a metric by a dimension its definition disallows', () => { + const req = parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['restricted'], + dimensions: ['segment'], + }); + try { + resolveAnalyticsQuery(req, resolution); + throw new Error('expected a rejection'); + } catch (e) { + expect((e as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + } + }); + + it('refuses a dataset that is not the authorised one', () => { + const req = parseAnalyticsQueryRequest({ dataset: 'ds_other', metrics: ['orders'] }); + try { + resolveAnalyticsQuery(req, resolution); + throw new Error('expected a rejection'); + } catch (e) { + expect((e as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + } + }); + + it('rejects a reversed time range instead of silently returning zero rows', () => { + expect(() => + parseAnalyticsQueryRequest({ ...base, timeRange: { from: '2026-02-01', to: '2026-01-01' } }), + ).toThrowError(SiaxError); + }); +}); + +// ── 2. one request, two targets, same meaning ────────────────────────────── + +describe('BI-P016 DoD — one request compiles to ClickHouse and to the semantic engine', () => { + const requests: Array<[string, unknown]> = [ + ['metric only', { dataset: 'ds_sales', metrics: ['orders'] }], + [ + 'two metrics, two dimensions', + { dataset: 'ds_sales', metrics: ['orders', 'revenue'], dimensions: ['country', 'channel'] }, + ], + [ + 'filters of every value operator', + { + dataset: 'ds_sales', + metrics: ['revenue'], + dimensions: ['country'], + filters: [ + { dimension: 'country', operator: 'eq', values: ['SE'] }, + { dimension: 'channel', operator: 'in', values: ['web', 'retail'] }, + { dimension: 'segment', operator: 'not_in', values: ['test'] }, + { dimension: 'channel', operator: 'contains', values: ['we'] }, + ], + }, + ], + [ + 'time range with grain and ordering', + { + dataset: 'ds_sales', + metrics: ['revenue'], + dimensions: ['country'], + timeRange: { from: '2026-01-01T00:00:00Z', to: '2026-02-01T00:00:00Z', grain: 'day' }, + orderBy: [{ ref: 'revenue', direction: 'desc' }], + limit: 50, + }, + ], + [ + 'presence filters', + { + dataset: 'ds_sales', + metrics: ['orders'], + filters: [ + { dimension: 'segment', operator: 'is_null' }, + { dimension: 'channel', operator: 'is_not_null' }, + ], + }, + ], + ]; + + it.each(requests)('%s — both targets agree', (_label, raw) => { + const req = parseAnalyticsQueryRequest(raw); + const resolved = resolveAnalyticsQuery(req, resolution); + const ch = compileRequestToClickHouse(resolved, { + tenant_id: 't_alpha', + project_id: 'p_main', + database: 'opt_analytics', + maxResultRows: 10_000, + }); + const sem = compileRequestToSemantic(resolved); + + const fromSql = shapeOfCompiledSql(ch); + const fromSemantic = shapeOfSemanticSpec(sem); + + expect(fromSql.dataset_id).toBe(fromSemantic.dataset_id); + expect(fromSql.metrics).toEqual(fromSemantic.metrics); + expect(fromSql.dimensions).toEqual(fromSemantic.dimensions); + expect(fromSql.filters).toEqual(fromSemantic.filters); + expect(fromSql.order).toEqual(fromSemantic.order); + expect(fromSql.limit).toBe(fromSemantic.limit); + expect(fromSql.timeFrom).toBe(fromSemantic.timeFrom); + expect(fromSql.timeTo).toBe(fromSemantic.timeTo); + expect(fromSql.grain).toBe(fromSemantic.grain); + }); + + it('carries the canonical metric_id into the semantic spec so the engine is replaceable', () => { + const req = parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['revenue'] }); + const spec = compileRequestToSemantic(resolveAnalyticsQuery(req, resolution)); + expect(spec.metrics[0].metric_id).toBe('m_revenue'); + expect(spec.semantic_model_version).toBe(7); + }); + + it('never puts a SQL predicate string in the semantic spec', () => { + const req = parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['orders'], + filters: [{ dimension: 'country', operator: 'eq', values: ["SE' OR '1'='1"] }], + }); + const spec = compileRequestToSemantic(resolveAnalyticsQuery(req, resolution)); + // The hostile value survives as DATA — structured, not concatenated. + expect(spec.where[0].values).toEqual(["SE' OR '1'='1"]); + expect(JSON.stringify(spec)).not.toMatch(/SELECT|WHERE\s/i); + }); +}); + +// ── 3. tenant forcing survives every request shape ───────────────────────── + +describe('BI-P011 forced tenant predicate survives the BI-P016 contract', () => { + it.each(requests())('compiled SQL forces the tenant predicate first: %s', (_label, raw) => { + const c = compile(parseAnalyticsQueryRequest(raw)); + const where = /\sWHERE\s(.*?)(?:\sGROUP BY\s|\sORDER BY\s|\sLIMIT\s)/.exec(c.sql); + expect(where).not.toBeNull(); + expect(splitConditions((where as RegExpExecArray)[1])[0]).toBe(FORCED_TENANT_PREDICATE); + expect(c.params.param_tenant).toBe('t_alpha'); + }); + + it('refuses to compile with a blank tenant rather than emitting no predicate', () => { + const resolved = resolveAnalyticsQuery( + parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['orders'] }), + resolution, + ); + for (const blank of ['', ' ']) { + expect(() => + compileRequestToClickHouse(resolved, { + tenant_id: blank, + database: 'opt_analytics', + maxResultRows: 100, + }), + ).toThrowError(SiaxError); + } + }); + + it('puts no caller-supplied value anywhere in the SQL text', () => { + const hostile = "'; DROP TABLE opt_analytics_event; --"; + const c = compile( + parseAnalyticsQueryRequest({ + dataset: 'ds_sales', + metrics: ['orders'], + filters: [{ dimension: 'country', operator: 'eq', values: [hostile] }], + }), + ); + expect(c.sql).not.toContain(hostile); + expect(c.sql).not.toContain('DROP'); + expect(Object.values(c.params)).toContain(hostile); + }); + + it('clamps the limit down to the governance ceiling and never up', () => { + const resolved = resolveAnalyticsQuery( + parseAnalyticsQueryRequest({ dataset: 'ds_sales', metrics: ['orders'], limit: 1_000_000 }), + resolution, + ); + const c = compileRequestToClickHouse(resolved, { + tenant_id: 't_alpha', + database: 'opt_analytics', + maxResultRows: 500, + }); + expect(c.effectiveLimit).toBe(500); + expect(c.sql).toContain('LIMIT 500'); + }); +}); + +function requests(): Array<[string, unknown]> { + return [ + ['bare', { dataset: 'ds_sales', metrics: ['orders'] }], + ['grouped', { dataset: 'ds_sales', metrics: ['orders'], dimensions: ['country'] }], + [ + 'filtered', + { + dataset: 'ds_sales', + metrics: ['orders'], + filters: [{ dimension: 'country', operator: 'in', values: ['SE', 'NO'] }], + }, + ], + [ + 'time-bucketed', + { dataset: 'ds_sales', metrics: ['revenue'], timeRange: { from: '2026-01-01', grain: 'month' } }, + ], + [ + 'everything at once', + { + dataset: 'ds_sales', + metrics: ['orders', 'revenue'], + dimensions: ['country', 'channel'], + filters: [ + { dimension: 'segment', operator: 'ne', values: ['internal'] }, + { dimension: 'channel', operator: 'is_not_null' }, + ], + timeRange: { from: '2026-01-01', to: '2026-03-01', grain: 'week' }, + orderBy: [{ ref: 'revenue', direction: 'desc' }], + limit: 25, + format: 'arrow_ipc', + }, + ], + ]; +} diff --git a/tests/analytics/resource-governance.test.ts b/tests/analytics/resource-governance.test.ts index 19854d6..1f4ef55 100644 --- a/tests/analytics/resource-governance.test.ts +++ b/tests/analytics/resource-governance.test.ts @@ -4,7 +4,7 @@ * real server and lives in * tests/integration/analytics-resource-governance.integration.test.ts. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect } from 'vitest'; import { applyQuota, BUDGET_TIERS, @@ -13,43 +13,43 @@ import { concurrencyProfileSql, DEFAULT_BUDGETS, REQUIRED_GOVERNANCE_SETTINGS, -} from "@/src/analytics/engine/resource-governance"; +} from '@/src/analytics/engine/resource-governance'; -describe("BI-P012 resource governance", () => { - it("every setting the program names has a value somewhere in the enforcement path", () => { +describe('BI-P012 resource governance', () => { + it('every setting the program names has a value somewhere in the enforcement path', () => { const settings = clickhouseSettings(DEFAULT_BUDGETS.INTERACTIVE); - const profile = concurrencyProfileSql("opt_t_alpha", DEFAULT_BUDGETS.INTERACTIVE).join(" "); + const profile = concurrencyProfileSql('opt_t_alpha', DEFAULT_BUDGETS.INTERACTIVE).join(' '); for (const name of REQUIRED_GOVERNANCE_SETTINGS) { - const present = name in settings || profile.includes(name) || name === "max_concurrent_queries"; + const present = name in settings || profile.includes(name) || name === 'max_concurrent_queries'; expect(present, `${name} is not enforced anywhere`).toBe(true); } // The five per-statement ones must be on the STATEMENT, not only in a // profile a future operator might forget to attach. for (const name of [ - "max_execution_time", - "max_result_rows", - "max_rows_to_read", - "max_bytes_to_read", - "max_memory_usage", + 'max_execution_time', + 'max_result_rows', + 'max_rows_to_read', + 'max_bytes_to_read', + 'max_memory_usage', ]) { expect(settings[name], `${name} missing from per-statement settings`).toBeDefined(); } }); - it("readonly=1 rides on every SELECT so the query text cannot re-set the limits", () => { + it('readonly=1 rides on every SELECT so the query text cannot re-set the limits', () => { for (const tier of BUDGET_TIERS) { - expect(clickhouseSettings(DEFAULT_BUDGETS[tier]).readonly).toBe("1"); + expect(clickhouseSettings(DEFAULT_BUDGETS[tier]).readonly).toBe('1'); } }); - it("overflow modes are THROW, not BREAK — a partial result must never look complete", () => { + it('overflow modes are THROW, not BREAK — a partial result must never look complete', () => { const s = clickhouseSettings(DEFAULT_BUDGETS.INTERACTIVE); - expect(s.result_overflow_mode).toBe("throw"); - expect(s.read_overflow_mode).toBe("throw"); - expect(s.timeout_overflow_mode).toBe("throw"); + expect(s.result_overflow_mode).toBe('throw'); + expect(s.read_overflow_mode).toBe('throw'); + expect(s.timeout_overflow_mode).toBe('throw'); }); - it("a CL0UD quota can only TIGHTEN a tier, never loosen it", () => { + it('a CL0UD quota can only TIGHTEN a tier, never loosen it', () => { const base = DEFAULT_BUDGETS.INTERACTIVE; const loosened = applyQuota(base, { maxExecutionTimeSec: 100000, @@ -67,7 +67,7 @@ describe("BI-P012 resource governance", () => { expect(tightened.maxExecutionTimeSec).toBe(base.maxExecutionTimeSec); }); - it("a nonsense quota (zero, negative, NaN) falls back to the tier rather than to no limit", () => { + it('a nonsense quota (zero, negative, NaN) falls back to the tier rather than to no limit', () => { const base = DEFAULT_BUDGETS.INTERACTIVE; expect(applyQuota(base, { maxResultRows: 0 }).maxResultRows).toBe(base.maxResultRows); expect(applyQuota(base, { maxResultRows: -5 }).maxResultRows).toBe(base.maxResultRows); @@ -75,44 +75,48 @@ describe("BI-P012 resource governance", () => { expect(applyQuota(base, undefined)).toEqual(base); }); - it("tiers get progressively more generous, and INTERACTIVE is the mean default", () => { - expect(DEFAULT_BUDGETS.INTERACTIVE.maxExecutionTimeSec).toBeLessThan(DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec); - expect(DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec).toBeLessThan(DEFAULT_BUDGETS.BATCH.maxExecutionTimeSec); + it('tiers get progressively more generous, and INTERACTIVE is the mean default', () => { + expect(DEFAULT_BUDGETS.INTERACTIVE.maxExecutionTimeSec).toBeLessThan( + DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec, + ); + expect(DEFAULT_BUDGETS.ANALYTICAL.maxExecutionTimeSec).toBeLessThan( + DEFAULT_BUDGETS.BATCH.maxExecutionTimeSec, + ); expect(DEFAULT_BUDGETS.INTERACTIVE.maxRowsToRead).toBeLessThan(DEFAULT_BUDGETS.ANALYTICAL.maxRowsToRead); }); - it("the concurrency gate throttles ONE tenant without touching another", () => { - const gate = new ConcurrencyGate((t) => (t === "t_noisy" ? 2 : 5)); - const a = gate.acquire("t_noisy"); - const b = gate.acquire("t_noisy"); - expect(gate.inFlightFor("t_noisy")).toBe(2); - expect(() => gate.acquire("t_noisy")).toThrowError(/queries in flight/); + it('the concurrency gate throttles ONE tenant without touching another', () => { + const gate = new ConcurrencyGate((t) => (t === 't_noisy' ? 2 : 5)); + const a = gate.acquire('t_noisy'); + const b = gate.acquire('t_noisy'); + expect(gate.inFlightFor('t_noisy')).toBe(2); + expect(() => gate.acquire('t_noisy')).toThrowError(/queries in flight/); // The quiet tenant is unaffected — BI-P012's "without degrading unrelated // tenant" in the smallest testable form. - const quiet = gate.acquire("t_quiet"); - expect(gate.inFlightFor("t_quiet")).toBe(1); + const quiet = gate.acquire('t_quiet'); + expect(gate.inFlightFor('t_quiet')).toBe(1); a(); - expect(gate.inFlightFor("t_noisy")).toBe(1); - expect(() => gate.acquire("t_noisy")).not.toThrow(); + expect(gate.inFlightFor('t_noisy')).toBe(1); + expect(() => gate.acquire('t_noisy')).not.toThrow(); b(); quiet(); }); - it("releasing twice does not free a slot that was never taken", () => { + it('releasing twice does not free a slot that was never taken', () => { const gate = new ConcurrencyGate(() => 1); - const release = gate.acquire("t_x"); + const release = gate.acquire('t_x'); release(); release(); - expect(gate.inFlightFor("t_x")).toBe(0); + expect(gate.inFlightFor('t_x')).toBe(0); expect(gate.totalInFlight()).toBe(0); }); it("the settings profile SQL carries max_concurrent_queries' server-side half", () => { - const stmts = concurrencyProfileSql("opt_t_alpha", DEFAULT_BUDGETS.INTERACTIVE); - expect(stmts.join(" ")).toContain("CREATE SETTINGS PROFILE IF NOT EXISTS opt_t_alpha"); - expect(stmts.join(" ")).toContain("readonly = 1"); - expect(stmts.join(" ")).toContain("CREATE QUOTA IF NOT EXISTS opt_t_alpha_quota"); + const stmts = concurrencyProfileSql('opt_t_alpha', DEFAULT_BUDGETS.INTERACTIVE); + expect(stmts.join(' ')).toContain('CREATE SETTINGS PROFILE IF NOT EXISTS opt_t_alpha'); + expect(stmts.join(' ')).toContain('readonly = 1'); + expect(stmts.join(' ')).toContain('CREATE QUOTA IF NOT EXISTS opt_t_alpha_quota'); }); }); diff --git a/tests/analytics/tenant-predicate-regression.test.ts b/tests/analytics/tenant-predicate-regression.test.ts index 2c1b04a..a378b3e 100644 --- a/tests/analytics/tenant-predicate-regression.test.ts +++ b/tests/analytics/tenant-predicate-regression.test.ts @@ -28,16 +28,16 @@ * the real server would receive, so asserting on it cannot be faked by the * test double. */ -import { describe, it, expect, afterEach, vi } from "vitest"; -import { ClickHouseAnalytics } from "@/src/analytics/clickhouse-analytics"; +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { ClickHouseAnalytics } from '@/src/analytics/clickhouse-analytics'; import { compileMetricQuery, compileProjectionQuery, dimensionCatalogFrom, FORCED_TENANT_PREDICATE, -} from "@/src/analytics/engine/query-compiler"; +} from '@/src/analytics/engine/query-compiler'; -const catalog = dimensionCatalogFrom({ severity: "severity", make: "vehicle_make" }); +const catalog = dimensionCatalogFrom({ severity: 'severity', make: 'vehicle_make' }); /** Captures every URL the adapter fetches, returning an empty JSON result. */ function captureFetch(): { urls: string[]; restore: () => void } { @@ -54,91 +54,98 @@ afterEach(() => { vi.restoreAllMocks(); }); -describe("BI-P009 regression guard: the forced tenant predicate", () => { - it("PRESERVED ClickHouseAnalytics still emits tenant_id = {tenant:String} on the wire", async () => { +describe('BI-P009 regression guard: the forced tenant predicate', () => { + it('PRESERVED ClickHouseAnalytics still emits tenant_id = {tenant:String} on the wire', async () => { const cap = captureFetch(); try { - const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123", database: "opt_test" }); - await analytics.queryMetric({ tenant_id: "t_alpha", metric: "rows", aggregate: "sum" }); + const analytics = new ClickHouseAnalytics({ baseUrl: 'http://ch.invalid:8123', database: 'opt_test' }); + await analytics.queryMetric({ tenant_id: 't_alpha', metric: 'rows', aggregate: 'sum' }); expect(cap.urls).toHaveLength(1); // URLSearchParams decodes `+` as a space, which raw decodeURIComponent // does not — read the statement the way the server will. - const query = new URL(cap.urls[0]).searchParams.get("query") ?? ""; - expect(query).toContain("tenant_id = {tenant:String}"); + const query = new URL(cap.urls[0]).searchParams.get('query') ?? ''; + expect(query).toContain('tenant_id = {tenant:String}'); expect(query).toMatch(/WHERE tenant_id = \{tenant:String\}/); // The VALUE travels as a bound parameter, never inside the statement. - expect(query).not.toContain("t_alpha"); - expect(new URL(cap.urls[0]).searchParams.get("param_tenant")).toBe("t_alpha"); + expect(query).not.toContain('t_alpha'); + expect(new URL(cap.urls[0]).searchParams.get('param_tenant')).toBe('t_alpha'); } finally { cap.restore(); } }); - it("PRESERVED ClickHouseAnalytics still refuses a blank tenant_id", async () => { - const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123" }); - await expect(analytics.queryMetric({ tenant_id: " ", metric: "x", aggregate: "sum" })).rejects.toMatchObject({ - code: "INVALID_REQUEST", + it('PRESERVED ClickHouseAnalytics still refuses a blank tenant_id', async () => { + const analytics = new ClickHouseAnalytics({ baseUrl: 'http://ch.invalid:8123' }); + await expect( + analytics.queryMetric({ tenant_id: ' ', metric: 'x', aggregate: 'sum' }), + ).rejects.toMatchObject({ + code: 'INVALID_REQUEST', }); }); - it("PRESERVED ClickHouseAnalytics still refuses a hostile group_by identifier", async () => { - const analytics = new ClickHouseAnalytics({ baseUrl: "http://ch.invalid:8123" }); + it('PRESERVED ClickHouseAnalytics still refuses a hostile group_by identifier', async () => { + const analytics = new ClickHouseAnalytics({ baseUrl: 'http://ch.invalid:8123' }); await expect( analytics.queryMetric({ - tenant_id: "t_alpha", - metric: "x", - aggregate: "sum", + tenant_id: 't_alpha', + metric: 'x', + aggregate: 'sum', group_by: "sev'; DROP TABLE opt_metric_events; --", }), - ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); + ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); }); - it("every statement the NEW compiler produces contains the forced predicate", () => { + it('every statement the NEW compiler produces contains the forced predicate', () => { const shapes = [ - { metric: "m", aggregate: "sum" as const }, - { metric: "m", aggregate: "count" as const, group_by: ["severity"] }, - { metric: "m", aggregate: "avg" as const, filters: { make: "volvo" }, dataset_id: "ds_1" }, - { metric: "m", aggregate: "max" as const, from_time: "2026-01-01", to_time: "2026-02-01" }, - { metric: "m", aggregate: "count_distinct" as const, group_by: ["severity", "make"], limit: 5 }, + { metric: 'm', aggregate: 'sum' as const }, + { metric: 'm', aggregate: 'count' as const, group_by: ['severity'] }, + { metric: 'm', aggregate: 'avg' as const, filters: { make: 'volvo' }, dataset_id: 'ds_1' }, + { metric: 'm', aggregate: 'max' as const, from_time: '2026-01-01', to_time: '2026-02-01' }, + { metric: 'm', aggregate: 'count_distinct' as const, group_by: ['severity', 'make'], limit: 5 }, ]; for (const shape of shapes) { - const compiled = compileMetricQuery({ tenant_id: "t_alpha", ...shape }, "opt_analytics", catalog, 1000); + const compiled = compileMetricQuery({ tenant_id: 't_alpha', ...shape }, 'opt_analytics', catalog, 1000); expect(compiled.sql).toContain(FORCED_TENANT_PREDICATE); // The predicate must be in the WHERE, not in a comment or an alias. expect(compiled.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); - expect(compiled.params.param_tenant).toBe("t_alpha"); - expect(compiled.sql).not.toContain("t_alpha"); + expect(compiled.params.param_tenant).toBe('t_alpha'); + expect(compiled.sql).not.toContain('t_alpha'); } const projection = compileProjectionQuery( - { tenant_id: "t_alpha", dataset_id: "ds_1", entity_keys: ["a", "b"] }, - "opt_analytics", + { tenant_id: 't_alpha', dataset_id: 'ds_1', entity_keys: ['a', 'b'] }, + 'opt_analytics', 1000, ); expect(projection.sql).toMatch(/WHERE tenant_id = \{tenant:String\}/); - expect(projection.sql).not.toContain("t_alpha"); + expect(projection.sql).not.toContain('t_alpha'); }); - it("the NEW compiler fails closed on a blank/absent tenant instead of emitting an unfiltered query", () => { - for (const tenant of ["", " "]) { + it('the NEW compiler fails closed on a blank/absent tenant instead of emitting an unfiltered query', () => { + for (const tenant of ['', ' ']) { expect(() => - compileMetricQuery({ tenant_id: tenant, metric: "m", aggregate: "sum" }, "opt_analytics", catalog, 1000), + compileMetricQuery( + { tenant_id: tenant, metric: 'm', aggregate: 'sum' }, + 'opt_analytics', + catalog, + 1000, + ), ).toThrowError(/tenant_id is required/); } // The undefined case, which TypeScript would normally forbid, is what a // JS caller or a loosely-typed HTTP boundary can actually produce. expect(() => compileMetricQuery( - { tenant_id: undefined as unknown as string, metric: "m", aggregate: "sum" }, - "opt_analytics", + { tenant_id: undefined as unknown as string, metric: 'm', aggregate: 'sum' }, + 'opt_analytics', catalog, 1000, ), ).toThrowError(/tenant_id is required/); }); - it("the constant itself is the predicate, so deleting the forcing cannot pass silently", () => { + it('the constant itself is the predicate, so deleting the forcing cannot pass silently', () => { // If someone redefines FORCED_TENANT_PREDICATE to something harmless // (e.g. "1 = 1") to make the substring assertions above pass, this fails. - expect(FORCED_TENANT_PREDICATE).toBe("tenant_id = {tenant:String}"); + expect(FORCED_TENANT_PREDICATE).toBe('tenant_id = {tenant:String}'); }); }); diff --git a/tests/duckdb/duckdb-provider.live.test.ts b/tests/duckdb/duckdb-provider.live.test.ts new file mode 100644 index 0000000..41c4cfc --- /dev/null +++ b/tests/duckdb/duckdb-provider.live.test.ts @@ -0,0 +1,396 @@ +/** + * tests/duckdb/duckdb-provider.live.test.ts + * ========================================== + * BI-P014 — the DuckDB provider against a real DuckDB. + * + * Two things are proven here that a unit test cannot: + * + * 1. BI-P014's DoD — "Queries work against Parquet / Arrow / Iceberg snapshot + * fixtures" — for the formats this environment can actually build. Parquet + * and CSV run from committed fixtures. The Iceberg case runs when a Python + * interpreter carrying `pyiceberg` is pointed at by PYICEBERG_PYTHON, + * because pyiceberg embeds ABSOLUTE paths in its manifests, so a committed + * Iceberg table would only work from the machine that made it. The recorded + * result of a real Iceberg run is committed as evidence instead, and + * tests/analytics/duckdb-provider.test.ts pins it. + * + * 2. The security claim, WITH LAYER 1 REMOVED. Every bypass attempt is sent + * straight to DuckDB, skipping the mount registry entirely, and layer 2 — + * the session sandbox — must refuse it. Which matters more than usual here, + * because layer 2 does not work the way it looks like it works: see the + * `allowed_directories` finding in duckdb-provider.ts. + * + * Gated on DUCKDB_PYTHON, an interpreter with the `duckdb` package. With it + * set and DuckDB broken, this suite FAILS; it does not degrade to a skip. + * + * ── WHY THIS IS NOT UNDER tests/integration/ ────────────────────────────── + * It was, and CI's integration job failed — correctly. That job provides a + * live OpenSearch and a live ClickHouse and then asserts that NOTHING in + * tests/integration/ skipped, on the stated principle that "a skipped suite + * must never masquerade as a green integration gate". A DuckDB suite needing + * a Python interpreter that job does not provide is not one of the suites that + * job is a gate for, so it skipped, and the guard fired. + * + * Moving the file respects that guard rather than weakening it. Loosening the + * guard to tolerate skips would have traded a real gate for one green job — + * which is the trade this estate keeps regretting. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + ArtifactMountRegistry, + DuckDbAnalyticsProvider, + duckDbSessionBootstrap, + type DuckDbRunner, +} from '@/src/analytics/providers/duckdb-provider'; + +const PY = process.env.DUCKDB_PYTHON ?? ''; +/** An interpreter carrying pyiceberg, used to REBUILD the non-portable fixture. */ +const PY_ICEBERG = process.env.PYICEBERG_PYTHON ?? PY; +const suite = PY ? describe : describe.skip; +const FIXTURES = path.join(process.cwd(), 'tests/fixtures/duckdb'); +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p014/duckdb-sandbox.json'); +const IDENTITY = { tenant_id: 't_alpha', principal_id: 'u_1' }; + +/** + * Runs statements through a real DuckDB. The bootstrap is applied FIRST and in + * order, exactly as the provider emits it. + */ +function runDuck( + bootstrap: readonly string[], + sql: string, + loadIceberg = false, +): { ok: boolean; out: string } { + const script = [ + 'import duckdb, json, sys', + 'c = duckdb.connect()', + ...(loadIceberg ? ['c.execute("INSTALL iceberg; LOAD iceberg")'] : []), + // The bootstrap is inside the try as well: a statement that tries to + // RE-OPEN a locked sandbox throws, and that throw is a RESULT to report, + // not a crash of the harness. The first version of this file let it kill + // the subprocess, which turned a meaningful refusal into an unreadable + // "Command failed". + 'try:', + ...bootstrap.map((s) => ` c.execute(${JSON.stringify(s)})`), + ` rows = c.execute(${JSON.stringify(sql)}).fetchall()`, + ' print(json.dumps({"ok": True, "rows": [[str(v) for v in r] for r in rows]}))', + 'except Exception as e:', + ' print(json.dumps({"ok": False, "error": str(e)[:300]}))', + ].join('\n'); + const out = execFileSync(PY, ['-c', script], { encoding: 'utf8', timeout: 60_000 }); + const parsed = JSON.parse(out.trim()) as { ok: boolean; rows?: string[][]; error?: string }; + return { ok: parsed.ok, out: JSON.stringify(parsed.rows ?? parsed.error) }; +} + +const runner: DuckDbRunner = { + engine: 'duckdb', + async run({ bootstrap, sql }) { + const result = runDuck(bootstrap, sql, sql.includes('iceberg_scan')); + if (!result.ok) + throw new SiaxError('PERMANENT_FAILURE', 'duckdb', `query failed: ${result.out.slice(0, 160)}`); + return (JSON.parse(result.out) as string[][]).map((r, i) => ({ _row: i, values: r })); + }, +}; + +const evidence: Record = {}; +let registry: ArtifactMountRegistry; +let provider: DuckDbAnalyticsProvider; + +beforeAll(() => { + if (!PY) return; + const version = execFileSync(PY, ['-c', 'import duckdb; print(duckdb.__version__)'], { + encoding: 'utf8', + }).trim(); + evidence.duckdb_version = version; + evidence.recorded_at = new Date().toISOString(); + + expect(fs.existsSync(path.join(FIXTURES, 'sales.parquet')), 'parquet fixture is missing').toBe(true); + + // Rebuild the Iceberg fixture, which is deliberately not committed: + // pyiceberg writes ABSOLUTE paths into its metadata and Avro manifests, so a + // committed table resolves only on the machine that made it. + if (PY_ICEBERG && !fs.existsSync(path.join(FIXTURES, 'iceberg_metadata_location.txt'))) { + const build = [ + 'import os, shutil, pyarrow as pa', + 'from pyiceberg.catalog.sql import SqlCatalog', + `root = ${JSON.stringify(FIXTURES)}`, + "wh = os.path.join(root, 'iceberg')", + 'shutil.rmtree(wh, ignore_errors=True); os.makedirs(wh)', + "cat = SqlCatalog('local', **{'uri': f'sqlite:///{wh}/catalog.db', 'warehouse': f'file://{wh}'})", + "cat.create_namespace_if_not_exists('bi')", + "d = pa.table({'tenant_id': ['t_alpha']*3, 'country': ['SE','NO','SE'], 'amount': [10.0,20.0,30.0]})", + "t = cat.create_table('bi.sales_iceberg', schema=d.schema); t.append(d)", + "open(os.path.join(root,'iceberg_metadata_location.txt'),'w').write(t.metadata_location)", + "print('built')", + ].join('\n'); + try { + execFileSync(PY_ICEBERG, ['-c', build], { encoding: 'utf8', timeout: 120_000 }); + } catch { + // pyiceberg absent. The Iceberg case then falls back to the committed + // evidence, and it asserts against it rather than passing vacuously. + } + } + + registry = new ArtifactMountRegistry(FIXTURES); + registry.register({ + artifact_id: 'a_sales_parquet', + tenant_id: 't_alpha', + st0re_ref: 'st0re://t_alpha/sales.parquet', + relative_path: 'sales.parquet', + format: 'parquet', + size_bytes: fs.statSync(path.join(FIXTURES, 'sales.parquet')).size, + }); + registry.register({ + artifact_id: 'a_customers_csv', + tenant_id: 't_alpha', + st0re_ref: 'st0re://t_alpha/customers.csv', + relative_path: 'customers.csv', + format: 'csv', + size_bytes: fs.statSync(path.join(FIXTURES, 'customers.csv')).size, + }); + // Another tenant's artifact, so the cross-tenant refusal is a real refusal + // and not "the id does not exist". + registry.register({ + artifact_id: 'a_beta_parquet', + tenant_id: 't_beta', + st0re_ref: 'st0re://t_beta/sales.parquet', + relative_path: 'sales.parquet', + format: 'parquet', + size_bytes: 1, + }); + + provider = new DuckDbAnalyticsProvider(registry, runner); +}, 60_000); + +suite('BI-P014 DoD — queries work against the fixtures', () => { + it('aggregates a Parquet fixture through the closed request', async () => { + const rows = await provider.query(IDENTITY, { + artifact_id: 'a_sales_parquet', + group_by: ['country'], + aggregate: { fn: 'sum', column: 'amount', alias: 'total' }, + }); + const totals = Object.fromEntries( + rows.map((r) => { + const [country, total] = (r as { values: string[] }).values; + return [country, Number(total)]; + }), + ); + expect(totals).toEqual({ SE: 394.5, NO: 105.5 }); + evidence.parquet = totals; + }); + + it('reads a CSV fixture, including a value containing a quote', async () => { + const rows = await provider.query(IDENTITY, { + artifact_id: 'a_customers_csv', + select: ['customer_id', 'name'], + }); + const names = rows.map((r) => (r as { values: string[] }).values[1]); + expect(names).toContain("O'Brien Ltd"); + evidence.csv = { rows: rows.length, includes_quoted_value: true }; + }); + + it('filters with a hostile value without it becoming SQL', async () => { + const rows = await provider.query(IDENTITY, { + artifact_id: 'a_customers_csv', + select: ['customer_id'], + filters: { name: "O'Brien Ltd" }, + }); + expect(rows).toHaveLength(1); + + // The injection attempt: a value that would close the literal and append. + const hostile = await provider.query(IDENTITY, { + artifact_id: 'a_customers_csv', + select: ['customer_id'], + filters: { name: "x' OR '1'='1" }, + }); + // It matched nothing, because it was DATA. Had the escaping failed, the + // OR would have matched all three rows. + expect(hostile).toHaveLength(0); + evidence.injection_attempt = { matched_rows: hostile.length, note: 'escaped as data; 0 rows, not 3' }; + }); + + it('reads an ICEBERG SNAPSHOT by its exact metadata file', async () => { + const metaFile = path.join(FIXTURES, 'iceberg_metadata_location.txt'); + if (!fs.existsSync(metaFile)) { + // No pyiceberg here. Carry the previously RECORDED result forward, marked + // as carried forward, and assert against it — the first version of this + // branch was `expect(evidence.iceberg ?? "unbuilt").toBeDefined()`, which + // can never fail and quietly erased the recorded block from the evidence + // file while both suites stayed green. + const prior = fs.existsSync(EVIDENCE_PATH) + ? (JSON.parse(fs.readFileSync(EVIDENCE_PATH, 'utf8')) as { + iceberg?: { totals: Record }; + }) + : {}; + expect( + prior.iceberg?.totals, + 'no pyiceberg available AND no recorded Iceberg result — the DoD is unproven, not skipped', + ).toEqual({ NO: 20, SE: 40 }); + evidence.iceberg = { ...prior.iceberg, carried_forward: true }; + return; + } + const meta = fs + .readFileSync(metaFile, 'utf8') + .trim() + .replace(/^file:\/\//, ''); + const bootstrap = duckDbSessionBootstrap([FIXTURES]); + const res = runDuck( + bootstrap, + `SELECT country, sum(amount) AS s FROM iceberg_scan('${meta}') GROUP BY country ORDER BY country`, + true, + ); + expect(res.ok, `iceberg_scan failed: ${res.out}`).toBe(true); + const rows = JSON.parse(res.out) as string[][]; + expect(Object.fromEntries(rows.map((r) => [r[0], Number(r[1])]))).toEqual({ NO: 20, SE: 40 }); + evidence.iceberg = { + read_by: 'iceberg_scan on the exact metadata JSON', + note: + 'an exact metadata file is a SNAPSHOT reference; DuckDB refuses to glob for the latest ' + + 'version by default, and that refusal is correct — globbing can read uncommitted data', + totals: { NO: 20, SE: 40 }, + }; + }); +}); + +suite('BI-P014 — the mount registry (layer 1)', () => { + it('refuses an artifact belonging to another tenant, as AUTHORIZATION_DENIED not NOT_FOUND', async () => { + let thrown: unknown; + try { + await provider.query(IDENTITY, { artifact_id: 'a_beta_parquet' }); + } catch (e) { + thrown = e; + } + expect((thrown as SiaxError).code).toBe('AUTHORIZATION_DENIED'); + }); + + it('refuses an unmounted artifact', async () => { + await expect(provider.query(IDENTITY, { artifact_id: 'a_nope' })).rejects.toBeInstanceOf(SiaxError); + }); + + it('refuses a SYMLINK that escapes the mount root', () => { + const link = path.join(FIXTURES, 'escape.parquet'); + fs.rmSync(link, { force: true }); + fs.symlinkSync('/etc/passwd', link); + try { + expect(() => + registry.register({ + artifact_id: 'a_escape', + tenant_id: 't_alpha', + st0re_ref: 'st0re://x', + relative_path: 'escape.parquet', + format: 'parquet', + size_bytes: 1, + }), + ).toThrowError(SiaxError); + evidence.symlink_escape_refused = true; + } finally { + fs.rmSync(link, { force: true }); + } + }); + + it('refuses a traversing relative path', () => { + expect(() => + registry.register({ + artifact_id: 'a_trav', + tenant_id: 't_alpha', + st0re_ref: 'st0re://x', + relative_path: '../../../../etc/passwd', + format: 'csv', + size_bytes: 1, + }), + ).toThrowError(SiaxError); + }); +}); + +suite('BI-P014 — layer 2 alone, WITH THE MOUNT REGISTRY REMOVED', () => { + /** Attempts sent straight to DuckDB. Layer 1 is not in the picture. */ + const attempts: Array<{ id: string; sql: string }> = [ + { id: 'read-etc-passwd', sql: "SELECT * FROM read_csv('/etc/passwd')" }, + { id: 'read-etc-shadow', sql: "SELECT * FROM read_csv('/etc/shadow')" }, + { id: 'traversal-from-mount', sql: `SELECT * FROM read_csv('${FIXTURES}/../../../../etc/passwd')` }, + { id: 'glob-root', sql: "SELECT * FROM read_csv('/etc/*')" }, + { id: 'http-exfiltration', sql: "SELECT * FROM read_csv('https://example.com/a.csv')" }, + { id: 's3-read', sql: "SELECT * FROM read_parquet('s3://bucket/a.parquet')" }, + // Not a SELECT: ATTACH is the other way a local engine touches a file. + // The first version of this corpus had `SELECT 1 FROM (SELECT 1) t` here, + // which is not an attack at all — it succeeded and was reported as a leak. + // A corpus entry that cannot fail is as useless as a check that cannot. + { id: 'attach-external-db', sql: "ATTACH '/tmp/duckdb_escape.db' AS evil" }, + { id: 'attach-etc', sql: "ATTACH '/etc/passwd' AS evil2 (READ_ONLY)" }, + { id: 'copy-out-of-mount', sql: "COPY (SELECT 1 AS a) TO '/tmp/duckdb_escape.csv' (FORMAT CSV)" }, + { id: 'read-parquet-outside', sql: "SELECT * FROM read_parquet('/root/x.parquet')" }, + { id: 'json-outside', sql: "SELECT * FROM read_json_auto('/etc/hosts')" }, + ]; + + it('no attempt reads anything outside the mount, and the file inside still works', () => { + const bootstrap = duckDbSessionBootstrap([FIXTURES]); + const results: Array> = []; + const leaks: string[] = []; + + for (const attempt of attempts) { + const res = runDuck(bootstrap, attempt.sql); + // A leak is a SUCCESSFUL read. `s3://` and `https://` may fail for other + // reasons too, but the assertion is on success, not on the message. + if (res.ok) leaks.push(attempt.id); + results.push({ id: attempt.id, succeeded: res.ok, engine_message: res.out.slice(0, 140) }); + } + + // The counterfactual: the SAME session reads the mounted file fine, so + // "nothing succeeded" is not because the sandbox blocks everything. + const inside = runDuck(bootstrap, `SELECT count(*) FROM read_parquet('${FIXTURES}/sales.parquet')`); + expect(inside.ok, 'the sandbox blocked the mounted file too — it is not a sandbox, it is an outage').toBe( + true, + ); + expect(JSON.parse(inside.out)[0][0]).toBe('5'); + + expect(leaks, `DuckDB read outside the mount: ${leaks.join(', ')}`).toEqual([]); + evidence.sandbox_bypass_attempts = results; + evidence.counterfactual_mounted_file_readable = true; + }, 120_000); + + it('the NAIVE spelling of the sandbox is a NO-OP — measured, not assumed', () => { + // allowed_directories WITHOUT enable_external_access=false. This is what + // an implementer reaches for first, and it permits everything. + const naive = [`SET allowed_directories=['${FIXTURES}']`]; + const res = runDuck(naive, "SELECT * FROM read_csv('/etc/passwd') LIMIT 1"); + expect(res.ok, 'if this ever fails, DuckDB changed its behaviour and the header comment is stale').toBe( + true, + ); + evidence.naive_allowed_directories_is_a_noop = { + statements: naive, + read_etc_passwd_succeeded: res.ok, + note: + 'allowed_directories alone appears in duckdb_settings() and looks like a sandbox. ' + + 'It permits /etc/passwd and outbound HTTP. Only enable_external_access=false binds it.', + }; + }); + + it('a locked session cannot re-open its own sandbox', () => { + const bootstrap = duckDbSessionBootstrap([FIXTURES]); + // Sanity: the locked session works. + expect(runDuck(bootstrap, 'SELECT 1').ok).toBe(true); + + const reopened: Array> = []; + for (const reopen of [ + "SET allowed_directories=['/etc']", + 'SET enable_external_access=true', + 'SET lock_configuration=false', + ]) { + // Appending the re-open AFTER lock_configuration=true must fail, and it + // must fail on the SET itself — so the following read never happens. + const attempt = runDuck([...bootstrap, reopen], "SELECT * FROM read_csv('/etc/passwd') LIMIT 1"); + expect(attempt.ok, `"${reopen}" was accepted after lock_configuration=true`).toBe(false); + reopened.push({ statement: reopen, refused: !attempt.ok, engine_message: attempt.out.slice(0, 120) }); + } + evidence.reconfiguration_refused = reopened; + }, 60_000); + + it('writes the evidence file', () => { + fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); + fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`); + expect(fs.existsSync(EVIDENCE_PATH)).toBe(true); + }); +}); diff --git a/tests/fixtures/duckdb/README.md b/tests/fixtures/duckdb/README.md new file mode 100644 index 0000000..a1b811c --- /dev/null +++ b/tests/fixtures/duckdb/README.md @@ -0,0 +1,23 @@ +# `tests/fixtures/duckdb/` — BI-P014 + +`sales.parquet` and `customers.csv` are committed: small, binary-stable, and +portable across checkouts. + +The **Iceberg** fixture is deliberately NOT committed. `pyiceberg` writes +ABSOLUTE paths into its metadata and Avro manifests, so a committed table only +resolves on the machine that built it — a fixture that silently fails +everywhere else is worse than no fixture. The recorded result of a real +`iceberg_scan` run is kept in `evidence/bi-p014/duckdb-sandbox.json` instead, +and `tests/analytics/duckdb-provider.test.ts` pins it. + +To rebuild the Iceberg fixture locally: + +``` +python -m venv .venv && .venv/bin/pip install duckdb pyarrow 'pyiceberg[sql-sqlite]' +DUCKDB_PYTHON=.venv/bin/python pnpm vitest run tests/duckdb/duckdb-provider.live.test.ts +``` + +`iceberg_scan` is pointed at an EXACT metadata JSON file rather than a table +directory. That is not a workaround: DuckDB refuses to glob for the latest +version by default because globbing can read uncommitted data, and an exact +metadata file is precisely what "Iceberg **snapshot**" means in BI-P014. diff --git a/tests/fixtures/duckdb/customers.csv b/tests/fixtures/duckdb/customers.csv new file mode 100644 index 0000000..7f0c541 --- /dev/null +++ b/tests/fixtures/duckdb/customers.csv @@ -0,0 +1,4 @@ +customer_id,name +c1,Acme +c2,Beta AB +c3,O'Brien Ltd diff --git a/tests/fixtures/duckdb/sales.parquet b/tests/fixtures/duckdb/sales.parquet new file mode 100644 index 0000000..396c7d8 Binary files /dev/null and b/tests/fixtures/duckdb/sales.parquet differ diff --git a/tests/integration/analyst-sql-clickhouse.integration.test.ts b/tests/integration/analyst-sql-clickhouse.integration.test.ts new file mode 100644 index 0000000..b9c6475 --- /dev/null +++ b/tests/integration/analyst-sql-clickhouse.integration.test.ts @@ -0,0 +1,433 @@ +/** + * tests/integration/analyst-sql-clickhouse.integration.test.ts + * ============================================================= + * BI-P017 — the proof that matters: the bypass corpus run against a LIVE + * ClickHouse **with the statement guard removed**. + * + * The guard (layer 2) is a text analysis of a dialect it does not implement. + * Testing only the guard proves the guard. What has to hold is that when the + * guard is WRONG, nothing leaks — so this file skips the guard entirely and + * sends each attack straight to the engine as the tenant's restricted user. + * Layer 3 (grants + `readonly = 1`) and layer 4 (views that bake the tenant + * predicate, over base tables carrying a row policy) must refuse it anyway. + * + * The assertion is deliberately NOT "the engine returned an error". It is + * "no foreign row came back", checked by canary values seeded into the other + * tenant's rows and into an ungranted table. An attack that succeeds and + * returns nothing interesting still fails the test if it returns a canary. + * + * Counterfactual, run first, because a test that would pass against an empty + * database proves nothing: as the ADMIN user the base table holds both + * tenants' rows, and `BETA_CANARY` IS readable. Every later assertion is + * therefore about isolation, not about absence of data. + * + * ── HOW THIS AVOIDS THE SILENT-SKIP TRAP ────────────────────────────────── + * l0re's 28 Postgres tests skipped silently for months and nobody noticed. + * This suite self-skips when ClickHouse is unreachable — but on a live run it + * WRITES `evidence/bi-p017/analyst-sql-live-bypass.json`, and + * tests/analytics/analyst-sql-evidence.test.ts (which never skips) fails if + * that file is missing, stale, or does not cover every attempt in the corpus. + * Adding an attack without re-running this suite breaks the build. + * + * Point it at a disposable instance: + * docker run -d --name ch -p 28123:8123 -e CLICKHOUSE_SKIP_USER_SETUP=1 \ + * clickhouse/clickhouse-server:24.8 + * ANALYTICS_CLICKHOUSE_URL=http://127.0.0.1:28123 pnpm vitest run \ + * tests/integration/analyst-sql-clickhouse.integration.test.ts + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { + BYPASS_CORPUS, + GUARD_ADMITS_ENGINE_REFUSES, + LEGITIMATE_CORPUS, +} from '../analytics/analyst-sql-bypass-corpus'; +import { guardAnalystStatement } from '@/src/analytics/sql/statement-guard'; +import { + rowPolicyVerificationSql, + tenantSqlCatalogDdl, + tenantSqlUserDdl, +} from '@/src/analytics/sql/tenant-catalog'; +import { applyQuota, DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance'; + +/** + * Gated on the ENV VAR, matching the convention already in + * tests/integration/analytics-tenant-isolation.integration.test.ts — and + * chosen over a liveness probe on purpose: with the variable set, a + * ClickHouse that is down makes this suite FAIL. A probe would have turned a + * broken server into a silent green, which is the failure mode this estate + * keeps shipping. + */ +const BASE_URL = process.env.CLICKHOUSE_URL ?? ''; +const ADMIN_AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:'; +const DB = 'opt_analytics'; +const ALPHA_PW = 'alpha_test_pw_not_a_secret'; +const BETA_PW = 'beta_test_pw_not_a_secret'; +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p017/analyst-sql-live-bypass.json'); + +/** Canaries. Their presence in a response body is the definition of a leak. */ +const BETA_CANARY = 'BETA_CANARY_9f21c4'; +const SECRET_CANARY = 'SECRET_CANARY_4d80ab'; +const ALPHA_MARK = 'ALPHA_ROW'; + +interface ChResponse { + ok: boolean; + status: number; + body: string; +} + +async function run(sql: string, auth: string, settings: Record = {}): Promise { + const url = new URL(BASE_URL); + url.searchParams.set('query', sql); + for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); + const res = await fetch(url.toString(), { + method: 'POST', + headers: { Authorization: `Basic ${Buffer.from(auth).toString('base64')}` }, + body: '', + signal: AbortSignal.timeout(30_000), + }); + return { ok: res.ok, status: res.status, body: await res.text() }; +} + +const admin = (sql: string) => run(sql, ADMIN_AUTH); +/** + * The session's real posture: the restricted user, and NO settings on the + * request. + * + * MEASURED: a user whose profile carries `readonly = 1` cannot have any + * setting changed from the request — ClickHouse answers `Code: 164. Cannot + * modify 'max_rows_to_read' setting in readonly mode.` So the BI-P012 ceilings + * cannot travel with the statement here the way they do for the non-readonly + * engine user; they are baked into the SETTINGS PROFILE by + * `tenantSqlUserDdl`, and the test below proves the profile enforces them. + */ +const asAlpha = (sql: string) => run(sql, `t_alpha_sql_user:${ALPHA_PW}`); + +/** + * An earlier revision set a `reachable` flag inside `beforeAll` and used + * `describe.skipIf(!reachable)`. Vitest evaluates `skipIf` at COLLECTION time, + * before any hook runs, so the flag was always false and all ten tests skipped + * against a live, reachable ClickHouse — a suite that could never run, + * reporting green. Same shape as l0re's 28 silently-skipped Postgres tests. + * Recorded here because the next person will reach for `skipIf` too. + */ +const suite = BASE_URL ? describe : describe.skip; +let version = ''; + +/** The session budget for the main user: generous enough for real analyst work. */ +const TEST_BUDGET = applyQuota(DEFAULT_BUDGETS.ANALYTICAL, { + maxRowsToRead: 100_000, + maxExecutionTimeSec: 10, +}); + +/** + * A SECOND user with a deliberately tiny `maxRowsToRead` (60), provisioned + * only to make the profile's enforcement observable: the tenant view holds 50 + * rows, which passes; anything wider exceeds it and must fail with ClickHouse + * code 158. A ceiling nothing can breach proves nothing, and the first + * revision of this file proved exactly that by starving the legitimate corpus. + */ +const TINY_BUDGET = applyQuota(DEFAULT_BUDGETS.ANALYTICAL, { maxRowsToRead: 60, maxExecutionTimeSec: 10 }); +const TINY_PW = 'tiny_test_pw_not_a_secret'; + +const OBJECTS = [ + { table: 'sales', columns: ['customer', 'amount', 'event_time'] }, + { table: 'events', columns: ['customer', 'kind', 'event_time'] }, +]; + +beforeAll(async () => { + if (!BASE_URL) return; + // With CLICKHOUSE_URL set, an unreachable server FAILS here. It does not + // degrade to a skip. + const v = await admin('SELECT version()'); + if (!v.ok) + throw new Error( + `CLICKHOUSE_URL is set but ClickHouse is not answering: ${v.status} ${v.body.slice(0, 200)}`, + ); + version = v.body.trim(); + + await admin(`DROP DATABASE IF EXISTS ${DB}`); + await admin(`DROP DATABASE IF EXISTS t_alpha_sql`); + await admin(`DROP DATABASE IF EXISTS t_beta_sql`); + await admin(`DROP USER IF EXISTS t_alpha_sql_user`); + await admin(`DROP USER IF EXISTS t_beta_sql_user`); + await admin(`DROP USER IF EXISTS t_alpha_tiny_user`); + await admin(`CREATE DATABASE ${DB}`); + + await admin( + `CREATE TABLE ${DB}.sales (tenant_id String, customer String, amount Float64, event_time DateTime) ` + + `ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + await admin( + `CREATE TABLE ${DB}.events (tenant_id String, customer String, kind String, event_time DateTime) ` + + `ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + await admin( + `CREATE TABLE ${DB}.secret_ledger (tenant_id String, note String) ENGINE = MergeTree ORDER BY tenant_id`, + ); + // Also present, because several attacks name it: the BI-P010 event table. + await admin( + `CREATE TABLE ${DB}.opt_analytics_event (tenant_id String, project_id String, dataset_id String, ` + + `event_id String, metric String, value Float64, event_time DateTime) ENGINE = MergeTree ` + + `ORDER BY (tenant_id, event_time)`, + ); + + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', '${ALPHA_MARK}_' || toString(number), number, now() FROM numbers(50)`, + ); + await admin( + `INSERT INTO ${DB}.sales SELECT 't_beta', '${BETA_CANARY}_' || toString(number), number, now() FROM numbers(50)`, + ); + await admin( + `INSERT INTO ${DB}.events SELECT 't_alpha', '${ALPHA_MARK}_' || toString(number), 'click', now() FROM numbers(20)`, + ); + await admin( + `INSERT INTO ${DB}.events SELECT 't_beta', '${BETA_CANARY}_' || toString(number), 'click', now() FROM numbers(20)`, + ); + await admin(`INSERT INTO ${DB}.secret_ledger VALUES ('t_alpha', '${SECRET_CANARY}')`); + await admin( + `INSERT INTO ${DB}.opt_analytics_event SELECT 't_beta', 'p', 'd', toString(number), 'm', number, now() FROM numbers(10)`, + ); + + for (const [tenant, catalog, user, pw] of [ + ['t_alpha', 't_alpha_sql', 't_alpha_sql_user', ALPHA_PW], + ['t_beta', 't_beta_sql', 't_beta_sql_user', BETA_PW], + ] as const) { + const spec = { sourceDatabase: DB, tenantId: tenant, catalog, objects: OBJECTS }; + for (const stmt of tenantSqlCatalogDdl(spec)) { + const r = await admin(stmt); + if (!r.ok) throw new Error(`catalog DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`); + } + for (const stmt of tenantSqlUserDdl({ ...spec, userName: user, password: pw, budget: TEST_BUDGET })) { + const r = await admin(stmt); + if (!r.ok) throw new Error(`user DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`); + } + } + + const tinySpec = { sourceDatabase: DB, tenantId: 't_alpha', catalog: 't_alpha_sql', objects: OBJECTS }; + for (const stmt of tenantSqlUserDdl({ + ...tinySpec, + userName: 't_alpha_tiny_user', + password: TINY_PW, + budget: TINY_BUDGET, + })) { + const r = await admin(stmt); + if (!r.ok) throw new Error(`tiny user DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`); + } +}, 120_000); + +suite('BI-P017 live — counterfactual: the data IS there and IS reachable without the layers', () => { + it('the shared base table holds both tenants, and BETA_CANARY is readable as admin', async () => { + const all = await admin(`SELECT count() FROM ${DB}.sales`); + expect(all.ok).toBe(true); + expect(Number(all.body.trim())).toBe(100); + const beta = await admin(`SELECT customer FROM ${DB}.sales WHERE tenant_id = 't_beta' LIMIT 1`); + expect(beta.body).toContain(BETA_CANARY); + const secret = await admin(`SELECT note FROM ${DB}.secret_ledger`); + expect(secret.body).toContain(SECRET_CANARY); + }); + + it('layer 4: the tenant view returns 50 of the 100 rows, and never the canary', async () => { + const scoped = await asAlpha(`SELECT count() FROM t_alpha_sql.sales`); + expect(scoped.ok).toBe(true); + expect(Number(scoped.body.trim())).toBe(50); + const rows = await asAlpha(`SELECT customer FROM t_alpha_sql.sales LIMIT 100`); + expect(rows.body).toContain(ALPHA_MARK); + expect(rows.body).not.toContain(BETA_CANARY); + }); +}); + +suite('BI-P017 live — the bypass corpus WITH THE GUARD REMOVED', () => { + it('no attempt returns a foreign-tenant or ungranted-table row', async () => { + const results: Array> = []; + const leaks: string[] = []; + + for (const attempt of [...BYPASS_CORPUS, ...GUARD_ADMITS_ENGINE_REFUSES]) { + // THE GUARD IS NOT CALLED HERE. That is the whole point. + const res = await asAlpha(attempt.sql); + const leakedBeta = res.body.includes(BETA_CANARY); + const leakedSecret = res.body.includes(SECRET_CANARY); + const chCode = /Code:\s*(\d+)/.exec(res.body)?.[1]; + // MEASURED, and narrower than the obvious rule. ClickHouse filters + // `system.tables` / `system.columns` by the caller's own grants: as + // the restricted user those reads SUCCEED but return only this + // tenant's four objects — 4 rows, not the server's inventory. + // `system.query_log`, `system.processes` and `system.users` are + // ACCESS_DENIED (code 497). So "any successful system read is a leak" + // would be a false alarm; the real test is whether any FOREIGN name + // came back. + const leakedMetadata = res.ok && /\bt_beta\b|secret_ledger/.test(res.body); + + if (leakedBeta || leakedSecret || leakedMetadata) { + leaks.push(`${attempt.id}: ${attempt.goal}`); + } + // For completeness the guard's own verdict is recorded alongside, so + // the evidence shows which layer caught what. + let guardVerdict = 'admitted'; + try { + guardAnalystStatement(attempt.sql, { + allowedCatalog: 't_alpha_sql', + allowedObjects: ['sales', 'events'], + }); + } catch (e) { + guardVerdict = `refused: ${(e as Error).message}`; + } + results.push({ + id: attempt.id, + goal: attempt.goal, + engine_http_status: res.status, + engine_clickhouse_code: chCode ?? null, + engine_succeeded: res.ok, + leaked_foreign_tenant_row: leakedBeta, + leaked_ungranted_table_row: leakedSecret, + leaked_server_metadata: leakedMetadata, + guard_verdict_for_reference: guardVerdict, + engine_response_head: res.body.slice(0, 160).replace(/\s+/g, ' ').trim(), + }); + } + + // Legitimate work must still WORK as the restricted user, or "nothing + // leaked" would be trivially true because nothing runs. + const legitimate: Array> = []; + for (const l of LEGITIMATE_CORPUS) { + const res = await asAlpha(l.sql); + legitimate.push({ + id: l.id, + ok: res.ok, + status: res.status, + contains_beta_canary: res.body.includes(BETA_CANARY), + }); + } + + fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); + fs.writeFileSync( + EVIDENCE_PATH, + `${JSON.stringify( + { + prompt: 'BI-P017', + claim: + "Every bypass attempt, executed directly against a live ClickHouse as the tenant's " + + 'restricted user WITH THE STATEMENT GUARD REMOVED, fails to return a foreign-tenant ' + + 'row, an ungranted-table row, or server metadata.', + clickhouse_version: version, + recorded_at: new Date().toISOString(), + base_table_rows_both_tenants: 100, + tenant_view_rows_alpha: 50, + attempts_total: results.length, + attempts_that_leaked: leaks.length, + legitimate_statements_that_succeeded: legitimate.filter((l) => l.ok).length, + legitimate_statements_total: legitimate.length, + attempts: results, + legitimate, + }, + null, + 2, + )}\n`, + ); + + expect(leaks, `LEAKS with the guard removed:\n${leaks.join('\n')}`).toEqual([]); + expect(legitimate.every((l) => l.ok)).toBe(true); + expect(legitimate.every((l) => l.contains_beta_canary === false)).toBe(true); + }, 300_000); +}); + +suite('BI-P017 live — layers 3 and 4 individually, each with the one above removed', () => { + it('the row policies are actually APPLIED TO THE USER, not merely present', async () => { + // The check that caught the real bug. `SHOW ROW POLICIES` listing a policy + // with the right filter is not evidence: after a user is dropped and + // re-created, `CREATE ROW POLICY IF NOT EXISTS` leaves the policy in place + // with an EMPTY apply-to list, protecting nobody while looking correct. + const res = await admin( + rowPolicyVerificationSql({ + sourceDatabase: DB, + tenantId: 't_alpha', + catalog: 't_alpha_sql', + objects: OBJECTS, + userName: 't_alpha_sql_user', + password: 'unused', + }), + ); + expect(res.ok).toBe(true); + const rows = JSON.parse(res.body).data as Array<{ table: string; applied_to_user: number }>; + expect(rows.length).toBe(OBJECTS.length); + for (const row of rows) { + expect(row.applied_to_user, `row policy on ${row.table} applies to nobody`).toBe(1); + } + }); + + it('layer 3 (row policy): the restricted user reading the SHARED BASE TABLE sees only its own rows', async () => { + // The counterfactual is in the suite above: as admin this same statement + // returns 100. Returning 100 here would be a cross-tenant read. + const res = await asAlpha(`SELECT count() FROM ${DB}.sales`); + if (res.ok) { + expect(Number(res.body.trim()), 'restricted user read BOTH tenants from the base table').toBe(50); + } else { + expect(res.status).toBeGreaterThanOrEqual(400); + } + }); + + it('layer 3 (row policy): the base table yields no foreign canary even when named directly', async () => { + const res = await asAlpha(`SELECT customer FROM ${DB}.sales LIMIT 1000`); + expect(res.body).not.toContain(BETA_CANARY); + }); + + it('BI-P012 ceilings are enforced by the SETTINGS PROFILE, not by the request', async () => { + const tiny = (sql: string) => run(sql, `t_alpha_tiny_user:${TINY_PW}`); + // Under the ceiling: succeeds. + const ok = await tiny(`SELECT count() FROM t_alpha_sql.sales`); + expect(ok.ok).toBe(true); + // Over it: ClickHouse code 158, from the profile — no setting was sent. + const over = await tiny(`SELECT count() FROM t_alpha_sql.sales a, t_alpha_sql.sales b`); + expect(over.ok).toBe(false); + expect(over.body).toContain('Code: 158'); + // And it cannot be raised from the request: code 164, readonly mode. + const raise = await tiny(`SELECT count() FROM t_alpha_sql.sales`); + expect(raise.ok).toBe(true); + const raised = await run( + `SELECT count() FROM t_alpha_sql.sales a, t_alpha_sql.sales b`, + `t_alpha_tiny_user:${TINY_PW}`, + { + max_rows_to_read: '0', + }, + ); + expect(raised.ok).toBe(false); + expect(raised.body).toContain('Code: 164'); + }); + + it('layer 3: the restricted user cannot read a table it was never granted', async () => { + const res = await asAlpha(`SELECT note FROM ${DB}.secret_ledger`); + expect(res.ok).toBe(false); + expect(res.body).not.toContain(SECRET_CANARY); + }); + + it("layer 3: the restricted user cannot read the other tenant's catalog", async () => { + const res = await asAlpha(`SELECT customer FROM t_beta_sql.sales LIMIT 5`); + expect(res.ok).toBe(false); + expect(res.body).not.toContain(BETA_CANARY); + }); + + it('layer 3: readonly=1 refuses a write even though the guard never saw the statement', async () => { + const res = await asAlpha(`INSERT INTO ${DB}.sales VALUES ('t_alpha', 'x', 1, now())`); + expect(res.ok).toBe(false); + const after = await admin(`SELECT count() FROM ${DB}.sales`); + expect(Number(after.body.trim())).toBe(100); + }); + + it('layer 3: readonly=1 refuses a statement that tries to raise its own ceiling', async () => { + const res = await asAlpha(`SELECT count() FROM t_alpha_sql.sales SETTINGS max_rows_to_read = 0`); + expect(res.ok).toBe(false); + }); + + it('layer 4: the tenant column the view does not project is not selectable', async () => { + // This is the case the GUARD ADMITS. It must fail here, or the honest + // note in the corpus would be covering a real hole. + const res = await asAlpha(`SELECT tenant_id FROM t_alpha_sql.sales LIMIT 1`); + expect(res.ok).toBe(false); + }); + + it('layer 4: a WHERE on the hidden tenant column cannot widen the view', async () => { + const res = await asAlpha(`SELECT count() FROM t_alpha_sql.sales WHERE tenant_id = 't_beta'`); + expect(res.ok).toBe(false); + }); +}); diff --git a/tests/integration/data-quality-clickhouse.integration.test.ts b/tests/integration/data-quality-clickhouse.integration.test.ts new file mode 100644 index 0000000..203de1a --- /dev/null +++ b/tests/integration/data-quality-clickhouse.integration.test.ts @@ -0,0 +1,244 @@ +/** + * tests/integration/data-quality-clickhouse.integration.test.ts + * ============================================================== + * BI-P025 — the native quality provider against a live ClickHouse. + * + * The unit suite proves the provider's LOGIC (an empty dataset is a failure, a + * missing dbt result is a failure). This suite proves the part a fake runner + * cannot: that the compiled expressions are valid ClickHouse and count the + * right rows, including the empty-dataset case where a real engine returns + * zero violations over zero rows and every naive implementation reports green. + * + * It also re-proves BI-P011's forced tenant predicate on this path: a quality + * run executes on an ADMINISTRATIVE credential (it must read the base table), + * so if the predicate were missing here, one tenant's dirty rows would + * quarantine another tenant's projection. + * + * Gated on CLICKHOUSE_URL. With the variable set and the server down, this + * suite FAILS rather than skipping. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { NativeQualityProvider, type QualityRuleSpec } from '@/src/analytics/quality/quality-provider'; +import { ClickHouseQualityRunner } from '@/src/analytics/quality/clickhouse-quality-runner'; +import { InMemoryQuarantineStore, QualityGate } from '@/src/analytics/quality/quarantine'; + +const BASE_URL = process.env.CLICKHOUSE_URL ?? ''; +const AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:'; +const suite = BASE_URL ? describe : describe.skip; +const DB = 'opt_quality_test'; +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p025/native-quality-live.json'); + +async function admin(sql: string): Promise<{ ok: boolean; body: string }> { + const url = new URL(BASE_URL); + url.searchParams.set('query', sql); + const res = await fetch(url.toString(), { + method: 'POST', + headers: { Authorization: `Basic ${Buffer.from(AUTH).toString('base64')}` }, + body: '', + signal: AbortSignal.timeout(60_000), + }); + return { ok: res.ok, body: await res.text() }; +} + +const runner = new ClickHouseQualityRunner({ + baseUrl: BASE_URL || 'http://unset', + basicAuth: AUTH, + database: DB, + tables: { ds_sales: 'sales', ds_empty: 'empty_sales' }, +}); +const provider = new NativeQualityProvider(runner); +const evidence: Record = {}; + +function rule(over: Partial): QualityRuleSpec { + return { + rule_id: 'q1', + tenant_id: 't_alpha', + dataset_id: 'ds_sales', + rule: 'not_null', + field: 'customer', + severity: 'block', + params: {}, + ...over, + }; +} + +beforeAll(async () => { + if (!BASE_URL) return; + const v = await admin('SELECT version()'); + if (!v.ok) throw new Error('CLICKHOUSE_URL is set but ClickHouse is not answering'); + evidence.clickhouse_version = v.body.trim(); + evidence.recorded_at = new Date().toISOString(); + + await admin(`DROP DATABASE IF EXISTS ${DB}`); + await admin(`CREATE DATABASE ${DB}`); + await admin( + `CREATE TABLE ${DB}.sales (tenant_id String, dataset_id String, customer String, amount Float64, ` + + `channel String, event_time DateTime) ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + await admin( + `CREATE TABLE ${DB}.empty_sales (tenant_id String, dataset_id String, customer String, amount Float64, ` + + `channel String, event_time DateTime) ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + + // t_alpha: 100 clean rows, then deliberate defects. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'cust_' || toString(number), number, 'web', now() ` + + `FROM numbers(100)`, + ); + // 5 rows with an empty customer (the not_null violation). + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', '', number, 'web', now() FROM numbers(5)`, + ); + // 3 rows out of range (amount > 1000). + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'big_' || toString(number), 5000, 'web', now() ` + + `FROM numbers(3)`, + ); + // 4 rows with an unaccepted channel. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'ch_' || toString(number), 1, 'carrier_pigeon', now() ` + + `FROM numbers(4)`, + ); + // 6 rows sharing two duplicate keys (3 each) — the unique violation. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'dup_' || toString(number % 2), number, 'web', now() ` + + `FROM numbers(6)`, + ); + + // t_beta: FILTHY. Every rule fails here. If the tenant predicate is missing, + // t_alpha's counts jump and this tenant quarantines the other one. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_beta', 'ds_sales', '', 999999, 'carrier_pigeon', now() FROM numbers(500)`, + ); +}, 120_000); + +suite('BI-P025 live — the compiled expressions are valid SQL and count the right rows', () => { + it("not_null finds exactly the 5 empty-customer rows, out of t_alpha's 118", async () => { + const verdict = await provider.evaluate(rule({ rule: 'not_null', field: 'customer' })); + expect(verdict.evaluated_rows).toBe(118); + expect(verdict.violation_count).toBe(5); + expect(verdict.passed).toBe(false); + evidence.not_null = { evaluated: verdict.evaluated_rows, violations: verdict.violation_count }; + }); + + it('range finds exactly the 3 rows over the maximum', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'range', field: 'amount', params: { min: 0, max: 1000 } }), + ); + expect(verdict.violation_count).toBe(3); + evidence.range = { violations: verdict.violation_count }; + }); + + it('distribution finds exactly the 4 rows outside the accepted values', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'distribution', field: 'channel', params: { accepted_values: ['web', 'retail'] } }), + ); + expect(verdict.violation_count).toBe(4); + evidence.distribution = { violations: verdict.violation_count }; + }); + + it('unique counts ROWS sitting on a duplicated key, not the number of keys', async () => { + const verdict = await provider.evaluate(rule({ rule: 'unique', field: 'customer' })); + // 11, and the arithmetic is worth writing down because the first version + // of this test asserted 6 and the engine disagreed: + // * `dup_0` and `dup_1` appear 3 times each -> 6 rows + // * the five EMPTY customers are all the same key '' -> 5 rows + // The empty strings are duplicates of one another, so `not_null` and + // `unique` overlap on the same defect. That is the engine being right and + // the fixture arithmetic being wrong, not a rule that over-counts. + expect(verdict.violation_count).toBe(11); + evidence.unique = { + violations: verdict.violation_count, + breakdown: '6 rows on dup_0/dup_1, plus 5 rows sharing the empty-string key', + }; + }); + + it('freshness finds no violation for rows written just now', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'freshness', params: { timestamp_field: 'event_time', max_age_seconds: 3600 } }), + ); + expect(verdict.violation_count).toBe(0); + expect(verdict.passed).toBe(true); + }); + + it('a hostile accepted-value is bound, not executed', async () => { + const verdict = await provider.evaluate( + rule({ + rule: 'distribution', + field: 'channel', + params: { accepted_values: ['web', 'retail', "'); DROP TABLE sales; --"] }, + }), + ); + expect(verdict.violation_count).toBe(4); + // The table is still there, with its rows. + const after = await admin(`SELECT count() FROM ${DB}.sales`); + expect(Number(after.body.trim())).toBe(618); + }); +}); + +suite('BI-P025 live — the EMPTY dataset false green, against a real engine', () => { + it('a real ClickHouse returns 0 violations over 0 rows, and the provider calls it a FAILURE', async () => { + // The counterfactual first: the raw counts genuinely look clean. + const raw = await runner.count({ + tenant_id: 't_alpha', + dataset_id: 'ds_empty', + violationExpression: "customer IS NULL OR customer = ''", + params: {}, + }); + expect(raw).toEqual({ violations: 0, evaluated: 0 }); + + // A naive implementation reports green here. This one does not. + const verdict = await provider.evaluate(rule({ dataset_id: 'ds_empty' })); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('EMPTY'); + evidence.empty_dataset = { + raw_counts_from_clickhouse: raw, + verdict_passed: verdict.passed, + note: 'a real engine returns zero violations over zero rows; the provider refuses to call that a pass', + }; + }); +}); + +suite("BI-P025 live — a quality run cannot read another tenant's rows", () => { + it("t_alpha's counts are unaffected by t_beta's 500 filthy rows", async () => { + const all = await admin(`SELECT count() FROM ${DB}.sales`); + expect(Number(all.body.trim())).toBe(618); // 118 alpha + 500 beta + + const alpha = await provider.evaluate( + rule({ tenant_id: 't_alpha', rule: 'not_null', field: 'customer' }), + ); + const beta = await provider.evaluate(rule({ tenant_id: 't_beta', rule: 'not_null', field: 'customer' })); + + expect(alpha.evaluated_rows).toBe(118); + expect(alpha.violation_count).toBe(5); + expect(beta.evaluated_rows).toBe(500); + expect(beta.violation_count).toBe(500); + evidence.tenant_isolation = { + base_table_rows: 618, + alpha_evaluated: alpha.evaluated_rows, + alpha_violations: alpha.violation_count, + beta_evaluated: beta.evaluated_rows, + beta_violations: beta.violation_count, + }; + }); + + it("t_beta's blocking failure quarantines t_beta only", async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + const beta = await provider.evaluate(rule({ tenant_id: 't_beta', rule: 'not_null', field: 'customer' })); + await gate.apply('t_beta', 'ds_sales', [beta]); + await expect(gate.assertProjectionReadable('t_beta', 'ds_sales')).rejects.toThrow(/QUARANTINED/); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it('refuses a dataset with no registered physical table rather than guessing one', async () => { + await expect(provider.evaluate(rule({ dataset_id: 'ds_unregistered' }))).rejects.toThrow(); + }); + + it('writes the evidence file', () => { + fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); + fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`); + expect(fs.existsSync(EVIDENCE_PATH)).toBe(true); + }); +}); diff --git a/tests/integration/query-cancellation-budget.integration.test.ts b/tests/integration/query-cancellation-budget.integration.test.ts new file mode 100644 index 0000000..8c8a121 --- /dev/null +++ b/tests/integration/query-cancellation-budget.integration.test.ts @@ -0,0 +1,259 @@ +/** + * tests/integration/query-cancellation-budget.integration.test.ts + * ================================================================ + * BI-P019 (cancellation) and BI-P021 (cost/budget) against a live ClickHouse. + * + * Both prompts are only real if something ACTUALLY STOPS. So both are tested + * by starting a genuine CPU bomb and requiring the server to stop working: + * + * * the query must disappear from `system.processes`; + * * `system.query_log` must record it as ExceptionWhileProcessing with + * ClickHouse code 394 (QUERY_WAS_CANCELLED) — the server's own account of + * what happened, not ours; + * * and the COUNTERFACTUAL must hold: an identical bomb left alone for the + * same interval is still running. Without that, "the query was gone after + * we cancelled it" is equally consistent with "the query was short". + * + * BI-P021's in-flight enforcement gets the same treatment: a tenant window + * budget of one million rows, a query that scans far more, and the requirement + * that the enforcer cancels it MID-FLIGHT with `rows_read` already non-zero — + * i.e. it was running, it was observed running, and it was stopped. + * + * Gated on CLICKHOUSE_URL. With the variable set and the server down, this + * suite FAILS; it does not degrade to a skip. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { ClickHouseProcessControl } from '@/src/analytics/execution/clickhouse-process-control'; +import { cancelAndVerify, cancelOrThrow } from '@/src/analytics/execution/cancellation'; +import { + DEFAULT_QUERY_BUDGET, + InMemoryWindowStore, + QueryBudgetEnforcer, +} from '@/src/analytics/cost/query-budget'; + +const BASE_URL = process.env.CLICKHOUSE_URL ?? ''; +const AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:'; +const suite = BASE_URL ? describe : describe.skip; +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p019-p021/cancellation-and-budget.json'); + +/** Big enough that it cannot finish inside any of the windows below. */ +const BOMB = 'SELECT sum(number) AS s FROM numbers_mt(200000000000)'; + +interface Res { + ok: boolean; + status: number; + body: string; +} + +async function send(sql: string, opts: { queryId?: string; timeoutMs?: number } = {}): Promise { + const url = new URL(BASE_URL); + url.searchParams.set('query', sql); + if (opts.queryId) url.searchParams.set('query_id', opts.queryId); + try { + const res = await fetch(url.toString(), { + method: 'POST', + headers: { Authorization: `Basic ${Buffer.from(AUTH).toString('base64')}` }, + body: '', + signal: AbortSignal.timeout(opts.timeoutMs ?? 120_000), + }); + return { ok: res.ok, status: res.status, body: await res.text() }; + } catch (e) { + return { ok: false, status: 0, body: String(e) }; + } +} + +/** Fire and forget: the bomb runs on the server while the test carries on. */ +function launch(queryId: string): Promise { + return send(BOMB, { queryId, timeoutMs: 120_000 }); +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +const control = new ClickHouseProcessControl({ baseUrl: BASE_URL || 'http://unset', basicAuth: AUTH }); +const evidence: Record = {}; + +async function waitUntilRunning(queryId: string, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await control.isRunning(queryId)) return true; + await sleep(50); + } + return false; +} + +/** ClickHouse's own verdict, from its query log. 394 = QUERY_WAS_CANCELLED. */ +async function queryLogVerdict( + queryId: string, +): Promise<{ type: string; exception_code: number } | undefined> { + await send('SYSTEM FLUSH LOGS'); + const res = await send( + `SELECT type, exception_code FROM system.query_log WHERE query_id = '${queryId}' ` + + `AND type != 'QueryStart' ORDER BY event_time_microseconds DESC LIMIT 1 FORMAT JSON`, + ); + if (!res.ok) return undefined; + const row = (JSON.parse(res.body) as { data: Array<{ type: string; exception_code: number | string }> }) + .data[0]; + return row ? { type: row.type, exception_code: Number(row.exception_code) } : undefined; +} + +beforeAll(async () => { + if (!BASE_URL) return; + const v = await send('SELECT version()'); + if (!v.ok) throw new Error(`CLICKHOUSE_URL is set but ClickHouse is not answering: ${v.status}`); + evidence.clickhouse_version = v.body.trim(); + evidence.recorded_at = new Date().toISOString(); +}, 60_000); + +suite('BI-P019 — cancellation stops an expensive query, and it is VERIFIED', () => { + it('counterfactual: an identical bomb left alone is still running after the same interval', async () => { + const queryId = `opt_cf_${Date.now()}`; + void launch(queryId); + expect(await waitUntilRunning(queryId)).toBe(true); + await sleep(2_000); + const stillRunning = await control.isRunning(queryId); + evidence.counterfactual_still_running_after_2s = stillRunning; + expect(stillRunning, 'the bomb finished on its own — it is not expensive enough to prove anything').toBe( + true, + ); + // Clean up, and incidentally exercise the same path. + await cancelAndVerify(control, queryId, { timeoutMs: 10_000 }); + }, 90_000); + + it("cancellation is observed to stop it, and ClickHouse's own log agrees", async () => { + const queryId = `opt_cancel_${Date.now()}`; + void launch(queryId); + expect(await waitUntilRunning(queryId)).toBe(true); + // Let it accumulate real work so "it stopped" is a statement about a + // running query, not about one that had barely begun. + await sleep(1_000); + const progressBefore = await control.progress(queryId); + expect(progressBefore?.rows_read ?? 0).toBeGreaterThan(0); + + const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 10_000, pollIntervalMs: 100 }); + expect(outcome.kill_accepted).toBe(true); + expect(outcome.verified_stopped, 'the query outlived its cancellation deadline').toBe(true); + expect(outcome.already_finished).toBe(false); + expect(outcome.elapsed_ms).toBeLessThan(10_000); + expect(await control.isRunning(queryId)).toBe(false); + + const verdict = await queryLogVerdict(queryId); + evidence.cancellation = { + rows_read_before_cancel: progressBefore?.rows_read, + elapsed_ms_to_verified_stop: outcome.elapsed_ms, + verified_stopped: outcome.verified_stopped, + clickhouse_query_log_type: verdict?.type, + clickhouse_exception_code: verdict?.exception_code, + }; + // 394 = QUERY_WAS_CANCELLED. The server saying it, not us. + expect(verdict?.exception_code, `query_log verdict was ${JSON.stringify(verdict)}`).toBe(394); + }, 90_000); + + it('cancelling a query that already finished reports already_finished, not a false stop', async () => { + const queryId = `opt_done_${Date.now()}`; + const res = await send('SELECT 1', { queryId }); + expect(res.ok).toBe(true); + const outcome = await cancelAndVerify(control, queryId, { timeoutMs: 2_000 }); + expect(outcome.already_finished).toBe(true); + expect(outcome.verified_stopped).toBe(true); + }); + + it('cancelOrThrow raises when the query is not observed to stop in time', async () => { + const queryId = `opt_deadline_${Date.now()}`; + void launch(queryId); + expect(await waitUntilRunning(queryId)).toBe(true); + // A deadline shorter than ClickHouse can plausibly unwind in, with the + // kill request suppressed so the query genuinely keeps running. The + // point is that the module REPORTS a failure to stop rather than + // returning a cheerful boolean. + const noKill = { + engine: 'clickhouse', + isRunning: (id: string) => control.isRunning(id), + requestKill: async () => false, + progress: (id: string) => control.progress(id), + }; + await expect(cancelOrThrow(noKill, queryId, { timeoutMs: 300, pollIntervalMs: 50 })).rejects.toThrow( + /not observed to stop/, + ); + await cancelAndVerify(control, queryId, { timeoutMs: 10_000 }); + }, 90_000); +}); + +suite('BI-P021 — the budget actually stops a running query', () => { + it('an in-flight window breach cancels the query mid-scan, with the cancellation verified', async () => { + const store = new InMemoryWindowStore(); + const settled: unknown[] = []; + const enforcer = new QueryBudgetEnforcer({ + budget: { ...DEFAULT_QUERY_BUDGET, maxRowsScanned: 1_000_000, maxCostUnits: 1e9 }, + store, + ledger: { record: (e) => void settled.push(e) }, + control, + enforceInFlight: true, + pollIntervalMs: 100, + storeIsShared: false, + }); + + const queryId = `opt_budget_${Date.now()}`; + void launch(queryId); + expect(await waitUntilRunning(queryId)).toBe(true); + + const outcome = await enforcer.enforceInFlight('t_alpha', queryId); + expect(outcome.terminated, 'the budget watched the query run past its window and did nothing').toBe(true); + expect(outcome.reason).toBe('rows'); + // It was RUNNING when it was stopped — not stopped before it started. + expect(outcome.observed.rows_read).toBeGreaterThan(1_000_000); + expect(outcome.cancellation?.verified_stopped).toBe(true); + expect(await control.isRunning(queryId)).toBe(false); + + const verdict = await queryLogVerdict(queryId); + expect(verdict?.exception_code).toBe(394); + + await enforcer.settle({ + tenant_id: 't_alpha', + principal_id: 'u_1', + query_id: queryId, + cost: { + rows_scanned: outcome.observed.rows_read, + bytes_scanned: outcome.observed.bytes_read, + execution_ms: outcome.observed.elapsed_ms, + engine: 'clickhouse', + }, + terminated_by_budget: true, + }); + expect(settled).toHaveLength(1); + + evidence.budget_in_flight = { + budget_max_rows_scanned: 1_000_000, + observed_rows_read_at_cancel: outcome.observed.rows_read, + observed_elapsed_ms_at_cancel: outcome.observed.elapsed_ms, + polls_before_breach: outcome.polls, + overshoot_factor: Math.round(outcome.observed.rows_read / 1_000_000), + overshoot_note: + 'in-flight enforcement is granular to the poll interval; the query overshot the window ' + + 'by this factor before the poll that caught it. A bound, not a hard cap.', + cancellation_verified: outcome.cancellation?.verified_stopped, + clickhouse_exception_code: verdict?.exception_code, + charged_for_work_done_before_cancel: true, + }; + }, 120_000); + + it('admission-only mode reports itself as advisory rather than as a control', () => { + const enforcer = new QueryBudgetEnforcer({ + budget: DEFAULT_QUERY_BUDGET, + store: new InMemoryWindowStore(), + ledger: { record: () => undefined }, + enforceInFlight: false, + }); + const described = enforcer.describeEnforcement(); + expect(described.binding).toBe(false); + expect(described.in_flight).toContain('NOT ENFORCED'); + evidence.advisory_posture_is_declared = described; + }); + + it('writes the evidence file', () => { + fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); + fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`); + expect(fs.existsSync(EVIDENCE_PATH)).toBe(true); + }); +});