/** * BI-P011 — CLICKHOUSE TENANT ISOLATION, proven against a LIVE server. * * The shape of this proof is the estate's own convention for isolation work: * DEMONSTRATE THE HOLE WITHOUT THE FIX, then demonstrate it CLOSED WITH the * fix. A suite that only exercises the fixed path proves that the fixed path * works, which is not the same claim. * * So each layer is tested twice: * * layer 2 (0pt query compiler forces tenant) * hole: a hand-written statement WITHOUT the predicate — the same query * minus the forcing — returns another tenant's rows. This is the * literal regression a future refactor would cause. * closed: the compiled statement returns only this tenant's rows. * * layer 3 (restricted database users + row policies) * hole: the UNRESTRICTED admin user, issuing a statement with NO tenant * predicate at all, sees every tenant. * closed: the RESTRICTED per-tenant user, issuing the SAME statement, * sees only its own tenant — the server filters, so the result * holds even if layer 2 were removed entirely. * * Gated on CLICKHOUSE_URL and SKIPS HONESTLY when unset, matching * tests/integration/analytics-clickhouse.integration.test.ts. CI's * `integration` job sets it from a service container and hard-fails if the * suites skip, so this is not a self-skipping green in practice. */ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; import { compileMetricQuery, dimensionCatalogFrom, restrictedTenantUserDdl, } from "@/src/analytics/engine/query-compiler"; import { ANALYTICS_TABLES, sortingKeySql } from "@/src/analytics/engine/schema"; import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; /** * Raw events carry a 30-day TTL (src/analytics/engine/schema.ts). Fixed * historical timestamps silently expire and make live-ClickHouse suites read * 0 rows, so seed rows are stamped relative to now (UTC, ClickHouse * DateTime64(3) text form). */ const SEED_EVENT_TIME = new Date(Date.now() - 2 * 3600_000).toISOString().replace("T", " ").replace("Z", ""); const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_isolation_${Date.now().toString(36)}`; const RESTRICTED_USER = `opt_iso_${Date.now().toString(36)}`; const RESTRICTED_PW = `iso_pw_${Math.random().toString(36).slice(2, 12)}`; const ALPHA: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; const BETA: AnalyticsIdentity = { tenant_id: "t_beta", project_id: "p_main", principal_id: "usr_beta" }; const catalog = dimensionCatalogFrom({ severity: "severity" }); const ALL_TABLES: readonly string[] = ANALYTICS_TABLES; /** Raw HTTP against the live server, as a named user. */ async function chQuery( sql: string, auth: string | undefined, params: Record = {}, settings: Record = {}, body?: string, ): Promise<{ ok: boolean; status: number; text: string }> { const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); const headers: Record = {}; if (auth) headers["Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`; const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); return { ok: res.ok, status: res.status, text: await res.text() }; } function rowsOf(text: string): { grp?: string; value: number }[] { if (!text.trim()) return []; const parsed = JSON.parse(text) as { data?: { grp?: string; value: number | string }[] }; return (parsed.data ?? []).map((r) => ({ ...(r.grp !== undefined ? { grp: r.grp } : {}), value: Number(r.value) })); } let engine: ClickHouseAnalyticsEngine; run("BI-P011 tenant isolation against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, dimensionCatalogs: { t_alpha: { severity: "severity" }, t_beta: { severity: "severity" } }, }); const health = await engine.health(); expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ""}`).toBe(true); await engine.ensureSchema(); // Two tenants, deliberately different magnitudes so a leak is unmistakable // in the NUMBER, not just in the row count. const rows: string[] = []; for (let i = 0; i < 50; i++) { rows.push( JSON.stringify({ tenant_id: "t_alpha", project_id: "p_main", dataset_id: "ds_shared", metric: "rows_ingested", event_id: `a${i}`, event_time: SEED_EVENT_TIME, value: 1, dim_names: ["severity"], dim_values: [i % 2 === 0 ? "warning" : "critical"], }), ); rows.push( JSON.stringify({ tenant_id: "t_beta", project_id: "p_main", dataset_id: "ds_shared", metric: "rows_ingested", event_id: `b${i}`, event_time: SEED_EVENT_TIME, value: 1000, dim_names: ["severity"], dim_values: ["critical"], }), ); } const ins = await chQuery( `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, CLICKHOUSE_AUTH, {}, {}, rows.join("\n"), ); expect(ins.ok, `seed insert failed: ${ins.text.slice(0, 200)}`).toBe(true); }, 120_000); afterAll(async () => { if (!CLICKHOUSE_URL) return; // Disposable by construction: drop the user, its policies, and the DB. for (const table of ANALYTICS_TABLES) { await chQuery(`DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, CLICKHOUSE_AUTH); } await chQuery(`DROP USER IF EXISTS ${RESTRICTED_USER}`, CLICKHOUSE_AUTH); await chQuery(`DROP DATABASE IF EXISTS ${DB}`, CLICKHOUSE_AUTH); }, 60_000); // ── layer 2: the compiler ──────────────────────────────────────────────── it("HOLE, layer 2 REMOVED: the same query WITHOUT the forced predicate returns t_beta's rows to t_alpha", async () => { // This is the exact statement the compiler would produce if someone // deleted the forcing — proof that the predicate is doing real work // against real data, not decorating a query that was safe anyway. const unsafeSql = `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event ` + `WHERE metric = {metric:String} LIMIT 1 FORMAT JSON`; const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: "rows_ingested" }); expect(res.ok).toBe(true); // 50*1 (alpha) + 50*1000 (beta) = 50050 — both tenants, one result. expect(rowsOf(res.text)[0].value).toBe(50050); }); it("CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000", async () => { const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum" }); expect(result.rows[0].value).toBe(50); const beta = await engine.executeMetric(BETA, { metric: "rows_ingested", aggregate: "sum" }); expect(beta.rows[0].value).toBe(50_000); }); it("tenant A querying tenant B's dataset id gets ZERO rows, not B's rows", async () => { const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count", dataset_id: "ds_shared", }); expect(result.rows[0].value).toBe(50); // only alpha's 50, though beta shares the dataset id }); it("a forged dataset id yields an empty result, never an error that leaks existence", async () => { const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count", dataset_id: "ds_that_does_not_exist_or_belongs_to_someone_else", }); expect(result.rows[0].value).toBe(0); }); it("a tenant with no data at all sees zero, not everyone's aggregate", async () => { const ghost: AnalyticsIdentity = { tenant_id: "t_gamma", project_id: "p_main", principal_id: "usr_g" }; const result = await engine.executeMetric(ghost, { metric: "rows_ingested", aggregate: "count" }); expect(result.rows[0].value).toBe(0); }); it("group-by results carry only this tenant's dimension values", async () => { const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count", group_by: ["severity"], }); const total = result.rows.reduce((a, r) => a + r.value, 0); expect(total).toBe(50); // t_beta's rows are all "critical" with value 1000; if any leaked, the // critical bucket would exceed alpha's 25. const critical = result.rows.find((r) => r.groups?.severity === "critical"); expect(critical?.value).toBe(25); }); it("SQL/dimension/filter injection through the live path changes nothing", async () => { await expect( engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum", group_by: ["severity'; SELECT tenant_id FROM system.tables; --"], }), ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); // A hostile VALUE reaches the server as a bound parameter and simply // matches nothing. const injected = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count", filters: { severity: "warning' OR 1=1 --" }, }); expect(injected.rows[0].value).toBe(0); // And the table is still there afterwards. const still = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count" }); expect(still.rows[0].value).toBe(50); }); it("raw SQL attempt: the analyst-SQL surface is refused, not silently executed", async () => { await expect( engine.executeQuery(ALPHA, { sql: `SELECT * FROM ${DB}.opt_analytics_event`, dataset_ids: [] }), ).rejects.toMatchObject({ code: "CAPABILITY_UNAVAILABLE" }); }); // ── layer 3: restricted users + row policies ───────────────────────────── it("HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant", async () => { const res = await chQuery( `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, CLICKHOUSE_AUTH, ); expect(res.ok).toBe(true); expect(rowsOf(res.text)[0].value).toBe(2); // t_alpha AND t_beta }); it("CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant", async () => { for (const stmt of restrictedTenantUserDdl({ db: DB, tenantId: "t_alpha", userName: RESTRICTED_USER, password: RESTRICTED_PW, tables: ALL_TABLES, })) { const r = await chQuery(stmt, CLICKHOUSE_AUTH); expect(r.ok, `DDL failed: ${stmt.slice(0, 80)} -> ${r.text.slice(0, 200)}`).toBe(true); } const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; // The IDENTICAL statement that leaked above, now as the restricted user. const res = await chQuery( `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, restrictedAuth, ); expect(res.ok, res.text.slice(0, 300)).toBe(true); expect(rowsOf(res.text)[0].value).toBe(1); const sum = await chQuery( `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, restrictedAuth, ); expect(rowsOf(sum.text)[0].value).toBe(50); // alpha's total, not 50050 }); it("the restricted user cannot write, and cannot reach a table it was not granted", async () => { const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; const write = await chQuery( `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, restrictedAuth, {}, {}, JSON.stringify({ tenant_id: "t_beta", project_id: "p", dataset_id: "d", metric: "m", event_id: "x", event_time: SEED_EVENT_TIME, value: 1, dim_names: [], dim_values: [] }), ); expect(write.ok).toBe(false); const foreign = await chQuery(`SELECT count() FROM system.users FORMAT JSON`, restrictedAuth); expect(foreign.ok).toBe(false); }); // ── the structural precondition both layers rely on ────────────────────── it("every analytics table's sorting key LEADS with tenant_id (what makes the predicate cheap)", async () => { for (const table of ALL_TABLES) { const res = await chQuery(sortingKeySql(DB, table), CLICKHOUSE_AUTH); expect(res.ok).toBe(true); expect(res.text.trim().startsWith("tenant_id"), `${table} sorting key: ${res.text.trim()}`).toBe(true); } }); it("the tenant value never appears in the statement text the server receives", async () => { const compiled = compileMetricQuery( { tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "sum" }, DB, catalog, DEFAULT_BUDGETS.INTERACTIVE.maxResultRows, ); expect(compiled.sql).not.toContain("t_alpha"); const res = await chQuery(compiled.sql, CLICKHOUSE_AUTH, compiled.params); expect(res.ok).toBe(true); expect(rowsOf(res.text)[0].value).toBe(50); }); });