0pt/tests/integration/analyst-sql-clickhouse.integration.test.ts

/**
 * tests/integration/analyst-sql-clickhouse.integration.test.ts
 * =============================================================
 * BI-P017 — the proof that matters: the bypass corpus run against a LIVE
 * ClickHouse **with the statement guard removed**.
 *
 * The guard (layer 2) is a text analysis of a dialect it does not implement.
 * Testing only the guard proves the guard. What has to hold is that when the
 * guard is WRONG, nothing leaks — so this file skips the guard entirely and
 * sends each attack straight to the engine as the tenant's restricted user.
 * Layer 3 (grants + `readonly = 1`) and layer 4 (views that bake the tenant
 * predicate, over base tables carrying a row policy) must refuse it anyway.
 *
 * The assertion is deliberately NOT "the engine returned an error". It is
 * "no foreign row came back", checked by canary values seeded into the other
 * tenant's rows and into an ungranted table. An attack that succeeds and
 * returns nothing interesting still fails the test if it returns a canary.
 *
 * Counterfactual, run first, because a test that would pass against an empty
 * database proves nothing: as the ADMIN user the base table holds both
 * tenants' rows, and `BETA_CANARY` IS readable. Every later assertion is
 * therefore about isolation, not about absence of data.
 *
 * ── HOW THIS AVOIDS THE SILENT-SKIP TRAP ──────────────────────────────────
 * l0re's 28 Postgres tests skipped silently for months and nobody noticed.
 * This suite self-skips when ClickHouse is unreachable — but on a live run it
 * WRITES `evidence/bi-p017/analyst-sql-live-bypass.json`, and
 * tests/analytics/analyst-sql-evidence.test.ts (which never skips) fails if
 * that file is missing, stale, or does not cover every attempt in the corpus.
 * Adding an attack without re-running this suite breaks the build.
 *
 * Point it at a disposable instance:
 *   docker run -d --name ch -p 28123:8123 -e CLICKHOUSE_SKIP_USER_SETUP=1 \
 *     clickhouse/clickhouse-server:24.8
 *   ANALYTICS_CLICKHOUSE_URL=http://127.0.0.1:28123 pnpm vitest run \
 *     tests/integration/analyst-sql-clickhouse.integration.test.ts
 */
import fs from 'node:fs';
import path from 'node:path';
import { beforeAll, describe, expect, it } from 'vitest';
import {
  BYPASS_CORPUS,
  GUARD_ADMITS_ENGINE_REFUSES,
  LEGITIMATE_CORPUS,
} from '../analytics/analyst-sql-bypass-corpus';
import { guardAnalystStatement } from '@/src/analytics/sql/statement-guard';
import {
  rowPolicyVerificationSql,
  tenantSqlCatalogDdl,
  tenantSqlUserDdl,
} from '@/src/analytics/sql/tenant-catalog';
import { applyQuota, DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance';

/**
 * Gated on the ENV VAR, matching the convention already in
 * tests/integration/analytics-tenant-isolation.integration.test.ts — and
 * chosen over a liveness probe on purpose: with the variable set, a
 * ClickHouse that is down makes this suite FAIL. A probe would have turned a
 * broken server into a silent green, which is the failure mode this estate
 * keeps shipping.
 */
const BASE_URL = process.env.CLICKHOUSE_URL ?? '';
const ADMIN_AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:';
const DB = 'opt_analytics';
const ALPHA_PW = 'alpha_test_pw_not_a_secret';
const BETA_PW = 'beta_test_pw_not_a_secret';
const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p017/analyst-sql-live-bypass.json');

/** Canaries. Their presence in a response body is the definition of a leak. */
const BETA_CANARY = 'BETA_CANARY_9f21c4';
const SECRET_CANARY = 'SECRET_CANARY_4d80ab';
const ALPHA_MARK = 'ALPHA_ROW';

interface ChResponse {
  ok: boolean;
  status: number;
  body: string;
}

async function run(sql: string, auth: string, settings: Record<string, string> = {}): Promise<ChResponse> {
  const url = new URL(BASE_URL);
  url.searchParams.set('query', sql);
  for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
  const res = await fetch(url.toString(), {
    method: 'POST',
    headers: { Authorization: `Basic ${Buffer.from(auth).toString('base64')}` },
    body: '',
    signal: AbortSignal.timeout(30_000),
  });
  return { ok: res.ok, status: res.status, body: await res.text() };
}

const admin = (sql: string) => run(sql, ADMIN_AUTH);
/**
 * The session's real posture: the restricted user, and NO settings on the
 * request.
 *
 * MEASURED: a user whose profile carries `readonly = 1` cannot have any
 * setting changed from the request — ClickHouse answers `Code: 164. Cannot
 * modify 'max_rows_to_read' setting in readonly mode.` So the BI-P012 ceilings
 * cannot travel with the statement here the way they do for the non-readonly
 * engine user; they are baked into the SETTINGS PROFILE by
 * `tenantSqlUserDdl`, and the test below proves the profile enforces them.
 */
const asAlpha = (sql: string) => run(sql, `t_alpha_sql_user:${ALPHA_PW}`);

/**
 * An earlier revision set a `reachable` flag inside `beforeAll` and used
 * `describe.skipIf(!reachable)`. Vitest evaluates `skipIf` at COLLECTION time,
 * before any hook runs, so the flag was always false and all ten tests skipped
 * against a live, reachable ClickHouse — a suite that could never run,
 * reporting green. Same shape as l0re's 28 silently-skipped Postgres tests.
 * Recorded here because the next person will reach for `skipIf` too.
 */
const suite = BASE_URL ? describe : describe.skip;
let version = '';

/** The session budget for the main user: generous enough for real analyst work. */
const TEST_BUDGET = applyQuota(DEFAULT_BUDGETS.ANALYTICAL, {
  maxRowsToRead: 100_000,
  maxExecutionTimeSec: 10,
});

/**
 * A SECOND user with a deliberately tiny `maxRowsToRead` (60), provisioned
 * only to make the profile's enforcement observable: the tenant view holds 50
 * rows, which passes; anything wider exceeds it and must fail with ClickHouse
 * code 158. A ceiling nothing can breach proves nothing, and the first
 * revision of this file proved exactly that by starving the legitimate corpus.
 */
const TINY_BUDGET = applyQuota(DEFAULT_BUDGETS.ANALYTICAL, { maxRowsToRead: 60, maxExecutionTimeSec: 10 });
const TINY_PW = 'tiny_test_pw_not_a_secret';

const OBJECTS = [
  { table: 'sales', columns: ['customer', 'amount', 'event_time'] },
  { table: 'events', columns: ['customer', 'kind', 'event_time'] },
];

beforeAll(async () => {
  if (!BASE_URL) return;
  // With CLICKHOUSE_URL set, an unreachable server FAILS here. It does not
  // degrade to a skip.
  const v = await admin('SELECT version()');
  if (!v.ok)
    throw new Error(
      `CLICKHOUSE_URL is set but ClickHouse is not answering: ${v.status} ${v.body.slice(0, 200)}`,
    );
  version = v.body.trim();

  await admin(`DROP DATABASE IF EXISTS ${DB}`);
  await admin(`DROP DATABASE IF EXISTS t_alpha_sql`);
  await admin(`DROP DATABASE IF EXISTS t_beta_sql`);
  await admin(`DROP USER IF EXISTS t_alpha_sql_user`);
  await admin(`DROP USER IF EXISTS t_beta_sql_user`);
  await admin(`DROP USER IF EXISTS t_alpha_tiny_user`);
  await admin(`CREATE DATABASE ${DB}`);

  await admin(
    `CREATE TABLE ${DB}.sales (tenant_id String, customer String, amount Float64, event_time DateTime) ` +
      `ENGINE = MergeTree ORDER BY (tenant_id, event_time)`,
  );
  await admin(
    `CREATE TABLE ${DB}.events (tenant_id String, customer String, kind String, event_time DateTime) ` +
      `ENGINE = MergeTree ORDER BY (tenant_id, event_time)`,
  );
  await admin(
    `CREATE TABLE ${DB}.secret_ledger (tenant_id String, note String) ENGINE = MergeTree ORDER BY tenant_id`,
  );
  // Also present, because several attacks name it: the BI-P010 event table.
  await admin(
    `CREATE TABLE ${DB}.opt_analytics_event (tenant_id String, project_id String, dataset_id String, ` +
      `event_id String, metric String, value Float64, event_time DateTime) ENGINE = MergeTree ` +
      `ORDER BY (tenant_id, event_time)`,
  );

  await admin(
    `INSERT INTO ${DB}.sales SELECT 't_alpha', '${ALPHA_MARK}_' || toString(number), number, now() FROM numbers(50)`,
  );
  await admin(
    `INSERT INTO ${DB}.sales SELECT 't_beta', '${BETA_CANARY}_' || toString(number), number, now() FROM numbers(50)`,
  );
  await admin(
    `INSERT INTO ${DB}.events SELECT 't_alpha', '${ALPHA_MARK}_' || toString(number), 'click', now() FROM numbers(20)`,
  );
  await admin(
    `INSERT INTO ${DB}.events SELECT 't_beta', '${BETA_CANARY}_' || toString(number), 'click', now() FROM numbers(20)`,
  );
  await admin(`INSERT INTO ${DB}.secret_ledger VALUES ('t_alpha', '${SECRET_CANARY}')`);
  await admin(
    `INSERT INTO ${DB}.opt_analytics_event SELECT 't_beta', 'p', 'd', toString(number), 'm', number, now() FROM numbers(10)`,
  );

  for (const [tenant, catalog, user, pw] of [
    ['t_alpha', 't_alpha_sql', 't_alpha_sql_user', ALPHA_PW],
    ['t_beta', 't_beta_sql', 't_beta_sql_user', BETA_PW],
  ] as const) {
    const spec = { sourceDatabase: DB, tenantId: tenant, catalog, objects: OBJECTS };
    for (const stmt of tenantSqlCatalogDdl(spec)) {
      const r = await admin(stmt);
      if (!r.ok) throw new Error(`catalog DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`);
    }
    for (const stmt of tenantSqlUserDdl({ ...spec, userName: user, password: pw, budget: TEST_BUDGET })) {
      const r = await admin(stmt);
      if (!r.ok) throw new Error(`user DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`);
    }
  }

  const tinySpec = { sourceDatabase: DB, tenantId: 't_alpha', catalog: 't_alpha_sql', objects: OBJECTS };
  for (const stmt of tenantSqlUserDdl({
    ...tinySpec,
    userName: 't_alpha_tiny_user',
    password: TINY_PW,
    budget: TINY_BUDGET,
  })) {
    const r = await admin(stmt);
    if (!r.ok) throw new Error(`tiny user DDL failed: ${stmt} -> ${r.body.slice(0, 200)}`);
  }
}, 120_000);

suite('BI-P017 live — counterfactual: the data IS there and IS reachable without the layers', () => {
  it('the shared base table holds both tenants, and BETA_CANARY is readable as admin', async () => {
    const all = await admin(`SELECT count() FROM ${DB}.sales`);
    expect(all.ok).toBe(true);
    expect(Number(all.body.trim())).toBe(100);
    const beta = await admin(`SELECT customer FROM ${DB}.sales WHERE tenant_id = 't_beta' LIMIT 1`);
    expect(beta.body).toContain(BETA_CANARY);
    const secret = await admin(`SELECT note FROM ${DB}.secret_ledger`);
    expect(secret.body).toContain(SECRET_CANARY);
  });

  it('layer 4: the tenant view returns 50 of the 100 rows, and never the canary', async () => {
    const scoped = await asAlpha(`SELECT count() FROM t_alpha_sql.sales`);
    expect(scoped.ok).toBe(true);
    expect(Number(scoped.body.trim())).toBe(50);
    const rows = await asAlpha(`SELECT customer FROM t_alpha_sql.sales LIMIT 100`);
    expect(rows.body).toContain(ALPHA_MARK);
    expect(rows.body).not.toContain(BETA_CANARY);
  });
});

suite('BI-P017 live — the bypass corpus WITH THE GUARD REMOVED', () => {
  it('no attempt returns a foreign-tenant or ungranted-table row', async () => {
    const results: Array<Record<string, unknown>> = [];
    const leaks: string[] = [];

    for (const attempt of [...BYPASS_CORPUS, ...GUARD_ADMITS_ENGINE_REFUSES]) {
      // THE GUARD IS NOT CALLED HERE. That is the whole point.
      const res = await asAlpha(attempt.sql);
      const leakedBeta = res.body.includes(BETA_CANARY);
      const leakedSecret = res.body.includes(SECRET_CANARY);
      const chCode = /Code:\s*(\d+)/.exec(res.body)?.[1];
      // MEASURED, and narrower than the obvious rule. ClickHouse filters
      // `system.tables` / `system.columns` by the caller's own grants: as
      // the restricted user those reads SUCCEED but return only this
      // tenant's four objects — 4 rows, not the server's inventory.
      // `system.query_log`, `system.processes` and `system.users` are
      // ACCESS_DENIED (code 497). So "any successful system read is a leak"
      // would be a false alarm; the real test is whether any FOREIGN name
      // came back.
      const leakedMetadata = res.ok && /\bt_beta\b|secret_ledger/.test(res.body);

      if (leakedBeta || leakedSecret || leakedMetadata) {
        leaks.push(`${attempt.id}: ${attempt.goal}`);
      }
      // For completeness the guard's own verdict is recorded alongside, so
      // the evidence shows which layer caught what.
      let guardVerdict = 'admitted';
      try {
        guardAnalystStatement(attempt.sql, {
          allowedCatalog: 't_alpha_sql',
          allowedObjects: ['sales', 'events'],
        });
      } catch (e) {
        guardVerdict = `refused: ${(e as Error).message}`;
      }
      results.push({
        id: attempt.id,
        goal: attempt.goal,
        engine_http_status: res.status,
        engine_clickhouse_code: chCode ?? null,
        engine_succeeded: res.ok,
        leaked_foreign_tenant_row: leakedBeta,
        leaked_ungranted_table_row: leakedSecret,
        leaked_server_metadata: leakedMetadata,
        guard_verdict_for_reference: guardVerdict,
        engine_response_head: res.body.slice(0, 160).replace(/\s+/g, ' ').trim(),
      });
    }

    // Legitimate work must still WORK as the restricted user, or "nothing
    // leaked" would be trivially true because nothing runs.
    const legitimate: Array<Record<string, unknown>> = [];
    for (const l of LEGITIMATE_CORPUS) {
      const res = await asAlpha(l.sql);
      legitimate.push({
        id: l.id,
        ok: res.ok,
        status: res.status,
        contains_beta_canary: res.body.includes(BETA_CANARY),
      });
    }

    fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true });
    fs.writeFileSync(
      EVIDENCE_PATH,
      `${JSON.stringify(
        {
          prompt: 'BI-P017',
          claim:
            "Every bypass attempt, executed directly against a live ClickHouse as the tenant's " +
            'restricted user WITH THE STATEMENT GUARD REMOVED, fails to return a foreign-tenant ' +
            'row, an ungranted-table row, or server metadata.',
          clickhouse_version: version,
          recorded_at: new Date().toISOString(),
          base_table_rows_both_tenants: 100,
          tenant_view_rows_alpha: 50,
          attempts_total: results.length,
          attempts_that_leaked: leaks.length,
          legitimate_statements_that_succeeded: legitimate.filter((l) => l.ok).length,
          legitimate_statements_total: legitimate.length,
          attempts: results,
          legitimate,
        },
        null,
        2,
      )}\n`,
    );

    expect(leaks, `LEAKS with the guard removed:\n${leaks.join('\n')}`).toEqual([]);
    expect(legitimate.every((l) => l.ok)).toBe(true);
    expect(legitimate.every((l) => l.contains_beta_canary === false)).toBe(true);
  }, 300_000);
});

suite('BI-P017 live — layers 3 and 4 individually, each with the one above removed', () => {
  it('the row policies are actually APPLIED TO THE USER, not merely present', async () => {
    // The check that caught the real bug. `SHOW ROW POLICIES` listing a policy
    // with the right filter is not evidence: after a user is dropped and
    // re-created, `CREATE ROW POLICY IF NOT EXISTS` leaves the policy in place
    // with an EMPTY apply-to list, protecting nobody while looking correct.
    const res = await admin(
      rowPolicyVerificationSql({
        sourceDatabase: DB,
        tenantId: 't_alpha',
        catalog: 't_alpha_sql',
        objects: OBJECTS,
        userName: 't_alpha_sql_user',
        password: 'unused',
      }),
    );
    expect(res.ok).toBe(true);
    const rows = JSON.parse(res.body).data as Array<{ table: string; applied_to_user: number }>;
    expect(rows.length).toBe(OBJECTS.length);
    for (const row of rows) {
      expect(row.applied_to_user, `row policy on ${row.table} applies to nobody`).toBe(1);
    }
  });

  it('layer 3 (row policy): the restricted user reading the SHARED BASE TABLE sees only its own rows', async () => {
    // The counterfactual is in the suite above: as admin this same statement
    // returns 100. Returning 100 here would be a cross-tenant read.
    const res = await asAlpha(`SELECT count() FROM ${DB}.sales`);
    if (res.ok) {
      expect(Number(res.body.trim()), 'restricted user read BOTH tenants from the base table').toBe(50);
    } else {
      expect(res.status).toBeGreaterThanOrEqual(400);
    }
  });

  it('layer 3 (row policy): the base table yields no foreign canary even when named directly', async () => {
    const res = await asAlpha(`SELECT customer FROM ${DB}.sales LIMIT 1000`);
    expect(res.body).not.toContain(BETA_CANARY);
  });

  it('BI-P012 ceilings are enforced by the SETTINGS PROFILE, not by the request', async () => {
    const tiny = (sql: string) => run(sql, `t_alpha_tiny_user:${TINY_PW}`);
    // Under the ceiling: succeeds.
    const ok = await tiny(`SELECT count() FROM t_alpha_sql.sales`);
    expect(ok.ok).toBe(true);
    // Over it: ClickHouse code 158, from the profile — no setting was sent.
    const over = await tiny(`SELECT count() FROM t_alpha_sql.sales a, t_alpha_sql.sales b`);
    expect(over.ok).toBe(false);
    expect(over.body).toContain('Code: 158');
    // And it cannot be raised from the request: code 164, readonly mode.
    const raise = await tiny(`SELECT count() FROM t_alpha_sql.sales`);
    expect(raise.ok).toBe(true);
    const raised = await run(
      `SELECT count() FROM t_alpha_sql.sales a, t_alpha_sql.sales b`,
      `t_alpha_tiny_user:${TINY_PW}`,
      {
        max_rows_to_read: '0',
      },
    );
    expect(raised.ok).toBe(false);
    expect(raised.body).toContain('Code: 164');
  });

  it('layer 3: the restricted user cannot read a table it was never granted', async () => {
    const res = await asAlpha(`SELECT note FROM ${DB}.secret_ledger`);
    expect(res.ok).toBe(false);
    expect(res.body).not.toContain(SECRET_CANARY);
  });

  it("layer 3: the restricted user cannot read the other tenant's catalog", async () => {
    const res = await asAlpha(`SELECT customer FROM t_beta_sql.sales LIMIT 5`);
    expect(res.ok).toBe(false);
    expect(res.body).not.toContain(BETA_CANARY);
  });

  it('layer 3: readonly=1 refuses a write even though the guard never saw the statement', async () => {
    const res = await asAlpha(`INSERT INTO ${DB}.sales VALUES ('t_alpha', 'x', 1, now())`);
    expect(res.ok).toBe(false);
    const after = await admin(`SELECT count() FROM ${DB}.sales`);
    expect(Number(after.body.trim())).toBe(100);
  });

  it('layer 3: readonly=1 refuses a statement that tries to raise its own ceiling', async () => {
    const res = await asAlpha(`SELECT count() FROM t_alpha_sql.sales SETTINGS max_rows_to_read = 0`);
    expect(res.ok).toBe(false);
  });

  it('layer 4: the tenant column the view does not project is not selectable', async () => {
    // This is the case the GUARD ADMITS. It must fail here, or the honest
    // note in the corpus would be covering a real hole.
    const res = await asAlpha(`SELECT tenant_id FROM t_alpha_sql.sales LIMIT 1`);
    expect(res.ok).toBe(false);
  });

  it('layer 4: a WHERE on the hidden tenant column cannot widen the view', async () => {
    const res = await asAlpha(`SELECT count() FROM t_alpha_sql.sales WHERE tenant_id = 't_beta'`);
    expect(res.ok).toBe(false);
  });
});