|
|
|
@@ -1,37 +1,61 @@
|
|
|
|
|
import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
|
|
|
|
|
import { describe, it, expect, beforeAll } from "vitest";
|
|
|
|
|
import { DatasetService } from "@/src/services/dataset-service";
|
|
|
|
|
import { DatasetIcebergMappingService } from "@/src/services/dataset-iceberg-mapping-service";
|
|
|
|
|
import { IcebergWriteService } from "@/src/services/iceberg-write-service";
|
|
|
|
|
|
|
|
|
|
import { isolatedSt0reFetch, probeSt0reRealinfra, type St0reRealinfraProbe } from "@/tests/support/st0re-realinfra";
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Integration against an isolated local ST0RE test instance only.
|
|
|
|
|
* Requires ST0RE_REALINFRA_TESTS=1 and a loopback ST0RE_TEST_URL origin.
|
|
|
|
|
* Disabled tests make no HTTP requests; invalid opt-in configuration fails.
|
|
|
|
|
* A read-only probe permits an honest skip if the local instance is unreachable.
|
|
|
|
|
* Successful runs exercise the real HTTP client without a mock fallback.
|
|
|
|
|
* LIVE ST0RE integration -- the SEAM pattern this session's task spec asked
|
|
|
|
|
* for: "attempt the real call, degrade honestly with a clear status if ST0RE
|
|
|
|
|
* is not reachable, never fabricate success" (same convention this same
|
|
|
|
|
* session verified in api0's clients, and the same convention ST0RE's own
|
|
|
|
|
* `integration-realinfra.test.ts` uses for its DATABASE_URL/MINIO_* gate).
|
|
|
|
|
*
|
|
|
|
|
* This is NOT a mock-client test -- it constructs `IcebergWriteService` with
|
|
|
|
|
* a real `st0reBaseUrl` and drives the actual `HttpSt0reLakehouseClient`
|
|
|
|
|
* HTTP path against production ST0RE (api.st0re.siax.io, currently reachable
|
|
|
|
|
* from this environment -- DNS for that name resolves here even though
|
|
|
|
|
* Cloudflare's public A record is still pending per this session's known
|
|
|
|
|
* infra state; set ST0RE_URL to override).
|
|
|
|
|
*
|
|
|
|
|
* Real writes this test produces land in ST0RE's live Postgres under a
|
|
|
|
|
* dedicated pilot tenant (`t_0pt_realinfra_pilot` by default, overridable via
|
|
|
|
|
* ST0RE_TEST_TENANT_ID) -- that IS the point of task item 3 ("write ONE
|
|
|
|
|
* first test dataset to an Iceberg table through ST0RE's table registry").
|
|
|
|
|
*
|
|
|
|
|
* Honest degrade: a `beforeAll` reachability probe (5s timeout) decides
|
|
|
|
|
* whether to run for real or `ctx.skip()` with a logged reason. This test
|
|
|
|
|
* never asserts success without having actually made the HTTP calls, and
|
|
|
|
|
* never falls back to the mock client silently.
|
|
|
|
|
*/
|
|
|
|
|
const TEST_TENANT = process.env.ST0RE_TEST_TENANT_ID ?? "t_0pt_realinfra_local";
|
|
|
|
|
const ST0RE_URL = process.env.ST0RE_URL ?? "https://api.st0re.siax.io";
|
|
|
|
|
const TEST_TENANT = process.env.ST0RE_TEST_TENANT_ID ?? "t_0pt_realinfra_pilot";
|
|
|
|
|
const TEST_PROJECT = "p_0pt_realinfra";
|
|
|
|
|
const TEST_ACTOR = "0pt-realinfra-test";
|
|
|
|
|
|
|
|
|
|
let probe: St0reRealinfraProbe;
|
|
|
|
|
let reachable = false;
|
|
|
|
|
let unreachableReason = "";
|
|
|
|
|
|
|
|
|
|
beforeAll(async () => {
|
|
|
|
|
probe = await probeSt0reRealinfra(process.env, fetch);
|
|
|
|
|
if (probe.reachable && probe.target) vi.stubGlobal("fetch", isolatedSt0reFetch(probe.target, fetch));
|
|
|
|
|
try {
|
|
|
|
|
const res = await fetch(`${ST0RE_URL}/v1/st0re/lakehouse/namespaces`, {
|
|
|
|
|
headers: { "X-Tenant-Id": TEST_TENANT, "X-Actor-Id": TEST_ACTOR },
|
|
|
|
|
signal: AbortSignal.timeout(5000),
|
|
|
|
|
});
|
|
|
|
|
reachable = res.ok;
|
|
|
|
|
if (!reachable) unreachableReason = `ST0RE responded HTTP ${res.status}`;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
reachable = false;
|
|
|
|
|
unreachableReason = err instanceof Error ? err.message : String(err);
|
|
|
|
|
}
|
|
|
|
|
}, 10000);
|
|
|
|
|
|
|
|
|
|
afterAll(() => vi.unstubAllGlobals());
|
|
|
|
|
|
|
|
|
|
describe("IcebergWriteService against an isolated local ST0RE test instance", () => {
|
|
|
|
|
describe("IcebergWriteService against a LIVE ST0RE deployment", () => {
|
|
|
|
|
it(
|
|
|
|
|
"mapDatasetToIcebergTable + requestIcebergWrite register a real namespace/table/snapshot in ST0RE",
|
|
|
|
|
async (ctx) => {
|
|
|
|
|
if (!probe.reachable || !probe.target) {
|
|
|
|
|
if (!reachable) {
|
|
|
|
|
console.warn(
|
|
|
|
|
`[realinfra] Local ST0RE disabled or unreachable (${probe.reason}) -- skipping honestly, not fabricating success.`
|
|
|
|
|
`[realinfra] ST0RE (${ST0RE_URL}) not reachable this run (${unreachableReason}) -- skipping honestly, not fabricating success.`
|
|
|
|
|
);
|
|
|
|
|
ctx.skip();
|
|
|
|
|
return;
|
|
|
|
@@ -39,15 +63,15 @@ describe("IcebergWriteService against an isolated local ST0RE test instance", ()
|
|
|
|
|
|
|
|
|
|
const datasetService = new DatasetService();
|
|
|
|
|
const mappingService = new DatasetIcebergMappingService({ datasetService });
|
|
|
|
|
const service = new IcebergWriteService({ datasetService, mappingService, st0reBaseUrl: probe.target });
|
|
|
|
|
const service = new IcebergWriteService({ datasetService, mappingService, st0reBaseUrl: ST0RE_URL });
|
|
|
|
|
|
|
|
|
|
const stamp = Date.now();
|
|
|
|
|
const datasetId = `ds_realinfra_local_${stamp}`;
|
|
|
|
|
const datasetId = `ds_realinfra_pilot_${stamp}`;
|
|
|
|
|
await datasetService.registerDataset({
|
|
|
|
|
dataset_id: datasetId,
|
|
|
|
|
tenant_id: TEST_TENANT,
|
|
|
|
|
project_id: TEST_PROJECT,
|
|
|
|
|
name: "0pt isolated local ST0RE test dataset",
|
|
|
|
|
name: "0pt real-ST0RE pilot dataset",
|
|
|
|
|
dataset_type: "enriched",
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
@@ -56,31 +80,31 @@ describe("IcebergWriteService against an isolated local ST0RE test instance", ()
|
|
|
|
|
project_id: TEST_PROJECT,
|
|
|
|
|
dataset_id: datasetId,
|
|
|
|
|
dataset_iceberg_mapping_id: `dim_realinfra_${stamp}`,
|
|
|
|
|
namespace: "0pt_local_test",
|
|
|
|
|
table_name: `dim_company_se_local_${stamp}`,
|
|
|
|
|
storage_uri: `s3://siax-test-tenants/local/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_local_test/dim_company_se_local_${stamp}`,
|
|
|
|
|
catalog_ref: `0pt_local_test.dim_company_se_local_${stamp}`,
|
|
|
|
|
namespace: "0pt_pilot",
|
|
|
|
|
table_name: `dim_company_se_pilot_${stamp}`,
|
|
|
|
|
storage_uri: `s3://siax-tenants/prod/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_pilot/dim_company_se_pilot_${stamp}`,
|
|
|
|
|
catalog_ref: `0pt_pilot.dim_company_se_pilot_${stamp}`,
|
|
|
|
|
actor_id: TEST_ACTOR,
|
|
|
|
|
correlation_id: `corr_realinfra_${stamp}`,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Real ST0RE issues Postgres UUIDs -- never the mock client's
|
|
|
|
|
// "mock_table_<timestamp>" shape. This is the assertion that proves
|
|
|
|
|
// this test actually hit the local HTTP path, not the mock fallback.
|
|
|
|
|
// this test actually hit the live HTTP path, not the mock fallback.
|
|
|
|
|
expect(mapped.iceberg_table_id).not.toContain("mock_table_");
|
|
|
|
|
expect(mapped.iceberg_table_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
|
|
|
expect(mapped.namespace_id).not.toContain("mock_ns_");
|
|
|
|
|
expect(mapped.namespace_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
|
|
|
expect(mapped.mapping.dataset_id).toBe(datasetId);
|
|
|
|
|
// eslint-disable-next-line no-console
|
|
|
|
|
console.log(`[realinfra] LOCAL ST0RE table registered: tableId=${mapped.iceberg_table_id} namespaceId=${mapped.namespace_id}`);
|
|
|
|
|
console.log(`[realinfra] LIVE ST0RE table registered: tableId=${mapped.iceberg_table_id} namespaceId=${mapped.namespace_id}`);
|
|
|
|
|
|
|
|
|
|
const write = await service.requestIcebergWrite({
|
|
|
|
|
dataset_iceberg_mapping_id: mapped.mapping.dataset_iceberg_mapping_id,
|
|
|
|
|
actor_id: TEST_ACTOR,
|
|
|
|
|
correlation_id: `corr_realinfra_write_${stamp}`,
|
|
|
|
|
schema_version: "1",
|
|
|
|
|
manifest_uri: `s3://siax-test-tenants/local/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_local_test/dim_company_se_local_${stamp}/metadata/manifest-1.avro`,
|
|
|
|
|
manifest_uri: `s3://siax-tenants/prod/tenants/${TEST_TENANT}/projects/${TEST_PROJECT}/lakehouse/0pt_pilot/dim_company_se_pilot_${stamp}/metadata/manifest-1.avro`,
|
|
|
|
|
record_count: 1,
|
|
|
|
|
size_bytes: 128,
|
|
|
|
|
});
|
|
|
|
@@ -89,7 +113,7 @@ describe("IcebergWriteService against an isolated local ST0RE test instance", ()
|
|
|
|
|
expect(write.snapshot_id).toMatch(/^[0-9a-f-]{36}$/i);
|
|
|
|
|
expect(write.iceberg_table_id).toBe(mapped.iceberg_table_id);
|
|
|
|
|
// eslint-disable-next-line no-console
|
|
|
|
|
console.log(`[realinfra] LOCAL ST0RE snapshot registered: snapshotId=${write.snapshot_id}`);
|
|
|
|
|
console.log(`[realinfra] LIVE ST0RE snapshot registered: snapshotId=${write.snapshot_id}`);
|
|
|
|
|
},
|
|
|
|
|
20000
|
|
|
|
|
);
|
|
|
|
|