94 lines
5.6 KiB
TypeScript
94 lines
5.6 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
const { judgeCandidates, WEIGHTS, jevFallback } = await import("../lib/jev.ts");
|
|
const { TypeSafeClient } = await import("@typesafe-ai/sdk");
|
|
|
|
const digest = { sql: "SELECT", bytes: 1, subtreeCost: 1, parallel: false, nonParallelReason: null, earlyAbort: null,
|
|
tables: [], rowGuessErrors: [], missingIndexes: [], warnings: [], waits: [],
|
|
topOperators: [{ id: "0", op: "Clustered Index Scan", cost: 4.1, estRows: 1000, actualRows: 1200000,
|
|
execs: 1, io: 40, cpu: 39, spillLevels: 2, parallel: false }] } as any;
|
|
const candidates = [
|
|
{ key: "a", title: "Covering index", diagnosis: "Scan of a million rows; no key exists on CustomerId.",
|
|
actions: [{ title: "Create index", detail: "CREATE INDEX", effort: "medium", requires_change_control: true }],
|
|
rewrite_sql: null, expected: "seek instead of scan" },
|
|
{ key: "b", title: "Add TOP 100", diagnosis: "Too many rows leave the scan, so cut them off.",
|
|
actions: [{ title: "Add TOP 100", detail: "TOP 100", effort: "low", requires_change_control: false }],
|
|
rewrite_sql: "SELECT TOP 100", expected: "less I/O" },
|
|
] as any;
|
|
|
|
const scored = (fit: number, safe: number, ease: number, cause: number) => ({
|
|
bottleneck_fit: { type: "score", score: fit, confidence: 0.9, legend: {}, probabilities: {} },
|
|
semantic_safety: { type: "score", score: safe, confidence: 0.8, legend: {}, probabilities: {} },
|
|
ease: { type: "score", score: ease, confidence: 0.7, legend: {}, probabilities: {} },
|
|
root_cause: { type: "noul", noul: cause } });
|
|
|
|
// The SDK's client takes an injectable fetch — no network in tests.
|
|
function fakeJev(map: any) {
|
|
return new TypeSafeClient({ apiKey: "test", fetch: (async (_u: any, init: any) => {
|
|
const body = JSON.parse(init.body);
|
|
const answers = body.questions.first_to_run ? map.cross : map.byKey[body.state.candidate.key];
|
|
return new Response(JSON.stringify({ model: "jev-1.13.0", usage: { input_tokens: 1, output_tokens: 1 }, answers }),
|
|
{ status: 200, headers: { "content-type": "application/json" } });
|
|
}) as any, retry: { maxRetries: 0 } });
|
|
}
|
|
const GOOD = { byKey: { a: scored(4, 4, 2, 1), b: scored(0.4, 0.5, 4, 0.1) },
|
|
cross: { first_to_run: { type: "choice", choice: "a", confidence: 0.82, probabilities: { a: 0.82, b: 0.18 } },
|
|
anything_worth_running: { type: "noul", noul: 0.9 } } };
|
|
|
|
test("weights sum to 1", () => assert.equal(Object.values(WEIGHTS).reduce((a, b) => a + b, 0), 1));
|
|
|
|
test("jev picks the winner; the unsafe option is flagged and cannot lead", async () => {
|
|
const v = await judgeCandidates(digest, candidates, fakeJev(GOOD));
|
|
assert.equal(v.source, "jev");
|
|
assert.equal(v.headline, "a");
|
|
assert.equal(v.order[0].key, "a");
|
|
assert.equal(v.agrees, true);
|
|
assert.equal(v.jev_confidence, 0.82);
|
|
assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics"));
|
|
assert.ok(v.order.find((r) => r.key === "a")!.flags.includes("change_control"));
|
|
assert.ok(v.order.find((r) => r.key === "a")!.dims.bottleneck_fit.value === 1);
|
|
});
|
|
test("a low-confidence jev pick defers to composite order", async () => {
|
|
const low = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "b", confidence: 0.2,
|
|
probabilities: { a: 0.4, b: 0.6 } }, anything_worth_running: { type: "noul", noul: 0.4 } } };
|
|
const v = await judgeCandidates(digest, candidates, fakeJev(low));
|
|
assert.equal(v.source, "composite");
|
|
assert.equal(v.headline, "a");
|
|
assert.equal(v.agrees, false);
|
|
assert.ok(v.flags.includes("low_confidence"));
|
|
assert.ok(v.flags.includes("nothing_clearly_worthwhile"));
|
|
});
|
|
test("an unknown jev choice label falls back to composite order", async () => {
|
|
const bogus = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "zzz", confidence: 0.99,
|
|
probabilities: { zzz: 0.99 } }, anything_worth_running: { type: "noul", noul: 0.9 } } };
|
|
const v = await judgeCandidates(digest, candidates, fakeJev(bogus));
|
|
assert.equal(v.source, "composite");
|
|
assert.equal(v.headline, "a");
|
|
assert.equal(v.jev_pick, null);
|
|
});
|
|
test("fallback ranks without Jev and says so", () => {
|
|
const v = jevFallback(candidates, "429 rate limited");
|
|
assert.equal(v.source, "composite");
|
|
assert.ok(v.flags.includes("jev_unavailable"));
|
|
assert.equal(v.order.length, 2);
|
|
});
|
|
test("a silent jev does not accuse every option of being unsafe", () => {
|
|
const v = jevFallback(candidates, "fetch failed");
|
|
// change_control is read off the candidate's own actions and stays; anything derived from a Jev score must go.
|
|
const derived = (flags: string[]) => flags.filter((f) => f !== "change_control");
|
|
assert.deepEqual(v.order.map((r) => derived(r.flags)), [[], []]);
|
|
});
|
|
test("verify_semantics fires where semantics can actually move", async () => {
|
|
// Real Jev scores an index create ~0.57 ("probably fine, no stated proof"), which a blanket <0.6 floor
|
|
// flagged as unsafe on every run. Index-only options must not shout; rewrites and bad scores must.
|
|
const idxOnly = [{ ...candidates[0], rewrite_sql: null }, candidates[1]];
|
|
const mid = { byKey: { a: scored(4, 2, 2, 1), b: scored(4, 2, 2, 1) }, cross: GOOD.cross };
|
|
const v = await judgeCandidates(digest, idxOnly, fakeJev(mid));
|
|
assert.ok(!v.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "index create flagged");
|
|
assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics"), "rewrite not flagged");
|
|
|
|
const bad = { byKey: { a: scored(4, 1, 2, 1), b: GOOD.byKey.b }, cross: GOOD.cross };
|
|
const v2 = await judgeCandidates(digest, idxOnly, fakeJev(bad));
|
|
assert.ok(v2.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "bad score not flagged");
|
|
});
|