From d8caa8e6dbb650142ce658b65af829c8d1cc8dc4 Mon Sep 17 00:00:00 2001 From: ahmed mohammed Date: Tue, 22 Sep 2026 07:53:03 +0300 Subject: [PATCH] feat: jev composite scoring and verdict Co-Authored-By: Claude Opus 4.7 --- lib/jev.ts | 108 ++++++++++++++++++++++++++++++++++++++++++++++ tests/jev.test.ts | 93 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 lib/jev.ts create mode 100644 tests/jev.test.ts diff --git a/lib/jev.ts b/lib/jev.ts new file mode 100644 index 0000000..59e1eab --- /dev/null +++ b/lib/jev.ts @@ -0,0 +1,108 @@ +import { TypeSafeClient, score, noul, choice } from "@typesafe-ai/sdk"; +import { digestForModel, type Digest } from "./digest.ts"; +import type { Candidate } from "./analyst.ts"; + +export type Dim = "bottleneck_fit" | "semantic_safety" | "ease" | "root_cause"; +export const WEIGHTS: Record = { bottleneck_fit: 0.4, semantic_safety: 0.3, ease: 0.15, root_cause: 0.15 }; +const SAFETY_FLOOR = 0.6; +const SAFETY_HARD_FLOOR = 0.45; +const CONFIDENCE_FLOOR = 0.35; + +const BOTTLENECK = [ + "Unrelated to the costly operator or the worst estimate error in the plan.", + "Touches a minor operator; the dominant cost is untouched.", + "Partly addresses the dominant cost; a scan or spill probably remains.", + "Directly removes the dominant cost for this query shape.", + "Removes the dominant cost and the pattern behind it, so the shape itself changes.", +] as const; +const SAFETY = [ + "Changes results: a predicate, join, grouping, or NULL handling is altered.", + "Likely changes results in edge cases: duplicates, ordering, tie-breaking.", + "Equivalent on most data but leans on unstated assumptions.", + "Clearly equivalent, with a stated reason.", + "Equivalent and checkable: ships a comparison or rollback plan.", +] as const; +const EASE = [ + "Needs a schema plus application redesign and a release.", + "Needs an application deploy with change control on the app tier.", + "DBA work with approval, e.g. an index create during a window.", + "One rewritten query or session setting, reviewable in minutes.", + "Copy-paste change with no server-side change at all.", +] as const; + +export type Ranked = { key: string; dims: Record; + composite: number; flags: string[] }; +export type Verdict = { order: Ranked[]; headline: string; source: "jev" | "composite"; jev_pick: string | null; + jev_confidence: number; agrees: boolean; anything_worth_running: number; weights: Record; flags: string[] }; + +export const makeJevClient = (apiKey: string, model = "jev-latest") => + new TypeSafeClient({ apiKey, defaultModel: model, timeout: 20_000 }); + +const norm = (s: number, levels: number) => Math.max(0, Math.min(1, s / (levels - 1))); + +function rank(c: Candidate, a: any): Ranked { + const dims = { + bottleneck_fit: { value: norm(a.bottleneck_fit.score, BOTTLENECK.length), confidence: a.bottleneck_fit.confidence }, + semantic_safety: { value: norm(a.semantic_safety.score, SAFETY.length), confidence: a.semantic_safety.confidence }, + ease: { value: norm(a.ease.score, EASE.length), confidence: a.ease.confidence }, + root_cause: { value: a.root_cause.noul, confidence: null }, // noul reports no confidence + } as Record; + const composite = (Object.keys(WEIGHTS) as Dim[]).reduce((s, k) => s + WEIGHTS[k] * dims[k].value, 0); + const confs = Object.values(dims).map((d) => d.confidence).filter((x): x is number => x !== null); + const flags: string[] = []; + // Live Jev scores an index create around 0.57 ("probably equivalent, no proof stated"). A blanket floor + // flagged every index option, so the warning would be noise. Flag where semantics can actually move: + // any rewrite below the soft floor, or anything at all below the hard floor. + const safety = dims.semantic_safety.value; + if (safety < SAFETY_HARD_FLOOR || (c.rewrite_sql !== null && safety < SAFETY_FLOOR)) flags.push("verify_semantics"); + if (confs.length && Math.min(...confs) < CONFIDENCE_FLOOR) flags.push("low_confidence"); + if (c.actions.some((x) => x.requires_change_control)) flags.push("change_control"); + return { key: c.key, dims, composite: Number(composite.toFixed(4)), flags }; +} + +export async function judgeCandidates(digest: Digest, candidates: Candidate[], client: TypeSafeClient): Promise { + const plan = digestForModel(digest); + const per = await Promise.all(candidates.map((c) => client.systemOne({ + state: { plan, candidate: c }, + questions: { + bottleneck_fit: score("How directly do the actions in `candidate` attack the dominant bottleneck in `plan` — the most expensive operator and the worst estimate error?", BOTTLENECK), + semantic_safety: score("If `candidate` is applied, how certain is it that the query still returns exactly the same rows as the original?", SAFETY), + ease: score("How easy is `candidate` to apply in a real shop, counting people, approvals, and downtime?", EASE), + root_cause: noul("The diagnosis in `candidate` names the actual cause of the cost in `plan`, not a symptom of it."), + }, + }))); + + const cross = await client.systemOne({ + state: { plan, options: candidates.map((c) => ({ key: c.key, title: c.title, diagnosis: c.diagnosis, expected: c.expected })) }, + questions: { + first_to_run: choice("Which single entry in `options` should the DBA run first for the biggest drop in the cost shown in `plan`?", + Object.fromEntries(candidates.map((c) => [c.key, `${c.title} — ${c.expected}`]))), + anything_worth_running: noul("At least one entry in `options` would meaningfully reduce the cost shown in `plan`."), + }, + }); + + const order = candidates.map((c, i) => rank(c, (per[i] as any).answers)).sort((a, b) => b.composite - a.composite); + const pick = (cross as any).answers.first_to_run; + const worth = (cross as any).answers.anything_worth_running.noul; + const valid = candidates.some((c) => c.key === pick.choice); + const useJev = valid && pick.confidence >= 0.5; // confidence-gated routing: act only when Jev is sure enough + const flags: string[] = []; + if (!useJev) flags.push(pick.confidence < CONFIDENCE_FLOOR ? "low_confidence" : "composite_order"); + if (worth < 0.5) flags.push("nothing_clearly_worthwhile"); + + return { order, source: useJev ? "jev" : "composite", headline: useJev ? pick.choice : order[0].key, + jev_pick: valid ? pick.choice : null, jev_confidence: pick.confidence, + agrees: valid && pick.choice === order[0].key, anything_worth_running: worth, weights: WEIGHTS, flags }; +} + +// Neutral answers: Jev never spoke, so nothing is claimed about safety or fit. The UI must say so. +export function jevFallback(candidates: Candidate[], why: string): Verdict { + // Scores are 0 because Jev never answered, not because the options are bad: drop every flag a score produced. + const order = candidates.map((c) => rank(c, { bottleneck_fit: { score: 0, confidence: null }, + semantic_safety: { score: 0, confidence: null }, ease: { score: 0, confidence: null }, + root_cause: { noul: 0 } })) + .map((r) => ({ ...r, flags: r.flags.filter((f) => f === "change_control") })) + .sort((a, b) => b.composite - a.composite); + return { order, source: "composite", headline: order[0]?.key ?? "", jev_pick: null, jev_confidence: 0, + agrees: false, anything_worth_running: 0, weights: WEIGHTS, flags: ["jev_unavailable", why.slice(0, 200)] }; +} diff --git a/tests/jev.test.ts b/tests/jev.test.ts new file mode 100644 index 0000000..741538b --- /dev/null +++ b/tests/jev.test.ts @@ -0,0 +1,93 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +const { judgeCandidates, WEIGHTS, jevFallback } = await import("../lib/jev.ts"); +const { TypeSafeClient } = await import("@typesafe-ai/sdk"); + +const digest = { sql: "SELECT", bytes: 1, subtreeCost: 1, parallel: false, nonParallelReason: null, earlyAbort: null, + tables: [], rowGuessErrors: [], missingIndexes: [], warnings: [], waits: [], + topOperators: [{ id: "0", op: "Clustered Index Scan", cost: 4.1, estRows: 1000, actualRows: 1200000, + execs: 1, io: 40, cpu: 39, spillLevels: 2, parallel: false }] } as any; +const candidates = [ + { key: "a", title: "Covering index", diagnosis: "Scan of a million rows; no key exists on CustomerId.", + actions: [{ title: "Create index", detail: "CREATE INDEX", effort: "medium", requires_change_control: true }], + rewrite_sql: null, expected: "seek instead of scan" }, + { key: "b", title: "Add TOP 100", diagnosis: "Too many rows leave the scan, so cut them off.", + actions: [{ title: "Add TOP 100", detail: "TOP 100", effort: "low", requires_change_control: false }], + rewrite_sql: "SELECT TOP 100", expected: "less I/O" }, +] as any; + +const scored = (fit: number, safe: number, ease: number, cause: number) => ({ + bottleneck_fit: { type: "score", score: fit, confidence: 0.9, legend: {}, probabilities: {} }, + semantic_safety: { type: "score", score: safe, confidence: 0.8, legend: {}, probabilities: {} }, + ease: { type: "score", score: ease, confidence: 0.7, legend: {}, probabilities: {} }, + root_cause: { type: "noul", noul: cause } }); + +// The SDK's client takes an injectable fetch — no network in tests. +function fakeJev(map: any) { + return new TypeSafeClient({ apiKey: "test", fetch: (async (_u: any, init: any) => { + const body = JSON.parse(init.body); + const answers = body.questions.first_to_run ? map.cross : map.byKey[body.state.candidate.key]; + return new Response(JSON.stringify({ model: "jev-1.13.0", usage: { input_tokens: 1, output_tokens: 1 }, answers }), + { status: 200, headers: { "content-type": "application/json" } }); + }) as any, retry: { maxRetries: 0 } }); +} +const GOOD = { byKey: { a: scored(4, 4, 2, 1), b: scored(0.4, 0.5, 4, 0.1) }, + cross: { first_to_run: { type: "choice", choice: "a", confidence: 0.82, probabilities: { a: 0.82, b: 0.18 } }, + anything_worth_running: { type: "noul", noul: 0.9 } } }; + +test("weights sum to 1", () => assert.equal(Object.values(WEIGHTS).reduce((a, b) => a + b, 0), 1)); + +test("jev picks the winner; the unsafe option is flagged and cannot lead", async () => { + const v = await judgeCandidates(digest, candidates, fakeJev(GOOD)); + assert.equal(v.source, "jev"); + assert.equal(v.headline, "a"); + assert.equal(v.order[0].key, "a"); + assert.equal(v.agrees, true); + assert.equal(v.jev_confidence, 0.82); + assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics")); + assert.ok(v.order.find((r) => r.key === "a")!.flags.includes("change_control")); + assert.ok(v.order.find((r) => r.key === "a")!.dims.bottleneck_fit.value === 1); +}); +test("a low-confidence jev pick defers to composite order", async () => { + const low = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "b", confidence: 0.2, + probabilities: { a: 0.4, b: 0.6 } }, anything_worth_running: { type: "noul", noul: 0.4 } } }; + const v = await judgeCandidates(digest, candidates, fakeJev(low)); + assert.equal(v.source, "composite"); + assert.equal(v.headline, "a"); + assert.equal(v.agrees, false); + assert.ok(v.flags.includes("low_confidence")); + assert.ok(v.flags.includes("nothing_clearly_worthwhile")); +}); +test("an unknown jev choice label falls back to composite order", async () => { + const bogus = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "zzz", confidence: 0.99, + probabilities: { zzz: 0.99 } }, anything_worth_running: { type: "noul", noul: 0.9 } } }; + const v = await judgeCandidates(digest, candidates, fakeJev(bogus)); + assert.equal(v.source, "composite"); + assert.equal(v.headline, "a"); + assert.equal(v.jev_pick, null); +}); +test("fallback ranks without Jev and says so", () => { + const v = jevFallback(candidates, "429 rate limited"); + assert.equal(v.source, "composite"); + assert.ok(v.flags.includes("jev_unavailable")); + assert.equal(v.order.length, 2); +}); +test("a silent jev does not accuse every option of being unsafe", () => { + const v = jevFallback(candidates, "fetch failed"); + // change_control is read off the candidate's own actions and stays; anything derived from a Jev score must go. + const derived = (flags: string[]) => flags.filter((f) => f !== "change_control"); + assert.deepEqual(v.order.map((r) => derived(r.flags)), [[], []]); +}); +test("verify_semantics fires where semantics can actually move", async () => { + // Real Jev scores an index create ~0.57 ("probably fine, no stated proof"), which a blanket <0.6 floor + // flagged as unsafe on every run. Index-only options must not shout; rewrites and bad scores must. + const idxOnly = [{ ...candidates[0], rewrite_sql: null }, candidates[1]]; + const mid = { byKey: { a: scored(4, 2, 2, 1), b: scored(4, 2, 2, 1) }, cross: GOOD.cross }; + const v = await judgeCandidates(digest, idxOnly, fakeJev(mid)); + assert.ok(!v.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "index create flagged"); + assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics"), "rewrite not flagged"); + + const bad = { byKey: { a: scored(4, 1, 2, 1), b: GOOD.byKey.b }, cross: GOOD.cross }; + const v2 = await judgeCandidates(digest, idxOnly, fakeJev(bad)); + assert.ok(v2.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "bad score not flagged"); +});