feat: jev composite scoring and verdict

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

lib/jev.ts

@@ -0,0 +1,108 @@
+import { TypeSafeClient, score, noul, choice } from "@typesafe-ai/sdk";
+import { digestForModel, type Digest } from "./digest.ts";
+import type { Candidate } from "./analyst.ts";
+
+export type Dim = "bottleneck_fit" | "semantic_safety" | "ease" | "root_cause";
+export const WEIGHTS: Record<Dim, number> = { bottleneck_fit: 0.4, semantic_safety: 0.3, ease: 0.15, root_cause: 0.15 };
+const SAFETY_FLOOR = 0.6;
+const SAFETY_HARD_FLOOR = 0.45;
+const CONFIDENCE_FLOOR = 0.35;
+
+const BOTTLENECK = [
+  "Unrelated to the costly operator or the worst estimate error in the plan.",
+  "Touches a minor operator; the dominant cost is untouched.",
+  "Partly addresses the dominant cost; a scan or spill probably remains.",
+  "Directly removes the dominant cost for this query shape.",
+  "Removes the dominant cost and the pattern behind it, so the shape itself changes.",
+] as const;
+const SAFETY = [
+  "Changes results: a predicate, join, grouping, or NULL handling is altered.",
+  "Likely changes results in edge cases: duplicates, ordering, tie-breaking.",
+  "Equivalent on most data but leans on unstated assumptions.",
+  "Clearly equivalent, with a stated reason.",
+  "Equivalent and checkable: ships a comparison or rollback plan.",
+] as const;
+const EASE = [
+  "Needs a schema plus application redesign and a release.",
+  "Needs an application deploy with change control on the app tier.",
+  "DBA work with approval, e.g. an index create during a window.",
+  "One rewritten query or session setting, reviewable in minutes.",
+  "Copy-paste change with no server-side change at all.",
+] as const;
+
+export type Ranked = { key: string; dims: Record<Dim, { value: number; confidence: number | null }>;
+  composite: number; flags: string[] };
+export type Verdict = { order: Ranked[]; headline: string; source: "jev" | "composite"; jev_pick: string | null;
+  jev_confidence: number; agrees: boolean; anything_worth_running: number; weights: Record<Dim, number>; flags: string[] };
+
+export const makeJevClient = (apiKey: string, model = "jev-latest") =>
+  new TypeSafeClient({ apiKey, defaultModel: model, timeout: 20_000 });
+
+const norm = (s: number, levels: number) => Math.max(0, Math.min(1, s / (levels - 1)));
+
+function rank(c: Candidate, a: any): Ranked {
+  const dims = {
+    bottleneck_fit: { value: norm(a.bottleneck_fit.score, BOTTLENECK.length), confidence: a.bottleneck_fit.confidence },
+    semantic_safety: { value: norm(a.semantic_safety.score, SAFETY.length), confidence: a.semantic_safety.confidence },
+    ease: { value: norm(a.ease.score, EASE.length), confidence: a.ease.confidence },
+    root_cause: { value: a.root_cause.noul, confidence: null },  // noul reports no confidence
+  } as Record<Dim, { value: number; confidence: number | null }>;
+  const composite = (Object.keys(WEIGHTS) as Dim[]).reduce((s, k) => s + WEIGHTS[k] * dims[k].value, 0);
+  const confs = Object.values(dims).map((d) => d.confidence).filter((x): x is number => x !== null);
+  const flags: string[] = [];
+  // Live Jev scores an index create around 0.57 ("probably equivalent, no proof stated"). A blanket floor
+  // flagged every index option, so the warning would be noise. Flag where semantics can actually move:
+  // any rewrite below the soft floor, or anything at all below the hard floor.
+  const safety = dims.semantic_safety.value;
+  if (safety < SAFETY_HARD_FLOOR || (c.rewrite_sql !== null && safety < SAFETY_FLOOR)) flags.push("verify_semantics");
+  if (confs.length && Math.min(...confs) < CONFIDENCE_FLOOR) flags.push("low_confidence");
+  if (c.actions.some((x) => x.requires_change_control)) flags.push("change_control");
+  return { key: c.key, dims, composite: Number(composite.toFixed(4)), flags };
+}
+
+export async function judgeCandidates(digest: Digest, candidates: Candidate[], client: TypeSafeClient): Promise<Verdict> {
+  const plan = digestForModel(digest);
+  const per = await Promise.all(candidates.map((c) => client.systemOne({
+    state: { plan, candidate: c },
+    questions: {
+      bottleneck_fit: score("How directly do the actions in `candidate` attack the dominant bottleneck in `plan` — the most expensive operator and the worst estimate error?", BOTTLENECK),
+      semantic_safety: score("If `candidate` is applied, how certain is it that the query still returns exactly the same rows as the original?", SAFETY),
+      ease: score("How easy is `candidate` to apply in a real shop, counting people, approvals, and downtime?", EASE),
+      root_cause: noul("The diagnosis in `candidate` names the actual cause of the cost in `plan`, not a symptom of it."),
+    },
+  })));
+
+  const cross = await client.systemOne({
+    state: { plan, options: candidates.map((c) => ({ key: c.key, title: c.title, diagnosis: c.diagnosis, expected: c.expected })) },
+    questions: {
+      first_to_run: choice("Which single entry in `options` should the DBA run first for the biggest drop in the cost shown in `plan`?",
+        Object.fromEntries(candidates.map((c) => [c.key, `${c.title} — ${c.expected}`]))),
+      anything_worth_running: noul("At least one entry in `options` would meaningfully reduce the cost shown in `plan`."),
+    },
+  });
+
+  const order = candidates.map((c, i) => rank(c, (per[i] as any).answers)).sort((a, b) => b.composite - a.composite);
+  const pick = (cross as any).answers.first_to_run;
+  const worth = (cross as any).answers.anything_worth_running.noul;
+  const valid = candidates.some((c) => c.key === pick.choice);
+  const useJev = valid && pick.confidence >= 0.5;   // confidence-gated routing: act only when Jev is sure enough
+  const flags: string[] = [];
+  if (!useJev) flags.push(pick.confidence < CONFIDENCE_FLOOR ? "low_confidence" : "composite_order");
+  if (worth < 0.5) flags.push("nothing_clearly_worthwhile");
+
+  return { order, source: useJev ? "jev" : "composite", headline: useJev ? pick.choice : order[0].key,
+    jev_pick: valid ? pick.choice : null, jev_confidence: pick.confidence,
+    agrees: valid && pick.choice === order[0].key, anything_worth_running: worth, weights: WEIGHTS, flags };
+}
+
+// Neutral answers: Jev never spoke, so nothing is claimed about safety or fit. The UI must say so.
+export function jevFallback(candidates: Candidate[], why: string): Verdict {
+  // Scores are 0 because Jev never answered, not because the options are bad: drop every flag a score produced.
+  const order = candidates.map((c) => rank(c, { bottleneck_fit: { score: 0, confidence: null },
+    semantic_safety: { score: 0, confidence: null }, ease: { score: 0, confidence: null },
+    root_cause: { noul: 0 } }))
+    .map((r) => ({ ...r, flags: r.flags.filter((f) => f === "change_control") }))
+    .sort((a, b) => b.composite - a.composite);
+  return { order, source: "composite", headline: order[0]?.key ?? "", jev_pick: null, jev_confidence: 0,
+    agrees: false, anything_worth_running: 0, weights: WEIGHTS, flags: ["jev_unavailable", why.slice(0, 200)] };
+}

tests/jev.test.ts

@@ -0,0 +1,93 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+const { judgeCandidates, WEIGHTS, jevFallback } = await import("../lib/jev.ts");
+const { TypeSafeClient } = await import("@typesafe-ai/sdk");
+
+const digest = { sql: "SELECT", bytes: 1, subtreeCost: 1, parallel: false, nonParallelReason: null, earlyAbort: null,
+  tables: [], rowGuessErrors: [], missingIndexes: [], warnings: [], waits: [],
+  topOperators: [{ id: "0", op: "Clustered Index Scan", cost: 4.1, estRows: 1000, actualRows: 1200000,
+    execs: 1, io: 40, cpu: 39, spillLevels: 2, parallel: false }] } as any;
+const candidates = [
+  { key: "a", title: "Covering index", diagnosis: "Scan of a million rows; no key exists on CustomerId.",
+    actions: [{ title: "Create index", detail: "CREATE INDEX", effort: "medium", requires_change_control: true }],
+    rewrite_sql: null, expected: "seek instead of scan" },
+  { key: "b", title: "Add TOP 100", diagnosis: "Too many rows leave the scan, so cut them off.",
+    actions: [{ title: "Add TOP 100", detail: "TOP 100", effort: "low", requires_change_control: false }],
+    rewrite_sql: "SELECT TOP 100", expected: "less I/O" },
+] as any;
+
+const scored = (fit: number, safe: number, ease: number, cause: number) => ({
+  bottleneck_fit: { type: "score", score: fit, confidence: 0.9, legend: {}, probabilities: {} },
+  semantic_safety: { type: "score", score: safe, confidence: 0.8, legend: {}, probabilities: {} },
+  ease: { type: "score", score: ease, confidence: 0.7, legend: {}, probabilities: {} },
+  root_cause: { type: "noul", noul: cause } });
+
+// The SDK's client takes an injectable fetch — no network in tests.
+function fakeJev(map: any) {
+  return new TypeSafeClient({ apiKey: "test", fetch: (async (_u: any, init: any) => {
+    const body = JSON.parse(init.body);
+    const answers = body.questions.first_to_run ? map.cross : map.byKey[body.state.candidate.key];
+    return new Response(JSON.stringify({ model: "jev-1.13.0", usage: { input_tokens: 1, output_tokens: 1 }, answers }),
+      { status: 200, headers: { "content-type": "application/json" } });
+  }) as any, retry: { maxRetries: 0 } });
+}
+const GOOD = { byKey: { a: scored(4, 4, 2, 1), b: scored(0.4, 0.5, 4, 0.1) },
+  cross: { first_to_run: { type: "choice", choice: "a", confidence: 0.82, probabilities: { a: 0.82, b: 0.18 } },
+           anything_worth_running: { type: "noul", noul: 0.9 } } };
+
+test("weights sum to 1", () => assert.equal(Object.values(WEIGHTS).reduce((a, b) => a + b, 0), 1));
+
+test("jev picks the winner; the unsafe option is flagged and cannot lead", async () => {
+  const v = await judgeCandidates(digest, candidates, fakeJev(GOOD));
+  assert.equal(v.source, "jev");
+  assert.equal(v.headline, "a");
+  assert.equal(v.order[0].key, "a");
+  assert.equal(v.agrees, true);
+  assert.equal(v.jev_confidence, 0.82);
+  assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics"));
+  assert.ok(v.order.find((r) => r.key === "a")!.flags.includes("change_control"));
+  assert.ok(v.order.find((r) => r.key === "a")!.dims.bottleneck_fit.value === 1);
+});
+test("a low-confidence jev pick defers to composite order", async () => {
+  const low = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "b", confidence: 0.2,
+    probabilities: { a: 0.4, b: 0.6 } }, anything_worth_running: { type: "noul", noul: 0.4 } } };
+  const v = await judgeCandidates(digest, candidates, fakeJev(low));
+  assert.equal(v.source, "composite");
+  assert.equal(v.headline, "a");
+  assert.equal(v.agrees, false);
+  assert.ok(v.flags.includes("low_confidence"));
+  assert.ok(v.flags.includes("nothing_clearly_worthwhile"));
+});
+test("an unknown jev choice label falls back to composite order", async () => {
+  const bogus = { byKey: GOOD.byKey, cross: { first_to_run: { type: "choice", choice: "zzz", confidence: 0.99,
+    probabilities: { zzz: 0.99 } }, anything_worth_running: { type: "noul", noul: 0.9 } } };
+  const v = await judgeCandidates(digest, candidates, fakeJev(bogus));
+  assert.equal(v.source, "composite");
+  assert.equal(v.headline, "a");
+  assert.equal(v.jev_pick, null);
+});
+test("fallback ranks without Jev and says so", () => {
+  const v = jevFallback(candidates, "429 rate limited");
+  assert.equal(v.source, "composite");
+  assert.ok(v.flags.includes("jev_unavailable"));
+  assert.equal(v.order.length, 2);
+});
+test("a silent jev does not accuse every option of being unsafe", () => {
+  const v = jevFallback(candidates, "fetch failed");
+  // change_control is read off the candidate's own actions and stays; anything derived from a Jev score must go.
+  const derived = (flags: string[]) => flags.filter((f) => f !== "change_control");
+  assert.deepEqual(v.order.map((r) => derived(r.flags)), [[], []]);
+});
+test("verify_semantics fires where semantics can actually move", async () => {
+  // Real Jev scores an index create ~0.57 ("probably fine, no stated proof"), which a blanket <0.6 floor
+  // flagged as unsafe on every run. Index-only options must not shout; rewrites and bad scores must.
+  const idxOnly = [{ ...candidates[0], rewrite_sql: null }, candidates[1]];
+  const mid = { byKey: { a: scored(4, 2, 2, 1), b: scored(4, 2, 2, 1) }, cross: GOOD.cross };
+  const v = await judgeCandidates(digest, idxOnly, fakeJev(mid));
+  assert.ok(!v.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "index create flagged");
+  assert.ok(v.order.find((r) => r.key === "b")!.flags.includes("verify_semantics"), "rewrite not flagged");
+
+  const bad = { byKey: { a: scored(4, 1, 2, 1), b: GOOD.byKey.b }, cross: GOOD.cross };
+  const v2 = await judgeCandidates(digest, idxOnly, fakeJev(bad));
+  assert.ok(v2.order.find((r) => r.key === "a")!.flags.includes("verify_semantics"), "bad score not flagged");
+});