diff --git a/lib/digest.ts b/lib/digest.ts index 2e1e917..fbda422 100644 --- a/lib/digest.ts +++ b/lib/digest.ts @@ -3,7 +3,26 @@ import { XMLParser } from "fast-xml-parser"; export class PlanError extends Error {} export const MAX_XML_BYTES = 12_000_000; -const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "", removeNSPrefix: true }); +// maxNestedTags defaults to 128 and a spaghettified SQL Server plan blows straight through it: a real +// 2.7 MB production plan measured 2729 tags deep. The 12 MB input cap bounds the worst case. +const NAMED_ENTITIES: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; +// SSMS writes plan text with numeric references ( for each line break, 90 of them in a production +// plan) and fast-xml-parser leaves them alone, so the model was handed entity noise instead of SQL. +const decodeEntities = (v: string) => v.includes("&") + ? v.replace(/&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z]+);/g, (m, body) => { + if (body[0] === "#") { + const cp = /^#[xX]/.test(body) ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10); + return Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m; + } + return NAMED_ENTITIES[body] ?? m; + }) + : v; + +const parser = new XMLParser({ + ignoreAttributes: false, attributeNamePrefix: "", removeNSPrefix: true, maxNestedTags: 20000, + attributeValueProcessor: (_n: string, value: string) => decodeEntities(value), + tagValueProcessor: (_n: string, value: string) => decodeEntities(value), +}); export type Operator = { id: string; op: string; cost: number; estRows: number; actualRows: number | null; execs: number | null; io: number | null; cpu: number | null; spillLevels: number | null; parallel: boolean }; @@ -65,8 +84,11 @@ export function digestPlan(raw: string): Digest { // Database/Schema/Table arrive as separate attributes, each bracketed: rebuild the three-part name. const fullName = (n: any) => [n.Database, n.Schema, n.Table].map(unbrack).filter(Boolean).join("."); + // The names sit on (or its child), not on / itself: a production + // plan with 51 objects digested to zero tables until this reached into the child. + const scanName = (n: any) => fullName(n) || fullName(n?.Object ?? {}); const tables = [...new Set([...collect(doc, "IndexScan"), ...collect(doc, "TableScan"), ...collect(doc, "Table")] - .map(fullName).filter(Boolean))]; + .map(scanName).filter(Boolean))]; const missingIndexes: MissingIndex[] = collect(doc, "MissingIndexGroup").map((g: any) => { const mi = collect(g, "MissingIndex")[0] ?? {}; diff --git a/plan-cxpaket.sqlplan b/plan-cxpaket.sqlplan new file mode 100644 index 0000000..830e587 Binary files /dev/null and b/plan-cxpaket.sqlplan differ diff --git a/probe-tmp.mjs b/probe-tmp.mjs new file mode 100644 index 0000000..929b612 --- /dev/null +++ b/probe-tmp.mjs @@ -0,0 +1,40 @@ +import { chromium } from "@playwright/test"; +import { readFileSync } from "node:fs"; + +const BASE = "http://localhost:3000"; +const b = await chromium.launch(); +const p = await (await b.newContext({ baseURL: BASE, viewport: { width: 1400, height: 1000 } })).newPage(); + +await p.goto("/signup"); +await p.fill("#email", `bigplan${Date.now()}@qai.test`); +await p.fill("#password", "0123456789ab"); +await p.getByRole("button", { name: "Create account" }).click(); +await p.waitForURL(/setup/, { timeout: 20000 }); +await p.getByRole("link", { name: "Next" }).click(); +await p.waitForURL(/step=2/, { timeout: 20000 }); +await p.fill("#analyst_base_url", "http://192.168.88.131:8888/v1"); +await p.fill("#analyst_key", "EMPTY"); +await p.fill("#analyst_model", "qwen3.8-flash-next"); +await p.fill("#analyst_extra", '{"chat_template_kwargs":{"enable_thinking":false}}'); +await p.fill("#jev_key", process.env.JEV_API_KEY || ""); +await p.getByRole("button", { name: "Save" }).click(); +await p.waitForURL(/step=3/, { timeout: 20000 }); + +await p.goto("/app"); +const t0 = Date.now(); +await p.setInputFiles('input[type="file"]', "plan-cxpaket.sqlplan"); +console.log("UPLIFT chars in box:", (await p.locator("#xml").inputValue()).length); +await p.getByRole("button", { name: "Analyse" }).click(); +await p.waitForFunction(() => { + const btn = [...document.querySelectorAll("button")].find((x) => x.textContent === "Analyse"); + return !!btn && !btn.disabled; +}, null, { timeout: 420000 }); +const secs = Math.round((Date.now() - t0) / 1000); +const alert = await p.getByRole("alert").allInnerTexts(); +console.log("RUN", secs + "s", "| alert:", JSON.stringify(alert).slice(0, 200)); +console.log("REPORT headline:", JSON.stringify(await p.locator("#report h2").first().innerText().catch(() => "none"))); +console.log("REPORT cards:", await p.locator("#report .card").count(), + "| sql blocks:", await p.locator("#report pre.sql").count(), + "| verdict:", JSON.stringify((await p.locator("[data-verdict] .label").first().innerText().catch(() => "none")))); +console.log("TABLES MENTIONED:", await p.getByText(/OSUSR_MSQ|dbo\./).count()); +await b.close(); diff --git a/tests/digest.test.ts b/tests/digest.test.ts index 48d8b32..35e2b01 100644 --- a/tests/digest.test.ts +++ b/tests/digest.test.ts @@ -108,3 +108,43 @@ test("a parse failure says why instead of blaming the export", async () => { assert.ok(e.message.length > 40, `message too thin: ${e.message}`); assert.notEqual(e.message, "That XML will not parse — re-export the actual execution plan."); }); + +test("a deeply nested real plan parses (spaghettified plans are thousands of tags deep)", async () => { + // fast-xml-parser defaults to maxNestedTags: 128. A 2.7 MB production plan measured 2729 deep and died + // with "Maximum nested tags exceeded", which read like a corrupt export. + const depth = 3000; + const open = "".repeat(1); + const inner = "".repeat(depth) + "" + "".repeat(depth); + let err: unknown = null; + let out: any = null; + try { out = digestPlan(open + inner + ""); } catch (e) { err = e; } + assert.equal(err, null, String((err as Error)?.message)); + assert.equal(out.topOperators.length, 12, "digest should rank, not truncate"); +}); + +// ---- real-plan shapes measured on a 2.7 MB production plan (plan-cxpaket.sqlplan) ---- +test("table names come off the Object child, not the scan element", async () => { + // ShowPlan carries Database/Schema/Table on , so reading them off returned nothing: + // a 51-object production plan digested to zero tables and the model never saw a table name. + const xml = ` + + + + + + `; + assert.deepEqual(digestPlan(xml).tables, ["crm.dbo.Orders", "crm.dbo.Customers"]); +}); + +test("numeric character references in StatementText become the characters", () => { + // SSMS writes the plan text with line breaks: 90 of them in the production plan, so the + // analyst model was handed entity noise instead of SQL. + const xml = ` + + `; + const sql = digestPlan(xml).sql; + assert.match(sql, /FROM dbo\.T/); + assert.match(sql, /\n/); + assert.ok(!sql.includes("&#x"), sql); + assert.match(sql, /x > 1 & y = 2/); +});