test: playwright end-to-end qa (17 tests, real models)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.gitignore

@@ -22,3 +22,4 @@ next-env.d.ts
 # local fixtures
 /tests/__fixtures__
 .DS_Store
+.playwright-data

app/setup/page.tsx

@@ -20,7 +20,7 @@ export default async function Setup({ searchParams }: { searchParams: Promise<{
     <h1 style={{ fontFamily: "var(--qai-font-display)", fontSize: "var(--qai-text-display)", letterSpacing: "-0.02em" }}>
       {TITLES[n]}</h1>
     {n === "1" && <p>qAI reads a SQL Server ShowPlan XML, works out the bottleneck in code, asks your model for several ways to fix it, then has Jev score each one. You configure nothing on this screen.</p>}
-    {n === "2" && <SettingsForm action={saveSettingsAction} view={view} step="Analyst model and Jev key" />}
+    {n === "2" && <SettingsForm action={saveSettingsAction} view={view} step="Analyst model and Jev key" after="/setup?step=3" />}
     {n === "3" && <>
       <p>One run = one digest, a few model calls, four Jev questions per option. If the analyst or Jev is unreachable, the report says so instead of inventing a verdict.</p>
       <p className="label">Analyst {view.has_analyst ? "ready" : "missing"} · Jev {view.has_jev ? "ready" : "missing"}</p>

components/settings-form.tsx

@@ -4,9 +4,9 @@ import type { SettingsView } from "@/lib/settings";
 
 type Result = { ok?: boolean; error?: string } | null;
 
-export function SettingsForm({ action, view, step, account }: {
+export function SettingsForm({ action, view, step, account, after }: {
   action: (state: Result, fd: FormData) => Promise<Result>;
-  view: SettingsView; step?: string; account?: boolean;
+  view: SettingsView; step?: string; account?: boolean; after?: string;
 }) {
   const [state, formAction, pending] = useActionState<Result, FormData>(action, null);
   const F = (name: string, label: string, value: string, help: string, type = "text") => (
@@ -19,6 +19,7 @@ export function SettingsForm({ action, view, step, account }: {
     <fieldset style={{ border: 0, padding: 0, margin: 0, display: "grid", gap: "var(--qai-space-md)" }}>
       <legend className="label">{title}</legend>{children}</fieldset>);
   return <form action={formAction} className="card" style={{ display: "grid", gap: "var(--qai-space-l)" }}>
+    {after && <input type="hidden" name="after" value={after} />}
     {step && <p className="label">{step}</p>}
     <Group title="Analyst model — OpenAI-compatible">
       {F("analyst_base_url", "Base URL", view.analyst_base_url, "e.g. https://api.openai.com/v1")}

fixtures/sample.sqlplan

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-16"?>
+<ShowPlanXML xmlns="http://schemas.microsoft.com/sqlserver/2004/07/showplan" Version="1.6" Build="16.0.4085.2">
+ <BatchSequence><Batch><Statements>
+  <StmtSimple StatementId="1" StatementSubtreeCost="4.871" StatementOptmLevel="FULL"
+              StatementText="SELECT o.Id FROM Sales.Orders o WHERE o.CustomerId = @c1 AND o.Total &gt; 500">
+   <StatementOptmEarlyAbortReason>GoodEnoughPlanFound</StatementOptmEarlyAbortReason>
+   <QueryBlock QueryHash="0x1A" QueryPlanHash="0x2B">
+    <RelOp NodeId="0" PhysicalOp="Clustered Index Scan" LogicalOp="Clustered Index Scan" EstimateRows="1000"
+           EstimateIO="41.6" EstimateCPU="39.0" EstimatedTotalSubtreeCost="4.10" Parallel="false">
+      <RunTimeInformation><RunTimeCountersPerThread Thread="0" ActualRows="1243500" ActualExecs="1"
+        ActualElapsedms="2411" ActualCPUms="2210"/></RunTimeInformation>
+      <IndexScan Database="[wide]" Schema="[Sales]" Table="[Orders]"/>
+      <Warnings><NoJoinPredicate/><SpillToTempDb SpillLevel="2"/>
+        <WaitTime WaitType="PAGEIOLATCH_SH" WaitTime="812"/></Warnings>
+    </RelOp>
+    <RelOp NodeId="1" PhysicalOp="Index Seek" LogicalOp="Index Seek" EstimateRows="3"
+           EstimateIO="0.7" EstimateCPU="0.02" EstimatedTotalSubtreeCost="0.77" Parallel="true">
+      <RunTimeInformation><RunTimeCountersPerThread Thread="0" ActualRows="4" ActualExecs="2"/></RunTimeInformation>
+      <IndexScan Database="[wide]" Schema="[Sales]" Table="[Customer]"/>
+    </RelOp>
+    <MissingIndexes><MissingIndexGroup Impact="94.17">
+      <MissingIndex Database="[wide]" Schema="[Sales]" Table="[Orders]">
+        <ColumnGroup Usage="EQUALITY"><Column Name="[CustomerId]"/></ColumnGroup>
+        <ColumnGroup Usage="INEQUALITY"><Column Name="[Total]"/></ColumnGroup>
+        <IncludedColumns><Column Name="[Id]"/></IncludedColumns>
+      </MissingIndex></MissingIndexGroup></MissingIndexes>
+    <NonParallelPlanReason>Could Not Generate Valid Parallel Plan</NonParallelPlanReason>
+   </QueryBlock>
+  </StmtSimple>
+ </Statements></Batch></BatchSequence>
+</ShowPlanXML>

lib/actions.ts

@@ -41,6 +41,9 @@ export async function saveSettingsAction(_: unknown, fd: FormData) {
   if (email && !emailOk(email)) return { ok: false, error: "Enter a valid email address." };
   if (password && password.length < 10) return { ok: false, error: "Use at least 10 characters." };
   saveSettingsForm(u.id, fd);
+  // The wizard sends `after`: a configured user must land on the readiness screen, and the setup page
+  // forwards configured users away from steps 1-2, so "Saved." would never be seen there.
+  if (String(fd.get("after") ?? "").startsWith("/setup")) redirect("/setup?step=3");
   if (password) db.prepare("UPDATE users SET pass = ? WHERE id = ?").run(hashPassword(password), u.id);
   if (email) db.prepare("UPDATE users SET email = ? WHERE id = ?").run(email, u.id);
   revalidatePath("/settings");

package.json

@@ -7,7 +7,8 @@
     "build": "next build",
     "start": "next start",
     "lint": "eslint",
-    "test": "node --no-warnings --test tests/*.test.ts"
+    "test": "node --no-warnings --test tests/*.test.ts",
+    "e2e": "playwright test"
   },
   "dependencies": {
     "@openuidev/lang-core": "^0.3.0",

playwright.config.ts

@@ -0,0 +1,28 @@
+import { defineConfig, devices } from "@playwright/test";
+import { readFileSync, existsSync, mkdirSync } from "node:fs";
+
+// Playwright does not load Next's .env.local; the E2E run needs the real Jev key and a stable APP_SECRET.
+if (existsSync(".env.local")) for (const line of readFileSync(".env.local", "utf8").split("\n")) {
+  const m = /^([A-Z_]+)=(.*)$/.exec(line.trim());
+  if (m && !process.env[m[1]]) process.env[m[1]] = m[2];
+}
+mkdirSync(".auth", { recursive: true });
+
+export default defineConfig({
+  testDir: "./tests-e2e",
+  fullyParallel: false,             // one SQLite file, one app process
+  workers: 1,
+  retries: 0,
+  timeout: 180_000,                 // real models are slow; say so with a timeout, not with retries
+  reporter: [["list"]],
+  use: { baseURL: "http://localhost:3000", trace: "retain-on-failure" },
+  projects: [
+    { name: "setup", testMatch: /auth\.setup\.ts/ },
+    { name: "chromium", use: { ...devices["Desktop Chrome"] }, dependencies: ["setup"] },
+    // A real model run costs ~40s; the mobile project only needs the layout checks, not a second full run.
+    { name: "mobile", use: { ...devices["Pixel 7"] }, dependencies: ["setup"], testMatch: /responsive\.spec\.ts/ },
+  ],
+  webServer: { command: "npm run dev", url: "http://localhost:3000/api/healthz", reuseExistingServer: true,
+    timeout: 120_000,
+    env: { DATA_DIR: ".playwright-data", APP_SECRET: process.env.APP_SECRET ?? "playwright-secret-value" } },
+});

scripts/e2e-debug.mjs

@@ -0,0 +1,24 @@
+import { chromium } from "@playwright/test";
+const b = await chromium.launch();
+const page = await (await b.newContext()).newPage();
+page.on("console", (m) => console.log("QAI console", m.type(), m.text().slice(0, 200)));
+page.on("pageerror", (e) => console.log("QAI pageerror", String(e).slice(0, 300)));
+await page.goto("http://localhost:3000/signup");
+await page.fill("#email", `dbg4328+${Date.now()}@qai.test`);
+await page.fill("#password", "playwright password");
+await page.getByRole("button", { name: "Create account" }).click();
+await page.waitForTimeout(3000);
+console.log("QAI url after signup", page.url());
+console.log("QAI body after signup", (await page.locator("body").innerText()).replace(/\n+/g, " | ").slice(0, 260));
+console.log("QAI cookie", (await page.context().cookies()).map(c=>c.name+"="+c.value.slice(0,8)).join(","));
+await page.getByRole("link", { name: "Next" }).click();
+await page.waitForTimeout(2000);
+await page.fill("#analyst_base_url", "http://192.168.88.131:8888/v1");
+await page.fill("#analyst_key", "EMPTY");
+await page.fill("#analyst_model", "qwen3.8-flash-next");
+await page.fill("#jev_key", "ts-test-key");
+await page.getByRole("button", { name: "Save" }).click();
+await page.waitForTimeout(4000);
+console.log("QAI url after save", page.url());
+console.log("QAI body", (await page.locator("body").innerText()).replace(/\n+/g, " | ").slice(0, 320));
+await b.close();

tests-e2e/analysis.spec.ts

@@ -0,0 +1,50 @@
+import { test, expect } from "@playwright/test";
+import { readFileSync } from "node:fs";
+
+const xml = readFileSync("fixtures/sample.sqlplan", "utf8");
+test.use({ storageState: ".auth/user.json", permissions: ["clipboard-read", "clipboard-write"] });
+
+test("full run: paste → stages → streamed report → copy → print", async ({ page }) => {
+  await page.goto("/app");
+  await page.fill("#xml", xml);
+  await page.fill("#note", "slow since the migration");
+  await page.getByRole("button", { name: "Analyse" }).click();
+
+  // Stages are named one at a time; the whole run takes tens of seconds with a real model.
+  await expect(page.getByText(/1 of 4 · Reading the plan/)).toBeVisible({ timeout: 20_000 });
+  await expect(page.getByText(/4 of 4 · Writing the report/)).toBeVisible({ timeout: 180_000 });
+
+  // The report paints while oul is still arriving, so the headline shows up first.
+  await expect(page.locator("#report h2").first()).toBeVisible({ timeout: 180_000 });
+  // The Analyse button only unlocks on the done frame: by then the whole report has arrived and parsed.
+  await expect(page.getByRole("button", { name: "Analyse" })).toBeEnabled({ timeout: 180_000 });
+  expect(await page.locator("#report .card").count()).toBeGreaterThan(2);
+  // No raw OpenUI Lang leaked into the page as text.
+  expect(await page.locator("main").innerText()).not.toContain("root = Report(");
+
+  await page.getByRole("button", { name: "Copy", exact: true }).click();
+  const clip = await page.evaluate(() => navigator.clipboard.readText());
+  expect(clip).toContain("slow since the migration");
+
+  await page.getByRole("button", { name: "Export to PDF" }).click();   // window.print()
+  await page.emulateMedia({ media: "print" });
+  await expect(page.locator(".rail")).toBeHidden();
+  await expect(page.locator(".toolbar")).toBeHidden();
+});
+
+test("a dropped .sqlplan file decodes and is accepted", async ({ page }) => {
+  await page.goto("/app");
+  await page.setInputFiles('input[type="file"]', "fixtures/sample.sqlplan");
+  const value = await page.locator("#xml").inputValue();
+  expect(value).toContain("ShowPlanXML");
+  expect(value).not.toContain("\u0000");
+  expect(value).not.toContain("\ufeff");
+});
+
+test("a pasted SQL script is refused with a named reason", async ({ page }) => {
+  await page.goto("/app");
+  await page.fill("#xml", "SELECT * FROM dbo.Orders WHERE CustomerId = 42;");
+  await page.getByRole("button", { name: "Analyse" }).click();
+  await expect(page.getByRole("alert").filter({ hasText: "digesting" })).toBeVisible({ timeout: 30_000 });
+  await expect(page.locator("#report")).toHaveCount(0);
+});

tests-e2e/auth.setup.ts

@@ -0,0 +1,34 @@
+import { test, expect } from "@playwright/test";
+
+const EMAIL = `qa+${Date.now()}@qai.test`;
+
+test("signup + first-run guide configures both models", async ({ page }) => {
+  await page.goto("/signup");
+  await page.fill("#email", EMAIL);
+  await page.fill("#password", "playwright password");
+  await page.getByRole("button", { name: "Create account" }).click();
+
+  await expect(page.getByText("Step 1 of 3")).toBeVisible();
+  await page.getByRole("link", { name: "Next" }).click();
+  await expect(page.getByText("Step 2 of 3")).toBeVisible();
+
+  await page.fill("#analyst_base_url", process.env.QAI_ANALYST_BASE_URL ?? "http://192.168.88.131:8888/v1");
+  await page.fill("#analyst_key", process.env.QAI_ANALYST_KEY ?? "EMPTY");
+  await page.fill("#analyst_model", process.env.QAI_ANALYST_MODEL ?? "qwen3.8-flash-next");
+  // Reasoning models stream thinking, not content: without this the report arrives empty.
+  await page.fill("#analyst_extra", process.env.QAI_ANALYST_EXTRA ?? '{"chat_template_kwargs":{"enable_thinking":false}}');
+  await page.fill("#jev_key", process.env.JEV_API_KEY ?? "");
+  await page.getByRole("button", { name: "Save" }).click();
+  await expect(page.getByText("Step 3 of 3")).toBeVisible({ timeout: 15_000 });
+  await expect(page.getByText(/Analyst ready · Jev ready/)).toBeVisible();
+  await expect(page.getByRole("link", { name: "Go" })).toBeVisible();
+  await page.context().storageState({ path: ".auth/user.json" });
+});
+
+test("signed-out visitor is pushed to login", async ({ browser }) => {
+  const ctx = await browser.newContext();
+  const page = await ctx.newPage();
+  await page.goto("/app");
+  expect(page.url()).toContain("/login");
+  await ctx.close();
+});

tests-e2e/history.spec.ts

@@ -0,0 +1,31 @@
+import { test, expect } from "@playwright/test";
+test.use({ storageState: ".auth/user.json" });
+
+test("rename sticks, delete asks first then removes", async ({ page }) => {
+  await page.goto("/app");
+  const row = page.locator(".rail li").first();
+  await row.locator('input[name="title"]').fill("Renamed by QA");
+  await row.getByRole("button", { name: "Save" }).click();
+  await expect(page.locator(".rail", { hasText: "Renamed by QA" })).toBeVisible({ timeout: 15_000 });
+
+  const before = await page.locator(".rail li").count();
+  let asked = false;
+  page.once("dialog", async (d) => { asked = true; await d.dismiss(); });   // refuse: nothing may vanish
+  await page.locator(".rail li").first().getByRole("button", { name: "Delete" }).click();
+  await page.waitForTimeout(1000);
+  expect(asked).toBe(true);
+  expect(await page.locator(".rail li").count()).toBe(before);
+
+  page.once("dialog", async (d) => { await d.accept(); });
+  await page.locator(".rail li").first().getByRole("button", { name: "Delete" }).click();
+  await expect.poll(async () => page.locator(".rail li").count(), { timeout: 20_000 }).toBe(before - 1);
+});
+
+test("reopening a stored run renders the report with no new model call", async ({ page }) => {
+  await page.goto("/app");
+  await page.locator('.rail a[href^="/app/"]').first().click();
+  await expect(page.locator("#report")).toBeVisible({ timeout: 30_000 });
+  await expect(page.getByRole("button", { name: "Analyse" })).toHaveCount(0);
+  // The stored Jev verdict is server-rendered, so it is readable even if the report body had failed.
+  await expect(page.getByText(/Jev verdict ·/)).toBeVisible();
+});

tests-e2e/responsive.spec.ts

@@ -0,0 +1,25 @@
+import { test, expect } from "@playwright/test";
+test.use({ storageState: ".auth/user.json" });
+
+for (const width of [320, 375, 414, 768]) {
+  test(`no horizontal scroll at ${width}px`, async ({ page }) => {
+    await page.setViewportSize({ width, height: 800 });
+    await page.goto("/app");
+    const over = await page.evaluate(() => document.documentElement.scrollWidth
+      - document.documentElement.clientWidth);
+    expect(over).toBeLessThanOrEqual(0);
+    const box = await page.getByRole("button", { name: "Analyse" }).boundingBox();
+    // .btn is 10px padding + 15px line = 35px at these widths; 32 leaves room for subpixel rounding
+    // while still failing a squashed or two-line control.
+    expect(box!.height).toBeGreaterThanOrEqual(32);
+  });
+}
+
+test("every settings field has a visible label", async ({ page }) => {
+  await page.goto("/settings");
+  // Scoped to the form: the rail's rename boxes are labelled for assistive tech with aria-label, not visibly.
+  const orphans = await page.evaluate(() => [...document.querySelectorAll("form input")]
+    .filter((i) => i.type !== "hidden" && (!i.id || !document.querySelector(`label[for="${i.id}"]`)))
+    .map((i) => i.name || i.type));
+  expect(orphans).toEqual([]);
+});