test(e2e): drive the assistant and Improve against a scripted provider

A local OpenAI-compatible stub plays a short conversation: read the document, propose an edit, ask a question when asked, and stream slowly enough to stop. The spec connects it in place, accepts an edit and checks it was saved, answers a question, stops a reply, improves a line, asks from the command palette and follows the old /agent links. The E2E workflow allows the loopback base URL the stub needs.
2026-10-03 10:13:47 +10:00 · 2026-09-29 00:11:21 +02:00
parent e1d5b9ba9b
commit 3ef8eaeb26
4 changed files with 297 additions and 1 deletions
@@ -24,7 +24,7 @@ Build the production app:

 Run tests:

-`APP_URL=http://localhost:3000 PORT=3000 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres FLAG_DISABLE_SIGNUPS=false FLAG_DISABLE_EMAIL_AUTH=false FLAG_DISABLE_API_RATE_LIMIT=true LOCAL_STORAGE_PATH=/workspace/data/e2e pnpm test:e2e`
+`APP_URL=http://localhost:3000 PORT=3000 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres FLAG_DISABLE_SIGNUPS=false FLAG_DISABLE_EMAIL_AUTH=false FLAG_DISABLE_API_RATE_LIMIT=true FLAG_ALLOW_UNSAFE_AI_BASE_URL=true LOCAL_STORAGE_PATH=/workspace/data/e2e pnpm test:e2e`

 ## Coverage

@@ -33,6 +33,7 @@ Run tests:
 - Builder section editing, autosave/navigation, and locking.
 - JSON export/import.
 - Public sharing for anonymous visitors.
+- The assistant and inline Improve, against a scripted OpenAI-compatible provider (`fixtures/ai-stub.ts`). It needs `FLAG_ALLOW_UNSAFE_AI_BASE_URL=true` so the server may call the stub on 127.0.0.1, and skips without it.

 Visual regression, PDF/DOCX rasterization parity, thumbnail resolution, and import-fixture reproduction are
 intentionally outside the PR gate to keep it fast; the opt-in geometry, offline-font, and root-resume suites stay
@@ -0,0 +1,177 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { AddressInfo } from "node:net";
+import { createServer } from "node:http";
+
+/**
+ * A scripted OpenAI-compatible provider for the assistant specs. It answers the connection test, and plays a small
+ * conversation: read the document, propose an edit to its first passage, then say what changed. A message asking
+ * for "a question" makes it ask one first; "slowly" streams a long reply that can be stopped. Improve gets the line
+ * back with "Led" in front.
+ */
+
+type ChatMessage = { role: string; content?: unknown; tool_call_id?: string; tool_calls?: unknown[] };
+type ChatRequest = { messages: ChatMessage[]; tools?: Array<{ function: { name: string } }>; stream?: boolean };
+
+const text = (content: unknown) =>
+	typeof content === "string"
+		? content
+		: Array.isArray(content)
+			? content.map((part) => (part as { text?: string }).text ?? "").join("")
+			: "";
+
+function chunk(response: ServerResponse, delta: Record<string, unknown>, finish: string | null = null) {
+	const body = {
+		id: "chatcmpl-stub",
+		object: "chat.completion.chunk",
+		created: 0,
+		model: "stub",
+		choices: [{ index: 0, delta, finish_reason: finish }],
+	};
+	response.write(`data: ${JSON.stringify(body)}\n\n`);
+}
+
+const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+async function streamText(response: ServerResponse, words: string[], delayMs = 0) {
+	chunk(response, { role: "assistant", content: "" });
+	for (const word of words) {
+		if (response.destroyed) return;
+		chunk(response, { content: word });
+		if (delayMs) await wait(delayMs);
+	}
+	chunk(response, {}, "stop");
+	response.end("data: [DONE]\n\n");
+}
+
+function streamToolCall(response: ServerResponse, name: string, args: unknown) {
+	chunk(response, {
+		role: "assistant",
+		content: null,
+		tool_calls: [
+			{
+				index: 0,
+				id: `call_${name}_${Date.now()}`,
+				type: "function",
+				function: { name, arguments: JSON.stringify(args) },
+			},
+		],
+	});
+	chunk(response, {}, "tool_calls");
+	response.end("data: [DONE]\n\n");
+}
+
+function reply(request: ChatRequest, response: ServerResponse) {
+	const messages = request.messages;
+	const tools = new Set((request.tools ?? []).map((tool) => tool.function.name));
+	const lastUser = [...messages].reverse().find((message) => message.role === "user");
+	const said = text(lastUser?.content).toLowerCase();
+	const sinceUser = messages.slice(messages.lastIndexOf(lastUser as ChatMessage) + 1);
+	const lastTool = [...sinceUser].reverse().find((message) => message.role === "tool");
+	const readTool = tools.has("read_letter") ? "read_letter" : "read_resume";
+
+	response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
+
+	if (said.includes("slowly"))
+		return streamText(
+			response,
+			Array.from({ length: 60 }, (_, index) => `word${index} `),
+			150,
+		);
+	if (!tools.has("propose_edits")) return streamText(response, ["You ", "left ", "the ", "document ", "out."]);
+
+	const asked = sinceUser.some((message) =>
+		(message.tool_calls as Array<{ function?: { name?: string } }> | undefined)?.some(
+			(call) => call.function?.name === "ask_user_question",
+		),
+	);
+	if (said.includes("question") && !asked)
+		return streamToolCall(response, "ask_user_question", {
+			question: "The posting mentions accessibility. Have you done accessibility work?",
+			choices: ["Yes, I have", "No, skip it"],
+		});
+
+	// The tool that produced the last result: read first, then propose, then say what changed.
+	const calls = sinceUser.flatMap(
+		(message) => (message.tool_calls as Array<{ id: string; function: { name: string } }> | undefined) ?? [],
+	);
+	const lastToolName = calls.find((call) => call.id === lastTool?.tool_call_id)?.function.name;
+	if (!lastTool || lastToolName === "ask_user_question") return streamToolCall(response, readTool, {});
+
+	let result: { data?: { passages?: Array<{ id: string; text: string }> } } = {};
+	try {
+		result = JSON.parse(text(lastTool.content) || "{}");
+	} catch {
+		// Not a read result.
+	}
+	const passage =
+		lastToolName === readTool ? result.data?.passages?.find((item) => item.text && item.text !== "(empty)") : undefined;
+	if (passage)
+		return streamToolCall(response, "propose_edits", {
+			title: "Tighten the first passage",
+			edits: [
+				{ passageId: passage.id, text: `${passage.text} (tightened by the stub)`, why: "Leads with the outcome." },
+			],
+		});
+
+	return streamText(response, [
+		"I ",
+		"tightened ",
+		"one ",
+		"passage. ",
+		"Accept ",
+		"it ",
+		"if ",
+		"it ",
+		"reads ",
+		"right.",
+	]);
+}
+
+function handle(request: IncomingMessage, response: ServerResponse) {
+	let body = "";
+	request.on("data", (data) => {
+		body += data;
+	});
+	request.on("end", () => {
+		if (!request.url?.endsWith("/chat/completions")) {
+			response.writeHead(404).end();
+			return;
+		}
+		const parsed = JSON.parse(body || "{}") as ChatRequest;
+		// The connection test asks for a single character, without streaming; Improve asks for one line as JSON.
+		if (!parsed.stream) {
+			const improving = text(parsed.messages[0]?.content).includes("You improve one line");
+			const line = text(parsed.messages.at(-1)?.content).match(/<<<LINE_START>>>\n([\s\S]*?)\n<<<LINE_END>>>/)?.[1];
+			const content = improving
+				? JSON.stringify({
+						text: `Led ${line ?? ""}`.trim(),
+						why: "Uses a verb that shows ownership.",
+						addsFacts: false,
+					})
+				: "1";
+			response.writeHead(200, { "content-type": "application/json" });
+			response.end(
+				JSON.stringify({
+					id: "chatcmpl-stub",
+					object: "chat.completion",
+					created: 0,
+					model: "stub",
+					choices: [{ index: 0, message: { role: "assistant", content }, finish_reason: "stop" }],
+					usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+				}),
+			);
+			return;
+		}
+		void reply(parsed, response);
+	});
+}
+
+export async function startAiStub(port = 0) {
+	const server = createServer(handle);
+	await new Promise<void>((resolve) => server.listen(port, "127.0.0.1", resolve));
+	const { port: bound } = server.address() as AddressInfo;
+	return {
+		baseURL: `http://127.0.0.1:${bound}/v1`,
+		close: () => new Promise<void>((resolve) => server.close(() => resolve())),
+	};
+}
@@ -0,0 +1,116 @@
+import type { Page } from "@playwright/test";
+import { startAiStub } from "../fixtures/ai-stub";
+import { createSampleResumeFromDashboard } from "../fixtures/resume";
+import { expect, test } from "../fixtures/test";
+
+// The stub is a local provider, so the server must allow a loopback base URL.
+test.skip(process.env.FLAG_ALLOW_UNSAFE_AI_BASE_URL !== "true", "Needs FLAG_ALLOW_UNSAFE_AI_BASE_URL=true");
+
+let stub: Awaited<ReturnType<typeof startAiStub>>;
+test.beforeAll(async () => {
+	stub = await startAiStub();
+});
+test.afterAll(async () => {
+	await stub?.close();
+});
+
+const resumeIdOf = (page: Page) => new URL(page.url()).pathname.split("/").at(-1) ?? "";
+
+async function resumeJson(page: Page, resumeId: string) {
+	const response = await page.request.get(`/api/openapi/resumes/${resumeId}`);
+	return JSON.stringify(await response.json());
+}
+
+/** D1: connects the stub through the assistant's inline setup. */
+async function connectStub(page: Page) {
+	const assistant = page.getByRole("region", { name: "Assistant" });
+	await assistant.getByRole("button", { name: "Other · OpenAI-compatible" }).click();
+	await assistant.getByLabel("Base URL").fill(stub.baseURL);
+	await assistant.getByLabel("API key").fill("stub-key");
+	await assistant.getByLabel("Model").fill("stub");
+	await assistant.getByRole("button", { name: "Connect" }).click();
+	await expect(assistant.getByLabel("Message the assistant")).toBeVisible({ timeout: 20_000 });
+}
+
+async function send(page: Page, text: string) {
+	// Send turns into Stop while a reply streams.
+	await expect(page.getByRole("button", { name: "Send" }).last()).toBeVisible({ timeout: 20_000 });
+	const composer = page.getByLabel("Message the assistant");
+	await composer.fill(text);
+	await composer.press("Enter");
+}
+
+test("proposes edits that change nothing until accepted, asks before assuming, and stops", async ({
+	authPage: page,
+}, testInfo) => {
+	test.setTimeout(90_000);
+	await createSampleResumeFromDashboard(page, testInfo);
+	const resumeId = resumeIdOf(page);
+
+	await page.getByRole("button", { name: "Assistant", exact: true }).click();
+	await connectStub(page);
+
+	await send(page, "Tighten my resume");
+	const edits = page.getByRole("list", { name: "Proposed edits" });
+	await expect(edits).toBeVisible({ timeout: 20_000 });
+	await expect(page.getByText("1 proposed", { exact: true })).toBeVisible();
+	await expect(page.getByText(/proposed edit on this page/)).toBeVisible();
+	expect(await resumeJson(page, resumeId)).not.toContain("(tightened by the stub)");
+
+	await edits.getByRole("button", { name: "Accept" }).click();
+	await expect(edits.getByText("Applied")).toBeVisible();
+	await expect.poll(() => resumeJson(page, resumeId), { timeout: 15_000 }).toContain("(tightened by the stub)");
+	await expect(page.getByText("1 proposed", { exact: true })).toBeHidden();
+
+	// A clarifying question: the answer continues the reply.
+	await send(page, "Ask me a question first");
+	await page.getByRole("button", { name: "Yes, I have" }).click();
+	await expect(page.getByText("You answered: Yes, I have")).toBeVisible();
+	await expect(page.getByRole("list", { name: "Proposed edits" })).toHaveCount(2, { timeout: 20_000 });
+	await expect(page.getByText("I tightened one passage. Accept it if it reads right.")).toHaveCount(2);
+
+	// Stopping mid-reply keeps what arrived and proposes nothing.
+	await send(page, "Reply slowly please");
+	await expect(page.getByText("word3", { exact: false })).toBeVisible({ timeout: 20_000 });
+	await page.getByRole("button", { name: "Stop" }).click();
+	await expect(page.getByText("Stopped. No edits were proposed.")).toBeVisible();
+	await expect(page.getByRole("button", { name: "Continue" })).toBeVisible();
+});
+
+test("improves one line in place, and asks from ⌘K; old assistant links redirect", async ({
+	authPage: page,
+}, testInfo) => {
+	test.setTimeout(90_000);
+	await createSampleResumeFromDashboard(page, testInfo);
+	const resumeId = resumeIdOf(page);
+
+	await page.getByRole("button", { name: "Assistant", exact: true }).click();
+	await connectStub(page);
+	await page.getByRole("button", { name: "Close the assistant" }).click();
+
+	// Improve works on the line holding the caret, and changes it only on Replace.
+	await page.getByRole("button", { name: "Open Summary" }).click();
+	const summary = page.getByRole("textbox", { name: "Summary" });
+	await summary.click();
+	await page.getByRole("button", { name: "Improve", exact: true }).click();
+	await page.getByRole("button", { name: "Stronger verb" }).click();
+	await expect(page.getByText("Uses a verb that shows ownership.")).toBeVisible({ timeout: 20_000 });
+	await expect(summary).not.toContainText("Led ");
+	await page.getByRole("button", { name: "Replace" }).click();
+	await expect(summary).toContainText("Led ");
+
+	// ⌘K → Ask opens the document edited last, with the question sent.
+	await page.goto("/dashboard");
+	await page.getByRole("button", { name: /Search or run/ }).click();
+	await page.keyboard.type("What should I cut?");
+	await page.getByRole("option", { name: /Ask the assistant/ }).click();
+	await page.waitForURL(new RegExp(`/builder/${resumeId}`));
+	await expect(page.getByText("What should I cut?")).toBeVisible();
+	await expect(page.getByRole("list", { name: "Proposed edits" })).toBeVisible({ timeout: 20_000 });
+
+	await page.goto("/agent");
+	await page.waitForURL(/\/dashboard$/);
+	await page.goto(`/agent/new?resumeId=${resumeId}`);
+	await page.waitForURL(new RegExp(`/builder/${resumeId}`));
+	await expect(page.getByRole("region", { name: "Assistant" })).toBeVisible();
+});