diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f6fd778bb..7b4513d8a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -17,6 +17,8 @@ env: FLAG_DISABLE_EMAIL_AUTH: "false" FLAG_DISABLE_API_RATE_LIMIT: "true" LOCAL_STORAGE_PATH: /tmp/reactive-resume-e2e-storage + # The assistant spec talks to a scripted provider on 127.0.0.1. + FLAG_ALLOW_UNSAFE_AI_BASE_URL: "true" jobs: e2e: diff --git a/tests/e2e/README.md b/tests/e2e/README.md index fd26072a3..5b8ca1a6f 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -24,7 +24,7 @@ Build the production app: Run tests: -`APP_URL=http://localhost:3000 PORT=3000 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres FLAG_DISABLE_SIGNUPS=false FLAG_DISABLE_EMAIL_AUTH=false FLAG_DISABLE_API_RATE_LIMIT=true LOCAL_STORAGE_PATH=/workspace/data/e2e pnpm test:e2e` +`APP_URL=http://localhost:3000 PORT=3000 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres FLAG_DISABLE_SIGNUPS=false FLAG_DISABLE_EMAIL_AUTH=false FLAG_DISABLE_API_RATE_LIMIT=true FLAG_ALLOW_UNSAFE_AI_BASE_URL=true LOCAL_STORAGE_PATH=/workspace/data/e2e pnpm test:e2e` ## Coverage @@ -33,6 +33,7 @@ Run tests: - Builder section editing, autosave/navigation, and locking. - JSON export/import. - Public sharing for anonymous visitors. +- The assistant and inline Improve, against a scripted OpenAI-compatible provider (`fixtures/ai-stub.ts`). It needs `FLAG_ALLOW_UNSAFE_AI_BASE_URL=true` so the server may call the stub on 127.0.0.1, and skips without it. Visual regression, PDF/DOCX rasterization parity, thumbnail resolution, and import-fixture reproduction are intentionally outside the PR gate to keep it fast; the opt-in geometry, offline-font, and root-resume suites stay diff --git a/tests/e2e/fixtures/ai-stub.ts b/tests/e2e/fixtures/ai-stub.ts new file mode 100644 index 000000000..d60076916 --- /dev/null +++ b/tests/e2e/fixtures/ai-stub.ts @@ -0,0 +1,177 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createServer } from "node:http"; + +/** + * A scripted OpenAI-compatible provider for the assistant specs. It answers the connection test, and plays a small + * conversation: read the document, propose an edit to its first passage, then say what changed. A message asking + * for "a question" makes it ask one first; "slowly" streams a long reply that can be stopped. Improve gets the line + * back with "Led" in front. + */ + +type ChatMessage = { role: string; content?: unknown; tool_call_id?: string; tool_calls?: unknown[] }; +type ChatRequest = { messages: ChatMessage[]; tools?: Array<{ function: { name: string } }>; stream?: boolean }; + +const text = (content: unknown) => + typeof content === "string" + ? content + : Array.isArray(content) + ? content.map((part) => (part as { text?: string }).text ?? "").join("") + : ""; + +function chunk(response: ServerResponse, delta: Record, finish: string | null = null) { + const body = { + id: "chatcmpl-stub", + object: "chat.completion.chunk", + created: 0, + model: "stub", + choices: [{ index: 0, delta, finish_reason: finish }], + }; + response.write(`data: ${JSON.stringify(body)}\n\n`); +} + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function streamText(response: ServerResponse, words: string[], delayMs = 0) { + chunk(response, { role: "assistant", content: "" }); + for (const word of words) { + if (response.destroyed) return; + chunk(response, { content: word }); + if (delayMs) await wait(delayMs); + } + chunk(response, {}, "stop"); + response.end("data: [DONE]\n\n"); +} + +function streamToolCall(response: ServerResponse, name: string, args: unknown) { + chunk(response, { + role: "assistant", + content: null, + tool_calls: [ + { + index: 0, + id: `call_${name}_${Date.now()}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }); + chunk(response, {}, "tool_calls"); + response.end("data: [DONE]\n\n"); +} + +function reply(request: ChatRequest, response: ServerResponse) { + const messages = request.messages; + const tools = new Set((request.tools ?? []).map((tool) => tool.function.name)); + const lastUser = [...messages].reverse().find((message) => message.role === "user"); + const said = text(lastUser?.content).toLowerCase(); + const sinceUser = messages.slice(messages.lastIndexOf(lastUser as ChatMessage) + 1); + const lastTool = [...sinceUser].reverse().find((message) => message.role === "tool"); + const readTool = tools.has("read_letter") ? "read_letter" : "read_resume"; + + response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + + if (said.includes("slowly")) + return streamText( + response, + Array.from({ length: 60 }, (_, index) => `word${index} `), + 150, + ); + if (!tools.has("propose_edits")) return streamText(response, ["You ", "left ", "the ", "document ", "out."]); + + const asked = sinceUser.some((message) => + (message.tool_calls as Array<{ function?: { name?: string } }> | undefined)?.some( + (call) => call.function?.name === "ask_user_question", + ), + ); + if (said.includes("question") && !asked) + return streamToolCall(response, "ask_user_question", { + question: "The posting mentions accessibility. Have you done accessibility work?", + choices: ["Yes, I have", "No, skip it"], + }); + + // The tool that produced the last result: read first, then propose, then say what changed. + const calls = sinceUser.flatMap( + (message) => (message.tool_calls as Array<{ id: string; function: { name: string } }> | undefined) ?? [], + ); + const lastToolName = calls.find((call) => call.id === lastTool?.tool_call_id)?.function.name; + if (!lastTool || lastToolName === "ask_user_question") return streamToolCall(response, readTool, {}); + + let result: { data?: { passages?: Array<{ id: string; text: string }> } } = {}; + try { + result = JSON.parse(text(lastTool.content) || "{}"); + } catch { + // Not a read result. + } + const passage = + lastToolName === readTool ? result.data?.passages?.find((item) => item.text && item.text !== "(empty)") : undefined; + if (passage) + return streamToolCall(response, "propose_edits", { + title: "Tighten the first passage", + edits: [ + { passageId: passage.id, text: `${passage.text} (tightened by the stub)`, why: "Leads with the outcome." }, + ], + }); + + return streamText(response, [ + "I ", + "tightened ", + "one ", + "passage. ", + "Accept ", + "it ", + "if ", + "it ", + "reads ", + "right.", + ]); +} + +function handle(request: IncomingMessage, response: ServerResponse) { + let body = ""; + request.on("data", (data) => { + body += data; + }); + request.on("end", () => { + if (!request.url?.endsWith("/chat/completions")) { + response.writeHead(404).end(); + return; + } + const parsed = JSON.parse(body || "{}") as ChatRequest; + // The connection test asks for a single character, without streaming; Improve asks for one line as JSON. + if (!parsed.stream) { + const improving = text(parsed.messages[0]?.content).includes("You improve one line"); + const line = text(parsed.messages.at(-1)?.content).match(/<<>>\n([\s\S]*?)\n<<>>/)?.[1]; + const content = improving + ? JSON.stringify({ + text: `Led ${line ?? ""}`.trim(), + why: "Uses a verb that shows ownership.", + addsFacts: false, + }) + : "1"; + response.writeHead(200, { "content-type": "application/json" }); + response.end( + JSON.stringify({ + id: "chatcmpl-stub", + object: "chat.completion", + created: 0, + model: "stub", + choices: [{ index: 0, message: { role: "assistant", content }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + ); + return; + } + void reply(parsed, response); + }); +} + +export async function startAiStub(port = 0) { + const server = createServer(handle); + await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)); + const { port: bound } = server.address() as AddressInfo; + return { + baseURL: `http://127.0.0.1:${bound}/v1`, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} diff --git a/tests/e2e/specs/assistant.spec.ts b/tests/e2e/specs/assistant.spec.ts new file mode 100644 index 000000000..92d26cc7f --- /dev/null +++ b/tests/e2e/specs/assistant.spec.ts @@ -0,0 +1,116 @@ +import type { Page } from "@playwright/test"; +import { startAiStub } from "../fixtures/ai-stub"; +import { createSampleResumeFromDashboard } from "../fixtures/resume"; +import { expect, test } from "../fixtures/test"; + +// The stub is a local provider, so the server must allow a loopback base URL. +test.skip(process.env.FLAG_ALLOW_UNSAFE_AI_BASE_URL !== "true", "Needs FLAG_ALLOW_UNSAFE_AI_BASE_URL=true"); + +let stub: Awaited>; +test.beforeAll(async () => { + stub = await startAiStub(); +}); +test.afterAll(async () => { + await stub?.close(); +}); + +const resumeIdOf = (page: Page) => new URL(page.url()).pathname.split("/").at(-1) ?? ""; + +async function resumeJson(page: Page, resumeId: string) { + const response = await page.request.get(`/api/openapi/resumes/${resumeId}`); + return JSON.stringify(await response.json()); +} + +/** D1: connects the stub through the assistant's inline setup. */ +async function connectStub(page: Page) { + const assistant = page.getByRole("region", { name: "Assistant" }); + await assistant.getByRole("button", { name: "Other · OpenAI-compatible" }).click(); + await assistant.getByLabel("Base URL").fill(stub.baseURL); + await assistant.getByLabel("API key").fill("stub-key"); + await assistant.getByLabel("Model").fill("stub"); + await assistant.getByRole("button", { name: "Connect" }).click(); + await expect(assistant.getByLabel("Message the assistant")).toBeVisible({ timeout: 20_000 }); +} + +async function send(page: Page, text: string) { + // Send turns into Stop while a reply streams. + await expect(page.getByRole("button", { name: "Send" }).last()).toBeVisible({ timeout: 20_000 }); + const composer = page.getByLabel("Message the assistant"); + await composer.fill(text); + await composer.press("Enter"); +} + +test("proposes edits that change nothing until accepted, asks before assuming, and stops", async ({ + authPage: page, +}, testInfo) => { + test.setTimeout(90_000); + await createSampleResumeFromDashboard(page, testInfo); + const resumeId = resumeIdOf(page); + + await page.getByRole("button", { name: "Assistant", exact: true }).click(); + await connectStub(page); + + await send(page, "Tighten my resume"); + const edits = page.getByRole("list", { name: "Proposed edits" }); + await expect(edits).toBeVisible({ timeout: 20_000 }); + await expect(page.getByText("1 proposed", { exact: true })).toBeVisible(); + await expect(page.getByText(/proposed edit on this page/)).toBeVisible(); + expect(await resumeJson(page, resumeId)).not.toContain("(tightened by the stub)"); + + await edits.getByRole("button", { name: "Accept" }).click(); + await expect(edits.getByText("Applied")).toBeVisible(); + await expect.poll(() => resumeJson(page, resumeId), { timeout: 15_000 }).toContain("(tightened by the stub)"); + await expect(page.getByText("1 proposed", { exact: true })).toBeHidden(); + + // A clarifying question: the answer continues the reply. + await send(page, "Ask me a question first"); + await page.getByRole("button", { name: "Yes, I have" }).click(); + await expect(page.getByText("You answered: Yes, I have")).toBeVisible(); + await expect(page.getByRole("list", { name: "Proposed edits" })).toHaveCount(2, { timeout: 20_000 }); + await expect(page.getByText("I tightened one passage. Accept it if it reads right.")).toHaveCount(2); + + // Stopping mid-reply keeps what arrived and proposes nothing. + await send(page, "Reply slowly please"); + await expect(page.getByText("word3", { exact: false })).toBeVisible({ timeout: 20_000 }); + await page.getByRole("button", { name: "Stop" }).click(); + await expect(page.getByText("Stopped. No edits were proposed.")).toBeVisible(); + await expect(page.getByRole("button", { name: "Continue" })).toBeVisible(); +}); + +test("improves one line in place, and asks from ⌘K; old assistant links redirect", async ({ + authPage: page, +}, testInfo) => { + test.setTimeout(90_000); + await createSampleResumeFromDashboard(page, testInfo); + const resumeId = resumeIdOf(page); + + await page.getByRole("button", { name: "Assistant", exact: true }).click(); + await connectStub(page); + await page.getByRole("button", { name: "Close the assistant" }).click(); + + // Improve works on the line holding the caret, and changes it only on Replace. + await page.getByRole("button", { name: "Open Summary" }).click(); + const summary = page.getByRole("textbox", { name: "Summary" }); + await summary.click(); + await page.getByRole("button", { name: "Improve", exact: true }).click(); + await page.getByRole("button", { name: "Stronger verb" }).click(); + await expect(page.getByText("Uses a verb that shows ownership.")).toBeVisible({ timeout: 20_000 }); + await expect(summary).not.toContainText("Led "); + await page.getByRole("button", { name: "Replace" }).click(); + await expect(summary).toContainText("Led "); + + // ⌘K → Ask opens the document edited last, with the question sent. + await page.goto("/dashboard"); + await page.getByRole("button", { name: /Search or run/ }).click(); + await page.keyboard.type("What should I cut?"); + await page.getByRole("option", { name: /Ask the assistant/ }).click(); + await page.waitForURL(new RegExp(`/builder/${resumeId}`)); + await expect(page.getByText("What should I cut?")).toBeVisible(); + await expect(page.getByRole("list", { name: "Proposed edits" })).toBeVisible({ timeout: 20_000 }); + + await page.goto("/agent"); + await page.waitForURL(/\/dashboard$/); + await page.goto(`/agent/new?resumeId=${resumeId}`); + await page.waitForURL(new RegExp(`/builder/${resumeId}`)); + await expect(page.getByRole("region", { name: "Assistant" })).toBeVisible(); +});