8000 test(clicks): journeys for regenerating, copying, exporting and choosing tools in a chat by adeelahmad · Pull Request #541 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
761 changes: 586 additions & 175 deletions ci/clicks/baseline-phone.json

Large diffs are not rendered by default.

731 changes: 555 additions & 176 deletions ci/clicks/baseline.json

Large diffs are not rendered by default.

36 changes: 33 additions & 3 deletions ci/clicks/fake-model.mjs
Original file line number Diff line number Diff line change
@@ -1,10 +1,40 @@
// A stand-in OpenAI-compatible model server, so the journeys that need a model (the install wizard finding one, the
// setup wizard testing one, the assistant answering) run without a real one. Every answer is the same short reply.
// setup wizard testing one, the assistant answering) run without a real one. Every answer is the same short reply,
// except that the assistant asked the same again (an answer regenerated) says which version it's writing, and a
// conversation's title (asked for after its first answer) is its first question's words, as a real model's would be.
import http from "node:http";

const REPLY = "This is the survey's stand-in model answering.";
const MODELS = ["survey-chat", "survey-embed"];

/** How many times the assistant (a call with tools, or streamed) was asked each question after the same conversation:
* the same again is an answer regenerated, whose reply reads "...answering, version 2." so the versions differ. Other
* calls get REPLY. */
const asked = new Map();

/** A title for a conversation: the first question's words, as Lens sends them ("Person: ...") without the question
* mark, so the journeys find the conversation by what was asked, as a person would. */
function title(payload) {
const said = payload.messages.find((m) => m.role === "user")?.content ?? "";
const question = String(said).match(/^Person: (.*)$/m)?.[1] ?? said;
return question.trim().replace(/[?.!]+$/, "").split(/\s+/).slice(0, 8).join(" ") || "A conversation";
}

function reply(payload) {
const system = (payload.messages ?? []).find((m) => m.role === "system")?.content;
if (typeof system === "string" && system.startsWith("You title conversations")) return title(payload);
if (!payload.tools && !payload.stream) return REPLY;
// what was said, without the system prompt or the excerpts sent with the question ("Excerpts: ... Question: ..."),
// which are marked afresh on each call and grow as more is imported
const said = (payload.messages ?? [])
.filter((m) => m.role !== "system")
.map((m) => [m.role, typeof m.content === "string" ? m.content.split("\n\nQuestion: ").at(-1) : m.content]);
const key = JSON.stringify([Boolean(payload.tools), Boolean(payload.stream), said]);
const n = (asked.get(key) ?? 0) + 1;
asked.set(key, n);
return n === 1 ? REPLY : `${REPLY.slice(0, -1)}, version ${n}.`;
}

export function startFakeModel(port) {
const server = http.createServer((req, res) => {
let body = "";
Expand Down Expand Up @@ -38,12 +68,12 @@ export function startFakeModel(port) {
res.writeHead(200, { "Content-Type": "text/event-stream" });
const chunk = (delta, finish = null) =>
`data: ${JSON.stringify({ id: "survey", object: "chat.completion.chunk", model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`;
res.write(chunk({ role: "assistant", content: REPLY }));
res.write(chunk({ role: "assistant", content: reply(payload) }));
res.write(chunk({}, "stop"));
return res.end("data: [DONE]\n\n");
}
// structured output: an empty object satisfies most schemas' optional fields
const content = payload.response_format ? "{}" : REPLY;
const content = payload.response_format ? "{}" : reply(payload);
return json(200, {
id: "survey",
object: "chat.completion",
Expand Down
99 changes: 96 additions & 3 deletions ci/clicks/journeys.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ async function save({ page, meter, phone }) {

/** The model-backed answer the stand-in model gives (fake-model.mjs). */
const ANSWER = /stand-in model answering/;
/** The stand-in model's answer, in the newest answer on the page: not anywhere on it, where the list of conversations
* beside it carries titles, and an earlier answer may read the same. */
const answered = (page) => page.getByRole("article", { name: "Answer" }).last().getByText(ANSWER);

const TRANSCRIPT = [
"Host A: Welcome back. Today we talk about local models on small computers.",
Expand Down Expand Up @@ -169,11 +172,11 @@ export const JOURNEYS = [
// The tour opens Chat with the question in the composer, not sent
const composer = page.getByRole("textbox", { name: "Your question" });
await composer.waitFor();
if (!(await page.getByText(ANSWER).first().isVisible().catch(() => false))) {
if (!(await answered(page).isVisible().catch(() => false))) {
await page.waitForTimeout(1500);
if ((await composer.inputValue()).trim()) await meter.press("Enter", "Enter again: the question waits in Chat's composer");
}
await page.getByText(ANSWER).first().waitFor({ timeout: 60000 });
await answered(page).waitFor({ timeout: 60000 });
},
},

Expand Down Expand Up @@ -248,7 +251,7 @@ export const JOURNEYS = [
await meter.click(page.getByRole("button", { name: "New conversation" }), "New conversation");
await meter.type(question, "Who said what about Raspberry Pi?", "question");
await meter.press("Enter", "Enter to send");
await page.getByText(ANSWER).first().waitFor({ timeout: 60000 });
await answered(page).waitFor({ timeout: 60000 });
},
},
{
Expand Down Expand Up @@ -702,6 +705,96 @@ export const JOURNEYS = [
await page.getByText("Merge undone").first().waitFor();
},
},
// --- Chat: versions, copying, exporting and tools. The new journeys go here, before cloud-model, which has to stay
// last: these need the stand-in model to answer ---------------------------------------------------------------------
{
id: "chat-regenerate",
ns: "",
module: "Chat",
feature: "Regenerate an answer, then switch back to the first version",
note: "In the conversation chat-ask started: Regenerate writes another answer beside it (2 / 2), and the arrow before the count shows the first again.",
from: "/",
async run(ctx) {
const { page, meter } = ctx;
await go(ctx, "Chat");
await page.waitForURL(/\/chat$/);
// a computer's Chat lists the conversations beside a new one; a phone's opens on the list. Lens titles each
// after its first answer (the stand-in model from the question, fake-model.mjs)
const conversation = page
.getByRole("navigation", { name: "Conversations" })
.getByRole("link", { name: /Who said what about Raspberry Pi/ })
.first();
await meter.click(conversation, "the conversation");
await page.waitForURL(/\/chat\/\d+/);
await answered(page).waitFor();
const answer = page.getByRole("article", { name: "Answer" }).last();
await meter.click(answer.getByRole("button", { name: "Regenerate", exact: true }), "Regenerate");
// the stand-in model says which version it's writing (fake-model.mjs)
await page.getByRole("article", { name: "Answer" }).last().getByText(/answering, version 2/).waitFor({ timeout: 60000 });
const versions = page.getByRole("group", { name: "Versions of this answer" }).last();
await versions.getByText("2 / 2").waitFor();
await meter.click(versions.getByRole("button", { name: "Previous version of this answer" }), "Previous version (‹)");
await versions.getByText("1 / 2").waitFor();
await page.getByRole("article", { name: "Answer" }).last().getByText(/answering, version 2/).waitFor({ state: "hidden" });
},
},
{
id: "chat-copy",
module: "Chat",
feature: "Copy an answer",
note: "Continues chat-regenerate, in its conversation. Copy copies it as Markdown, its citations as footnotes; the arrow beside it has Copy text and Export.",
async run(ctx) {
const { page, meter, base } = ctx;
await page.context().grantPermissions(["clipboard-read", "clipboard-write"], { origin: base });
await page.evaluate(() => navigator.clipboard.writeText(""));
const answer = page.getByRole("article", { name: "Answer" }).last();
await meter.click(answer.getByRole("button", { name: "Copy", exact: true }), "Copy");
await page.getByText("Copied as Markdown").first().waitFor();
const copied = await page.evaluate(() => navigator.clipboard.readText());
if (!ANSWER.test(copied)) throw new Error(`copied ${JSON.stringify(copied.slice(0, 80))}, not the answer`);
},
},
{
id: "chat-export-pdf",
module: "Chat",
feature: "Export a conversation as a PDF",
note: "Continues chat-copy. The download icon in the conversation's header, then PDF. Where the server can make the PDF (Chromium, anytopdf or LibreOffice) the browser saves it; where it can't, the browser's print dialog opens on the conversation and does the saving (one click there, counted).",
async run(ctx) {
const { page, meter } = ctx;
await page.evaluate(() => {
window.__printed = false;
window.print = () => (window.__printed = true);
});
// anytopdf is fetched the first time a PDF is made, so the first can take a while
const wait = 180000;
const saved = page.waitForEvent("download", { timeout: wait }).then((d) => d, () => null);
await meter.click(page.getByRole("button", { name: "Export conversation" }), "Export conversation (the header's download icon)");
await meter.click(page.getByRole("menuitem", { name: "PDF" }), "PDF");
const printed = page.waitForFunction(() => window.__printed, null, { timeout: wait }).then(() => "printed", () => null);
const got = await Promise.race([saved, printed]);
if (got === "printed") {
await page.getByText("Printing it instead").first().waitFor();
meter.record("outside", "the print dialog: Save", 1, 0);
} else if (got) {
if (!got.suggestedFilename().endsWith(".pdf")) throw new Error(`downloaded ${got.suggestedFilename()}, not a PDF`);
} else throw new Error("neither a PDF nor the print dialog came");
},
},
{
id: "chat-tools",
module: "Chat",
feature: "Pick the tools and a tool-call limit for a conversation",
note: "Continues chat-export-pdf. The Tools chip beside the scope: switching one group off keeps the others (Auto goes off), and the limit starts at 10 calls. Each change is saved to the conversation as it's made, and new conversations start with it.",
async run(ctx) {
const { page, meter } = ctx;
await meter.click(page.getByRole("button", { name: /^Tools: Auto\. Choose the tools/ }), "the Tools chip (Tools: Auto)");
const picker = page.getByRole("dialog").filter({ has: page.getByRole("list", { name: "Tool groups" }) });
await meter.click(picker.getByRole("switch", { name: "The server", exact: true }), "switch off The server");
await meter.click(picker.getByRole("switch", { name: "Limit tool calls per answer" }), "Limit tool calls per answer (10)");
await meter.click(picker.getByRole("button", { name: "Done" }), "Done");
await page.getByRole("button", { name: /^Tools: \d+ picked · 10 calls\. / }).waitFor();
},
},
{
// Last: it points Lens at a cloud model, which the stand-in model behind the journeys above isn't
id: "cloud-model",
Expand Down
14 changes: 12 additions & 2 deletions ci/clicks/survey.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ async function surveyPages(page, base, phone) {
return seen.filter((el) => el.offsetParent !== null && !el.closest("[hidden], nav, details:not([open]) > :not(summary)")).length;
});
const settle = () => page.waitForTimeout(1200);
// A page's controls once it has drawn: a page still loading what it shows has none for a moment, so none is counted
// again for a few seconds before it's taken
const counted = async () => {
let n = await controls();
for (let tries = 0; n === 0 && tries < 12; tries++) {
await page.waitForTimeout(300);
n = await controls();
}
return n;
};
// The controls the floating Ask button covers with the page (and anything scrolling in it) at its end, where
// scrolling can't carry them clear: it should rise above them or step aside, so any is a failure.
const askCovers = async () => {
Expand Down Expand Up @@ -143,7 +153,7 @@ async function surveyPages(page, base, phone) {
await page.waitForLoadState("domcontentloaded");
await settle();
const key = pageKey(page.url());
await visit({ page: key, label: item.label, reach: 1 + menu, controls: await controls() });
await visit({ page: key, label: item.label, reach: 1 + menu, controls: await counted() });
// a second level: settings sections, and tabs that are links (Admin, Sensors, Pipelines)
const sub = await page
.locator("main")
Expand All @@ -155,7 +165,7 @@ async function surveyPages(page, base, phone) {
if (!s.href || pages.some((p) => p.page === pageKey(base + s.href))) continue;
await page.goto(base + s.href, { waitUntil: "domcontentloaded" });
await settle();
await visit({ page: pageKey(page.url()), label: `${item.label} › ${s.label}`, reach: 2 + menu, controls: await controls() });
await visit({ page: pageKey(page.url()), label: `${item.label} › ${s.label}`, reach: 2 + menu, controls: await counted() });
}
}
return pages;
Expand Down
6 changes: 3 additions & 3 deletions docs/assistant.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,9 @@ Exports carry each tool call's arguments and summary, never its raw result, whic
export the same way over MCP: `list_chats`, `read_chat` and `export_chat` ([MCP](mcp.md)) work on the person's own
conversations.

In the web app, an answer's copy button offers **Copy as Markdown** (its citations as footnotes linking back to Lens,
as in the Markdown export) and **Copy text** (without Markdown or citation numbers), then exports that answer with
its question as Markdown, PDF or JSON. A question has its own copy button, shown on hover, and each code block in an
In the web app, an answer's copy button copies it as Markdown in one click (its citations as footnotes linking back to
Lens, as in the Markdown export); the arrow beside it also offers **Copy text** (without Markdown or citation numbers)
and exports that answer with its question as Markdown, PDF or JSON. A question has its own copy button, shown on hover, and each code block in an
answer has one that copies the code as written. The download icon in a conversation's header exports the whole
conversation. When the server can't make a PDF, the browser's print dialog opens on the conversation (or that one
answer), without the app around it, so Save as PDF still works.
Expand Down
2 changes: 1 addition & 1 deletion docs/requirements.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ request, to another check that proves the same thing; loosening a requirement or
| It acts with tools, and changes wait for approval | [test_chats.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chats.py) `test_tools_approvals_and_checking`; [test_approvals.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_approvals.py) `test_the_assistant_lists_and_declines_but_never_approves` |
| Web search and reading pages, cited like the archive, off until an admin turns it on | [test_web_search.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_search.py) `test_an_answer_cites_the_web_and_the_conversation_keeps_it`, `test_reading_pages_and_pdfs`; [chat-web-citations.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-web-citations.test.tsx) |
| Python and bash in a sandbox of each conversation's own, off until an admin turns it on | [test_sandbox.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_sandbox.py) `test_python_and_bash_share_one_sandbox_per_conversation`, `test_the_sandbox_is_off_until_an_admin_turns_it_on_for_the_people_it_names` |
| Regenerating an answer or editing a question makes a new version beside the old one | [test_chat_versions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_versions.py) `test_regenerate_edit_switch_and_go_on`; [chat-versions.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-versions.test.tsx) `regenerating an answer`, `the version switch` |
| Regenerating an answer or editing a question makes a new version beside the old one | [test_chat_versions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_versions.py) `test_regenerate_edit_switch_and_go_on`; [chat-versions.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-versions.test.tsx) `regenerating an answer`, `the version switch`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `chat-regenerate`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `chat-regenerate` ≤ 4 clicks |
| Each answer keeps a record of what produced it: the model, its tools, how long it took and what it cost | [test_chat_records.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_records.py) `test_an_answer_records_what_produced_it`, `test_a_regenerated_answer_has_its_own_record`; [chat-details.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-details.test.tsx) `says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended` |
| Answers read aloud by a speech provider, macOS `say` or StyleTTS2 (opt-in) | [test_voice.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice.py) `test_answers_are_read_aloud_by_a_speech_model`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_macos_say_reads_aloud_on_a_mac`, `test_styletts2_reads_aloud_from_its_own_service`; [styletts2.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/styletts2.yml) `Read a sentence aloud` |
| Any message read aloud in one click, one at a time, its audio kept so a replay costs nothing | [chat-read-aloud.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-read-aloud.test.tsx) `plays an answer by the speech model in one click, stops on the next, and keeps its audio`, `reads one message at a time: starting another stops the first`, `reads a long answer piece by piece, asking for the next while one plays` |
Expand Down
20 changes: 12 additions & 8 deletions nextjs-frontend/__tests__/chat-copy-export.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -83,28 +83,32 @@ describe("copying an answer", () => {
expect(plainText("## Steps\n\n```bash\nls\n```")).toBe("Steps\n\nls");
});

it("copies from the answer's menu, and exports from it", async () => {
it("copies as Markdown in one click, and the rest from the arrow beside it", async () => {
const >
show(<AnswerMenu text={ANSWER} passages={PASSAGES} />);
await choose("Copy", "Copy as Markdown");
expect(writeText).toHaveBeenCalledWith(answerMarkdown(ANSWER, PASSAGES, ORIGIN));
fireEvent.click(screen.getByRole("button", { name: "Copy" }));
await waitFor(() => expect(writeText).toHaveBeenCalledWith(answerMarkdown(ANSWER, PASSAGES, ORIGIN)));
expect(await screen.findByText("Copied as Markdown")).toBeInTheDocument();

await choose("Copy", "Copy text");
const more = "More ways to copy and export";
await choose(more, "Copy text");
expect(writeText).toHaveBeenLastCalledWith(plainText(ANSWER));

await choose("Copy", "PDF");
await choose(more, "Copy as Markdown");
expect(writeText).toHaveBeenLastCalledWith(answerMarkdown(ANSWER, PASSAGES, ORIGIN));

await choose(more, "PDF");
await waitFor(() => expect(onExport).toHaveBeenCalledWith("pdf"));
});

it("says so when the browser won't copy", async () => {
writeText.mockImplementation(() => Promise.reject(new Error("denied")));
show(<AnswerMenu text="Hi" passages={[]} />);
await choose("Copy", "Copy text");
fireEvent.click(screen.getByRole("button", { name: "Copy" }));
expect(await screen.findByText("Couldn’t copy")).toBeInTheDocument();
// not saved yet: nothing to export
fireEvent.keyDown(screen.getByRole("button", { name: "Copy" }), { key: "Enter" });
expect(await screen.findByRole("menuitem", { name: "Copy as Markdown" })).toBeInTheDocument();
fireEvent.keyDown(screen.getByRole("button", { name: "More ways to copy" }), { key: "Enter" });
expect(await screen.findByRole("menuitem", { name: "Copy text" })).toBeInTheDocument();
expect(screen.queryByRole("menuitem", { name: "PDF" })).not.toBeInTheDocument();
});

Expand Down
Loading
Loading
0