8000 test(clicks): a deep research journey, and the switch keeps the cursor in the box by adeelahmad · Pull Request #561 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 121 additions & 43 deletions ci/clicks/baseline-phone.json

Large diffs are not rendered by default.

150 changes: 106 additions & 44 deletions ci/clicks/baseline.json

Large diffs are not rendered by default.

28 changes: 24 additions & 4 deletions ci/clicks/fake-model.mjs
10BC8
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
// A stand-in OpenAI-compatible model server, so the journeys that need a model (the install wizard finding one, the
// setup wizard testing one, the assistant answering) run without a real one. Every answer is the same short reply,
// except that the assistant asked the same again (an answer regenerated) says which version it's writing, and a
// conversation's title (asked for after its first answer) is its first question's words, as a real model's would be.
// except that the assistant asked the same again (an answer regenerated) says which version it's writing, a
// conversation's title (asked for after its first answer) is its first question's words, as a real model's would be,
// and deep research gets a plan for its goal.
import http from "node:http";

const REPLY = "This is the survey's stand-in model answering.";
Expand All @@ -20,6 +21,26 @@ function title(payload) {
return question.trim().replace(/[?.!]+$/, "").split(/\s+/).slice(0, 8).join(" ") || "A conversation";
}

/** Structured output: a research plan for a deep research goal ("Goal: ..."), with no questions to answer first, so
* it can be approved as it is; an empty object for anything else, which satisfies most schemas' optional fields. */
function structured(payload) {
const system = (payload.messages ?? []).find((m) => m.role === "system")?.content;
if (typeof system !== "string" || !system.startsWith("You plan deep research")) return "{}";
const said = String((payload.messages ?? []).find((m) => m.role === "user")?.content ?? "");
const goal = said.match(/^Goal: (.*)$/m)?.[1]?.trim() || "Research";
return JSON.stringify({
questions: [],
plan: {
title: goal.replace(/[?.!]+$/, ""),
goal,
sub_questions: ["What was said about it?", "Who said it?", "What is still open?"],
sources: ["archive"],
max_tool_calls: 3,
report_outline: ["What was said", "What is still open"],
},
});
}

function reply(payload) {
const system = (payload.messages ?? []).find((m) => m.role === "system")?.content;
if (typeof system === "string" && system.startsWith("You title conversations")) return title(payload);
Expand Down Expand Up @@ -72,8 +93,7 @@ export function startFakeModel(port) {
res.write(chunk({}, "stop"));
return res.end("data: [DONE]\n\n");
}
// structured output: an empty object satisfies most schemas' optional fields
const content = payload.response_format ? "{}" : reply(payload);
const content = payload.response_format ? structured(payload) : reply(payload);
return json(200, {
id: "survey",
object: "chat.completion",
Expand Down
27 changes: 27 additions & 0 deletions ci/clicks/journeys.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,33 @@ export const JOURNEYS = [
await page.getByRole("button", { name: /^Tools: \d+ picked · 10 calls\. / }).waitFor();
},
},
{
id: "chat-research",
ns: "podcasts",
module: "Chat",
feature: "Research a goal, approve its plan and save the report as a note",
note: "A new conversation in one namespace: the Deep research switch beside the composer, the goal and Enter; the plan comes back as a card to approve (Approve, or Enter in it), then the run's progress, then the cited report with Save as note. The stand-in model plans three sub-questions without asking anything first.",
from: "/",
async run(ctx) {
const { page, meter } = ctx;
await go(ctx, "Chat");
const question = page.getByRole("textbox", { name: "Your question" });
// A phone's Chat opens on the list of conversations once there is one
if (!(await question.waitFor({ timeout: 3000 }).then(() => true, () => false)))
await meter.click(page.getByRole("button", { name: "New conversation" }), "New conversation");
await meter.click(page.getByRole("button", { name: "Deep research", exact: true }), "Deep research");
await meter.type(question, "What was said about Raspberry Pi?", "the goal");
await meter.press("Enter", "Enter to plan it");
const plan = page.getByRole("form", { name: "Research plan" });
await plan.waitFor({ timeout: 60000 });
await meter.click(plan.getByRole("button", { name: "Approve", exact: true }), "Approve");
// it runs in the background: the progress card, then the report and what can be done with it
const actions = page.getByRole("group", { name: "Report actions" });
await actions.waitFor({ timeout: 120000 });
await meter.click(actions.getByRole("button", { name: "Save as note" }), "Save as note");
await page.getByText("Saved as a note in podcasts").first().waitFor();
},
},
{
// Last: it points Lens at a cloud model, which the stand-in model behind the journeys above isn't
id: "cloud-model",
Expand Down
7 changes: 5 additions & 2 deletions ci/clicks/survey.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,14 @@ async function surveyPages(page, base, phone) {
// On a phone the navigation sits behind the menu button: one more tap to reach every page
const menu = phone ? 1 : 0;
for (const item of items) {
// one of each kind of page: the notes in the navigation are one page, whichever notes the journeys left there
if (item.href && pages.some((p) => p.page === pageKey(base + item.href))) continue;
await page.goto(base + "/", { waitUntil: "domcontentloaded" });
await settle();
if (phone) await page.getByRole("button", { name: "Open navigation" }).tap();
const name = new RegExp(`^${item.label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}( \\d+)?$`);
const link = page.getByRole("navigation", { name: "Primary" }).getByRole("link", { name }).first();
// found by where it goes: a link's name can carry more than its label (a note the assistant wrote is "Written by
// the assistant" and its title)
const link = page.getByRole("navigation", { name: "Primary" }).locator(`a[href="${item.href}"]`).first();
await (phone ? link.tap() : link.click());
await page.waitForLoadState("domcontentloaded");
await settle();
Expand Down
9 changes: 5 additions & 4 deletions docs/assistant.md
Original file line number Diff line number Diff line change
Expand Up @@ -558,8 +558,8 @@ Built: the web app, the API, MCP, chat rooms and the assistant's tools.
Give the assistant a goal that needs many look-ups, and it comes back with a plan; once you approve it, the research
runs in the background and a cited report comes back in the conversation.

- **Asking.** Turn on the composer's Deep research switch (the telescope) and type the goal, start a question with
`/research` (`/research what changed in the capsid work this year?`), send it with
- **Asking.** Turn on the composer's Deep research switch (the telescope; the cursor stays in the box) and type the
goal, start a question with `/research` (`/research what changed in the capsid work this year?`), send it with
`research: true` (`POST /api/v1/chats/<id>/messages`), call `POST /api/v1/chats/<id>/research {goal}`, or ask the
assistant for research: its `start_research` tool (the `research` [tool group](#choosing-tools-namespaces-and-a-limit))
works from chat, voice and [chat rooms](chat-rooms.md). It needs a language model that uses tools.
Expand Down Expand Up @@ -626,8 +626,9 @@ runs in the background and a cited report comes back in the conversation.
checking are two or three calls. Every call is in the activity ledger for the conversation and `research:<id>`.

Refine later: sub-questions in parallel, research on a schedule, sharing a report with a namespace's members, research
as an automation action, a report template people can edit, and the report posted back into the chat room it was
asked in (today the room gets the plan, and the report is in the conversation in Lens).
as an automation action, a report template people can edit, the report posted back into the chat room it was asked
in (today the room gets the plan, and the report is in the conversation in Lens), and one way to read a report aloud
(today its Read aloud and the answer's speaker button both do).

## Third-party content and prompt injection

Expand Down
1 change: 1 addition & 0 deletions docs/requirements.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ request, to another check that proves the same thing; loosening a requirement or
| Web search and reading pages, cited like the archive, off until an admin turns it on | [test_web_search.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_web_search.py) `test_an_answer_cites_the_web_and_the_conversation_keeps_it`, `test_reading_pages_and_pdfs`; [chat-web-citations.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-web-citations.test.tsx) |
| An admin's assistant searches and reads connected mail, calendars and files, cites messages and events, and brings them in only once approved | [test_connection_tools.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_connection_tools.py) `test_only_admins_get_them_and_only_for_what_is_connected`, `test_searching_and_reading_gmail`, `test_calendar_events_between_two_times`, `test_finding_and_reading_files_in_storage`, `test_bringing_things_in_waits_for_the_persons_yes`; [chat-web-citations.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-web-citations.test.tsx) `a message from a connected mailbox` |
| Python and bash in a sandbox of each conversation's own, off until an admin turns it on | [test_sandbox.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_sandbox.py) `test_python_and_bash_share_one_sandbox_per_conversation`, `test_the_sandbox_is_off_until_an_admin_turns_it_on_for_the_people_it_names` |
| Deep research: a goal gets a plan to approve, runs in the background and comes back as a cited report, saved as a note or a podcast episode | [test_research.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_research.py) `test_a_goal_gets_a_plan_that_waits_for_approval_and_can_be_revised`, `test_an_approved_plan_runs_and_its_report_comes_back_checked`, `test_a_report_becomes_a_note_and_a_podcast`; [chat-research.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-research.test.tsx) `approves the plan with Enter, sending the edits`, `saves it as a note, asking where when it could go in several namespaces`, `turns the next message into a research goal, and says when it can't`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `chat-research`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `chat-research` ≤ 4 clicks |
| Regenerating an answer or editing a question makes a new version beside the old one | [test_chat_versions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_versions.py) `test_regenerate_edit_switch_and_go_on`; [chat-versions.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-versions.test.tsx) `regenerating an answer`, `the version switch`; [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `chat-regenerate`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `chat-regenerate` ≤ 4 clicks |
| Each answer keeps a record of what produced it: the model, its tools, how long it took and what it cost | [test_chat_records.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_records.py) `test_an_answer_records_what_produced_it`, `test_a_regenerated_answer_has_its_own_record`; [chat-details.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-details.test.tsx) `says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended` |
| Answers read aloud by a speech provider, macOS `say` or StyleTTS2 (opt-in) | [test_voice.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice.py) `test_answers_are_read_aloud_by_a_speech_model`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_macos_say_reads_aloud_on_a_mac`, `test_styletts2_reads_aloud_from_its_own_service`; [styletts2.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/styletts2.yml) `Read a sentence aloud` |
Expand Down
2 changes: 2 additions & 0 deletions nextjs-frontend/__tests__/chat-research.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,8 @@ describe("the Deep research switch", () => {
);
fireEvent.click(screen.getByRole("button", { name: "Deep research" }));
expect(onToggle).toHaveBeenCalled();
// the goal is typed next, without a click into the box
expect(screen.getByRole("textbox", { name: "Your question" })).toHaveFocus();
rerender(
<Composer
value=""
Expand Down
9 changes: 7 additions & 2 deletions nextjs-frontend/components/chat/composer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,11 @@ export function Composer({
research?: { on: boolean; onToggle: () => void; off?: string | null };
}) {
const ref = useRef<HTMLTextAreaElement>(null);
// the switch hands the cursor back to the box, so the goal (or the question) is typed next without a click
const toggleResearch = () => {
research?.onToggle();
ref.current?.focus();
};
const [over, setOver] = useState(false);
useEffect(() => {
const el = ref.current;
Expand Down Expand Up @@ -529,7 +534,7 @@ export function Composer({
Deep research: you approve a plan first
<button
type="button"
class="x x-first x-last">research.onToggle}
class="x x-first x-last">toggleResearch}
aria-label="Turn deep research off"
className="grid size-5 place-items-center rounded-full hover:bg-black/5"
>
Expand Down Expand Up @@ -564,7 +569,7 @@ export function Composer({
: "Research a goal: a plan to approve, then a cited report")
}
disabled={Boolean(research.off) && !research.on}
class="x x-first x-last">research.onToggle}
class="x x-first x-last">toggleResearch}
className={cn(
"grid size-9 shrink-0 place-items-center rounded-full transition-colors duration-fast disabled:cursor-not-allowed disabled:opacity-50",
research.on
Expand Down
Loading
0