8000 feat(chat): chat on any page, with the page and highlighted text as context by adeelahmad · Pull Request #61 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/assistant.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ then.
- **Choosing the model:** a conversation can use any model an admin offers (`llm.chat_models`, else whatever the model
server lists); Try another model asks a question again with a different one. Each answer records the model that
wrote it.
- **Chat on any page:** every page except Chat itself has an Ask button that opens a chat panel beside the page (full
screen on phones). Its conversation follows you between pages, and across visits in the same browser, until New
chat; it is an ordinary conversation, also listed in Chat. Each question can share the page's text (the page chip,
on by default) and text highlighted on the page (Ask about this, above any selection). The model reads them with the
question (up to 12,000 characters of page text and 4,000 of the highlight) as what the person is looking at, not as
citable excerpts. The question keeps the page's address, title and highlight, shown above it; the page's text isn't
stored. A recording's transcript and document keep their own selection toolbar, whose Ask in chat goes to the
recording's chat.
- **No model configured:** chat returns the best-matching passages instead. Anyone signed in can see whether a model
is set up, and which (`GET /chats/capabilities`), so the app says so before the first question.

Expand Down
12 changes: 8 additions & 4 deletions fastapi_backend/app/api/v1/routes/chats.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ def _ev(name: str, data: Any) -> str:
responses={200: {"content": {"text/event-stream": {}}, "description": "server-sent events"}},
)
async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db: Db, cfg: Cfg, request: Request) -> StreamingResponse:
8000 """Ask a question. Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST
"""Ask a question, optionally from a page (`context`: the page, its text and any highlighted part, which the model
reads with the question). Streams events: step, approval, notice, passages, token (answer text), error, stopped (POST
/chats/{cid}/stop: what came before is saved, marked stopped), done (the saved message id)."""
c = await run_in_threadpool(_own_chat, db, cid, user)
q = body.content.strip()[:4000]
Expand All @@ -183,7 +184,9 @@ async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db
q = "I attached " + ("this file." if len(files) == 1 else "these files.")
if not q:
raise HTTPException(400, "ask something")
asked = q + chat.attached_note(files) # what the model reads: the question, and the files it can import
page = body.context.model_dump() if body.context else None
# what the model reads: the question with the page it was asked from, and the files it can import
asked = chat.with_context(q, page) + chat.attached_note(files)
# this answer's model: the one asked for (400 if it isn't offered), else the conversation's while it's still offered
model = await run_in_threadpool(_model, cfg, body.model) if body.model else c.get("model")
if model and not body.model and model not in await run_in_threadpool(chat.model_choices, cfg):
Expand All @@ -192,7 +195,7 @@ async def send_message(cid: int, body: MessageCreate, user: Writer, acl: Acl, db

def prepare() -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
past = chat.history(db, cid)
chat.add(db, cid, "user", q, attachments=files)
chat.add(db, cid, "user", q, attachments=files, context=chat.shared_context(page))
if not past and c["title"] == "New conversation":
db.q("UPDATE $r SET title = $t", r=R("chat", cid), t=q[:80])
return past, chat.retrieve(db, q, readable, c.get("scope"), cfg=cfg)
Expand Down Expand Up @@ -239,7 +242,8 @@ def answer(on: chat.Answering) -> Iterator[str]:
yield _ev("approval", appr)
yield from stopped("", [])
return
elif kind == "direct" and passages and not setup: # a model that skips the tools hasn't seen the archive
# a model that skips the tools hasn't seen the archive (but may answer from the page it was asked on)
elif kind == "direct" and passages and not setup and not (page and page.get("text")):
break
else:
answer = data
Expand Down
64 changes: 57 additions & 7 deletions fastapi_backend/app/domain/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,58 @@ def retrieve(db, question, spaces, scope=None, k=8, cfg=None):
return out


PAGE_TEXT, PAGE_SELECTION = 12000, 4000


def shared_context(ctx):
"""What a question keeps of the page it was asked from: where, its title, the highlighted text, and whether the
page's text was shared (the text itself isn't kept)."""
if not ctx or not ctx.get("url"):
return None
return store.clean(
{
"url": str(ctx["url"])[:2000],
"title": (ctx.get("title") or "").strip()[:300] or None,
"selection": (ctx.get("selection") or "").strip()[:PAGE_SELECTION] or None,
"page": bool((ctx.get("text") or "").strip()) or None,
}
)


def with_context(question, ctx):
"""The question as the model reads it when it was asked from a page: the page, the highlighted text and (when
shared) the page's text come first, marked as what the person is looking at in Lens rather than archive excerpts."""
if not ctx or not ctx.get("url"):
return question
title = (ctx.get("title") or "").strip()[:300]
parts = [f"The person is looking at the Lens page {title + ' ' if title else ''}({str(ctx['url'])[:2000]})."]
sel = (ctx.get("selection") or "").strip()[:PAGE_SELECTION]
if sel:
parts.append(f'They highlighted t 8000 his part of it:\n"""\n{sel}\n"""')
text = (ctx.get("text") or "").strip()
if text:
cut = text[:PAGE_TEXT]
more = " (cut short)" if len(text) > PAGE_TEXT else ""
parts.append(f'The page\'s text{more}:\n"""\n{cut}\n"""')
parts.append(f"Use the page to understand the question; it is not an archive excerpt, so don't cite it with [n]. Question: {question}")
return "\n\n".join(parts)


def past_turns(history, n=6):
"""The conversation's last messages for the model, with the files sent with each; a question asked about
highlighted text keeps (the start of) that text, so a follow-up still knows what "it" was."""
out = []
for m in list(history)[-n:]:
sel = ((m.get("context") or {}).get("selection") or "")[:500] if m["role"] == "user" else ""
said = f'(About the highlighted text: "{sel}") {m["content"]}' if sel else m["content"]
out.append({"role": m["role"], "content": said + attached_note(m.get("attachments"))})
return out


def messages_for(question, passages, history=()):
ctx = "\n\n".join(f"[{p['n']}] {p['title']} · {(p.get('recorded_at') or '')[:10]} · {p['time']}\n{p['text']}" for p in passages)
msgs = [{"role": "system", "content": SYSTEM}]
msgs += [{"role": m["role"], "content": m["content"] + attached_note(m.get("attachments"))} for m in list(history)[-6:]]
msgs += past_turns(history)
msgs.append({"role": "user", "content": f"Excerpts:\n\n{ctx or '(nothing in the archive matched)'}\n\nQuestion: {question}"})
return msgs

Expand Down Expand Up @@ -274,13 +322,16 @@ def get(db, cid, account):
def history(db, cid):
return db.rows(
"SELECT record::id(id) AS id, role, content, passages, created_at, stopped ?? false AS stopped, steps ?? [] AS steps, "
"attachments ?? [] AS attachments, notice, error, check, model FROM chat_message WHERE chat = $c ORDER BY id",
"attachments ?? [] AS attachments, notice, error, check, model, context FROM chat_message WHERE chat = $c ORDER BY id",
c=cid,
)


def add(db, cid, role, content, passages=None, stopped=False, steps=None, notice=None, error=None, model=None, attachments=None):
"""Save a message; an answer keeps the tool steps it took, any notice (e.g. the model can't use tools) and error."""
def add(
db, cid, role, content, passages=None, stopped=False, steps=None, notice=None, error=None, model=None, attachments=None, context=None
):
"""Save a message; an answer keeps the tool steps it took, any notice (e.g. the model can't use tools) and error; a
question asked from a page keeps what it shared of it (shared_context)."""
mid = db.next_id("chat_message")
db.q(
"CREATE $r CONTENT $d",
Expand All @@ -298,6 +349,7 @@ def add(db, cid, role, content, passages=None, stopped=False, steps=None, notice
"notice": notice,
"error": error,
"model": model,
"context": context,
}
),
)
Expand Down Expand Up @@ -386,9 +438,7 @@ def tool_answer(cfg, toolbox, question, history=(), max_steps=6, model=None, set
answered without looking anything up (it never saw the archive, so the caller can answer from a search instead).
Raises llm.ToolsUnsupported."""
system = TOOL_SYSTEM + ("\n\n" + SETUP_SYSTEM if setup else "")
msgs = [{"role": "system", "content": system}] + [
{"role": m["role"], "content": m["content"] + attached_note(m.get("attachments"))} for m in list(history)[-6:]
]
msgs = [{"role": "system", "content": system}] + past_turns(history)
msgs.append({"role": "user", "content": question})
for step in range(max_steps):
msg = llm.chat_message(cfg, msgs, tools=toolbox.specs(), model=model)
Expand Down
26 changes: 26 additions & 0 deletions fastapi_backend/app/schemas/chats.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,30 @@ class ToolStep(ResponseModel):
summary: str = ""


class PageContext(RequestModel):
"""The page a question was asked from (chat on any page): where it is, and what of it the person shared."""

url: str = Field(max_length=2000, pattern=r"^/([^/]|$)", description="the page's path in Lens, e.g. /resources/12?t=30")
title: str | None = Field(None, max_length=300)
text: str | None = Field(
None,
max_length=200_000,
description="the page's visible text, when the person shares the page (only the first 12,000 characters are used)",
)
selection: str | None = Field(
None, max_length=50_000, description="text the person highlighted on it (only the first 4,000 characters are used)"
)


class SharedContext(ResponseModel):
"""What a question shared from the page it was asked on. The page's text itself isn't kept."""

url: str
title: str | None = None
selection: str | None = None
page: bool = Field(False, description="the page's text was shared")


class Attachment(ResponseModel):
id: str = Field(description="the upload")
filename: str
Expand All @@ -93,6 +117,7 @@ class ChatMessage(ResponseModel):
id: int
role: Literal["user", "assistant"]
content: str
context: SharedContext | None = Field(None, description="a question asked from a page: the page, and any highlighted text")
attachments: list[Attachment] = Field(default_factory=list, description="files sent with it (POST /uploads with hold)")
passages: list[Passage] | None = None
created_at: str | None = None
Expand Down Expand Up @@ -127,6 +152,7 @@ class Chat(ChatSummary):

class MessageCreate(RequestModel):
content: str = Field("", description="the question (up to 4000 characters); may be empty when files are attached")
context: PageContext | None = Field(None, description="asked from a page: the page, its text and any highlighted part")
attachments: list[str] = Field(
default_factory=list,
max_length=20,
Expand Down
66 changes: 66 additions & 0 deletions fastapi_backend/tests/api/test_chats.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,72 @@ def test_tools_stay_in_scope_and_say_what_is_missing(app, db, cfg, folder, new_c
assert box.approvals == []


def test_asking_from_a_page(plain, client, db, cfg, folder, llm):
"""Chat on any page: the model reads the page and the highlighted part with the question; the question keeps where
it was asked and what was highlighted, not the page's text."""
seed(db, cfg, folder)
make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor", "calls": "editor"})
h = login(client, "ed@x.io", "editor password 1")
cid = client.post("/api/v1/chats", headers=h, json={}).json()["id"]
page = {"url": "/library?ns=calls", "title": "Library", "text": "Shipment report " + "x" * 20000, "selection": " Dyno shipment "}
llm.seen.clear()
ev = sse(client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "When does it leave?", "context": page}).text)
assert "done" in ev
asked = llm.seen[-1]["messages"][-1]["content"]
assert "Lens page Library (/library?ns=calls)" in asked and '"""\nDyno shipment\n"""' in asked
assert "Shipment report" in asked and "(cut short)" in asked and len(asked) < 14000 and asked.endswith("Question: When does it leave?")
msgs = client.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"]
assert msgs[0]["content"] == "When does it leave?"
assert msgs[0]["context"] == {"url": "/library?ns=calls", "title": "Library", "selection": "Dyno shipment", "page": True}
assert client.get("/api/v1/chats", headers=h).json()[0]["title"] == "When does it leave?"

# only the highlighted part, no page text; then a question with nothing shared reads as before
sse(
client.post(
f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Who?", "context": {"url": "/x", "selection": "Alice"}}
).text
)
sse(client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "And then?"}).text)
assert "page's text" not in llm.seen[-2]["messages"][-1]["content"] and llm.seen[-1]["messages"][-1]["content"].endswith(
"Question: And then?"
)
msgs = client.get(f"/api/v1/chats/{cid}", headers=h).json()["messages"]
assert (msgs[2]["context"], msgs[4]["context"]) == ({"url": "/x", "title": None, "selection": "Alice", "page": False}, None)
for bad in (
{"title": "no url"},
{"url": "https://elsewhere.example"},
{"url": "//elsewhere.example/x"},
{"url": "javascript:alert(1)"},
):
assert client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "?", "context": bad}).status_code == 422

# a follow-up still knows what the highlighted text was
llm.seen.clear()
sse(client.post(f"/api/v1/chats/{cid}/messages", headers=h, json={"content": "Say more"}).text)
turns = [m["content"] for m in llm.seen[-1]["messages"]]
assert '(About the highlighted text: "Alice") Who?' in turns


def test_page_context_reaches_the_tools_too(app, db, cfg, folder, new_client, llm):
seed(db, cfg, folder)
make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"})
c = new_client()
h = login(c, "ed@x.io", "editor password 1")
cid = c.post("/api/v1/chats", headers=h, json={}).json()["id"]
llm.tool_script[:] = [{"content": "It's the capsid one."}]
llm.seen.clear()
ev = sse(
c.post(
f"/api/v1/chats/{cid}/messages",
headers=h,
json={"content": "Which?", "context": {"url": "/resources/1", "title": "Ep 1", "text": "Capsid talk"}},
).text
)
assert ev["token"][0]["text"] == "It's the capsid one."
asked = llm.seen[0]["messages"][-1]["content"]
assert "Lens page Ep 1 (/resources/1)" in asked and "Capsid talk" in asked and asked.endswith("Question: Which?")


def test_a_model_that_skips_the_tools_still_answers_from_the_archive(app, db, cfg, folder, new_client, llm):
s = Assist(app, db, cfg, folder, new_client)
c, h = s.cl["editor"]
Expand Down
Loading
Loading
0