8000 feat(assistant): give each namespace its own assistant that remembers by adeelahmad · Pull Request #154 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/assistant.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,31 @@ When the configured model supports function calling, chat becomes an agent.
with `import_files` (`attachments` on `POST /api/v1/chats/<id>/messages`).
- **Keep typing:** what you send while an answer is being written waits its turn and goes next.

## A namespace's own assistant

Each namespace can have its own assistant that remembers from one conversation to the next. It is off until an owner
of the namespace turns it on (`PATCH /api/v1/namespaces/<name>/assistant` with `enabled`, `name` and `instructions`).
Then every conversation scoped to that namespace alone, picked by hand or chosen from the first question, talks to it.
It is the same assistant with the same tools, extensions and approvals; what it adds:

- **A name and instructions** from the namespace's owners, read with every question.
- **A memory:** short facts it keeps with `remember` when it learns something lasting (a decision, a preference, a
plan), each with where it came from: the exact moment of the excerpt it was shown, or the conversation it was told
in. Remembering is routine, so it happens at once and shows as a step; people who can edit the namespace can also
write memories themselves. Viewers' conversations read the memory but don't add to it.
- **Cited:** the newest 30 memories (pinned ones first) are read with every question. A memory from a moment is
numbered like an excerpt, so an answer from memory cites [n] and opens the recording at that moment; older ones are
found with `recall`. A memory from a recording the asker can't read is left out.
- **Asks first:** forgetting from chat waits for the person's approval, like every tool that changes data.
- **Yours to edit:** `GET /api/v1/namespaces/<name>/assistant/memories` (`q` searches them), `POST` to add one,
`PATCH .../memories/<id>` to correct or pin one, `DELETE .../memories/<id>` to forget it, and owners `DELETE
.../memories` to forget everything.

It makes no model calls of its own and keeps no index, so it costs nothing extra on a Raspberry Pi.

Refine later: a decision model choosing what's worth remembering, chat rooms answered by a namespace's assistant,
its settings and memories in the web app, similar memories merged.

## Extending the assistant: tools, skills, hooks and plugins

Anyone who can edit can add to what the assistant does. Each addition is an *extension*: versioned, kept private or
Expand Down
2 changes: 2 additions & 0 deletions fastapi_backend/app/api/v1/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
notebook,
notes,
notifications,
ns_assistant,
oauth,
passkeys,
pipelines,
Expand Down Expand Up @@ -72,6 +73,7 @@
fedora,
setup,
namespaces,
ns_assistant,
hierarchy,
recordings,
notes,
Expand Down
5 changes: 3 additions & 2 deletions fastapi_backend/app/api/v1/routes/chats.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,9 @@ def answer(on: chat.Answering) -> Iterator[str]:
yield _ev("approval", appr)
yield from stopped("", [])
return
# a model that skips the tools hasn't seen the archive (but may answer from the page it was asked on)
elif kind == "direct" and passages and not setup and not (page and page.get("text")):
# a model that skips the tools hasn't seen the archive (but may answer from the page it was asked on,
# or from what the namespace's assistant remembers, citing it)
elif kind == "direct" and passages and not setup and not (page and page.get("text")) and not box.cited(data):
break
else:
answer = data
Expand Down
107 changes: 107 additions & 0 deletions fastapi_backend/app/api/v1/routes/ns_assistant.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""Each namespace's own assistant (ns_assistant.py): owners turn it on and give it a name and instructions; people who
edit the namespace read, write, pin and forget what it remembers, and those who read it see the memories."""

from __future__ import annotations

from fastapi import APIRouter, HTTPException, Query

from app.api.deps import Acl, Db, Writer
from app.domain import auth, ns_assistant
from app.schemas.common import Created, Ok
from app.schemas.ns_assistant import (
AssistantMemory,
AssistantMemoryCreate,
AssistantMemoryUpdate,
NamespaceAssistant,
NamespaceAssistantUpdate,
)

router = APIRouter(prefix="/namespaces", tags=["namespaces"])


def _view(db: Db, sid: int) -> NamespaceAssistant:
return NamespaceAssistant(**ns_assistant.profile(db, sid), memories=ns_assistant.count(db, sid))


@router.get("/{name}/assistant")
def get_namespace_assistant(name: str, acl: Acl, db: Db) -> NamespaceAssistant:
"""The namespace's assistant: whether it's on, its name and instructions, and how much it remembers."""
return _view(db, acl.namespace(name))


@router.patch("/{name}/assistant")
def update_namespace_assistant(name: str, body: NamespaceAssistantUpdate, acl: Acl, user: Writer, db: Db) -> NamespaceAssistant:
"""Owners: turn the assistant on or off, name it, or change its instructions."""
sid = acl.namespace(name, "owner")
try:
ns_assistant.save_profile(db, sid, body.enabled, body.name, body.instructions, user.email)
except ValueError as e:
raise HTTPException(400, str(e)) from None
auth.audit(db, user.as_audit(), "namespace.assistant", name, body.model_dump(exclude_unset=True))
return _view(db, sid)


@router.get("/{name}/assistant/memories")
def list_assistant_memories(
name: str, acl: Acl, db: Db, q: str | None = Query(None, description="only memories with these words, best first")
) -> list[AssistantMemory]:
"""What the namespace's assistant remembers: pinned first, then newest first (or by `q`)."""
sid = acl.namespace(name)
rows = ns_assistant.recall(db, sid, q, 200) if q else ns_assistant.memories(db, sid)
ok = set(acl.roles)
spaces = {
r["id"]: r["space"]
for r in db.rows(
"SELECT record::id(id) AS id, space FROM recording WHERE id IN $ids",
ids=[ns_assistant.R("recording", m["recording"]) for m in rows if m.get("recording") is not None],
)
}
# a memory from a recording you can't read (moved since) is left out, like a citation would be
rows = [m for m in rows if m.get("recording") is None or spaces.get(m["recording"], sid) in ok]
return [AssistantMemory.model_validate(m) for m in ns_assistant.labelled(db, rows)]


@router.post("/{name}/assistant/memories")
def add_assistant_memory(name: str, body: AssistantMemoryCreate, acl: Acl, user: Writer, db: Db) -> Created:
"""Editors: tell the assistant something to remember."""
sid = acl.namespace(name, "editor")
try:
mid = ns_assistant.remember(db, sid, body.text, user.id, author="person")
if body.pinned:
ns_assistant.update(db, sid, mid, pinned=True)
except ValueError as e:
raise HTTPException(400, str(e)) from None
return Created(id=mid)


@router.patch("/{name}/assistant/memories/{mid}")
def update_assistant_memory(name: str, mid: int, body: AssistantMemoryUpdate, acl: Acl, user: Writer, db: Db) -> AssistantMemory:
"""Editors: correct a memory, or pin it so it's read with every question."""
sid = acl.namespace(name, "editor")
try:
m = ns_assistant.update(db, sid, mid, body.text, body.pinned)
except KeyError:
raise HTTPException(404, "not found") from None
except ValueError as e:
raise HTTPException(400, str(e)) from None
return AssistantMemory.model_validate(ns_assistant.labelled(db, [m])[0])


@router.delete("/{name}/assistant/memories/{mid}")
def forget_assistant_memory(name: str, mid: int, acl: Acl, user: Writer, db: Db) -> Ok:
"""Editors: forget one memory."""
sid = acl.namespace(name, "editor")
try:
ns_assistant.forget(db, sid, mid)
except KeyError:
raise HTTPException(404, "not found") from None
return Ok()


@router.delete("/{name}/assistant/memories 8BD0 ")
def forget_all_assistant_memories(name: str, acl: Acl, user: Writer, db: Db) -> Ok:
"""Owners: forget everything the assistant remembers."""
sid = acl.namespace(name, "owner")
ns_assistant.forget_all(db, sid)
auth.audit(db, user.as_audit(), "namespace.assistant.forget_all", name)
return Ok()
108 changes: 105 additions & 3 deletions fastapi_backend/app/domain/ai_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
entity_setup,
graph_history,
notebook,
ns_assistant,
ops_tools,
recsets,
render,
Expand Down Expand Up @@ -275,9 +276,37 @@
]


# a namespace's own assistant (ns_assistant.py), in conversations scoped to that namespace while it's on: remembering
# is routine and made at once (by people who edit the namespace), forgetting waits for the person's yes
MEMORY_TOOLS = [
(
"remember",
"Keep a short fact about this namespace for later conversations: a decision, a preference, a plan or a fact "
"people will ask about again. source_ref: the [n] of the excerpt it came from, so it stays cited to that moment.",
{"fact": _S, "source_ref": _I},
["fact"],
True,
),
(
"recall",
"Find what you remember about this namespace beyond the memories you were given, by words.",
{"query": _S, "limit": _I},
["query"],
False,
),
(
"forget",
"Forget something you remember (by its memory id from recall), when it's wrong or no longer true. Needs the person's approval.",
{"memory_id": _I},
["memory_id"],
True,
),
]


def builtin_names():
"""The assistant's own tool names, which extensions can't take."""
return {t[0] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS} | {"use_skill"}
return {t[0] for t in TOOLS + ops_tools.ADMIN_TOOLS + ops_tools.FILE_TOOLS + AUTHOR_TOOLS + MEMORY_TOOLS} | {"use_skill"}


class Toolbox(ops_tools.OpsTools):
Expand All @@ -299,17 +328,49 @@ def __init__(self, db, cfg, user, readable, editable, scope, chat_id, base=None,
self.me = extensions.who(user["id"], user.get("email"), admin, roles)
self.ext = extensions.Active(db, self.me) if cfg["ai"].get("extensions", True) else None
self.hooking = False
# the namespace's own assistant, when the conversation is about that namespace alone and its assistant is on
home = ns_assistant.for_scope(db, self.scope)
self.home = home if home and home[0] in self.readable else None

def system_note(self):
"""What the model is told besides its usual instructions: the skills it can follow, and context that hooks add
for this question."""
out = self._home_note()
if not self.ext:
return ""
out = self.ext.system_note()
return out
out += self.ext.system_note()
for h in self.ext.matching("message", text=self.said):
out += self._hook(h, {"said": self.said})
return out

def _home_note(self):
"""Who the assistant is in this namespace and what it remembers; memories from a moment are numbered like
excerpts. One from a recording the asker can't read is left out."""
if not self.home:
return ""
sid, name, p = self.home
rows = ns_assistant.labelled(self.db, ns_assistant.memories(self.db, sid, ns_assistant.PROMPT_MAX))
spaces = {
r["id"]: r["space"]
for r in self.db.rows(
"SELECT record::id(id) AS id, space FROM recording WHERE id IN $ids",
ids=[R("recording", m["recording"]) for m in rows if m.get("recording") is not None],
)
}
lines = []
for m in rows:
day = (m.get("created_at") or "")[:10]
rid = m.get("recording")
if rid is not None and rid in spaces:
if spaces[rid] not in self.readable:
continue
n = self.ref(rid, m.get("t0") or 0, m["text"], title=m.get("title"), source="memory")
lines.append(f"[{n}] {m['text']} (memory {m['id']}, kept {day}, from {m.get('title') or 'a recording'} at {m.get('time')})")
else:
told = "told in a conversation" if m.get("author") == "assistant" else "written by a person"
lines.append(f"- {m['text']} (memory {m['id']}, {told}, {day})")
return ns_assistant.prompt(name, p, lines)

def after_answer(self, text):
"""Run the hooks for an answer that was written."""
for h in self.ext.matching("answer", text=text) if self.ext else ():
Expand Down Expand Up @@ -338,6 +399,8 @@ def specs(self):
tools += ops_tools.FILE_TOOLS
if self.cfg["ai"].get("extensions", True):
tools += [t for t in AUTHOR_TOOLS if can_act or self.admin or not t[4]]
if self.home:
tools += [t for t in MEMORY_TOOLS if self.home[0] in self.editable or not t[4]]
if self.ext:
tools += self.ext.tool_specs(can_act or self.admin)
return [
Expand Down Expand Up @@ -469,6 +532,39 @@ def t_switch_extension(self, name, enabled):
what = f"Switch {g['kind']} {name} {'on' if enabled else 'off'}"
return self._approval("switch_extension", {"extension": g["id"], "name": name, "enabled": bool(enabled)}, what)

# ---- the namespace's memory (ns_assistant.py) ----
def t_remember(self, fact, source_ref=None):
sid, name, _ = self.home
rid = t0 = None
if source_ref is not None:
n = int(source_ref)
if not 1 <= n <= len(self.refs):
raise ValueError(f"there's no excerpt [{n}] in this conversation")
r = self.refs[n - 1]
rid, t0 = r["recording_id"], r.get("t0")
mid = ns_assistant.remember(self.db, sid, fact, self.user["id"], rid, t0, self.chat)
return {"memory_id": mid, "remembered": fact}, f"Remembered: {fact[:80]}"

def t_recall(self, query, limit=10):
sid, name, _ = self.home
rows = ns_assistant.labelled(self.db, ns_assistant.recall(self.db, sid, query, min(int(limit or 10), 30)))
out = []
for m in rows:
item = {"memory_id": m["id"], "fact": m["text"], "kept": (m.get("created_at") or "")[:10]}
if m.get("recording") is not None:
try:
self._ok(m["recording"])
except ValueError:
continue
item["ref"] = self.ref(m["recording"], m.get("t0") or 0, m["text"], title=m.get("title"), source="memory")
out.append(item)
return {"memories": out}, f'Recalled "{query}": {len(out)} memor{"y" if len(out) == 1 else "ies"}'

def t_forget(self, memory_id):
sid, name, _ = self.home
m = ns_assistant.get(self.db, sid, int(memory_id))
return self._approval("forget_memory", {"namespace": name, "memory_id": m["id"]}, f"Forget “{m['text'][:120]}”")

def _run(self, name, fn, args):
try:
out, summary = fn(**{k: v for k, v in (args or {}).items() if v is not None})
Expand Down Expand Up @@ -994,6 +1090,12 @@ def _carry_out(db, cfg, a, user, editable, decision, base, admin, readable):
result = author_extension(db, a["tool"], args, user, admin, readable if readable is not None else editable, editable)
elif a["tool"] in ("change_settings", "create_namespace", "import_files"):
result = ops_tools.apply(db, cfg, base or cfg, a["tool"], args, user, editable, admin)
elif a["tool"] == "forget_memory":
sid = {v: k for k, v in store.space_names(db).items()}.get(args["namespace"])
if sid not in editable:
raise PermissionError("you can't change what that namespace's assistant remembers")
ns_assistant.forget(db, sid, args["memory_id"])
result = {"forgot": args["memory_id"]}
elif a["tool"] == "run_template":
steps, label = batches.steps_for(db, {"template": args["template_id"]})
ids = [i for i in args["recordings"] if (db.one("SELECT space FROM $r", r=R("recording", i)) or {}).get("space") in editable]
Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/app/domain/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def answer(db, cfg, base, account, cid, q):
for kind, data in chat.tool_answer(cfg, box, q, past, cfg["ai"].get("max_steps") or 6):
if kind == "step":
steps.append(data)
elif kind == "direct" and passages:
elif kind == "direct" and passages and not box.cited(data): # unless it cites what it remembers
break
else:
text = data
Expand Down
Loading
Loading
0