adeelahmad · Oct 5, 2026 · Oct 5, 2026 · Oct 5, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,14 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
 
 ## Unreleased
 
+- **Word search no longer weighs down the database.** Searching words in transcripts, text on screen, files, objects
+  and descriptions now uses a small SQLite index next to the database instead of SurrealDB's own full-text index. On
+  10,000 transcript lines the database used 64 MB of memory instead of 380 MB, and writing them took half the time.
+  Nothing changes in how search, the assistant or IIIF search behave. Existing installs build the new index on their
+  first start and give back the old one's memory. Settings › Search › **Word search** switches back to SurrealDB's
+  index, or to an **OpenSearch** cluster on another machine for a big archive (then Reindex now); the `opensearch`
+  compose profile runs one next to Lens.
+
 - **A much smaller web app image.** The production image runs Next.js's standalone server with only the packages it
   uses: 465 MB instead of 2.56 GB. Cloudron and `pnpm build` keep `next start`.
 

diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -128,10 +128,26 @@ services:
       - fedora-data:/fcrepo-home
     networks: [lens]
 
+  # OpenSearch (docs/database.md), for word search on a big archive when this machine can spare the memory, or as a
+  # pattern for a cluster on another machine. Opt-in: `docker compose --profile opensearch up`, then Settings → Search
+  # → Word search: OpenSearch at http://opensearch:9200, and Reindex now. Reachable only inside Lens's network.
+  opensearch:
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.19.1}
+    profiles: [opensearch]
+    environment:
+      discovery.type: single-node
+      DISABLE_SECURITY_PLUGIN: "true"
+      DISABLE_INSTALL_DEMO_CONFIG: "true"
+      OPENSEARCH_JAVA_OPTS: -Xms${OPENSEARCH_HEAP:-512m} -Xmx${OPENSEARCH_HEAP:-512m}
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    networks: [lens]
+
 volumes:
   surreal-data:
   archive-data:
   fedora-data:
+  opensearch-data:
 
 networks:
   lens:

diff --git a/docs/database.md b/docs/database.md
@@ -41,8 +41,25 @@ that are queried. Graph edges are real relations: `mentions` (segment → entity
 SELECT ->mentions->entity.name FROM segment WHERE recording = 12;
 ```
 
-Full-text indexes use 3.x's `FULLTEXT` syntax and fall back to 2.x's `SEARCH`, with a `snowball(english)` analyser
-unless `search.stemming: none` (then run `lens reindex`).
+Word search (`search.engine`) is kept out of SurrealDB by default. With `sqlite`, the default, the text of transcript
+lines, text on screen, file lines, objects and descriptions is indexed in a SQLite FTS5 file per database
+(`<data_dir>/search/`, BM25 ranking, Porter stemming, highlights; `app/domain/textindex.py`). An event on each of
+those tables notes changed rows in `text_change`, and the index catches up before each search, so writers need do
+nothing. SurrealDB still decides which rows a person may see. On 10,000 transcript lines against a 3.2.4 server
+(`surrealkv` storage) this used 64 MB of server memory instead of 380 MB, and writing and indexing them took about 5 s
+instead of 10 s; the SQLite file was 4 MB.
+
+With `opensearch`, the same words go to an OpenSearch (or Elasticsearch-compatible) cluster at
+`search.opensearch_url` (with `opensearch_user`, `opensearch_password` and `opensearch_verify` when it needs them):
+an index per table named `lens-<namespace>-<database>-<table>`, with the `english` analyser unless
+`search.stemming: none`. It suits an archive too big for one small machine, with OpenSearch on another one. When the
+cluster can't be reached, searches read the tables instead and say nothing is wrong, so Lens keeps working. The
+`opensearch` compose profile (`docker compose --profile opensearch up`) runs one next to Lens at
+`http://opensearch:9200`.
+
+With `surrealdb`, full-text indexes use 3.x's `FULLTEXT` syntax and fall back to 2.x's `SEARCH`, with a
+`snowball(english)` analyser unless `search.stemming: none`. After changing either setting, run `lens reindex` (or
+Settings › Search › Reindex now): it switches engines, taking the other one's index down.
 
 Passages for search by meaning (`passage`) carry their vector in `embedding`, under an HNSW index (`passage_vec`,
 cosine; servers only) defined when the first vector is stored, since its `DIMENSION` is the model's; `embedding_state:current` says

diff --git a/docs/get-started.md b/docs/get-started.md
@@ -104,7 +104,7 @@ lens worker --steps transcribe,diarize     # a worker that only transcribes (e.g
 lens search "capsid" --ns podcasts
 lens search "money worries" --mode semantic   # by meaning (with an embedding model; see configuration.md)
 lens embed                                 # index what isn't yet searchable by meaning
-lens reindex                               # after changing search.stemming
+lens reindex                               # after changing search.engine or search.stemming
 lens migrations                            # database upgrades: run, pending or failed
 lens backup                                # back the database up into <data_dir>/backups
 ```

diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py
@@ -47,12 +47,21 @@ def __init__(self, cfg: dict[str, Any] | None = None, db: store.DB | None = None
     def current(self) -> dict[str, Any]:
         return self.settings.current()
 
+    def _index_text(self) -> None:
+        try:
+            self.db.textindex.sync()
+        except Exception:  # noqa: BLE001 - the first search tries again
+            log.warning("couldn't bring the full-text index up to date", exc_info=True)
+
     def prepare(self) -> None:
         templates.seed(self.db)
         content_types.seed(self.db)
         routines.seed(self.db)
-        # Pay for the embedded engine's full-text repair at startup rather than in someone's first search.
+        # Pay for the embedded engine's full-text repair, or for building the SQLite index, at startup rather than
+        # in someone's first search.
         self.db.ready_fulltext()
+        if getattr(self.db, "textindex", None) is not None:
+            threading.Thread(target=self._index_text, name="textindex", daemon=True).start()
         if auth.account_count(self.db) == 0:
             setup.mark_fresh(self.db, self.base)  # a fresh install: the web app walks the first admin through setup
         setup.apply_env(self.db)

diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py
@@ -13,7 +13,7 @@
 import time
 from collections import Counter, defaultdict
 
-from . import llm, recsets, render, semantic, store
+from . import llm, recsets, render, semantic, store, textindex
 
 R = store.R
 STOP = set(
@@ -74,8 +74,10 @@ def retrieve(db, question, spaces, scope=None, k=8, cfg=None):
     where, p = scope_filter(db, spaces, scope)
     hits = {}
     for w in words:
-        rows = None
-        if db.ready_fulltext():
+        rows = textindex.rows(db, "segment", textindex.words(w), "record::id(id) AS id, recording, idx", f" AND {where}", p, 50)
+        for r in rows or []:
+            r["s"] = r.pop("s1")
+        if rows is None and db.ready_fulltext():
             try:
                 rows = db.rows(
                     f"SELECT record::id(id) AS id, recording, idx, search::score(1) AS s FROM segment WHERE text @1@ $w AND {where} LIMIT 50",
@@ -175,11 +177,15 @@ def retrieve(db, question, spaces, scope=None, k=8, cfg=None):
         where_o = where.replace("speaker IN $spk", "true")
         for w in words:
             try:
-                rows = db.rows(
-                    f"SELECT record::id(id) AS id, recording, t0, text, space FROM ocr_span WHERE text @1@ $w AND {where_o} LIMIT 20",
-                    w=w,
-                    **p,
+                rows = textindex.rows(
+                    db, "ocr_span", textindex.words(w), "record::id(id) AS id, recording, t0, text, space", f" AND {where_o}", p, 20
                 )
+                if rows is None:
+                    rows = db.rows(
+                        f"SELECT record::id(id) AS id, recording, t0, text, space FROM ocr_span WHERE text @1@ $w AND {where_o} LIMIT 20",
+                        w=w,
+                        **p,
+                    )
             except Exception:  # noqa: BLE001
                 rows = [
                     r

diff --git a/fastapi_backend/app/domain/iiif.py b/fastapi_backend/app/domain/iiif.py
@@ -37,6 +37,7 @@
     settings,
     speakers as spk,
     store,
+    textindex,
 )
 
 R = store.R
@@ -681,13 +682,23 @@ def search(db, base, q, rids, page_url, page=0):
     if not words or not rids:
         return empty
     try:
-        if not db.ready_fulltext():
-            raise LookupError("no full-text index on this engine")
-        rows = db.rows(
-            "SELECT recording, idx, t0, t1, text, page, box FROM segment WHERE text @1@ $q AND recording IN $r LIMIT 2000",
-            q=" ".join(words),
-            r=list(rids),
+        rows = textindex.rows(
+            db,
+            "segment",
+            textindex.words(" ".join(words)),
+            "recording, idx, t0, t1, text, page, box",
+            " AND recording IN $r",
+            {"r": list(rids)},
+            2000,
         )
+        if rows is None:
+            if not db.ready_fulltext():
+                raise LookupError("no full-text index on this engine")
+            rows = db.rows(
+                "SELECT recording, idx, t0, t1, text, page, box FROM segment WHERE text @1@ $q AND recording IN $r LIMIT 2000",
+                q=" ".join(words),
+                r=list(rids),
+            )
     except Exception:  # noqa: BLE001 - no full-text index on this engine
         rows = [
             s

diff --git a/fastapi_backend/app/domain/search.py b/fastapi_backend/app/domain/search.py
@@ -11,7 +11,7 @@
 import re
 from collections import Counter
 
-from . import semantic, store
+from . import semantic, store, textindex
 
 log = logging.getLogger(__name__)
 
@@ -134,10 +134,8 @@ def search(
     base_params = dict(params)
     cap = min(1000, (offset + limit) * 3 + 50)
     fields = "record::id(id) AS id, recording, idx, t0, t1, emotion, speaker, space, text, page, box"
-    rows = None
-    if used == "semantic":
-        rows = []
-    elif db.ready_fulltext():
+    rows = [] if used == "semantic" else textindex.rows(db, "segment", groups, fields, where_f, params, cap, (M0, M1))
+    if rows is None and db.ready_fulltext():
         conds, sel = [], []
         for k, g in enumerate(groups, 1):
             params[f"q{k}"] = " ".join(g["words"] + g["phrases"])
@@ -436,8 +434,8 @@ def _matches(db, groups, table, fields, where_f, base):
     params = dict(base)
     phrased = any(g["phrases"] for g in groups)
     cols = fields + (", text" if phrased else "")
-    rows = None
-    if db.ready_fulltext():
+    rows = textindex.rows(db, table, groups, cols, where_f, params, FACET_CAP + 1)
+    if rows is None and db.ready_fulltext():
         conds = []
         for k, g in enumerate(groups, 1):
             params[f"q{k}"] = " ".join(g["words"] + g["phrases"])
@@ -575,8 +573,8 @@ def _layer(db, groups, where_f, cap, base, table, extra, source):
     """Hits in another table of text with times (`extra` names the fields it adds), marked as from `source`."""
     params = {k: v for k, v in base.items() if k in ("m0", "m1", "sp", "allowed", "also", "rec") or k.startswith("q")}
     fields = f"record::id(id) AS id, recording, t0, t1, space, text, {extra}"
-    rows = None
-    if db.ready_fulltext():
+    rows = textindex.rows(db, table, groups, fields, where_f, params, cap, (M0, M1))
+    if rows is None and db.ready_fulltext():
         conds = [f"text @{k}@ $q{k}" for k in range(1, len(groups) + 1)]
         sel = [f"search::highlight($m0, $m1, {k}) AS h{k}, search::score({k}) AS s{k}" for k in range(1, len(groups) + 1)]
         try:

diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py
@@ -92,6 +92,7 @@
     "documents": ("page_pixels", "thumb_pixels", "ocr_below_chars", "max_pages", "convert_seconds", "attachment_resources"),
 }
 SECRETS = {
+    "search": ("opensearch_password",),
     "llm": ("api_key",),
     "embeddings": ("api_key",),
     "decisions": ("api_key",),
@@ -111,6 +112,7 @@
     ("speakers", "cross_namespace"): {"suggest", "off"},
     ("analysis", "entities"): {"rules", "spacy"},
     ("search", "stemming"): {"english", "none"},
+    ("search", "engine"): set(store.SEARCH_ENGINES),
     ("reports", "audio"): {"link", "embed", "none"},
     ("video", "ocr_engine"): {"auto", "tesseract", "apple-vision", "rapidocr", "doctr", "none"},
     ("video", "face_engine"): {"opencv", "insightface", "none"},
@@ -897,6 +899,10 @@ def save(db, base, section, changes, user=None):
         if not data.get("connection"):
             raise ValueError("choose the storage connection files are kept on")
         blobs.check_connection(db, data["connection"])
+    if section == "search" and data.get("engine") == "opensearch":
+        url = data.get("opensearch_url", (base.get("search") or {}).get("opensearch_url"))
+        if not (isinstance(url, str) and re.match(r"https?://[^/\s]+", url)):
+            raise ValueError("search.engine opensearch needs search.opensearch_url (http:// or https://)")
     if section == "tunnel":
         from . import tunnel
 

diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
@@ -162,7 +162,14 @@ def labels():
         "describe_max": 50,  # pages or shots of a resource described at most
     },
     "graph": {"max_nodes": 150, "min_edge_weight": 2},
-    "search": {"stemming": "english"},
+    "search": {
+        "stemming": "english",
+        "engine": "sqlite",
+        "opensearch_url": "",
+        "opensearch_user": "",
+        "opensearch_password": None,
+        "opensearch_verify": True,
+    },
     # search by meaning (app/domain/semantic.py): an OpenAI-compatible embeddings server (null: the LLM provider's),
     # the model, how long passages are, and how alike a passage must be to a query (null: what suits the model)
     "embeddings": {
@@ -480,6 +487,8 @@ def load_config(path=None, overrides=None):
         cfg["documents"]["web_networks"] = [*(cfg["documents"].get("web_networks") or []), *extra.replace(",", " ").split()]
     if cfg["search"]["stemming"] not in ("english", "none"):
         raise SystemExit("search.stemming must be english or none")
+    if cfg["search"]["engine"] not in SEARCH_ENGINES:
+        raise SystemExit(f"search.engine must be one of {', '.join(SEARCH_ENGINES)}")
     cfg["_path"] = str(p)
     return cfg
 
@@ -566,6 +575,7 @@ def __init__(self, cfg, pool_size=None):
         self._target = (os.environ.get("SURREAL_NS") or d["namespace"], os.environ.get("SURREAL_DB") or d["database"])
         size = 1 if self.embedded else max(1, int(pool_size or os.environ.get("SURREAL_POOL_SIZE") or d.get("pool_size") or 8))
         self.fulltext = None  # FULLTEXT (3.x), SEARCH (2.x) or None; set by connect()
+        self.textindex = None  # the SQLite full-text index (textindex.py) when search.engine is sqlite
         self._text_ready, self._text_lock = not self.embedded, threading.Lock()
         self._pool: queue.LifoQueue = queue.LifoQueue()
         self._all = []
@@ -752,6 +762,8 @@ def ping(self):
         return True
 
     def close(self):
+        if self.textindex is not None:
+            self.textindex.close()
         for c in self._all:
             try:
                 c.close()
@@ -1128,6 +1140,9 @@ def _analyzer(cfg):
 )
 
 
+SEARCH_ENGINES = ("sqlite", "surrealdb", "opensearch")
+
+
 def _text_index(db):
     # SurrealDB 3 spells full-text indexes FULLTEXT; 2.x (and the embedded engine) spell them SEARCH.
     found = None
@@ -1141,13 +1156,51 @@ def _text_index(db):
     return found
 
 
+def search_settings(db, cfg):
+    """search.*, with what was saved in the app over archive.yaml: the engine has to be known before the settings are."""
+    from . import settings
+
+    return settings.effective(db, cfg)["search"]
+
+
+def _search_engine(db, cfg, rebuild=False):
+    """Set up word search for search.engine: the words in a SQLite file or an OpenSearch cluster, kept in step by
+    events (textindex.py), or SurrealDB's own index. SurrealDB's is taken down when it isn't used, so its memory and
+    disk are given back."""
+    from . import textindex
+
+    search = search_settings(db, cfg)
+    old = getattr(db, "textindex", None)
+    if old is not None:
+        old.close()
+        db.textindex = None
+    index = textindex.open_index(db, {**cfg, "search": search})
+    if index is None:
+        db.run(
+            [f"REMOVE EVENT IF EXISTS {table}_textindex ON {table}" for _, table in TEXT_INDEXES] + ["DELETE text_change"],
+            transaction=False,
+        )
+        db.fulltext = _text_index(db)
+        return
+    db.run(
+        [f"REMOVE INDEX IF EXISTS {name} ON {table}" for name, table in TEXT_INDEXES]
+        + ["DEFINE TABLE IF NOT EXISTS text_change SCHEMALESS"]
+        + [textindex.event(table) for _, table in TEXT_INDEXES],
+        transaction=False,
+    )
+    db.fulltext = None
+    db.textindex = index
+    if rebuild:
+        db.textindex.rebuild()
+
+
 def connect(cfg, upgrade=True):
     """Open the database, apply the schema and, unless upgrade=False (for `lens migrations`), bring its data up to
     date (domain/migrations.py)."""
     db = DB(cfg)
     # one round trip, not one per statement: against a server that was over a second for every test's database
     db.run([_analyzer(cfg), *SCHEMA], transaction=False)
-    db.fulltext = _text_index(db)
+    _search_engine(db, cfg)
     for name, spec in cfg["namespaces"].items():
         sid = ns_id(db, name)
         db.q("UPDATE $r SET graph = $g", r=R("space", sid), g=spec["graph"])
@@ -1164,15 +1217,20 @@ def migrate(db):
 
 
 def reindex(db, cfg):
-    """Rebuild the full-text index, e.g. after changing search.stemming."""
-    for s in [f"REMOVE INDEX IF EXISTS {index} ON {table}" for index, table in TEXT_INDEXES] + ["REMOVE ANALYZER IF EXISTS archive_text"]:
-        try:
-            db.q(s)
-        except Exception:  # noqa: BLE001
-            pass
-    db.q(_analyzer(cfg))
-    db.fulltext = _text_index(db)
-    db._text_ready = True  # just built from the current data
+    """Rebuild the full-text index for the search settings saved now (search.engine and search.stemming)."""
+    if search_settings(db, cfg)["engine"] == "surrealdb":
+        for s in [f"REMOVE INDEX IF EXISTS {index} ON {table}" for index, table in TEXT_INDEXES] + [
+            "REMOVE ANALYZER IF EXISTS archive_text"
+        ]:
+            try:
+                db.q(s)
+            except Exception:  # noqa: BLE001
+                pass
+        db.q(_analyzer({**cfg, "search": search_settings(db, cfg)}))
+        _search_engine(db, cfg)
+        db._text_ready = True  # just built from the current data
+    else:
+        _search_engine(db, cfg, rebuild=True)
 
 
 def ns_id(db, name, create=True):