adeelahmad · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,9 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version
 
 ## Unreleased
 
+- **Search photos by what they show.** Turn on Settings › Search › Search photos by what they show (off by default)
+  and Lens downloads anytopdf's CLIP model (about 600 MB). Search then also shows the photos most like what you typed,
+  such as "a bus on a city street", whatever words are on them ([anytopdf](docs/configuration.md#anytopdf)).
 - **anytopdf can find objects.** Settings › Video › Object detector › anytopdf runs Lens's YOLOX model with
   anytopdf's objects plugin (sandboxed, no network), so a lean server or a Raspberry Pi finds objects without ONNX
   Runtime. Lens fetches anytopdf and the model on first use. YOLOX on ONNX Runtime stays the default

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -261,6 +261,13 @@ term becomes a place), and the namespace's entity setup still decides what is ke
 resource's Details tab, for the namespace's members only: visitors, embeds and share links don't see where a photo was
 taken. If anytopdf can't read something, analysis goes on without hints. Lens downloads anytopdf for this the same way.
 
+**Search photos by what they show.** With `embeddings.photos` on (Settings › Search › Search photos by what they
+show; off by default), Lens keeps anytopdf's CLIP plugin beside the program and downloads its model, OpenAI's CLIP
+ViT-B/32 (about 600 MB, into `data_dir/models/clip`). The `embed` step then gives each image resource a CLIP vector
+(once per file, on the CPU, offline), and a search also shows the photos most like it, such as "a bus on a city
+street", in the namespaces and collections the person may see (`GET /api/v1/search/photos`). It needs no embeddings
+server, and its vectors are kept apart from search by meaning's. On a Raspberry Pi it works, slowly.
+
 Neither may reach anything while converting: Chromium goes through a proxy inside Lens that serves the page and refuses
 every other request (the page also allows no scripts), and LibreOffice is given a proxy address that isn't there.
 Chromium runs with its sandbox where it can, and without it as root or where the container lacks what the sandbox

diff --git a/fastapi_backend/app/api/v1/routes/search.py b/fastapi_backend/app/api/v1/routes/search.py
@@ -9,9 +9,9 @@
 from app.api.deps import Acl, Cfg, CurrentUser, Db
 from app.api.media import sign_urls
 from app.domain import graph as graphmod
-from app.domain import graph_history, render, store
+from app.domain import graph_history, hierarchy, photos, render, store
 from app.domain import search as searchmod
-from app.schemas.search import Graph, Mention, SearchResults, TermSuggestion
+from app.schemas.search import Graph, Mention, PhotoResults, SearchResults, TermSuggestion
 
 router = APIRouter(tags=["search"])
 
@@ -72,6 +72,51 @@ def search_transcripts(
     return sign_urls(res, full=True)
 
 
+@router.get("/search/photos")
+def search_photos(
+    acl: Acl,
+    user: CurrentUser,
+    db: Db,
+    cfg: Cfg,
+    q: str = Query(min_length=1, max_length=300, description='what the photo shows, such as "a bus on a city street"'),
+    ns: str | None = None,
+    limit: int = Query(24, ge=1, le=100),
+) -> PhotoResults:
+    """Photos that show what was asked for, best first, in the namespaces you can read and the collections you were
+    given a role on, whatever words are on them: anytopdf's CLIP plugin compares the query with each photo
+    (Settings → Search → Search photos by what they show; off by default)."""
+    if not photos.enabled(cfg):
+        raise HTTPException(409, "searching photos by what they show is off (Settings → Search)")
+    whole, part = acl.scope(ns)
+    spaces, also = set(whole), hierarchy.recordings_in(db, [c for cols in part.values() for c in cols])
+    try:
+        hits = photos.search(db, cfg, q, spaces, also, limit)
+    except photos.Unavailable as e:
+        raise HTTPException(503, str(e)) from None
+    except ValueError as e:
+        raise HTTPException(502, str(e)) from None
+    rids = [h["recording"] for h in hits]
+    recs = {
+        r["id"]: r
+        for r in db.rows("SELECT record::id(id) AS id, title, space FROM recording WHERE id IN $r", r=[R("recording", x) for x in rids])
+    }
+    thumbs = {
+        x["recording"]: x.get("thumb") for x in db.rows("SELECT recording, thumb FROM page WHERE recording IN $r AND idx = 0", r=rids)
+    }
+    names = store.space_names(db)
+    out = [
+        {
+            **h,
+            "title": recs.get(h["recording"], {}).get("title"),
+            "namespace": names.get(recs.get(h["recording"], {}).get("space")),
+            "thumb": f"{store.API}/recordings/{h['recording']}/frames/{thumbs[h['recording']]}" if thumbs.get(h["recording"]) else None,
+        }
+        for h in hits
+        if h["recording"] in recs
+    ]
+    return PhotoResults.model_validate(sign_urls({"hits": out}, full=True))
+
+
 @router.get("/search/terms")
 def suggest_terms(
     acl: Acl,

diff --git a/fastapi_backend/app/domain/anytopdf.py b/fastapi_backend/app/domain/anytopdf.py
@@ -58,7 +58,7 @@
 MAX_PDF = 2 << 30  # the largest PDF taken from a conversion node
 JOB_ID = re.compile(r"[A-Za-z0-9_.:-]{1,128}")
 # the plugins Lens keeps beside the program, for analyze(): the rest of the release's plugins stay behind
-PLUGINS = ("anytopdf-plugin-faces", "anytopdf-plugin-face-id", "anytopdf-plugin-objects")
+PLUGINS = ("anytopdf-plugin-faces", "anytopdf-plugin-face-id", "anytopdf-plugin-objects", "anytopdf-plugin-clip")
 
 
 class Unavailable(RuntimeError):
@@ -113,15 +113,16 @@ def mode(cfg):
 
 def fetch(cfg, say=None):
     """Download this machine's anytopdf release, check it against its checksum, and keep only the program and its
-    face and object plugins (PLUGINS)."""
+    face, object and CLIP plugins (PLUGINS)."""
     import hashlib
 
     found = archive()
     if not found:
         raise Unavailable(f"anytopdf has no build for {platform.system()} {platform.machine()}")
     name, sha = found
     dest = fetched_path(cfg)
-    if dest.is_file() and (dest.parent / "plugins").is_dir():  # fetched before plugins were kept: fetched again
+    asked = dest.parent / "plugins" / ".asked"  # the plugins it was fetched for: fetched again when Lens keeps more
+    if dest.is_file() and asked.is_file() and set(PLUGINS) <= set(asked.read_text().split()):
         return str(dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     if say:
@@ -154,8 +155,9 @@ def fetch(cfg, say=None):
                 out.chmod(0o755)
         if (dest.parent / "plugins").exists():
             shutil.rmtree(dest.parent / "plugins")
-        if (got / "plugins").exists():
-            (got / "plugins").replace(dest.parent / "plugins")
+        (got / "plugins").mkdir(exist_ok=True)
+        (got / "plugins" / ".asked").write_text("\n".join(PLUGINS) + "\n")
+        (got / "plugins").replace(dest.parent / "plugins")
         (got / "anytopdf").replace(dest)
     return str(dest)
 

diff --git a/fastapi_backend/app/domain/components.py b/fastapi_backend/app/domain/components.py
@@ -434,6 +434,27 @@ def fetch(self, cfg, m, say):
         anytopdf.fetch(cfg, say)
 
 
+class ClipModel(Component):
+    """anytopdf's CLIP plugin and its model (photos.py), fetched when Settings → Search asks to search photos."""
+
+    def needed(self, cfg, m):
+        from . import anytopdf, photos
+
+        return photos.enabled(cfg) and bool(anytopdf.archive() or photos.plugin(cfg))
+
+    def present(self, cfg):
+        from . import photos
+
+        return bool(photos.plugin(cfg)) and photos.model_ready(cfg)
+
+    def fetch(self, cfg, m, say):
+        from . import anytopdf, photos
+
+        if not photos.plugin(cfg):
+            anytopdf.fetch(cfg, say)
+        photos.fetch_model(cfg, say)
+
+
 class Msg(Component):
     def needed(self, cfg, m):
         return False  # only when asked for (components.also): it's GPL-3.0
@@ -566,6 +587,14 @@ def detail(self, cfg):
         size_mb=50,
         license="MIT OR Apache-2.0",
     ),
+    ClipModel(
+        "clip",
+        "CLIP model",
+        "searches photos by what they show (embeddings.photos), with anytopdf's CLIP plugin",
+        steps={"embed"},
+        size_mb=600,
+        license="MIT",
+    ),
     PYTORCH,
     SENSEVOICE,
     WHISPER,

diff --git a/fastapi_backend/app/domain/deletion.py b/fastapi_backend/app/domain/deletion.py
@@ -49,6 +49,7 @@
     "file_line",
     "page",
     "passage",
+    "photo_vector",
 )
 
 

diff --git a/fastapi_backend/app/domain/jobs.py b/fastapi_backend/app/domain/jobs.py
@@ -72,8 +72,13 @@ def _analyze(db, cfg, rid, say, spec=None):
 
 
 def _embed(db, cfg, rid, say, spec=None):
-    from . import semantic
+    from . import photos, semantic
 
+    if photos.enabled(cfg):  # a photo's CLIP vector: apart from the embeddings server, and never in its way
+        try:
+            photos.index_recording(db, cfg, rid, say)
+        except (photos.Unavailable, ValueError, OSError) as e:
+            say(f"not indexed for searching photos: {e}")
     if not semantic.configured(cfg):
         raise Skip("search by meaning is off, or has no embeddings server (Settings → Search)")
     try:

diff --git a/fastapi_backend/app/domain/photos.py b/fastapi_backend/app/domain/photos.py
@@ -0,0 +1,138 @@
+"""Search photos by what they show (embeddings.photos, off by default; docs/configuration.md#anytopdf).
+
+anytopdf's CLIP plugin (anytopdf-plugin-clip, kept beside the program by anytopdf.fetch) embeds each image resource
+with OpenAI's CLIP ViT-B/32, on the CPU, offline. Its model (about 600 MB) is fetched once into
+data_dir/models/clip, which is why this is off by default: a Raspberry Pi can run it, slowly. A query is embedded the
+same way, as text, and compared with every photo the person may see, so "a bus on a city street" finds the photo of
+one whatever words are on it. These vectors are CLIP's, not the embeddings server's, so they're kept apart from
+search by meaning's passages (photo_vector, one per image resource).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import pathlib
+import tempfile
+
+from . import anytopdf, ingest, netguard, store
+
+log = logging.getLogger(__name__)
+R = store.R
+PLUGIN = "anytopdf-plugin-clip"
+MODEL_FILES = ("visual.onnx", "textual.onnx", "bpe_simple_vocab_16e6.txt.gz")
+SECONDS = 120
+# how alike a photo must be to a query; CLIP scores text against pictures low: a good match is about 0.28 to 0.32
+MIN_SIMILARITY = 0.24  # (a plain blue picture scores 0.21 to 0.24 against most queries)
+SPREAD = 0.05  # and no more than this less alike than the best
+
+
+class Unavailable(RuntimeError):
+    """The plugin or its model isn't here."""
+
+
+def enabled(cfg):
+    return bool((cfg.get("embeddings") or {}).get("photos"))
+
+
+def model_dir(cfg):
+    return pathlib.Path(cfg["data_dir"]) / "models" / "clip"
+
+
+def plugin(cfg):
+    """The CLIP plugin beside the anytopdf program Lens fetched (or beside documents.anytopdf), or None."""
+    exe = anytopdf.binary(cfg)
+    for d in [anytopdf.fetched_path(cfg).parent / "plugins"] + ([pathlib.Path(exe).parent / "plugins"] if exe else []):
+        p = d / PLUGIN
+        if p.is_file() and os.access(p, os.X_OK):
+            return str(p)
+    return None
+
+
+def model_ready(cfg):
+    return all((model_dir(cfg) / f).is_file() for f in MODEL_FILES)
+
+
+def ready(cfg):
+    return enabled(cfg) and bool(plugin(cfg)) and model_ready(cfg)
+
+
+def fetch_model(cfg, say=None):
+    """Download the CLIP model with the plugin (it checks each file's checksum)."""
+    from . import convert
+
+    exe = plugin(cfg)
+    if not exe:
+        raise Unavailable("anytopdf's CLIP plugin isn't here")
+    d = model_dir(cfg)
+    d.mkdir(parents=True, exist_ok=True)
+    if say:
+        say("downloading the CLIP model (about 600 MB)")
+    r = convert._run([exe, "--fetch-model", str(d)], 3600)
+    if r.returncode != 0 or not model_ready(cfg):
+        raise RuntimeError(f"the CLIP model wasn't fetched ({convert._last_said(r.stderr or r.stdout) or f'exit {r.returncode}'})")
+
+
+def _encode(cfg, flag, value):
+    """CLIP's vector for a picture (--encode-image) or a text (--encode-text): (vector, model id)."""
+    from . import convert
+
+    exe = plugin(cfg)
+    if not exe or not model_ready(cfg):
+        raise Unavailable("search photos by what they show needs anytopdf's CLIP plugin and its model")
+    with tempfile.TemporaryDirectory(prefix="lens-clip-") as tmp:
+        env = netguard.nowhere_env({**os.environ, "HOME": tmp, "TMPDIR": tmp})
+        env = {k: v for k, v in env.items() if not k.startswith("ANYTOPDF_")}
+        env["ANYTOPDF_CLIP_MODEL_DIR"] = str(model_dir(cfg))
+        r = convert._run([exe, flag, value], SECONDS, env=env, cwd=tmp)
+    try:
+        out = json.loads(r.stdout or "")
+        vec = [float(x) for x in out["embedding"]]
+    except (ValueError, KeyError, TypeError):
+        raise ValueError(f"CLIP couldn't read it ({convert._last_said(r.stderr) or f'exit {r.returncode}'})") from None
+    return vec, out.get("model") or "clip"
+
+
+def index_recording(db, cfg, rid, say=print):
+    """An image resource's CLIP vector, made once per file. Returns whether one was made."""
+    rec = db.one("SELECT source, path, remote, space, collection, fp_key FROM $r", r=R("recording", rid)) or {}
+    if rec.get("source") != "image":
+        return False
+    fp = rec.get("fp_key") or rec.get("path")
+    have = db.one("SELECT fp FROM $r", r=R("photo_vector", rid)) or {}
+    if fp and have.get("fp") == fp:
+        return False
+    path = ingest.audio_path(db, cfg, rec)
+    if not path or not os.path.exists(path):
+        return False
+    vec, model = _encode(cfg, "--encode-image", str(path))
+    db.q(
+        "UPSERT $r CONTENT $d",
+        r=R("photo_vector", rid),
+        d=store.clean(
+            {"recording": rid, "space": rec["space"], "collection": rec.get("collection"), "vector": vec, "model": model, "fp": fp}
+        ),
+    )
+    say("indexed for searching photos by what they show")
+    return True
+
+
+def search(db, cfg, q, spaces, also=(), limit=24):
+    """The photos most like `q` in namespaces `spaces` (and recordings `also`), best first: [{recording, similarity}]."""
+    q = (q or "").strip()
+    if not q:
+        return []
+    vec, model = _encode(cfg, "--encode-text", q[:300])
+    rows = db.rows(
+        "SELECT recording, vector::similarity::cosine(vector, $v) AS similarity FROM photo_vector "
+        "WHERE model = $m AND (space IN $sp OR recording IN $also) ORDER BY similarity DESC LIMIT $k",
+        v=vec,
+        m=model,
+        sp=sorted(spaces),
+        also=sorted(also),
+        k=int(limit),
+    )
+    rows = [r for r in rows if r.get("similarity") is not None]
+    least = max([MIN_SIMILARITY] + [r["similarity"] - SPREAD for r in rows[:1]])
+    return [{"recording": r["recording"], "similarity": round(float(r["similarity"]), 4)} for r in rows if r["similarity"] >= least]
diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py
@@ -185,6 +185,7 @@ def labels():
         "neighbours": 40,
         "min_similarity": None,
         "timeout": 60,
+        "photos": False,  # search photos by what they show, with anytopdf's CLIP plugin (photos.py; a 600 MB model)
     },
     "server": {
         "host": "127.0.0.1",
@@ -1101,6 +1102,9 @@ def close(self):
     "DEFINE INDEX IF NOT EXISTS passage_rec ON passage FIELDS recording",
     "DEFINE INDEX IF NOT EXISTS passage_space ON passage FIELDS space",
     "DEFINE TABLE IF NOT EXISTS embedding_state SCHEMALESS",
+    # each image resource's CLIP vector, for searching photos by what they show (app/domain/photos.py): photo_vector:<rid>
+    "DEFINE TABLE IF NOT EXISTS photo_vector SCHEMALESS",
+    "DEFINE INDEX IF NOT EXISTS photo_vector_space ON photo_vector FIELDS space",
     # collections, batch runs, assistant approvals
     "DEFINE TABLE IF NOT EXISTS saved_collection SCHEMALESS",
     # saved views of the Library (app/domain/views.py)

diff --git a/fastapi_backend/app/schemas/search.py b/fastapi_backend/app/schemas/search.py
@@ -112,3 +112,17 @@ class Mention(ResponseModel):
     text: str | None = None
     speaker: str | None = None
     recorded_at: str = ""
+
+
+class PhotoHit(ResponseModel):
+    """A photo like what was asked for (embeddings.photos)."""
+
+    recording: int
+    title: str | None = None
+    namespace: str | None = None
+    similarity: float = Field(description="CLIP's cosine similarity of the photo and the query; a good match is about 0.3")
+    thumb: str | None = Field(None, description="a signed link to it, small")
+
+
+class PhotoResults(ResponseModel):
+    hits: list[PhotoHit] = Field(default_factory=list)
diff --git a/fastapi_backend/tests/api/test_anytopdf.py b/fastapi_backend/tests/api/test_anytopdf.py
@@ -167,7 +167,8 @@ def do_POST(self):  # noqa: N802
             self.send_header("Content-Length", "0")
             self.end_headers()
             return
-        if _Hostile.mode == "long":
+        if _Hostile.mode == "long":  # the upload read first: answering over it resets the connection on some runners
+            self.rfile.read(int(self.headers["Content-Length"]))
             return self._send(202, b"{" + b" " * (anytopdf.MAX_JSON + 10) + b"}")
         if _Hostile.mode == "odd id":
             return self._send(202, {"job_id": "../../admin", "state": "queued"})
@@ -319,13 +320,14 @@ def test_it_is_fetched_when_asked_for(cfg, monkeypatch, tmp_path):
     got = anytopdf.fetch(cfg)
     assert got == str(anytopdf.fetched_path(cfg)) and os.access(got, os.X_OK)
     kept = sorted(str(p.relative_to(anytopdf.fetched_path(cfg).parent)) for p in anytopdf.fetched_path(cfg).parent.rglob("*"))
-    assert kept == ["anytopdf", "plugins", "plugins/anytopdf-plugin-faces"]  # the program and its face and object plugins
+    # the program and its face, object and CLIP plugins (those the release has), and which it was fetched for
+    assert kept == ["anytopdf", "plugins", "plugins/.asked", "plugins/anytopdf-plugin-faces"]
     assert comp.present({**cfg, "documents": {**cfg["documents"], "anytopdf": None}})
 
     monkeypatch.setattr(anytopdf, "archive", lambda: (f"{name}.tar.gz", "0" * 64))
     assert anytopdf.fetch(cfg) == got  # already here
-    shutil.rmtree(anytopdf.fetched_path(cfg).parent / "plugins")
-    with pytest.raises(RuntimeError, match="checksum"):  # fetched before plugins were kept: fetched again
+    (anytopdf.fetched_path(cfg).parent / "plugins" / ".asked").write_text("anytopdf-plugin-faces\n")
+    with pytest.raises(RuntimeError, match="checksum"):  # fetched before Lens kept the plugins it keeps now: again
         anytopdf.fetch(cfg)
     anytopdf.fetched_path(cfg).unlink()
     with pytest.raises(RuntimeError, match="checksum"):
-Original file line number
+Diff line change
@@ Expand Up / @@ -49,6 +49,7 @@ @@
         "file_line",
         "page",
         "passage",
+        "photo_vector",
     )
@@ Expand Down @@