adeelahmad · Oct 7, 2026 · Oct 7, 2026
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -224,7 +224,7 @@ Set at startup only (the config file; the web app can't choose what the server r
 that makes documents, photos and media into searchable PDFs. Lens uses it as a converter, two ways:
 
 - **Here.** With `documents.converter` set to `anytopdf`, Lens downloads release 0.4.0 for this machine on first use
-  (Linux x86-64 and arm64, macOS), checks it against the release's checksum and keeps only the program in
+  (Linux x86-64 and arm64, macOS), checks it against the release's checksum and keeps only the program and its face and object plugins in
   `data_dir/models/anytopdf-0.4.0/`. It makes the PDF of text, Markdown, web pages and emails from the same cleaned page
   Lens would print with Chromium, so a server without Chromium reads them, and it reads images too: a photographed page
   is found, straightened and flattened before OCR. Office files still need LibreOffice beside it.
@@ -237,6 +237,10 @@ It runs with no network, no runtime plugins and no config file of its own, and i
 holds only the document's text, so what Lens reads is what the document says. Faces, objects and speech stay Lens's
 own steps. The resource's rendition says `anytopdf` made it.
 
+Lens can also ask anytopdf's face and object plugins what is in a picture (`anytopdf.analyze`). They run sandboxed
+(`--plugin-sandbox strict`: no network, writing only to their own workspace, reading only the picture and the models
+Lens names), and the objects plugin runs only when it's given a model, such as Lens's YOLOX file.
+
 Neither may reach anything while converting: Chromium goes through a proxy inside Lens that serves the page and refuses
 every other request (the page also allows no scripts), and LibreOffice is given a proxy address that isn't there.
 Chromium runs with its sandbox where it can, and without it as root or where the container lacks what the sandbox

diff --git a/fastapi_backend/app/domain/anytopdf.py b/fastapi_backend/app/domain/anytopdf.py
@@ -13,6 +13,9 @@
 every document and image, `lens` never. Run here, it has no network, no runtime plugins (faces, objects and speech
 stay Lens's own steps) and no config file of its own; its PDF has no provenance page, and its search layer holds the
 document's text only, so what Lens reads from it is what the document says.
+
+analyze() asks its face and object plugins, kept beside the program, what is in a picture: sandboxed, with no network,
+reading only the picture and the models Lens names.
 """
 
 from __future__ import annotations
@@ -44,6 +47,8 @@
 # its search layer: the document's own text, not the dates, colours and places it would add
 QUIET = ["--no-provenance-page", "--no-entities", "--colors", "off", "--location", "off"]
 POLL_SECONDS = 1.0
+# the plugins Lens keeps beside the program, for analyze(): the rest of the release's plugins stay behind
+PLUGINS = ("anytopdf-plugin-faces", "anytopdf-plugin-face-id", "anytopdf-plugin-objects")
 
 
 class Unavailable(RuntimeError):
@@ -97,15 +102,16 @@ def mode(cfg):
 
 
 def fetch(cfg, say=None):
-    """Download this machine's anytopdf release, check it against its checksum, and keep only the program."""
+    """Download this machine's anytopdf release, check it against its checksum, and keep only the program and its
+    face and object plugins (PLUGINS)."""
     import hashlib
 
     found = archive()
     if not found:
         raise Unavailable(f"anytopdf has no build for {platform.system()} {platform.machine()}")
     name, sha = found
     dest = fetched_path(cfg)
-    if dest.is_file():
+    if dest.is_file() and (dest.parent / "plugins").is_dir():  # fetched before plugins were kept: fetched again
         return str(dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     if say:
@@ -118,17 +124,29 @@ def fetch(cfg, say=None):
                 f.write(chunk)
         if h.hexdigest() != sha:
             raise RuntimeError(f"{name} didn't match its checksum")
-        want = f"{name.removesuffix('.tar.gz')}/anytopdf"
+        top = name.removesuffix(".tar.gz")
+        wanted = {f"{top}/anytopdf": "anytopdf", **{f"{top}/plugins/{p}": f"plugins/{p}" for p in PLUGINS}}
+        got = pathlib.Path(tmp) / "got"
         with tarfile.open(part) as t:
-            member = t.getmember(want)  # the program only: its plugins stay behind, and nothing else is unpacked
-            if not member.isfile():
-                raise RuntimeError(f"{want} in {name} isn't a file")
-            src = t.extractfile(member)
-            out = pathlib.Path(tmp) / "anytopdf"
-            with open(out, "wb") as f:
-                shutil.copyfileobj(src, f)
-        out.chmod(0o755)
-        out.replace(dest)
+            for want, to in wanted.items():  # these only: nothing else in the archive is unpacked
+                try:
+                    member = t.getmember(want)
+                except KeyError:
+                    if to == "anytopdf":
+                        raise
+                    continue  # a plugin this release doesn't ship: analyze() goes without it
+                if not member.isfile():
+                    raise RuntimeError(f"{want} in {name} isn't a file")
+                out = got / to
+                out.parent.mkdir(parents=True, exist_ok=True)
+                with t.extractfile(member) as src, open(out, "wb") as f:
+                    shutil.copyfileobj(src, f)
+                out.chmod(0o755)
+        if (dest.parent / "plugins").exists():
+            shutil.rmtree(dest.parent / "plugins")
+        if (got / "plugins").exists():
+            (got / "plugins").replace(dest.parent / "plugins")
+        (got / "anytopdf").replace(dest)
     return str(dest)
 
 
@@ -218,3 +236,51 @@ def _remote(cfg, src, out):
         raise ValueError(f"the conversion node said {e.code}: {_said(e)}") from None
     except (urllib.error.URLError, OSError) as e:
         raise ValueError(f"the conversion node can't be reached ({getattr(e, 'reason', e)})") from None
+
+
+def analyze(cfg, picture, models=None, seconds=None):
+    """What anytopdf's plugins find in a picture, run here: [{"kind": "face" | "object", "label", "box": [x, y, w, h]
+    fractions, "score", "attributes"}], one per thing found where it is (their summaries left out).
+
+    `models` are the plugins' settings, such as {"ANYTOPDF_OBJECTS_MODEL": "/models/yolox_s.onnx"}: a plugin that
+    needs a model runs only when it's given one, and may read nothing else (the plugins run sandboxed, with no network
+    and no config file). Faces are found by the bundled YuNet whatever is given."""
+    from . import convert
+
+    exe = binary(cfg)
+    if not exe:
+        raise Unavailable("anytopdf isn't installed here (documents.anytopdf)")
+    models = {k: str(v) for k, v in (models or {}).items() if k.startswith("ANYTOPDF_") and v}
+    seconds = seconds or _opts(cfg).get("convert_seconds") or 300
+    with tempfile.TemporaryDirectory(prefix="lens-anytopdf-") as tmp:
+        pic = pathlib.Path(tmp) / f"picture{pathlib.Path(picture).suffix.lower() or '.jpg'}"
+        shutil.copyfile(picture, pic)
+        env = netguard.nowhere_env({**os.environ, "HOME": tmp, "TMPDIR": tmp})
+        env = {k: v for k, v in env.items() if not k.startswith("ANYTOPDF_")}
+        env.update(models, ANYTOPDF_DATA_DIR=f"{tmp}/data")
+        reads = [a for v in models.values() if os.path.exists(v) for a in ("--plugin-sandbox-allow-read", v)]
+        graph = pathlib.Path(tmp) / "graph.json"
+        argv = [exe, "--no-config", "--plugin-sandbox", "strict", *reads, "--plugin-timeout", str(int(seconds))]
+        argv += ["convert", str(pic), "-o", f"{tmp}/out.pdf", "--dump-graph", str(graph), "--ocr", "off", *QUIET]
+        r = convert._run(argv, seconds, env=env, cwd=tmp)
+        if r.returncode != 0 or not graph.is_file():
+            said = convert._last_said(r.stderr or r.stdout)
+            raise ValueError(f"anytopdf couldn't look at it ({said or f'exit {r.returncode}'})")
+        found = json.loads(graph.read_text())
+    out = []
+    for unit in found.get("units") or []:
+        for a in unit.get("annotations") or []:
+            box = a.get("region")
+            if a.get("kind") not in ("face", "object") or not box:
+                continue
+            attrs = a.get("attributes") or {}
+            out.append(
+                {
+                    "kind": a["kind"],
+                    "label": attrs.get("label") or a["kind"],
+                    "box": [float(box["x"]), float(box["y"]), float(box["width"]), float(box["height"])],
+                    "score": float(a.get("confidence") or 0.0),
+                    "attributes": attrs,
+                }
+            )
+    return out
diff --git a/fastapi_backend/tests/api/test_anytopdf.py b/fastapi_backend/tests/api/test_anytopdf.py
@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import os
+import pathlib
 import shutil
 import tarfile
 
@@ -188,6 +189,7 @@ def test_it_is_fetched_when_asked_for(cfg, monkeypatch, tmp_path):
     (tmp_path / name / "plugins").mkdir(parents=True)
     (tmp_path / name / "anytopdf").write_bytes(b"#!/bin/sh\necho anytopdf 0.4.0\n")
     (tmp_path / name / "plugins" / "anytopdf-plugin-faces").write_bytes(b"x")
+    (tmp_path / name / "plugins" / "anytopdf-plugin-sentiment").write_bytes(b"x")  # not one Lens keeps
     with tarfile.open(tmp_path / f"{name}.tar.gz", "w:gz") as t:
         t.add(tmp_path / name, arcname=name)
     raw = (tmp_path / f"{name}.tar.gz").read_bytes()
@@ -198,16 +200,50 @@ def test_it_is_fetched_when_asked_for(cfg, monkeypatch, tmp_path):
     monkeypatch.setattr(anytopdf.urllib.request, "urlopen", lambda url, timeout=None: io.BytesIO(raw))
     got = anytopdf.fetch(cfg)
     assert got == str(anytopdf.fetched_path(cfg)) and os.access(got, os.X_OK)
-    assert sorted(p.name for p in anytopdf.fetched_path(cfg).parent.iterdir()) == ["anytopdf"]
+    kept = sorted(str(p.relative_to(anytopdf.fetched_path(cfg).parent)) for p in anytopdf.fetched_path(cfg).parent.rglob("*"))
+    assert kept == ["anytopdf", "plugins", "plugins/anytopdf-plugin-faces"]  # the program and its face and object plugins
     assert comp.present({**cfg, "documents": {**cfg["documents"], "anytopdf": None}})
 
     monkeypatch.setattr(anytopdf, "archive", lambda: (f"{name}.tar.gz", "0" * 64))
+    assert anytopdf.fetch(cfg) == got  # already here
+    shutil.rmtree(anytopdf.fetched_path(cfg).parent / "plugins")
+    with pytest.raises(RuntimeError, match="checksum"):  # fetched before plugins were kept: fetched again
+        anytopdf.fetch(cfg)
     anytopdf.fetched_path(cfg).unlink()
     with pytest.raises(RuntimeError, match="checksum"):
         anytopdf.fetch(cfg)
     assert not anytopdf.fetched_path(cfg).exists()
 
 
+def test_its_plugins_look_at_a_picture(cfg, tmp_path):
+    log = tmp_path / "runs.jsonl"
+    cfg["documents"]["anytopdf"] = fake_anytopdf.make(tmp_path / "anytopdf", log)
+    pic = tmp_path / "bus.jpg"
+    pic.write_bytes(b"a picture")
+    model = tmp_path / "yolox_s.onnx"
+    model.write_bytes(b"a model")
+    assert anytopdf.analyze(cfg, pic) == [
+        {"kind": "face", "label": "face", "box": [0.1, 0.2, 0.05, 0.06], "score": 0.9, "attributes": {"face_index": "0"}}
+    ]  # faces are found whatever is given; their summary is left out
+    found = anytopdf.analyze(cfg, pic, {"ANYTOPDF_OBJECTS_MODEL": model, "HOME": "/elsewhere"})
+    assert [(f["kind"], f["label"], f["box"]) for f in found] == [
+        ("face", "face", [0.1, 0.2, 0.05, 0.06]),
+        ("object", "bus", [0.02, 0.2, 0.9, 0.5]),
+    ]
+    run = fake_anytopdf.runs(log)[-1]
+    args = run["args"]
+    assert args[args.index("--plugin-sandbox") + 1] == "strict" and args[args.index("--plugin-sandbox-allow-read") + 1] == str(model)
+    assert "--no-plugins" not in args and "--no-config" in args and args[args.index("--ocr") + 1] == "off"
+    assert run["anytopdf_env"] == ["ANYTOPDF_DATA_DIR", "ANYTOPDF_OBJECTS_MODEL"] and run["proxy"]  # only what it's given, no network
+
+    pic.write_bytes(b"unreadable")
+    with pytest.raises(ValueError, match="anytopdf couldn't look at it"):
+        anytopdf.analyze(cfg, pic)
+    cfg["documents"]["anytopdf"] = str(tmp_path / "gone")
+    with pytest.raises(anytopdf.Unavailable):
+        anytopdf.analyze(cfg, pic)
+
+
 @pytest.fixture
 def real(cfg):
     cfg["documents"]["anytopdf"] = REAL
@@ -224,3 +260,6 @@ def test_the_real_anytopdf(real, lean, client, env, db, cfg):
         rec = _rec(db, up["recording"])
         assert (rec["status"], rec["rendition"]["by"]) == ("analyzed", "anytopdf"), rec.get("error")
         assert words in _text(db, up["recording"]), _text(db, up["recording"])
+    pic = pathlib.Path(cfg["data_dir"]) / "page.png"
+    pic.write_bytes(_png(scan(["Galway harbour"])))
+    assert anytopdf.analyze(cfg, pic) == []  # its plugins ran, sandboxed, and found no face on a page of text
diff --git a/fastapi_backend/tests/fake_anytopdf.py b/fastapi_backend/tests/fake_anytopdf.py
@@ -1,5 +1,6 @@
 """An anytopdf for tests (anytopdf.py): `anytopdf ... convert IN -o OUT ...` writes a PDF of IN's text (a page of
-HTML's text without its markup; an image's, the words in its name), and logs how it was run, one JSON line each, to
+HTML's text without its markup; an image's, the words in its name) and, with --dump-graph, what its plugins found (a
+face, and a bus when it's given an objects model). It logs how it was run, one JSON line each, to
 the file named by its log. It's also a conversion node: `node(token)` serves `anytopdf queue serve`'s API from
 memory, converting each upload the same way."""
 
@@ -31,6 +32,13 @@
     print("WARNING [input.unreadable]: it can't be read", file=sys.stderr)
     sys.exit(3)
 pathlib.Path(out).write_bytes(pdf_of(src, open(src, "rb").read()))
+if "--dump-graph" in args:  # what its plugins found: a face, and a bus when given an objects model
+    found = [{"kind": "face", "text": "face", "confidence": 0.9, "region": {"x": 0.1, "y": 0.2, "width": 0.05, "height": 0.06}, "attributes": {"face_index": "0"}},
+             {"kind": "face", "text": "1 face", "confidence": None, "region": None, "attributes": {}}]
+    if os.environ.get("ANYTOPDF_OBJECTS_MODEL"):
+        found.append({"kind": "object", "text": "bus", "confidence": 0.93, "region": {"x": 0.02, "y": 0.2, "width": 0.9, "height": 0.5}, "attributes": {"label": "bus"}})
+    graph = {"sources": [], "units": [{"kind": "visual", "annotations": found}]}
+    pathlib.Path(args[args.index("--dump-graph") + 1]).write_text(json.dumps(graph))
 """