adeelahmad · Oct 4, 2026 · Oct 4, 2026 · Oct 4, 2026
diff --git a/DEVLOG.md b/DEVLOG.md
@@ -127,11 +127,12 @@ Todo:
       language, events and sentiment; provider speaker labels in speaker separation
 - [x] Voice chat: own speech-to-text engine; ElevenLabs and Deepgram text to speech
 - [x] Settings → Speech providers in the web app; new engines in Transcription and Speaker separation
-- [ ] Local LLM: GGUF catalog filtered by this machine, download from Hugging Face, llama.cpp server as the provider
+- [x] Local LLM: GGUF catalog filtered by this machine, download from Hugging Face, llama.cpp server as the provider
 
 Refine later: per-minute prices for provider calls in the ledger; AssemblyAI and Deepgram summaries, chapters and
 entities as optional imports next to Lens's own analysis; several endpoints per service; speaker labels kept across
-chunks of long OpenAI-compatible transcriptions.
+chunks of long OpenAI-compatible transcriptions; CUDA and Vulkan builds of llama.cpp; local embedding models.
+
 ## 2026-10-04 · Topics: a controlled vocabulary apart from entities
 
 Goal (Adeel): entities and topics are mixed (a topic is an entity of type TERM, which is also the fallback type for

diff --git a/docs/local-models.md b/docs/local-models.md
@@ -0,0 +1,51 @@
+# Local models
+
+Lens can run a chat model itself, so summaries, templates and the assistant work with no model server and nothing
+leaving the machine. It uses [llama.cpp](https://github.com/ggml-org/llama.cpp)'s server and GGUF files from
+Hugging Face.
+
+Status: built. Planned: GPU builds of llama.cpp (CUDA, Vulkan) fetched by Lens, local embedding models.
+
+## Use it
+
+**Settings → Local model**:
+
+1. Pick a model. The list shows each model's download size and the memory it needs (the weights, the context, and room
+   for Lens). Models this machine can't run, or has no disk for, are greyed out. Small ones (Qwen2.5 0.5B, Llama 3.2
+   1B, Qwen2.5 1.5B) run on a Raspberry Pi; 7B and 8B models want 16 GB or a GPU.
+2. Turn on **Run a model here** and save.
+
+Lens then, in the background:
+
+- fetches llama.cpp's server, unless `llama-server` is already on PATH (Homebrew: `brew install llama.cpp`) or
+  `local_llm.server` names it in archive.yaml. The release build for Linux x64, Linux arm64 and macOS (Metal on Apple
+  Silicon) goes into `data_dir/bin/llama.cpp`;
+- downloads the model into `data_dir/models/gguf`, resuming an interrupted download. Listed models are pinned to a
+  revision and checked against their SHA-256;
+- starts the server with its own random API key (not on the command line), and once it answers makes it the LLM
+  provider. The provider that was set before is kept and put back when you turn the local model off.
+
+The page shows each step (fetching llama.cpp, downloading with a percentage, starting, running) and the server's last
+lines when something goes wrong. Downloaded models can be deleted there to free the disk.
+
+Any other GGUF chat model on Hugging Face works too: set the model to `hf:<owner>/<repo>/<file>.gguf`. Gated models
+need `HF_TOKEN` in the environment.
+
+## Settings
+
+| Setting | Default | |
+| --- | --- | --- |
+| `local_llm.enabled` | off | |
+| `local_llm.model` | none | a listed id, or `hf:owner/repo/file.gguf` |
+| `local_llm.use_as_provider` | on | off: it runs, and you point things at it yourself |
+| `local_llm.context` | 4096 | tokens; more needs more memory |
+| `local_llm.threads` | auto | |
+| `local_llm.gpu_layers` | 999 | as many as fit on a GPU; 0 for CPU only |
+| `local_llm.port` | 8091 | |
+| `local_llm.host` | automatic | the address workers reach it at (in Docker, the container's address) |
+| `local_llm.server` | none | llama-server's path; archive.yaml only |
+
+One Lens process runs the server at a time (it holds a lease, like the Cloudflare tunnel). In Docker, workers in
+other containers reach it at the container's address on the compose network.
+
+Code: `fastapi_backend/app/domain/local_llm.py`.
diff --git a/fastapi_backend/app/api/v1/routes/admin.py b/fastapi_backend/app/api/v1/routes/admin.py
@@ -13,7 +13,7 @@
 from app import email
 from app.api.deps import AdminReader, AdminWriter, Cfg, Db, domain_errors
 from app.core.middleware import host_name
-from app.domain import auth, bridge, decide, jobs, llm, semantic, settings, sources, speech, store, telemetry, tunnel
+from app.domain import auth, bridge, decide, jobs, llm, local_llm, semantic, settings, sources, speech, store, telemetry, tunnel
 from app.schemas.admin import (
     AuditEntry,
     BridgeStatus,
@@ -24,7 +24,9 @@
     Health,
     IndexQueued,
     LlmTestResult,
+    LocalLlmStatus,
     MailTestResult,
+    Removed,
     SemanticStatus,
     SpeechTestResult,
     Started,
@@ -97,6 +99,25 @@ def bridge_status(user: AdminReader, cfg: Cfg, db: Db) -> BridgeStatus:
     return BridgeStatus(**bridge.status(db, cfg))
 
 
+@router.get("/settings/local-llm/status")
+def local_llm_status(user: AdminReader, cfg: Cfg, db: Db) -> LocalLlmStatus:
+    """The chat model Lens runs itself: the models this machine can run, which are downloaded, and how the server is
+    doing (fetching llama.cpp, downloading, starting, running)."""
+    return LocalLlmStatus.model_validate(local_llm.status(db, cfg))
+
+
+@router.delete("/settings/local-llm/models")
+def remove_local_model(user: AdminWriter, cfg: Cfg, db: Db, model: str = Query(..., max_length=400)) -> Removed:
+    """Delete a downloaded model's file to free the disk (not the one running)."""
+    try:
+        removed = local_llm.remove(cfg, model)
+    except local_llm.LocalModelError as e:
+        raise HTTPException(409, str(e)) from None
+    if removed:
+        auth.audit(db, user.as_audit(), "local_llm.remove", None, {"model": model})
+    return Removed(removed=removed)
+
+
 @router.get("/settings/tunnel/status")
 def tunnel_status(user: AdminReader, cfg: Cfg, db: Db) -> TunnelStatus:
     """How the Cloudflare tunnel (Settings › Remote access) is doing: its address, whether it's connected, and

diff --git a/fastapi_backend/app/cli.py b/fastapi_backend/app/cli.py
@@ -260,7 +260,7 @@ def platform_main(argv, config):
     import threading
     import time
 
-    from .domain import auth, bridge, jobs, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel
+    from .domain import auth, bridge, jobs, local_llm, notify, passkeys, routines, sensors, settings, sources, telemetry, tunnel
 
     ap = argparse.ArgumentParser(prog="lens")
     sub = ap.add_subparsers(dest="cmd", required=True)
@@ -366,6 +366,7 @@ def password():
                     sensors.start(db, C, stop, log=print, name=wk.name)  # and the sensor hub, while sensors are on
                     bridge.start(db, C, stop, lambda: cfg, name=wk.name, log_fn=print)  # and the chat-room bridge, while it's on
                     tunnel.start(db, C, stop, name=wk.name, log=print)  # and the Cloudflare tunnel, while it's on
+                    local_llm.start(db, C, stop, name=wk.name, log=print)  # and the local chat model, while it's on
                 _stop_on_term()
                 try:
                     wk.loop(stop)

diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py
@@ -17,6 +17,7 @@
     components,
     content_types,
     jobs,
+    local_llm,
     notify,
     routines,
     sensors,
@@ -78,6 +79,7 @@ def start_background(self) -> None:
         sensors.start(self.db, self.current, self.stop, log=log.info, name=f"api-{os.getpid()}")
         bridge.start(self.db, self.current, self.stop, lambda: self.base, name=f"api-{os.getpid()}", log_fn=log.warning)
         tunnel.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info)
+        local_llm.start(self.db, self.current, self.stop, name=f"api-{os.getpid()}", log=log.info)
 
     def close(self) -> None:
         self.stop.set()