adeelahmad · Oct 15, 2026 · Oct 10, 2026 · Oct 10, 2026 · Oct 10, 2026 · Oct 14, 2026
diff --git a/docs/updates.md b/docs/updates.md
@@ -1,8 +1,8 @@
 # Updates and backups
 
 Lens tells admins when a new release is out, backs the database up, and installs the release from the web app, so
-upgrading never needs a terminal. Status: **built** (the check, upgrades from the API and the agent, and backups in
-Settings › Backups); the web app's upgrade page is **in progress**.
+upgrading never needs a terminal. Status: **built**: the check, backups, one-click upgrades with their progress and a
+rollback when the new version doesn't start, in Settings → Updates and a banner for admins.
 
 ## The check
 
@@ -40,7 +40,21 @@ The containers see the control folder as `/control` (`LENS_CONTROL_DIR`). `POST
 3. the agent (`packaging/host/lens-upgrade.sh`) runs install.sh again for that release, exactly as running the install
    line by hand does: the secrets, the database, the archive, the port and the address are kept;
 4. it writes `upgrade-status` and `upgrade.log` next to the inbox, which the API reads back while Lens restarts on the
-   new version. The database upgrades itself on start ([Database](database.md#upgrades)).
+   new version, with the step it is on (download, build, start, check), which Settings → Updates shows. The database
+   upgrades itself on start ([Database](database.md#upgrades)).
+
+### When the new version doesn't start
+
+If install.sh fails for the new version (it didn't build, or didn't answer in time), the agent goes back by itself:
+
+1. it checks out the code that ran before and runs install.sh on it again (`LENS_KEEP_CODE=1`: no download), so the
+   previous version runs again (`rolling-back`, then `rolled-back`);
+2. it leaves `inbox/restore` naming the backup taken first. As the previous version starts, it looks at the database:
+   if the new version had changed its shape (ran upgrade steps the previous version doesn't know), the backup is put
+   back; otherwise the database is kept as it is, so nothing written since is lost. What it did is shown with the run.
+
+Only the API restores the backup; a worker running meanwhile may write to the database as it is put back, so its work
+from those seconds can be lost. If going back fails too, the run ends `failed` and says to run the install line.
 
 The agent installs only a release tag (`vX.Y.Z`) or the branch the install already follows, whoever writes the
 request, and the API refuses a release older than the one running (restore a backup instead).

diff --git a/fastapi_backend/app/core/runtime.py b/fastapi_backend/app/core/runtime.py
@@ -59,6 +59,8 @@ def _index_text(self) -> None:
             log.warning("couldn't bring the full-text index up to date", exc_info=True)
 
     def prepare(self) -> None:
+        # first: after a rollback to this version, the database as it was before the upgrade (docs/updates.md)
+        updates.restore_on_start(self.db, self.base)
         templates.seed(self.db)
         content_types.seed(self.db)
         routines.seed(self.db)

diff --git a/fastapi_backend/app/domain/tunnel.py b/fastapi_backend/app/domain/tunnel.py
@@ -268,6 +268,9 @@ def __init__(self, db, cfg_fn, name, log=None):
         self.fails = 0
         self.next_try = 0.0
         self.lock = threading.Lock()
+        # snapshot and write together: otherwise an older snapshot (tick's, taken as cloudflared starts) can reach the
+        # database after a newer one (the log reader's), and the row keeps saying there is no error
+        self.saving = threading.Lock()
 
     def _lease(self):
         now = _now()
@@ -282,16 +285,17 @@ def _lease(self):
         )
 
     def _save(self, **extra):
-        with self.lock:
-            d = {
-                "running": self.proc is not None and self.proc.poll() is None,
-                "connected": self.connected,
-                "url": self.url,
-                "error": self.error,
-                "log": json.dumps(self.lines[-LOG_LINES:]),
-                **extra,
-            }
-        self.db.q("UPDATE $r MERGE $d WHERE owner = $me", r=ROW, d=d, me=self.name)
+        with self.saving:
+            with self.lock:
+                d = {
+                    "running": self.proc is not None and self.proc.poll() is None,
+                    "connected": self.connected,
+                    "url": self.url,
+                    "error": self.error,
+                    "log": json.dumps(self.lines[-LOG_LINES:]),
+                    **extra,
+                }
+            self.db.q("UPDATE $r MERGE $d WHERE owner = $me", r=ROW, d=d, me=self.name)
 
     def _signature(self, cfg):
         t = conf(cfg)

diff --git a/fastapi_backend/app/domain/updates.py b/fastapi_backend/app/domain/updates.py
@@ -40,7 +40,8 @@
 TICK_SECONDS = 600
 NOTES_MAX = 20_000
 LOG_LINES = 60
-ACTIVE = ("backing-up", "queued", "running")
+ACTIVE = ("backing-up", "queued", "running", "rolling-back")
+STEPS = ("download", "build", "start", "check")  # what the agent reports it is on (after the backup)
 # How long a request may sit in the inbox before the web app says nobody picked it up
 PICKUP_SECONDS = 120
 
@@ -190,7 +191,9 @@ def run_state(db):
             for k in ("started_at", "finished_at", "version"):
                 if s.get(k):
                     run[k] = s[k]
-            if s.get("state") == "failed":
+            if s.get("step") in STEPS:
+                run["step"] = s["step"]
+            if s.get("state") in ("failed", "rolled-back"):
                 run["error"] = s.get("error") or f"install.sh stopped (exit {s.get('code', '?')}); see the log"
         if run.get("state") in ACTIVE or run.get("id") == s.get("id"):
             run["log"] = _log_tail(d)
@@ -280,7 +283,10 @@ def _hand_over(db, run, backup):
         inbox = control_dir() / "inbox"
         part = inbox / f".upgrade.{run['id']}"
         extra = f"https={1 if run['https'] else 0}\n" if run.get("https") is not None else ""
-        part.write_text(f"id={run['id']}\nref={run['ref']}\n{extra}by={run.get('by') or ''}\nat={_ts()}\n", encoding="utf-8")
+        part.write_text(
+            f"id={run['id']}\nref={run['ref']}\n{extra}backup={run.get('backup') or ''}\nby={run.get('by') or ''}\nat={_ts()}\n",
+            encoding="utf-8",
+        )
         part.replace(inbox / "upgrade")
         run.update(state="queued", queued_at=_ts())
     except Exception as e:  # noqa: BLE001 - the admin reads why
@@ -291,6 +297,46 @@ def _hand_over(db, run, backup):
     _put(db, upgrade=run)
 
 
+# ---------- after a rollback ----------
+def restore_on_start(db, cfg):
+    """Called as Lens starts. When the agent went back to this version after a new one didn't start, it leaves
+    inbox/restore naming the backup taken before the upgrade. If the new version had changed the database (ran steps
+    this version doesn't know, migrations.status "unknown"), that backup is put back; otherwise the database is kept as
+    it is, so nothing written since is lost. One process does it: the first to take the file."""
+    d = control_dir()
+    if not d:
+        return None
+    req, mine = d / "inbox" / "restore", d / "inbox" / f".restore.{os.getpid()}"
+    try:
+        req.replace(mine)
+    except OSError:
+        return None  # none waiting, or another process took it
+    want = _kv(mine)
+    mine.unlink(missing_ok=True)
+    name = want.get("backup") or ""
+    newer_steps = [m["name"] for m in migrations.status(db) if m["state"] == "unknown"]
+    if not newer_steps:
+        said = "the new version hadn't changed the database, so it was kept as it is"
+    else:
+        from . import backups
+
+        try:
+            backups.usable(db, name)
+            backups.restore(db, cfg, name)
+            said = f"the database was put back as it was before the upgrade ({name})"
+            log.warning("rolled back: %s; the new version had run %s", said, ", ".join(newer_steps))
+        except (ValueError, backups.BackupError) as e:
+            said = f"the new version had changed the database ({', '.join(newer_steps)}), but its backup {name!r} can't be used: {e}"[:500]
+            log.error("couldn't restore after the rollback: %s", said)
+        except Exception as e:  # noqa: BLE001 - the admin reads why
+            said = f"restoring {name} failed: {e}"[:500]
+            log.exception("couldn't restore after the rollback")
+    run = dict(_row(db).get("upgrade") or {})
+    if run.get("id") == want.get("id"):
+        _put(db, upgrade={**run, "restore": said})
+    return said
+
+
 # ---------- the background check ----------
 def _once(db, name):
     """True for the one process (and the one time) that claims `name`."""

diff --git a/fastapi_backend/app/schemas/updates.py b/fastapi_backend/app/schemas/updates.py
@@ -22,7 +22,13 @@ class UpgradeRun(ResponseModel):
     id: str
     ref: str = Field(description="the release tag (or branch) being installed; empty: the one running, set up again")
     https: bool | None = Field(None, description="the HTTPS ports opened (true) or closed (false) by this run (docs/https.md)")
-    state: Literal["backing-up", "queued", "running", "done", "failed"]
+    state: Literal["backing-up", "queued", "running", "rolling-back", "done", "failed", "rolled-back"] = Field(
+        description="rolling-back / rolled-back: the new version didn't start, so the agent went back to the one before"
+    )
+    step: Literal["download", "build", "start", "check"] | None = Field(
+        None, description="while running: what the agent is on (the backup comes before)"
+    )
+    restore: str | None = Field(None, description="after a rollback: whether the backup was put back, and why")
     by: str | None = None
     asked_at: str | None = None
     queued_at: str | None = None

diff --git a/fastapi_backend/tests/api/test_upgrade_rollback.py b/fastapi_backend/tests/api/test_upgrade_rollback.py
@@ -0,0 +1,81 @@
+"""Going back after a failed upgrade (docs/updates.md): the agent reports each step and the rollback, and the version
+it went back to restores the backup taken first only when the new version had changed the database."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.domain import store, updates
+from app.domain.store import R
+
+
+@pytest.fixture
+def control(tmp_path, monkeypatch):
+    d = tmp_path / "control"
+    (d / "inbox").mkdir(parents=True)
+    (d / "agent").write_text("kind=systemd\n")
+    monkeypatch.setenv("LENS_CONTROL_DIR", str(d))
+    return d
+
+
+@pytest.fixture
+def kept(folder, db):
+    """A database that can be backed up: the test server when there is one, else an embedded surrealkv one."""
+    if not db.embedded:
+        yield db
+        return
+    from tests.conftest import make_cfg
+
+    conn = store.connect(make_cfg(folder, url="surrealkv://" + str(folder / "data" / "surrealdb")))
+    yield conn
+    conn.close()
+
+
+def _upgraded_then_rolled_back(kept, cfg, control, newer_steps):
+    """An upgrade whose backup was taken; the new version then wrote data (and ran `newer_steps`) and didn't start, so
+    the agent went back and left the restore request."""
+    kept.q("CREATE $r SET title = 'before'", r=R("recording", 1))
+    run = updates.upgrade(kept, cfg, "v99.0.0", by="t", wait=True)
+    assert run["state"] == "queued" and run["backup"].startswith("before-upgrade-")
+    req = dict(line.split("=", 1) for line in (control / "inbox" / "upgrade").read_text().splitlines())
+    assert req["backup"] == run["backup"]  # the agent is told which backup to go back to
+    kept.q("UPDATE $r SET title = 'after'", r=R("recording", 1))
+    kept.q("CREATE recording SET title = 'new'")
+    for name in newer_steps:
+        kept.q("CREATE migration CONTENT $d", d={"name": name, "done_at": store.now()})
+    (control / "inbox" / "restore").write_text(f"id={run['id']}\nbackup={run['backup']}\n")
+    (control / "upgrade-status").write_text(
+        f"id={run['id']}\nref=v99.0.0\nstate=rolled-back\nstep=check\ncode=1\nerror=the new version didn't start\n"
+    )
+    return run
+
+
+def test_the_backup_comes_back_when_the_new_version_changed_the_database(kept, cfg, control):
+    _upgraded_then_rolled_back(kept, cfg, control, ["a-step-from-the-future"])
+    said = updates.restore_on_start(kept, cfg)
+    assert said.startswith("the database was put back"), said
+    assert kept.values("SELECT VALUE title FROM recording") == ["before"]
+    assert "a-step-from-the-future" not in kept.values("SELECT VALUE name FROM migration")
+    assert not (control / "inbox" / "restore").exists()
+    run = updates.run_state(kept)
+    assert run["state"] == "rolled-back" and run["step"] == "check" and run["restore"] == said
+    assert updates.restore_on_start(kept, cfg) is None  # once
+
+
+def test_the_data_is_kept_when_the_new_version_left_the_database_alone(kept, cfg, control):
+    _upgraded_then_rolled_back(kept, cfg, control, [])
+    said = updates.restore_on_start(kept, cfg)
+    assert "kept as it is" in said
+    assert sorted(kept.values("SELECT VALUE title FROM recording")) == ["after", "new"]
+
+
+def test_a_missing_backup_is_said(kept, cfg, control):
+    run = _upgraded_then_rolled_back(kept, cfg, control, ["a-step-from-the-future"])
+    (control / "inbox" / "restore").write_text(f"id={run['id']}\nbackup=../../etc/passwd\n")
+    assert "can't be used" in updates.restore_on_start(kept, cfg)
+    assert sorted(kept.values("SELECT VALUE title FROM recording")) == ["after", "new"]
+
+
+def test_no_control_folder_nothing_to_do(db, cfg, monkeypatch):
+    monkeypatch.delenv("LENS_CONTROL_DIR", raising=False)
+    assert updates.restore_on_start(db, cfg) is None
diff --git a/fastapi_backend/tests/domain/test_upgrade_agent.py b/fastapi_backend/tests/domain/test_upgrade_agent.py
@@ -0,0 +1,101 @@
+"""The machine's upgrade agent (packaging/host/lens-upgrade.sh), run for real: it takes the request from the inbox,
+gets the release, runs install.sh and reports each step; when the new version doesn't start it goes back to the code
+that ran before, starts it, and leaves the restore request for it (docs/updates.md).
+
+install.sh is a stand-in that gets the code as the real one does and prints its lines, failing for a broken release;
+the repository is a local one with three releases."""
+
+from __future__ import annotations
+
+import pathlib
+import shutil
+import subprocess
+
+import pytest
+
+ROOT = pathlib.Path(__file__).resolve().parents[3]
+
+FAKE_INSTALL = """#!/bin/sh
+set -eu
+cd "$LENS_DIR"
+if [ "${LENS_KEEP_CODE:-0}" != 1 ]; then
+  echo "Updating Lens in $LENS_DIR..."
+  git fetch -q --depth 1 origin "$LENS_REF" && git checkout -q -f -B "$LENS_REF" FETCH_HEAD
+fi
+echo "Building and starting Lens (the first build takes a few minutes)..."
+echo "Waiting for the web app..."
+echo "$(cat VERSION) $LENS_KEEP_CODE_SEEN" >>"$LENS_DIR/../started"
+if grep -q broken VERSION; then echo "Lens didn't answer within 10 minutes." && exit 1; fi
+echo "Lens is running: http://localhost:3000/setup?code=x"
+""".replace("$LENS_KEEP_CODE_SEEN", "${LENS_KEEP_CODE:-0}")
+
+pytestmark = pytest.mark.skipif(not shutil.which("git"), reason="needs git")
+
+
+def git(cwd, *args):
+    subprocess.run(
+        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-c", "init.defaultBranch=main", *args],
+        cwd=cwd,
+        check=True,
+        capture_output=True,
+    )
+
+
+@pytest.fixture
+def lens(tmp_path):
+    """An install on v0.6.0 whose origin also has v0.7.0 (works) and v0.8.0 (doesn't start)."""
+    origin = tmp_path / "origin"
+    (origin / "packaging" / "host").mkdir(parents=True)
+    git(origin, "init", "-q")
+    shutil.copy(ROOT / "packaging" / "host" / "lens-upgrade.sh", origin / "packaging" / "host")
+    (origin / "install.sh").write_text(FAKE_INSTALL)
+    (origin / "fastapi_backend").mkdir()
+    (origin / ".gitignore").write_text(".lens-control/\n.env\n")
+    for v in ("0.6.0", "0.7.0", "0.8.0-broken"):
+        (origin / "VERSION").write_text(v)
+        (origin / "fastapi_backend" / "pyproject.toml").write_text(f'[project]\nversion = "{v.split("-")[0]}"\n')
+        git(origin, "add", "-A")
+        git(origin, "commit", "-q", "-m", v)
+        git(origin, "tag", "v" + v.split("-")[0])
+    lens = tmp_path / "lens"
+    git(tmp_path, "clone", "-q", "--branch", "v0.6.0", str(origin), str(lens))
+    git(lens, "checkout", "-q", "-B", "v0.6.0")  # as install.sh leaves a release
+    (lens / ".lens-control" / "inbox").mkdir(parents=True)
+    return lens
+
+
+def upgrade(lens, ref, backup="before-upgrade-20261010-120000.surql.gz"):
+    ctl = lens / ".lens-control"
+    (ctl / "inbox" / "upgrade").write_text(f"id=r1\nref={ref}\nbackup={backup}\nby=t\n")
+    subprocess.run(["sh", str(lens / "packaging" / "host" / "lens-upgrade.sh"), str(lens)], check=True, timeout=60)
+    status = dict(line.split("=", 1) for line in (ctl / "upgrade-status").read_text().splitlines())
+    return status, (ctl / "upgrade.log").read_text()
+
+
+def test_a_release_is_installed(lens):
+    status, log = upgrade(lens, "v0.7.0")
+    assert status["state"] == "done" and status["version"] == "0.7.0"
+    assert (lens / "VERSION").read_text() == "0.7.0"
+    assert "Lens is running" in log
+    assert not (lens / ".lens-control" / "inbox" / "restore").exists()
+    assert not (lens / ".lens-control" / "inbox" / "upgrade").exists()
+
+
+def test_a_release_that_doesnt_start_is_rolled_back(lens):
+    status, log = upgrade(lens, "v0.8.0")
+    assert status["state"] == "rolled-back", log
+    assert status["version"] == "0.6.0" and "didn't start" in status["error"]
+    assert status["backup"] == "before-upgrade-20261010-120000.surql.gz"
+    assert (lens / "VERSION").read_text() == "0.6.0"
+    # the broken one was started, then the previous one again from the code already there
+    assert (lens.parent / "started").read_text().splitlines() == ["0.8.0-broken 0", "0.6.0 1"]
+    restore = (lens / ".lens-control" / "inbox" / "restore").read_text()
+    assert restore == "id=r1\nbackup=before-upgrade-20261010-120000.surql.gz\n"
+    # and a later upgrade from there works
+    assert upgrade(lens, "v0.7.0")[0]["state"] == "done"
+
+
+def test_only_releases_and_this_installs_branch(lens):
+    status, log = upgrade(lens, "evil-branch")
+    assert status["state"] == "failed" and "only a release tag" in status["error"]
+    assert (lens / "VERSION").read_text() == "0.6.0"
diff --git a/install.sh b/install.sh
@@ -31,6 +31,7 @@
 #                  first start builds everything, else 600)
 #   LENS_WIZARD    1: answer everything in the web installer, in the browser (the default for a first install at a
 #                  terminal, where Python 3 is there); 0: here in the terminal
+#   LENS_KEEP_CODE 1: build and start the code already in LENS_DIR, without getting it (the upgrade agent's rollback)
 set -eu
 
 # All in a function, so nothing runs until the whole script has arrived (curl | sh streams it).
@@ -285,7 +286,9 @@ fi
 # --- The code -----------------------------------------------------------------------------------------------------
 auth_url="https://github.com/$REPO.git"
 [ -n "${GITHUB_TOKEN:-}" ] && auth_url="https://x-access-token:$GITHUB_TOKEN@github.com/$REPO.git"
-if [ -d "$DIR/.git" ]; then
+if [ "${LENS_KEEP_CODE:-0}" = 1 ]; then
+  [ -f "$DIR/install.sh" ] || fail "LENS_KEEP_CODE=1, but there is no Lens in $DIR."
+elif [ -d "$DIR/.git" ]; then
   say "Updating Lens in $DIR..."
   git -C "$DIR" fetch -q --depth 1 "$auth_url" "$REF" && git -C "$DIR" checkout -q -B "$REF" FETCH_HEAD
 elif have git; then