8000 feat(storage): choose where files go in Settings → Storage and the setup wizard by adeelahmad · Pull Request #149 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
< 7FFF link rel="preload" as="image" href="https://github.githubassets.com/assets/copilot-e2b89335a66dba00.png"/>
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DEVLOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ Todo:
- [x] The file store (blobs.py, docs/storage.md): this machine or any storage connection through rclone (S3, Drive,
Dropbox, OneDrive, SFTP, SMB, WebDAV, a folder), keyring-encrypted before it leaves, optional rclone crypt; the
`files` settings, a check, the assistant's file_storage tool. Adeel said go 2026-10-05, with rclone and crypt
- [ ] Settings → Storage and the setup wizard step (pick a connection, folder, crypt, check); new installs choose
- [x] Settings → Storage and the setup wizard step (pick or add a connection, folder, crypt, check); new installs choose
- [ ] Attachments on notes in the file store; BlockSuite images and files
- [ ] Moving uploads and recordings' media to the file store (pipelines need local copies: a cache)
- [ ] A downloadable storage key (the server secret; opens every namespace that isn't a passkey vault)
Expand Down
7 changes: 4 additions & 3 deletions docs/storage.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# Where Lens keeps its files

Status: **built**: the file store (this machine, or any storage connection through rclone, optionally wrapped in rclone
crypt), its settings, a check, and the assistant's `file_storage` tool. **In progress**: Settings → Storage and the
setup wizard step, notes' attachments. **Planned**: moving uploads and recordings' media there too, and a downloadable
crypt), its settings, a check, the assistant's `file_storage` tool, Settings → Storage and the setup wizard's choice.
**In progress**: notes' attachments. **Planned**: moving uploads and recordings' media there too, and a downloadable
storage key.

The files Lens makes its own, starting with notes' attachments, go where **Settings → Storage** (the `files` section)
says:
says. The setup wizard's Storage step asks the same: on this machine, or on a storage connection, which can be added
right there (the same dialog as Sources) and is checked before it's saved.

| Setting | |
|---|---|
Expand Down
29 changes: 29 additions & 0 deletions nextjs-frontend/__tests__/file-store.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { crossErrors, FIELDS, parse } from "@/components/settings/model";
import { fileStoreTry, NOT_STORAGE } from "@/components/storage/file-store";

const spec = (key: string) => FIELDS.find((f) => f.section === "files" && f.key === key)!;

describe("where Lens keeps its own files", () => {
it("checks or saves what is chosen", () => {
expect(fileStoreTry("local", "3", "x", true)).toEqual({ store: "local", crypt: false });
expect(fileStoreTry("connection", "3", " my-bucket/lens ", 1)).toEqual({
store: "connection",
connection: 3,
folder: "my-bucket/lens",
crypt: true,
});
expect(fileStoreTry("connection", "", "", false)).toMatchObject({ connection: null });
});

it("parses the connection as an id, and asks for one", () => {
expect(parse(spec("connection"), "12")).toEqual({ value: 12 });
expect(parse(spec("connection"), "")).toEqual({ value: null });
expect(crossErrors({ "files.store": "connection", "files.connection": null })["files.connection"]).toBeTruthy();
expect(crossErrors({ "files.store": "local", "files.connection": null })["files.connection"]).toBeUndefined();
expect(crossErrors({ "files.folder": "a/../b" })["files.folder"]).toBeTruthy();
});

it("never offers email or calendars as storage", () => {
expect(NOT_STORAGE.has("imap") && NOT_STORAGE.has("ical") && !NOT_STORAGE.has("s3")).toBe(true);
});
});
77 changes: 76 additions & 1 deletion nextjs-frontend/components/settings/bodies.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,16 @@ import { PlugZap, RefreshCw, Terminal } from "lucide-react";
import Link from "next/link";
import { useEffect, useRef, type ReactNode } from "react";

import { Admin, Fedora, Metadata, Sensors } from "@/app/openapi-client";
import { Admin, Fedora, Metadata, Sensors, Sources } from "@/app/openapi-client";
import { ComponentsStatus } from "@/components/settings/components-status";
import { AllTokens } from "@/components/account/all-tokens";
import { ACCESS } from "@/components/iiif/metadata-model";
import { RIGHTS } from "@/components/iiif/rights";
import { hubText } from "@/components/sensors/sensor-model";
import { SecretSetting, SettingField, ZoneBar, type FieldState } from "@/components/settings/fields";
import { SignInProviders } from "@/components/settings/sign-in-providers";
import { AddConnection } from "@/components/storage/add-connection";
import { NOT_STORAGE, fileStoreTry } from "@/components/storage/file-store";
import {
AI_TOOLS,
SPEECH_PROVIDERS,
Expand Down Expand Up @@ -369,6 +371,8 @@ export function SectionBody({ ctx }: { ctx: BodyCtx }) {
</p>
</>
);
case "storage":
return <StorageBody ctx={ctx} />;
case "telemetry":
return <TelemetryBody ctx={ctx} />;
case "fedora":
Expand Down Expand Up @@ -566,6 +570,77 @@ function TelemetryBody({ ctx }: { ctx: BodyCtx }) {
);
}

function StorageBody({ ctx }: { ctx: BodyCtx }) {
const client = useApiClient();
const conns = useQuery({ queryKey: ["sources"], queryFn: () => data(Sources.listSources({ client })) });
const storage = (conns.data ?? []).filter((c) => !NOT_STORAGE.has(c.type));
const store = ctx.state("files.store").value;
const body = fileStoreTry(
store,
ctx.state("files.connection").value,
ctx.state("files.folder").value,
ctx.state("files.crypt").value,
);
const test = useMutation({ mutationFn: () => data(Admin.testFileStore({ client, body })) });
const saved = ctx.view.files?.values ?? {};
const savedConn = storage.find((c) => c.id === saved.connection);
return (
<>
<Banner
title={
saved.store === "connection"
? `Files go to ${savedConn?.name ?? "a connection"}${saved.folder ? `, under ${saved.folder}` : ""}${saved.crypt ? ", through rclone crypt" : ""}.`
: "Files are kept on this machine."
}
>
Each file is encrypted with its namespace’s key before it leaves the machine, so a connection only ever holds
ciphertext. Files already kept stay where they were put when this changes.
</Banner>
<F ctx={ctx} id="files.store" />
{store === "connection" && (
<>
{conns.isSuccess && !storage.length ? (
<Banner tone="error" title="No storage connection yet.">
Add one (S3, Google Drive, Dropbox, OneDrive, SFTP, SMB or WebDAV); it’s picked here once it works.
</Banner>
) : (
<F
ctx={ctx}
id="files.connection"
options={[
{ value: "", label: "Choose a connection" },
...storage.map((c) => ({ value: String(c.id), label: `${c.name} (${c.type})` })),
]}
/>
)}
<div>
<AddConnection => ctx.state("files.connection").onChange(String(id))} />
</div>
<F ctx={ctx} id="files.folder" />
<F ctx={ctx} id="files.crypt" />
</>
)}
<div className="flex flex-wrap items-center gap-2.5">
<Button size="sm" icon={<PlugZap />} => test.mutate()} disabled={test.isPending}>
{test.isPending ? "Checking…" : "Check it"}
</Button>
<span className="text-[12px] text-fg-muted">Writes a small file there, reads it back and removes it</span>
</div>
{test.data &&
(test.data.ok ? (
<Banner tone="success" title="It works.">
Written, read back and removed in {test.data.seconds} s.
</Banner>
) : (
<Banner tone="error" title="That didn’t work.">
{test.data.error}
</Banner>
))}
{test.isError && <Banner tone="error">{test.error.message}</Banner>}
</>
);
}

/** "Sent 12, 3 files, 140 unchanged" from the last sync's counts. */
export function fedoraCounts(c: Record<string, unknown> | null | undefined): string {
if (!c) return "";
Expand Down
54 changes: 54 additions & 0 deletions nextjs-frontend/components/settings/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ export type SectionId =
| "fedora"
| "sensors"
| "uploads"
| "storage"
| "documents"
| "tokens"
| "iiif"
Expand Down Expand Up @@ -249,6 +250,13 @@ export const SECTIONS: SectionSpec[] = [
description:
"Audio and video people upload in the web app: which types, how large, and how long an unfinished upload waits.",
},
{
id: "storage",
label: "Storage",
backend: ["files"],
description:
"Where Lens keeps the files it makes its own, such as notes’ attachments: this machine, or a storage connection (S3, Google Drive, Dropbox, OneDrive, SFTP, SMB, WebDAV). Files are encrypted before they leave the machine.",
},
{
id: "documents",
label: "Documents",
Expand Down Expand Up @@ -1811,6 +1819,46 @@ export const FIELDS: FieldSpec[] = [
kind: "checks",
options: UPLOAD_TYPES.map((e) => ({ value: e, label: e.slice(1) })),
},
// Storage (Lens's own files: blobs.py)
{
section: "files",
key: "store",
label: "Keep files",
kind: "cards",
options: [
{ value: "local", label: "On this machine", hint: "In the server’s data folder, under objects" },
{
value: "connection",
label: "On a storage connection",
hint: "S3, Google Drive, Dropbox, OneDrive, SFTP, SMB, WebDAV",
},
],
},
{
section: "files",
key: "connection",
label: "Connection",
kind: "select",
nullable: true,
options: [],
hint: "Connections are added under Sources",
},
{
section: "files",
key: "folder",
label: "Folder",
kind: "text",
mono: true,
placeholder: "lens",
hint: "Where on it: for S3, the bucket and a path (my-bucket/lens); for a folder on this machine, its full path",
},
{
section: "files",
key: "crypt",
label: "Also encrypt with rclone crypt",
kind: "switch",
hint: "Hides the names and sizes of the files there too. Lens makes and keeps its password; there is nothing to remember",
},
// Documents
{
section: "documents",
Expand Down Expand Up @@ -2045,6 +2093,8 @@ export function parse(f: FieldSpec, ui: unknown): Parsed {
}
case "select":
if (f.key === "frame_width") return { value: Number(ui) };
if (f.section === "files" && f.key === "connection")
return { value: ui === "" || ui == null ? null : Number(ui) };
return { value: ui === "" && f.nullable ? null : ui };
default: {
const s = String(ui ?? "").trim();
Expand Down Expand Up @@ -2115,6 +2165,10 @@ export function crossErrors(values: Record<string, unknown>): Record<string, str
e["server.embed_frame_ancestors"] = `“${badOrigin}” isn’t an origin like https://blog.example.org or 'self'`;
const types = values["uploads.extensions"] as string[] | undefined;
if (types && !types.length) e["uploads.extensions"] = "Pick at least one type";
if (values["files.store"] === "connection" && values["files.connection"] == null)
e["files.connection"] = "Choose the connection files go to";
const folder = values["files.folder"];
if (typeof folder === "string" && folder.split("/").includes("..")) e["files.folder"] = "A folder can’t contain ..";
const gaz = values["analysis.gazetteer"] as string[] | undefined;
const g = gaz ? gazetteerErrors(gaz) : null;
if (g) e["analysis.gazetteer"] = g;
Expand Down
2 changes: 2 additions & 0 deletions nextjs-frontend/components/settings/settings-shell.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
ChartNoAxesColumn,
Clapperboard,
Cpu,
Database,
HardDrive,
FileType,
Fingerprint,
Expand Down Expand Up @@ -80,6 +81,7 @@ const ICON: Record<AnySectionId, LucideIcon> = {
fedora: Archive,
sensors: RadioTower,
uploads: Upload,
storage: Database,
documents: FileType,
tokens: KeyRound,
iiif: Globe,
Expand Down
Loading
Loading
0