8000 fix(sources): let Google Drive sign in with a service account key file by adeelahmad · Pull Request #266 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/processing.md
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@ files, then Import (`POST /api/v1/import/source`). The listing marks files that
- Credentials are encrypted in the database (AES-GCM, key from `ARCHIVE_SECRET_KEY` or `data_dir/secret.key`) and given
to rclone in a private temporary config file per call. For Dropbox, Drive and OneDrive, paste the token from
`rclone authorize dropbox` (or drive, onedrive); tokens rclone refreshes are saved back.
- Google Drive can also sign in as a service account: choose Service account and upload its JSON key file
(`service_account_credentials`). Share the folders Lens should read with the account's address, or give a shared
drive ID (`team_drive`); in Google Workspace, `impersonate` reads a user's own Drive through domain-wide delegation.
- Folders on this machine can only be watched inside `sources.local_roots`, and the rclone binary can only be set in
archive.yaml: the web app can neither open up the server's disk nor choose what runs.

Expand Down
22 changes: 18 additions & 4 deletions fastapi_backend/app/domain/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
"dropbox": {"label": "Dropbox", "fields": {}, "secrets": ["token"], "oauth": "rclone authorize dropbox"},
"drive": {
"label": "Google Drive",
"fields": {"scope": "drive.readonly", "root_folder_id": ""},
"secrets": ["token"],
"fields": {"scope": "drive.readonly", "root_folder_id": "", "team_drive": "", "impersonate": ""},
"secrets": ["token", "service_account_credentials"], # signs in with a token, or as a service account
"oauth": "rclone authorize drive",
},
"onedrive": {
Expand Down Expand Up @@ -314,10 +314,24 @@ def _check_params(typ, params, secret_values):
return spec


def _secret(k, v):
"""A secret as it's stored: a service account's key file is checked and kept on one line, as rclone reads it."""
if k != "service_account_credentials":
return str(v)
try:
key = json.loads(v)
except ValueError:
raise ValueError("the service account key isn't JSON: upload the .json key file Google Cloud gave you") from None
if not isinstance(key, dict) or key.get("type") != "service_account" or not key.get("client_email") or not key.get("private_key"):
raise ValueError("that isn't a service account key: it needs type service_account, client_email and private_key")
return json.dumps(key, separators=(",", ":"))


def create(db, cfg, name, typ, params=None, secret_values=None, user=None):
spec = _check_params(typ, params, secret_values)
plain = {k: _secret(k, v) for k, v in (secret_values or {}).items() if isinstance(v, str) and v}
sid = db.next_id("storage_source")
sealed = {k: settings.seal(cfg, str(v), f"source:{sid}:{k}") for k, v in (secret_values or {}).items() if isinstance(v, str) and v}
sealed = {k: settings.seal(cfg, v, f"source:{sid}:{k}") for k, v in plain.items()}
db.q(
"CREATE $r CONTENT $d",
r=R("storage_source", sid),
Expand All @@ -343,7 +357,7 @@ def update(db, cfg, sid, name=None, params=None, secret_values=None):
if v is None or v == "":
sealed.pop(k, None)
elif isinstance(v, str):
sealed[k] = settings.seal(cfg, v, f"source:{sid}:{k}")
sealed[k] = settings.seal(cfg, _secret(k, v), f"source:{sid}:{k}")
db.q(
"UPDATE $r MERGE $d",
r=R("storage_source", sid),
Expand Down
37 changes: 37 additions & 0 deletions fastapi_backend/tests/api/test_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,40 @@ def test_importing_chosen_files_of_a_source(client, new_client, db, cfg, folder,
later = client.post(url, json={**body, "paths": [str(inbox / "later.txt")], "collection": shelf}, headers=h).json()["results"]
assert later[0]["status"] == "queued", later
assert db.one("SELECT collection FROM $r", r=R("recording", later[0]["recording"]))["collection"] == shelf


def test_google_drive_as_a_service_account(client, db, cfg, monkeypatch):
"""A Drive connection signs in with a service account's key file: checked, sealed, and handed to rclone on one line."""
monkeypatch.setattr(sources, "test", lambda *a, **k: {"ok": True}) # no Google to reach in tests
make_user(db, "root@x.io", "root password 1", admin=True)
h = login(client, "root@x.io", "root password 1")
assert "service_account_credentials" in client.get("/api/v1/sources/backends", headers=h).json()["drive"]["secrets"]
key = {
"type": "service_account",
"project_id": "lens-test",
"private_key": "-----BEGIN " + "PRIVATE KEY-----\nMIIEv...\n-----END " + "PRIVATE KEY-----\n", # split: not a real key
"client_email": "lens@lens-test.iam.gserviceaccount.com",
}
body = {
"name": "Drive",
"type": "drive",
"params": {"team_drive": "0ABC"},
"secrets": {"service_account_credentials": json.dumps(key, indent=2)},
}
r = client.post("/api/v1/sources", json=body, headers=h)
assert r.status_code == 200, r.text
sid = r.json()["id"]
drive = next(s for s in client.get("/api/v1/sources", headers=h).json() if s["id"] == sid)
assert drive["secrets"]["service_account_credentials"]["set"] and not drive["secrets"]["token"]["set"]
assert "lens-test.iam" not in str(db.rows("SELECT * FROM storage_source"))

_, conf = sources._config(cfg, sources.get(db, sid))
line = next(l for l in conf.splitlines() if l.startswith("service_account_credentials = "))
assert json.loads(line.split(" = ", 1)[1]) == key # one line rclone can read back, the key's own \n kept
assert "team_drive = 0ABC" in conf and "token" not in conf

for bad in ("not json", json.dumps({"type": "authorized_user", "client_email": "x", "private_key": "y"})):
r = client.post("/api/v1/sources", json={**body, "secrets": {"service_account_credentials": bad}}, headers=h)
assert r.status_code == 400 and "service account" in r.text
r = client.patch(f"/api/v1/sources/{sid}", json={"secrets": {"service_account_credentials": "{}"}}, headers=h)
assert r.status_code == 400
41 changes: 41 additions & 0 deletions nextjs-frontend/__tests__/sources-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
imapHost,
nextScan,
parseRcloneToken,
parseServiceAccountKey,
pickupText,
sourceSubtitle,
splitPatterns,
Expand Down Expand Up @@ -132,6 +133,46 @@ describe("connections", () => {
token: '{"access_token":"a"}',
});
});
it("signs Google Drive in with a token or a service account key file", () => {
const DRIVE: BackendSpec = {
label: "Google Drive",
fields: { scope: "drive.readonly", root_folder_id: "", team_drive: "", impersonate: "" },
secrets: ["token", "service_account_credentials"],
oauth: "rclone authorize drive",
};
const key = JSON.stringify({
type: "service_account",
client_email: "lens@p.iam.gserviceaccount.com",
private_key: "K",
});
expect(parseServiceAccountKey(key)).toEqual({ ok: true, email: "lens@p.iam.gserviceaccount.com" });
expect(parseServiceAccountKey 8000 ("nope").ok).toBe(false);
expect(parseServiceAccountKey('{"access_token":"a"}').ok).toBe(false);

const sa = { ...emptyForm(DRIVE), driveAuth: "service_account" as const };
expect(Object.keys(validateForm("drive", DRIVE, sa, false))).toEqual(["service_account_credentials"]);
const withKey = { ...sa, secrets: { service_account_credentials: key } };
expect(validateForm("drive", DRIVE, withKey, false)).toEqual({});
expect(validateForm("drive", DRIVE, sa, true, (k) => k === "service_account_credentials")).toEqual({});
expect(buildPayload(DRIVE, withKey, "drive").secrets).toEqual({ service_account_credentials: key, token: null });

// a pasted token replaces a saved key, and the form opens on whichever is saved
const tok = {
...emptyForm(DRIVE),
secrets: { service_account_credentials: key },
tokenText: '{"access_token":"a"}',
};
expect(buildPayload(DRIVE, tok, "drive").secrets).toEqual({
token: '{"access_token":"a"}',
service_account_credentials: null,
});
const saved = {
name: "d",
params: {},
secrets: { token: { secret: true, set: false }, service_account_credentials: { secret: true, set: true } },
};
expect(emptyForm(DRIVE, saved).driveAuth).toBe("service_account");
});
it("takes email accounts and calendar feeds", () => {
const IMAP: BackendSpec = {
label: "Email (IMAP)",
Expand Down
2 changes: 1 addition & 1 deletion nextjs-frontend/components/sources/connection-dialog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ export function ConnectionDialog({
? `Edit “${source?.name}”`
: TYPE_NAME[type] === "S3 / compatible"
? "S3 or S3-compatible"
: spec?.oauth
: spec?.oauth && type !== "drive"
? `${TYPE_NAME[type]} · paste a token`
: (spec?.label ?? TYPE_NAME[type])
: step === "test"
Expand Down
58 changes: 56 additions & 2 deletions nextjs-frontend/components/sources/connection-fields.tsx
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
type ConnForm,
type SourceType,
imapHost,
parseServiceAccountKey,
} from "@/components/sources/source-model";
import { TokenPaste } from "@/components/sources/token-paste";
import { Button } from "@/components/ui/button";
Expand Down Expand Up @@ -180,19 +181,72 @@ export function ConnectionFields({
);
case "dropbox":
return token;
case "drive":
case "drive": {
const keyText = form.secrets.service_account_credentials;
const key = keyText ? parseServiceAccountKey(keyText) : null;
const serviceAccount = (
<Field
label={FIELD_LABEL.service_account_credentials}
hint={
key?.ok
? `Signs in as ${key.email}. Share the folders Lens should read with that address.`
: editing && isSet("service_account_credentials") && keyText === undefined
? "A key is saved (never shown again). Choose a file to replace it."
: "The .json key file from Google Cloud (IAM › Service accounts › Keys). Never shown again after saving"
}
error={errors.service_account_credentials}
>
{({ id, describedBy }) => (
<input
id={id}
aria-describedby={describedBy}
type="file"
accept=".json,application/json"
className="text-[13px]"
(e) => {
const f = e.target.files?.[0];
setSecret("service_account_credentials", f ? await f.text() : undefined);
}}
/>
)}
</Field>
);
return (
<div className="flex flex-col gap-3">
{token}
<Segmented
className="w-full [&>button]:flex-1 [&>button]:justify-center"
value={form.driveAuth}
=> onChange({ ...form, driveAuth: v as ConnForm["driveAuth"] })}
items={[
{ value: "token", label: "Google account" },
{ value: "service_account", label: "Service account" },
]}
/>
{form.driveAuth === "service_account" ? serviceAccount : token}
{select("scope", DRIVE_SCOPES)}
{text("root_folder_id", {
mono: true,
label: "Root folder ID",
optional: true,
hint: "From the folder’s URL; limits the connection to that folder",
})}
{form.driveAuth === "service_account" && (
<div className="grid grid-cols-2 gap-2.5">
{text("team_drive", {
mono: true,
optional: true,
hint: "Read a shared drive the account is a member of",
})}
{text("impersonate", {
optional: true,
placeholder: "you@company.com",
hint: "Workspace only: needs domain-wide delegation",
})}
</div>
)}
</div>
);
}
case "onedrive":
return (
<div className="flex flex-col gap-3">
Expand Down
54 changes: 50 additions & 4 deletions nextjs-frontend/components/sources/source-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ export const FIELD_LABEL: Record<string, string> = {
token: "Token",
scope: "Scope",
root_folder_id: "Root folder ID",
team_drive: "Shared drive ID",
impersonate: "Act as user",
service_account_credentials: "Service account key",
drive_id: "Drive ID",
drive_type: "Drive type",
host: "Host",
Expand Down Expand Up @@ -410,8 +413,34 @@ export type ConnForm = {
sftpAuth: "pass" | "key_pem";
/** What was pasted from `rclone authorize` (OAuth types). */
tokenText: string;
/** Google Drive signs in with a token, or as a service account (its JSON key file). */
driveAuth: "token" | "service_account";
};

export type ServiceAccountKey = { ok: true; email: string } | { ok: false; error: string };

/** A Google Cloud service account key file: what it signs in as, or why it isn't one. */
export function parseServiceAccountKey(text: string): ServiceAccountKey {
let o: { type?: unknown; client_email?: unknown; private_key?: unknown } | null;
try {
o = JSON.parse(text);
} catch {
return { ok: false, error: "That file isn’t JSON. Choose the .json key file Google Cloud gave you." };
}
if (
!o ||
typeof o !== "object" ||
o.type !== "service_account" ||
typeof o.client_email !== "string" ||
!o.private_key
)
return {
ok: false,
error: "That isn’t a service account key. In Google Cloud: IAM › Service accounts › Keys › Add key › JSON.",
};
return { ok: true, email: o.client_email };
}

export function emptyForm(spec: BackendSpec, source?: Pick<Source, "name" | "params" | "secrets"> | null): ConnForm {
const params: Record<string, string> = {};
for (const [k, def] of Object.entries(spec.fields))
Expand All @@ -423,6 +452,8 @@ export function emptyForm(spec: BackendSpec, source?: Pick<Source, "name" | "par
secrets: {},
sftpAuth: keySet && !source?.secrets?.pass?.set ? "key_pem" : "pass",
tokenText: "",
driveAuth:
source?.secrets?.service_account_credentials?.set && !source?.secrets?.token?.set ? "service_account" : "token",
};
}

Expand Down Expand Up @@ -451,7 +482,14 @@ export function validateForm(
if (!str(form.params[k])) e[k] = msg;
};
const secret = (k: string) => (form.secrets[k] === undefined ? editing && isSet(k) : Boolean(form.secrets[k]));
if (spec.oauth) {
const serviceAccount = type === "drive" && form.driveAuth === "service_account";
if (serviceAccount) {
const k = form.secrets.service_account_credentials;
const key = k ? parseServiceAccountKey(k) : null;
if (key?.ok === false) e.service_account_credentials = key.error;
else if (!secret("service_account_credentials"))
e.service_account_credentials = "Choose the service account’s JSON key file.";
} else if (spec.oauth) {
const t = parseRcloneToken(form.tokenText);
if (t.ok === false) e.token = t.error;
else if (t.ok === null && !(editing && isSet("token"))) e.token = "Paste the token rclone printed.";
Expand Down Expand Up @@ -499,17 +537,25 @@ export function buildPayload(
for (const k of spec.secrets) {
if (k === "token") continue;
const v = form.secrets[k];
if (type === "sftp" && k !== form.sftpAuth) {
const other =
(type === "sftp" && k !== form.sftpAuth) ||
(type === "drive" && k === "service_account_credentials" && form.driveAuth !== "service_account");
if (other) {
// Signing in one way drops the other secret only when it was explicitly cleared.
if (v === "") secrets[k] = null;
continue;
}
if (v === undefined) continue;
secrets[k] = v === "" ? null : type === "sftp" && k === "key_pem" ? v.replace(/\r\n/g, "\n") : v;
}
if (spec.oauth) {
if (type === "drive" && form.driveAuth === "service_account") {
if (secrets.service_account_credentials) secrets.token = null; // rclone would sign in as the account anyway
} else if (spec.oauth) {
const t = parseRcloneToken(form.tokenText);
if (t.ok) secrets.token = t.token;
if (t.ok) {
secrets.token = t.token;
if (type === "drive") secrets.service_account_credentials = null; // or rclone keeps using the account
}
}
return { params, secrets };
}
Loading
0