8000 feat(documents): export a resource or a collection as an evidence PDF by adeelahmad · Pull Request #226 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version

## Unreleased

- **Export an evidence PDF.** A resource's Export menu, and a collection's page, download it as one PDF made by
anytopdf: each file's pages (audio and video with their transcript), searchable, then a provenance page with every
file's SHA-256, so it can be handed on and checked ([Evidence PDFs](docs/api.md#evidence-pdfs)).
- **anytopdf can find faces.** Settings › Video, OCR, faces and objects › Face engine › anytopdf: its plugins find
the faces in frames and pages and describe them with the same SFace model, so a server needs no OpenCV, and the
people a namespace already knows still match ([Video](docs/video.md)). Who someone is stays in Lens.
Expand Down
20 changes: 20 additions & 0 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@ DELETE /api/v1/resources/{rid}/share
GET /api/v1/resources/{rid}/shares
DELETE /api/v1/resources/{rid}/shares/{id}
GET /api/v1/resources/{rid}/export.{fmt}
GET /api/v1/resources/{rid}/evidence.pdf
PATCH /api/v1/resources/{rid}/segments/{idx}
POST /api/v1/resources/{rid}/segments/{idx}/split
POST /api/v1/resources/{rid}/segments/{idx}/merge
Expand Down Expand Up @@ -532,6 +533,22 @@ how many pages it's on (the namespace's faces don't count those as time on scree
Manifest with a Canvas per page ([IIIF](iiif.md)), and its public page shows its pages and its text page by page
([public](#public)).

### Evidence PDFs

`GET /resources/{rid}/evidence.pdf` and `GET /collections/{cid}/evidence.pdf` are a resource, or the ones in a
collection you can read (a list in its own order, a filter's oldest first; 50 at most), as one PDF to hand on, made by
[anytopdf](configuration.md#anytopdf). Each resource goes in as its own file: a document or an image as it is, a
captured web page as captured, audio or video with Lens's transcript of it (its speakers' names on each line), and a
transcript without media as its text. The PDF is searchable PDF/A-3 with its text and chunks embedded, and its last
page is a provenance page giving every file's SHA-256 and size, so whoever gets it can check each part against the
file it came from. It holds no paths of the server (anytopdf's `share` profile), is made when asked for, and nothing
of it is kept. Anyone who can read the resources can have it (share links and signed links can't).

anytopdf runs as it does for conversions (no network, plugins or config file of its own); if it isn't here it's
fetched on first use, unless Settings › Documents says never to use it (503). A conversion node takes one file per
job and its own options: a single document goes as it is, and more go as one zip, with audio and video as their
transcripts only.

## notes

```
Expand Down Expand Up @@ -1135,8 +1152,11 @@ POST /api/v1/collections
GET /api/v1/collections/{cid}
PATCH /api/v1/collections/{cid}
DELETE /api/v1/collections/{cid}
GET /api/v1/collections/{cid}/evidence.pdf
```

`GET /{cid}/evidence.pdf` is the collection as one [evidence PDF](#evidence-pdfs).

## views

```
Expand Down
37 changes: 37 additions & 0 deletions fastapi_backend/app/api/evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""Evidence PDFs as downloads (evidence.py), for a recording or a collection."""

from __future__ import annotations

import urllib.error
from typing import Any

from fastapi import HTTPException
from fastapi.responses import Response

from app.api.streaming import LOCKED
from app.domain import anytopdf, evidence, keyring
from app.domain.store import DB

PDF: dict[int | str, dict[str, Any]] = {200: {"content": {"application/pdf": {}}}}


def download(db: DB, cfg: dict[str, Any], rids: list[int], title: str) -> Response:
try:
data = evidence.make(db, cfg, rids)
except keyring.Locked:
raise HTTPException(423, LOCKED) from None
except FileNotFoundError as e:
raise HTTPException(404, str(e)) from None
except KeyError:
raise HTTPException(404, "not found") from None
except ValueError as e: # nothing in it, too much, or anytopdf couldn't read a file
raise HTTPException(400, str(e)) from None
except (evidence.Unavailable, anytopdf.Unavailable) as e:
raise HTTPException(503, str(e)) from None
except (urllib.error.URLError, OSError, RuntimeError) as e: # anytopdf couldn't be fetched
raise HTTPException(503, f"anytopdf couldn't be fetched ({getattr(e, 'reason', e)})") from None
return Response(
data,
media_type="application/pdf",
headers={"Content-Disposition": f'attachment; filename="{evidence.file_name(title)}"', "Cache-Control": "private, no-store"},
)
17 changes: 15 additions & 2 deletions fastapi_backend/app/api/v1/routes/collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
from typing import Any

from fastapi import APIRouter, HTTPException
from fastapi.responses import Response

from app.api.deps import Acl, CurrentUser, Db, Principal, Writer
from app.domain import recsets, store
from app.api import evidence as evidence_api
from app.api.deps import Acl, Cfg, CurrentUser, Db, Principal, Writer
from app.domain import evidence, recsets, store
from app.domain.store import DB
from app.schemas.collections import Collection, CollectionCreate, CollectionDetail, CollectionUpdate
from app.schemas.common import Created, Ok
Expand Down Expand Up @@ -62,6 +64,17 @@ def get_collection(cid: int, user: CurrentUser, acl: Acl, db: Db) -> CollectionD
)


@router.get("/{cid}/evidence.pdf", response_class=Response, responses=evidence_api.PDF)
def collection_evidence(cid: int, user: CurrentUser, acl: Acl, db: Db, cfg: Cfg) -> Response:
"""The collection's recordings you can read (a list in its order, a filter's oldest first) as one evidence PDF made by anytopdf (a download): each
one's pages, then a provenance page with every file's SHA-256 and size. At most 50 recordings."""
col = own_collection(db, cid, user)
ids = recsets.members(db, col, set(acl.roles))
if len(ids) > evidence.MAX:
raise HTTPException(400, f"an evidence PDF holds {evidence.MAX} recordings at most; this collection has {len(ids)}")
return evidence_api.download(db, cfg, ids if col.get("recordings") is not None else ids[::-1], col["name"])


@router.patch("/{cid}")
def update_collection(cid: int, body: CollectionUpdate, user: Writer, db: Db) -> Ok:
"""Change your collection. A filter collection takes a new filter; a fixed one a new list of recordings."""
Expand Down
10 changes: 10 additions & 0 deletions fastapi_backend/app/api/v1/routes/recordings.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from fastapi import APIRouter, BackgroundTasks, HTTPException, Path, Query, Request
from fastapi.responses import Response

from app.api import evidence as evidence_api
from app.api.deps import Access, Acl, Cfg, CurrentUser, Db, Writer, domain_errors
from app.api.media import sign_url, sign_urls
from app.api.streaming import LOCKED, file_response, range_response
Expand Down Expand Up @@ -649,6 +650,15 @@ def export_recording(rid: int, fmt: str, acl: Acl, db: Db) -> Response:
)


@router.get("/{rid}/evidence.pdf", response_class=Response, responses=evidence_api.PDF)
def evidence_pdf(rid: int, acl: Acl, db: Db, cfg: Cfg) -> Response:
"""The resource as an evidence PDF, made by anytopdf (a download): its own file's pages (audio and video with
their transcript), searchable, then a provenance page with the file's SHA-256 and size. 503 when anytopdf can't
run here (it's fetched on first use unless Settings › Documents says never to use it)."""
rec = acl.recording(rid)
return evidence_api.download(db, cfg, [rid], rec.get("title") or f"recording {rid}")


@router.patch("/{rid}/segments/{idx}")
def edit_segment(rid: int, idx: int, body: SegmentUpdate, acl: Acl, user: Writer, db: Db) -> JobQueued:
"""Correct a transcript line (text and/or speaker); the recording is re-analysed afterwards."""
Expand Down
51 changes: 48 additions & 3 deletions fastapi_backend/app/domain/anytopdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@
MACHINES = {"amd64": "x86_64", "arm64": "arm64", "aarch64": "aarch64"}
# its search layer: the document's own text, not the dates, colours and places it would add
QUIET = ["--no-provenance-page", "--no-entities", "--colors", "off", "--location", "off"]
# an evidence PDF to hand on: the same, with its provenance page, and no paths of this server in it
EVIDENCE = ["--profile", "share", *QUIET[1:]]
POLL_SECONDS = 1.0
CHUNKS = "anytopdf-chunks.json" # what it read, embedded in each PDF it makes (anytopdf.chunks/1)
MAX_JSON = 1 << 20 # the most of a conversion node's job answer that is read
Expand Down Expand Up @@ -320,12 +322,13 @@ def _said(e):
return getattr(e, "reason", str(e))


def _remote(cfg, src, out):
"""The file sent to the conversion node as a job, waited for, and its PDF fetched."""
def _remote(cfg, src, out, name=None):
"""The file sent to the conversion node as a job (named `name`, else document.<type>), waited for, and its PDF
fetched."""
seconds = _opts(cfg).get("convert_seconds") or 300
part = out.with_suffix(".part")
try:
with _call(cfg, "POST", "/v1/jobs", src.read_bytes(), {"filename": f"document{src.suffix.lower()}"}) as r:
with _call(cfg, "POST", "/v1/jobs", src.read_bytes(), {"filename": name or f"document{src.suffix.lower()}"}) as r:
job = _job(r)
jid, end = job["job_id"], time.monotonic() + seconds
while job.get("state") not in ("succeeded", "failed"):
Expand Down Expand Up @@ -429,6 +432,48 @@ def analyze(cfg, picture, models=None, seconds=None):
return out


def evidence(cfg, inputs, transcripts, out):
"""Make `out`, one evidence PDF of the files `inputs`, in order: each one's pages, then a provenance page giving
every source's SHA-256 and size, as a PDF/A-3 with its text and chunks embedded. `transcripts` are Lens's
transcripts (.srt) of the audio and video among them, by the same name, shown as each one's timed transcript.

Here, it's one run over them all. A conversion node takes one file per job and its own options, so there a single
document goes as it is, and anything more as one zip, each audio or video as its transcript only (when it has
one) rather than the media itself."""
where = available(cfg)
if not where:
raise Unavailable("anytopdf isn't installed here, and no conversion node is set (documents.anytopdf_url)")
inputs, transcripts, out = [pathlib.Path(x) for x in inputs], [pathlib.Path(x) for x in transcripts], pathlib.Path(out)
if where == "node":
if len(inputs) == 1 and not transcripts:
return _remote(cfg, inputs[0], out, name=inputs[0].name)
import zipfile

said = {t.stem: t for t in transcripts}
with tempfile.TemporaryDirectory(prefix="lens-evidence-") as tmp:
pack = pathlib.Path(tmp) / "evidence.zip"
with zipfile.ZipFile(pack, "w", zipfile.ZIP_DEFLATED) as z:
for f in inputs:
sent = said.get(f.stem, f)
z.write(sent, sent.name)
return _remote(cfg, pack, out, name="evidence.zip")
exe, seconds = binary(cfg), (_opts(cfg).get("convert_seconds") or 300) * max(1, len(inputs))
with tempfile.TemporaryDirectory(prefix="lens-evidence-") as tmp:
made = pathlib.Path(tmp) / "evidence.pdf"
argv = [exe, "--no-plugins", "--no-config", "convert", *map(str, inputs), "-o", str(made), *EVIDENCE]
argv += ["--ocr", "off"] if (cfg.get("video") or {}).get("ocr_engine") == "none" else []
for t in transcripts:
argv += ["--transcript", str(t)]
from . import convert

r = convert._run(argv, seconds, env=_env(tmp), cwd=tmp)
if r.returncode != 0 or not made.is_file() or made.stat().st_size == 0:
said = convert._last_said(r.stderr or r.stdout)
raise ValueError(f"anytopdf couldn't make the evidence PDF ({said or f'exit {r.returncode}'})")
out.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(made), out)


def face_plugins(cfg):
"""Whether the plugins that find and describe faces are beside the program (fetch() keeps them in plugins/)."""
exe = binary(cfg)
Expand Down
101 changes: 101 additions & 0 deletions fastapi_backend/app/domain/evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
"""Evidence PDFs (docs/api.md#evidence-pdfs): a recording, or a collection of them, as one PDF to hand on, made by
anytopdf (anytopdf.py). Each resource's own file goes in as it is (a document, an image, a captured web page, the audio
or video with Lens's transcript of it), and anytopdf ends the PDF with a provenance page that gives every source's
SHA-256 and size, so whoever gets it can check each page against the file it came from. Its text is searchable and its
chunks are embedded (PDF/A-3), so what an agent cites from it stays cited.

A resource without a file of its own (an imported transcript) goes in as its text. Nothing is kept: the PDF is made
when asked for, from plain copies in a folder of its own that is removed straight after.
"""

from __future__ import annotations

import pathlib
import re
import shutil
import tempfile

from . import anytopdf, ingest, keyring, render, store

R = store.R
MAX = 50 # recordings in one evidence PDF, at most


class Unavailable(RuntimeError):
"""Evidence PDFs can't be made here: anytopdf isn't allowed (documents.converter lens) or has no build for it."""


def _name(i, title):
"""A file name that keeps its place in the PDF and says what it is, with nothing a path could misread."""
plain = re.sub(r"[^\w .,()&'-]+", " ", title or "", flags=re.UNICODE)
plain = re.sub(r"\s+", " ", plain).strip(" .")[:80] or "recording"
return f"{i:02d} {plain}"


def ready(cfg, say=None):
"""Make sure anytopdf can run: here, fetched on first use unless documents.converter is `lens`."""
if anytopdf.available(cfg):
return
if anytopdf.mode(cfg) == "lens":
raise Unavailable("evidence PDFs are made by anytopdf, and Settings › Documents says never to use it")
if not anytopdf.archive():
raise Unavailable("anytopdf has no build for this server; set a conversion node in Settings › Documents")
anytopdf.fetch(cfg, say)


def _gather(db, cfg, rid, i, into):
"""Put one recording's file (and its transcript, for audio or video) into the folder: (input, transcript or None)."""
rec = db.one("SELECT space, path, remote, source, title, media FROM $r", r=R("recording", rid))
if not rec:
raise KeyError(rid)
kind, name = render.kind(rec), _name(i, rec.get("title"))
src = None
if kind != "transcript" and (rec.get("path") or rec.get("remote")):
try:
src = ingest.audio_path(db, cfg, rec)
except (FileNotFoundError, ValueError):
src = None
if not src or not pathlib.Path(src).is_file():
if kind in ("document", "image"):
raise FileNotFoundError(f"the file of “{rec.get('title') or rid}” is missing on the server")
src = None
d = render.player_data(db, rid)
if src is None: # its text is all there is
out = into / f"{name}.txt"
out.write_text(render.export_text(d, "txt") or "(no text)\n", encoding="utf-8")
return out, None
out = into / f"{name}{pathlib.PurePosixPath(str((rec.get('remote') or {}).get('path') or src)).suffix.lower()}"
shutil.copyfile(src, out)
said = None
if kind in ("audio", "video") and d["segments"]:
said = into / "transcripts" / f"{name}.srt"
said.parent.mkdir(exist_ok=True)
said.write_text(render.export_text(d, "srt"), encoding="utf-8")
return out, said


def make(db, cfg, rids):
"""The evidence PDF of these recordings, in this order, as bytes."""
rids = list(dict.fromkeys(int(r) for r in rids))
if not rids:
raise ValueError("there's nothing to put in it")
if len(rids) > MAX:
raise ValueError(f"an evidence PDF holds {MAX} recordings at most; this has {len(rids)}")
ready(cfg)
work = pathlib.Path(cfg["data_dir"]) / "tmp"
work.mkdir(parents=True, exist_ok=True)
with keyring.work(cfg), tempfile.TemporaryDirectory(dir=work, prefix="evidence-") as tmp:
into = pathlib.Path(tmp) / "in"
into.mkdir(mode=0o700)
inputs, transcripts = [], []
for i, rid in enumerate(rids, 1):
f, t = _gather(db, cfg, rid, i, into)
inputs.append(f)
transcripts += [t] if t else []
out = pathlib.Path(tmp) / "evidence.pdf"
anytopdf.evidence(cfg, inputs, transcripts, out)
return out.read_bytes()


def file_name(title):
return f"{render.slug(title)}-evidence.pdf"
Loading
Loading
0