8000 fix(jobs): read documents and images instead of transcribing them by adeelahmad · Pull Request #265 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -509,11 +509,11 @@ title when the caller may see it); an attachment file's `resource` is the resour
in the namespace (`documents.attachment_resources`). Transcribing an email again keeps and makes nothing twice.
`GET /uploads/limits` says what the server can convert (`convert: {office, pages, msg}`); an upload it can't read is
refused, saying what it needs, and the files that can be transcripts too (PDFs, Word, text and Markdown) can be imported
as transcripts instead (`POST /import`). Its pipeline's transcribe step draws its
pages and reads their text: a PDF's own text, block by block with where each block is on its page (poppler's
as transcripts instead (`POST /import`). Its pipeline's read step (its transcribe step, for a document or an image:
no speech model is fetched for one) draws its pages and reads their text: a PDF's own text, block by block with where each block is on its page (poppler's
pdftotext; pypdf without poppler, when there are no pages to look at), and by OCR (`video.ocr_engine`) the pages with
fewer than `documents.ocr_below_chars` characters of text, drawn sharper for it. An image is one page (a TIFF one per
frame), read by OCR. Diarize, shots and text on screen skip them; analyze, summarize and report run as for any text.
frame), read by OCR. Diarize is left out for them, shots and text on screen skip them; analyze, summarize and report run as for any text.
[Configuration](configuration.md#documents-and-images) has the settings.

The text becomes the resource's segments, one per block, so search, chat, entities, notes and corrections
Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/app/domain/components.py
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ def _serves(self, cfg):
detects = cfg["video"].get("object_engine") == "anytopdf" # its objects plugin runs here, node or not
faces = cfg["video"].get("face_engine") == "anytopdf"
enriches = bool((cfg.get("analysis") or {}).get("anytopdf"))
steps = {"transcribe": converts, "objects": detects, "faces": faces, "analyze": enriches}
steps = {"read": converts, "objects": detects, "faces": faces, "analyze": enriches}
return {step for step, wanted in steps.items() if wanted}

def needed(self, cfg, m):
Expand Down
40 changes: 33 additions & 7 deletions fastapi_backend/app/domain/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,20 @@ def _spec(s):
return {"type": s} if isinstance(s, str) else dict(s)


def _read(db, cfg, rid, say, spec=None):
"""A document's or an image's step: draw its pages and read their text (documents.py). It needs no speech model,
so a document never waits for one to be fetched."""
if _source(db, rid) not in ("document", "image"):
raise Skip("only documents and images are read; audio and video are transcribed")
from . import documents

return documents.transcribe(db, cfg, rid, say)


def _transcribe(db, cfg, rid, say, spec=None):
rec = db.one("SELECT source, engine, status, envelope, web, path, space, title FROM $r", r=R("recording", rid)) or {}
if rec.get("source") in ("document", "image"):
from . import documents

return documents.transcribe(db, cfg, rid, say)
if rec.get("source") in ("document", "image"): # a run queued before documents had a step of their own
return _read(db, cfg, rid, say, spec)
if rec.get("source") != "audio":
raise Skip("an imported transcript has nothing to transcribe")
if str(rec.get("engine") or "").startswith("import:") and rec.get("status") != "new" and not (spec or {}).get("force"):
Expand Down Expand Up @@ -170,6 +178,7 @@ def _describe(db, cfg, rid, say, spec=None):
VIDEO_STEPS = {"shots", "ocr", "faces", "objects", "describe"} # added after workers.steps lists were first written
STEPS = {
"transcribe": _transcribe,
"read": _read,
"diarize": _diarize,
"shots": _shots,
"ocr": _ocr,
Expand All @@ -187,17 +196,32 @@ def _describe(db, cfg, rid, say, spec=None):
}


def for_source(steps, source):
"""A pipeline's steps as they run on a resource of this source: a document or an image is read, not transcribed,
and has no voices to tell apart, so a speech or voice model is never fetched for it."""
if source not in ("document", "image"):
return steps
out = []
for s in steps:
if s.get("type") == "transcribe":
s = {**s, "type": "read"}
if s.get("type") == "diarize" or (s.get("type") == "read" and any(o.get("type") == "read" for o in out)):
continue
out.append(s)
return out


def enqueue(db, rid, steps=None, by=None, priority=0, pipeline=None, batch=None):
"""Queue a recording for these steps (default: its namespace's pipeline); one already queued or running keeps its job."""
rec = db.one("SELECT space FROM $r", r=R("recording", rid))
rec = db.one("SELECT space, source FROM $r", r=R("recording", rid))
if not rec:
raise KeyError(rid)
ref = None
if steps is None:
from . import content_types

steps, ref = pipelines.resolve(db, rec["space"], pipeline, None if pipeline else content_types.of_recording(db, rid)[0])
steps = [_spec(s) for s in steps]
steps = for_source([_spec(s) for s in steps], rec.get("source"))
if not steps or any(s.get("type") not in STEPS for s in steps):
raise ValueError(f"steps are {', '.join(STEPS)}")
live = db.values("SELECT VALUE record::id(id) FROM job WHERE recording = $r AND status IN $a", r=rid, a=ACTIVE)
Expand Down Expand Up @@ -233,7 +257,7 @@ def add_steps(db, rid, steps, by=None):
"""Run these steps on a recording after what its queued or running job still has to do, or as a new job when it
has none. Workers read a job's steps again before each step, and only finish a job whose steps are all done, so
steps added while it runs aren't missed."""
specs = [_spec(s) for s in steps]
specs = for_source([_spec(s) for s in steps], _source(db, rid))
for jid in db.values("SELECT VALUE record::id(id) FROM job WHERE recording = $r AND status IN $a", r=rid, a=ACTIVE):
if db.rows(
"UPDATE $j SET steps = array::concat(steps, $m), updated_at = $t WHERE status IN $a RETURN AFTER",
Expand Down Expand Up @@ -839,6 +863,8 @@ def __init__(self, db, cfg_fn, name=None, steps=None, log=None):
cfg_fn() # reads the saved settings now, so telemetry (if on) is set up before the first job's span
self.name = name or f"{socket.gethostname()}-{os.getpid()}"
self.can = set(steps or cfg_fn()["workers"]["steps"]) & set(STEPS)
if not steps and "transcribe" in self.can: # reading documents moved out of transcribe; older lists run it too
self.can.add("read")
if "llm" in self.can: # a workflow or a podcast needs what an llm step needs; older lists run them too
self.can |= {"workflow", "podcast"}
if not steps and "transcribe" in self.can and not self.can & VIDEO_STEPS:
Expand Down
8 changes: 5 additions & 3 deletions fastapi_backend/app/domain/pipelines.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Pipelines: named, versioned lists of steps a recording goes through. Each namespace can choose its default; without
one, recordings get the standard pipeline. Steps: transcribe, diarize, analyze, embed (passages for search by meaning,
one, recordings get the standard pipeline. Steps: transcribe (speech to text), read (a document's or an
image's pages and their text), diarize, analyze, embed (passages for search by meaning,
semantic.py), summarize, llm (a prompt template whose structured result is saved as a named output), report (built in,
or from a report template) and export (a template rendered to a file, optionally copied to a storage source) and
workflow (a workflow graph, workflows.py, that turns what the steps made into metadata). Any step can carry a
Expand All @@ -24,10 +25,11 @@
R = store.R
STANDARD = ["transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "report"]
TYPES = {
"transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "llm", "report", "export",
"transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "describe", "analyze", "embed", "summarize", "llm", "report",
"export",
"workflow",
} # fmt: skip
ASSET_STEPS = ("transcribe", "diarize", "shots", "ocr", "faces", "objects", "describe") # they make something of the media
ASSET_STEPS = ("transcribe", "read", "diarize", "shots", "ocr", "faces", "objects", "describe") # they make something of the media
KEYS = {"type", "name", "when", "template", "version", "key", "filename", "destination", "model", "force", "workflow"}
NODE_RX = re.compile(r"^[A-Za-z0-9_-]{1,40}$")
WHEN = {"min_minutes": (int, float), "max_minutes": (int, float), "source": str, "languages": list}
Expand Down
1 change: 1 addition & 0 deletions fastapi_backend/app/domain/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ def labels():
"max_attempts": 3,
"steps": [
"transcribe",
"read",
"diarize",
"shots",
"ocr",
Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/app/domain/video.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,7 @@ def _norm(text):
def step_ocr(db, cfg, rid, say):
rec = db.one("SELECT space, source, media, samples, sample_ms FROM $r", r=R("recording", rid)) or {}
if rec.get("source") in ("document", "image"):
raise jobs.Skip("its pages were read when it was transcribed")
raise jobs.Skip("its pages were read by the read step")
if (rec.get("media") or {}).get("kind") != "video":
raise jobs.Skip("it isn't a video")
engine, why = ocr_engine_why(cfg)
Expand Down
20 changes: 15 additions & 5 deletions fastapi_backend/tests/api/test_documents.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""Documents and images as resources (docs/api.md, Documents and images): uploaded like audio and video, their pages
drawn and read by the transcribe step (a PDF's own text, OCR for scans and images), then shown, listed, searched and
drawn and read by the read step (a PDF's own text, OCR for scans and images), then shown, listed, searched and
summarised by page."""

from __future__ import annotations
Expand Down Expand Up @@ -89,10 +89,10 @@ def test_a_pdf_becomes_a_document_with_its_pages_and_text(client, env, db, cfg):
assert (rec["status"], rec["engine"], rec.get("duration_ms")) == ("analyzed", "pdf", None)
assert rec["media"]["kind"] == "document" and rec["media"]["pages"] == 2 and rec["media"]["height"] == 2000
runs = _runs(client, he, up["job"])
assert runs["transcribe"] == ("done", "2 page(s), 3 block(s) of text")
assert runs["diarize"] == ("skipped", "a document has no voices")
assert runs["read"] == ("done", "2 page(s), 3 block(s) of text")
assert "transcribe" not in runs and "diarize" not in runs # read, not transcribed: no speech model is fetched
assert runs["shots"] == ("skipped", "it isn't a video")
assert runs["ocr"] == ("skipped", "its pages were read when it was transcribed")
assert runs["ocr"] == ("skipped", "its pages were read by the read step")
# summaries cite pages: the first point is on page 1, the follow-up on page 2
assert rec["summary"]["key_points"][0]["page"] == 0 and rec["summary"]["action_items"][0]["page"] == 1

Expand Down Expand Up @@ -192,7 +192,7 @@ def test_without_poppler_or_an_ocr_engine(client, env, db, cfg, monkeypatch):
assert [x["image"] for x in documents.pages(db, img["recording"])] == ["page-0001.jpg"]
assert db.rows("SELECT * FROM segment WHERE recording = $r", r=img["recording"]) == []
runs = _runs(client, he, img["job"])
assert runs["transcribe"] == ("done", "1 page(s), 0 block(s) of text")
assert runs["read"] == ("done", "1 page(s), 0 block(s) of text")
assert runs["summarize"] == ("skipped", "there's no text to summarise")
log = client.get(f"/api/v1/jobs/{img['job']}/log", headers=he).json()["lines"]
assert any("its text wasn't read: OCR is off (video.ocr_engine)" in x for x in log)
Expand Down Expand Up @@ -435,3 +435,13 @@ def watch(ns, **more):
assert watch("old", kinds="both") == [("call", "transcript"), ("ep1", "audio"), ("harbour", "transcript")]
assert watch("new") == [("call", "document"), ("ep1", "audio"), ("harbour", "document"), ("photo", "image")]
assert client.get("/api/v1/watches", headers=ha).json()[-1]["kinds"] == "all" # the default now


@POPPLER
def test_a_document_adds_to_the_graph(client, env, db, cfg):
"""A document's text is analysed like a transcript's: the people and places in it become entities in the graph."""
pdf = text_pdf([["Alice Johnson met Bob Smith at the harbour.", "Alice Johnson wrote to Microsoft."]])
up = _upload(client, env["he"], pdf, "minutes.pdf")
drain(db, cfg)
said = {m["text"] for m in db.rows("SELECT text FROM mentions WHERE recording = $r", r=up["recording"])}
assert {"Alice Johnson", "Bob Smith", "Microsoft"} <= said
2 changes: 1 addition & 1 deletion fastapi_backend/tests/api/test_enrich.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def test_fetched_for_analysis_and_only_waits_analysis(cfg):
assert c.needed(cfg, None) is bool(enrich.anytopdf.archive())
assert c.serves(cfg, None) == {"analyze"}
cfg["documents"]["converter"] = "anytopdf"
assert c.serves(cfg, None) == {"transcribe", "analyze"}
assert c.serves(cfg, None) == {"read", "analyze"}


@pytest.mark.skipif(not REAL, reason="LENS_TEST_ANYTOPDF names a real anytopdf 0.4.0")
Expand Down
23 changes: 23 additions & 0 deletions fastapi_backend/tests/api/test_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,3 +377,26 @@ def test_a_steps_list_from_before_the_video_steps_runs_them_too(db, cfg):
w = jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": [*old, "shots"]}}, name="chosen", log=said.append)
assert "ocr" not in w.can and "leaves out describe, faces, objects, ocr" in said[0]
assert jobs.Worker(db, lambda: cfg, name="mac", steps=["transcribe"], log=said.append).can == {"transcribe"} and len(said) == 1


def test_a_document_is_read_not_transcribed(db, cfg, folder):
"""The standard pipeline (or any pipeline) on a document or an image reads its pages: no transcribe or diarize
step is queued for it, so no speech or voice model is fetched and it never waits for one."""
_, _, _ = seed(db, cfg, folder)
rid = db.next_id("recording")
db.q(
"CREATE $r CONTENT {source: 'document', status: 'new', space: (SELECT VALUE space FROM recording LIMIT 1)[0]}",
r=store.R("recording", rid),
)
jid = jobs.enqueue(db, rid)
steps = [s["type"] for s in db.one("SELECT steps FROM $j", j=store.R("job", jid))["steps"]]
assert steps[0] == "read" and "transcribe" not in steps and "diarize" not in steps
assert db.values("SELECT VALUE next_step FROM job WHERE status = 'queued'") == ["read"]
# audio keeps transcribe; an explicit read on it has nothing to do
assert jobs.for_source([{"type": "transcribe"}, {"type": "diarize"}], "audio") == [{"type": "transcribe"}, {"type": "diarize"}]
assert jobs.for_source([{"type": "transcribe"}, {"type": "read"}, {"type": "report"}], "image") == [
{"type": "read"},
{"type": "report"},
]
# a worker whose steps list predates the read step runs it too
assert "read" in jobs.Worker(db, lambda: {**cfg, "workers": {**cfg["workers"], "steps": ["transcribe"]}}, name="old").can
1 change: 1 addition & 0 deletions nextjs-frontend/components/pipelines/pipeline-canvas.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import { cn } from "@/lib/utils";

const ICON: Record<string, LucideIcon> = {
transcribe: Captions,
read: ScanText,
diarize: AudioLines,
shots: Clapperboard,
ocr: ScanText,
Expand Down
1 change: 1 addition & 0 deletions nextjs-frontend/components/pipelines/pipeline-editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ import { cn } from "@/lib/utils";

const ICON: Record<string, LucideIcon> = {
transcribe: Captions,
read: ScanText,
diarize: AudioLines,
shots: Clapperboard,
ocr: ScanText,
Expand Down
10 changes: 6 additions & 4 deletions nextjs-frontend/components/pipelines/pipeline-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,19 @@ export const NEEDS: Record<string, string[]> = {
faces: ["shots"],
objects: ["shots"],
describe: ["shots"],
analyze: ["transcribe", "diarize"],
embed: ["transcribe", "describe"],
analyze: ["transcribe", "read", "diarize"],
embed: ["transcribe", "read", "describe"],
summarize: ["analyze"],
llm: ["transcribe", "diarize", "analyze"],
llm: ["transcribe", "read", "diarize", "analyze"],
report: ["analyze", "summarize", "llm"],
export: ["analyze", "summarize", "llm"],
workflow: ["transcribe", "diarize"],
workflow: ["transcribe", "read", "diarize"],
};

/** What a step makes available to later steps and templates. */
export const PROVIDES: Record<string, string> = {
transcribe: "transcript · segments[] · language",
read: "pages[] · segments[]",
diarize: "speakers[] · turns",
shots: "shots[] · frames",
ocr: "text on screen",
Expand All @@ -90,6 +91,7 @@ export const PROVIDES: Record<string, string> = {

export const DESCRIBE: Record<string, string> = {
transcribe: "Speech to text with word timings (Settings → Transcription)",
read: "Draws a document’s or an image’s pages and reads their text (Settings → Documents)",
diarize: "Who spoke when (Settings → Diarization)",
shots: "Cuts a video into shots and keyframes",
ocr: "Reads text on screen (Settings → Video)",
Expand Down
2 changes: 2 additions & 0 deletions nextjs-frontend/components/ui/loop.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { cn } from "@/lib/utils";
*/
export const STEP_TONE: Record<string, Tone> = {
transcribe: "intent",
read: "intent",
shots: "intent",
diarize: "red",
ocr: "red",
Expand All @@ -27,6 +28,7 @@ export const STEP_TONE: Record<string, Tone> = {

export const STEP_LABEL: Record<string, string> = {
transcribe: "Transcribe",
read: "Read pages",
diarize: "Diarize",
shots: "Shots",
ocr: "Text on screen",
Expand Down
Loading
0