8000 feat(documents): read camera RAW and HEIC photos with anytopdf by adeelahmad · Pull Request #233 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version

## Unreleased

- **Camera RAW and HEIC photos.** DNG, CR2, CR3, NEF, ARW, ORF, RAF, RW2, HEIC and HEIF can be uploaded and scanned
when anytopdf is here or a conversion node is set: anytopdf reads them, and RAW photos get the new Camera RAW photo
content type. Without anytopdf, Lens says what reading them needs ([anytopdf](docs/configuration.md#anytopdf)).
- **Capture many web pages at once.** Import › Web page › Many pages at once takes pasted links, a browser's
bookmark export or Chrome's Bookmarks file. Each link is captured like a single page, and its bookmark folders
become its tags. Links already captured in the namespace are left as they are
Expand Down
6 changes: 6 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,12 @@ that makes documents, photos and media into searchable PDFs. Lens uses it as a c
to 1 MB at most and its PDF to 2 GB, and a PDF that doesn't start like one is refused. **Check it** in Settings →
Documents asks the node for a job with Lens's token and says whether it answered and took it.

**Camera RAW and HEIC photos** (.dng, .cr2, .cr3, .nef, .arw, .orf, .raf, .rw2, .heic, .heif) are photos Lens can't
open itself, so anytopdf reads them wherever it is here or a node is set, whatever `documents.converter` says but
`lens`: RAW from the preview the camera embedded, HEIC where libheif or ImageMagick is beside anytopdf. Without
anytopdf, uploading one says what it needs, and folder scans pass them by. A RAW photo gets the Camera RAW photo
content type.

It runs with no network, no runtime plugins and no config file of its own, and its PDF has no provenance page and
holds only the document's text, so what Lens reads is what the document says. Faces, objects and speech stay Lens's
own steps. The resource's rendition says `anytopdf` made it.
Expand Down
2 changes: 2 additions & 0 deletions fastapi_backend/app/domain/content_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
("image", "image", "Image", "Any picture", {}),
("scan", "image", "Scanned page", "A page from a scanner or a phone", {"pattern": r"\bscan(s|ned|ning)?\b"}),
("photo", "image", "Photo", "A photograph", {"extensions": [".jpg", ".jpeg", ".heic"]}),
("raw_photo", "image", "Camera RAW photo", "Straight from a camera's sensor: DNG, CR2, NEF, ARW and the like",
{"extensions": [".arw", ".cr2", ".cr3", ".dng", ".nef", ".orf", ".raf", ".rw2"]}),
("text", "text", "Text", "Any text", {}),
("chat_export", "text", "Chat export", "A WhatsApp, Telegram, Slack or iMessage chat, by who said it", {"forms": ["chat"]}),
("structured", "text", "Records", "JSON or JSON Lines data, one record at a time", {"forms": ["records"]}),
Expand Down
12 changes: 10 additions & 2 deletions fastapi_backend/app/domain/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,10 @@ def needs(name):

def word(name):
"""What a document is, for people: "Word document", "email", "PDF"."""
return WORDS.get(ext_of(name), "document")
ext = ext_of(name)
if ext in store.CAMERA_EXT:
return "photo"
return WORDS.get(ext, "document")


def content_type(name):
Expand Down Expand Up @@ -144,6 +147,11 @@ def capabilities(cfg):
def unavailable(cfg, name):
"""Why this server can't read this document (None when it can): what it would need."""
ext = ext_of(name)
if ext in store.CAMERA_EXT:
if anytopdf.mode(cfg) == "lens" or not anytopdf.available(cfg):
kind = "camera RAW" if ext in store.RAW_EXT else "HEIC"
return f"reading {kind} photos needs anytopdf (Settings › Documents › Make PDFs with anytopdf, or a conversion node)"
return None
if ext not in store.DOCUMENT_EXT or ext == ".pdf":
return None
can = capabilities(cfg)
Expand Down Expand Up @@ -799,7 +807,7 @@ def keep_attachments(db, cfg, rid, attachments, say):
continue
ext = ext_of(f["name"])
kind = documents.kind_of(f["name"]) or ("audio" if ext in {e.lower() for e in cfg["audio"]["extensions"]} else None)
if not kind or (kind == "document" and unavailable(cfg, f["name"])):
if not kind or unavailable(cfg, f["name"]):
continue
made += _resource_of(db, cfg, rid, rec, f, kind)
if attachments[MAX_ATTACHMENTS:]:
Expand Down
11 changes: 11 additions & 0 deletions fastapi_backend/app/domain/documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
R = store.R
TYPES = {
".pdf": "application/pdf",
".heic": "image/heic",
".heif": "image/heif",
".dng": "image/x-adobe-dng",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
Expand Down Expand Up @@ -469,6 +472,14 @@ def transcribe(db, cfg, rid, say):
except convert.Unavailable as e:
raise ValueError(f"this {convert.word(path)} can't be read here: {e}") from None
say(f"made into a PDF by {BY[learnt['by']]}")
elif source == "image" and convert.ext_of(path) in store.CAMERA_EXT:
# a camera RAW or HEIC photo: Lens can't open it, anytopdf can; its PDF is the photo's rendition
why = convert.unavailable(cfg, path)
if why:
raise ValueError(f"this photo can't be read here: {why}")
learnt = {"by": anytopdf.to_pdf(cfg, path, convert.rendition_path(cfg, rid))}
pdf = convert.rendition_path(cfg, rid)
say("made into a PDF by anytopdf")
elif source == "image" and anytopdf.mode(cfg) == "anytopdf" and anytopdf.available(cfg):
# anytopdf flattens a photographed page and reads it: its PDF is the image's rendition, read like a document's
try:
Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/app/domain/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -582,7 +582,7 @@ def file_kind(cfg, name, documents_as="document"):
return "transcript"
return None if convert.unavailable(cfg, name) else "document"
if ext in store.IMAGE_EXT:
return "image"
return None if convert.unavailable(cfg, name) else "image" # a RAW photo needs anytopdf
return "transcript" if ext in TRANSCRIPT_EXT else None


Expand Down
5 changes: 4 additions & 1 deletion fastapi_backend/app/domain/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,10 @@ def labels():
EMAIL_EXT = (".eml", ".msg")
DOCUMENT_EXT = (".pdf", *OFFICE_EXT, *TEXT_EXT, *PAGE_EXT, *EMAIL_EXT)
STRUCTURED_EXT = (".json", ".jsonl") # read as records, a chat or a transcript (domain/records.py)
IMAGE_EXT = (".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp")
RAW_EXT = (".arw", ".cr2", ".cr3", ".dng", ".nef", ".orf", ".raf", ".rw2") # camera RAW
# photos Pillow can't open: anytopdf reads them (RAW from its embedded preview; HEIC where libheif is beside it)
CAMERA_EXT = (*RAW_EXT, ".heic", ".heif")
IMAGE_EXT = (".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", *CAMERA_EXT)
DEFAULTS = {
"data_dir": "./archive-data",
"database": {"url": None, "namespace": "archive", "database": "main", "user": "root", "password": "root"},
Expand Down
21 changes: 21 additions & 0 deletions fastapi_backend/tests/api/test_anytopdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,27 @@ def test_with_anytopdf_chosen_it_reads_images_too(here, client, env, db, cfg, fo
assert any("read without anytopdf: anytopdf couldn't read it (error: it broke)" in x for x in log), log


def test_camera_raw_photos_are_read_by_anytopdf(here, client, env, db, cfg):
"""A RAW photo is one Lens can't open: anytopdf reads it, whatever documents.converter says but `lens`."""
from app.domain import content_types

he, ha = env["he"], env["ha"]
up = _upload(client, he, b"NEF raw bytes", "DSC_0042.nef")
drain(db, settings.effective(db, cfg))
rec = _rec(db, up["recording"])
assert (rec["source"], rec["status"], rec["rendition"]) == ("image", "analyzed", {"from": ".nef", "by": "anytopdf"}), rec.get("error")
assert "Photographed page document" in _text(db, up["recording"])
assert "--scan-mode" not in fake_anytopdf.runs(here)[-1]["args"] # a photo, not a page to flatten
assert content_types.of_recording(db, up["recording"])[0]["key"] == "raw_photo"

# without anytopdf (or with Lens's converters chosen) a RAW photo isn't taken, and says what it needs
assert client.put("/api/v1/settings/documents", headers=ha, json={"converter": "lens"}).status_code == 200
r = _start(client, he, b"x" * 10, "DSC_0043.nef")
assert r.status_code == 400 and "camera RAW photos needs anytopdf" in r.json()["detail"], r.text
r = _start(client, he, b"x" * 10, "IMG_0001.heic")
assert r.status_code == 400 and "HEIC photos needs anytopdf" in r.json()["detail"], r.text


def test_with_a_browser_and_poppler_here_it_draws_web_pages(here, client, env, db, cfg, monkeypatch):
from app.domain import documents

Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/tests/fake_anytopdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def face_of(path):
def text_of(name, raw):
"""What the fake reads in a file: an image's name, a page of HTML's text, else its text."""
name = str(name)
if pathlib.Path(name).suffix.lower() in (".png", ".jpg", ".jpeg"):
if pathlib.Path(name).suffix.lower() in (".png", ".jpg", ".jpeg", ".nef", ".dng", ".heic"): # a camera RAW too
return [f"Photographed page {pathlib.Path(name).stem}"]
if pathlib.Path(name).suffix.lower() == ".docx": # its paragraphs, as LibreOffice on the node would read them
import io
Expand Down
7 changes: 6 additions & 1 deletion nextjs-frontend/components/import/files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ const TEXT_DOC_EXT = [".txt", ".text", ".md", ".markdown", ".mdx"];
const PAGE_EXT = [".html", ".htm"];
const EMAIL_EXT = [".eml", ".msg"];
export const DOCUMENT_EXT = [".pdf", ...OFFICE_EXT, ...TEXT_DOC_EXT, ...PAGE_EXT, ...EMAIL_EXT];
export const IMAGE_EXT = [".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp"];
/** Camera RAW and HEIC photos (the backend's CAMERA_EXT): read by anytopdf, so the server takes them only with it. */
const CAMERA_EXT = [".arw", ".cr2", ".cr3", ".dng", ".nef", ".orf", ".raf", ".rw2", ".heic", ".heif"];
export const IMAGE_EXT = [".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ...CAMERA_EXT];
/** What a document is, in words (the backend's convert.WORDS). */
const DOCUMENT_NAME: Record<string, string> = {
".pdf": "PDF document",
Expand Down Expand Up @@ -85,6 +87,9 @@ export const DEFAULT_LIMITS: UploadLimits = {
".webm",
".webp",
...DOCUMENT_EXT,
...CAMERA_EXT,
".json",
".jsonl",
].sort(),
chunk_mb: 8,
transcript_mb: 50,
Expand Down
21 changes: 20 additions & 1 deletion nextjs-frontend/components/sources/source-model.ts
91E6
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,26 @@ const DOCUMENT = [
".eml",
".msg",
];
const IMAGE = [".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp"];
const IMAGE = [
".jpg",
".jpeg",
".png",
".tif",
".tiff",
".webp",
".gif",
".bmp",
".arw",
".cr2",
".cr3",
".dng",
".nef",
".orf",
".raf",
".rw2",
".heic",
".heif",
];

export type SourceFileKind = "audio" | "transcript" | "document" | "image" | "other";

Expand Down
Loading
0