8000 fix(ingest): stop ffmpeg, antiword and pdftotext runs that never finish by adeelahmad · Pull Request #202 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions fastapi_backend/app/domain/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,23 @@ def recorded_at(path, mtime):
return dt.datetime.fromtimestamp(mtime).isoformat(timespec="seconds")


DECODE_SECONDS = 6 * 3600 # a recording is decoded far faster than it plays; past this ffmpeg is stuck
READ_SECONDS = 300 # antiword, catdoc, pdftotext


def decode(path, channels=1):
out = subprocess.run(
try:
out = _decode(path, channels)
except subprocess.TimeoutExpired:
raise RuntimeError(f"ffmpeg: decoding took longer than {DECODE_SECONDS} s") from None
if out.returncode != 0:
raise RuntimeError("ffmpeg: " + out.stderr.decode(errors="replace")[-300:])
x = np.frombuffer(out.stdout, dtype=np.float32)
return x.reshape(-1, channels) if channels > 1 else x


def _decode(path, channels):
return subprocess.run(
[
"ffmpeg",
"-nostdin",
Expand All @@ -142,11 +157,8 @@ def decode(path, channels=1):
"-",
],
capture_output=True,
timeout=DECODE_SECONDS,
)
if out.returncode != 0:
raise RuntimeError("ffmpeg: " + out.stderr.decode(errors="replace")[-300:])
x = np.frombuffer(out.stdout, dtype=np.float32)
return x.reshape(-1, channels) if channels > 1 else x


def envelope(x, bins=1600):
Expand Down Expand Up @@ -778,7 +790,10 @@ def read_doc(path):

for cmd in (["antiword", "-w", "0", str(path)], ["catdoc", "-w", str(path)]):
if shutil.which(cmd[0]):
out = subprocess.run(cmd, capture_output=True, text=True)
try:
out = subprocess.run(cmd, capture_output=True, text=True, timeout=READ_SECONDS)
except subprocess.TimeoutExpired:
continue
if out.returncode == 0 and out.stdout.strip():
return out.stdout
office = shutil.which("soffice") or shutil.which("libreoffice")
Expand All @@ -804,7 +819,10 @@ def read_pdf(path):

if not shutil.which("pdftotext"):
raise SystemExit("reading PDFs needs pypdf (pip install pypdf) or pdftotext") from None
text = subprocess.run(["pdftotext", "-layout", str(path), "-"], capture_output=True, text=True).stdout
try:
text = subprocess.run(["pdftotext", "-layout", str(path), "-"], capture_output=True, text=True, timeout=READ_SECONDS).stdout
except subprocess.TimeoutExpired:
raise SystemExit(f"reading {path} took longer than {READ_SECONDS} s") from None
if not text.strip():
raise SystemExit(f"{path} has no text layer (a scan?); run OCR first")
return re.sub(r"(\w)-\n(\w)", r"\1\2", text)
Expand Down
55 changes: 31 additions & 24 deletions fastapi_backend/app/domain/speech.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,37 +146,44 @@ def multipart(fields, files):
return bytes(out), f"multipart/form-data; boundary={b}"


ENCODE_SECONDS = 600 # a piece sent to a speech service encodes in seconds; past this ffmpeg is stuck


def encode(audio):
"""16 kHz mono samples as a small file to send: Ogg Opus, or FLAC where ffmpeg has no Opus. (bytes, name, type)."""
pcm = np.ascontiguousarray(audio, dtype=np.float32).tobytes()
for codec, fmt, name, ctype, extra in (
("libopus", "ogg", "audio.ogg", "audio/ogg", ["-b:a", "32k"]),
("flac", "flac", "audio.flac", "audio/flac", []),
):
out = subprocess.run(
[
"ffmpeg",
"-nostdin",
"-v",
"error",
"-f",
"f32le",
"-ar",
str(SR),
"-ac",
"1",
"-i",
"-",
"-c:a",
codec,
*extra,
"-f",
fmt,
"-",
],
input=pcm,
capture_output=True,
)
try:
out = subprocess.run(
[
"ffmpeg",
"-nostdin",
"-v",
"error",
"-f",
"f32le",
"-ar",
str(SR),
"-ac",
"1",
"-i",
"-",
"-c:a",
codec,
*extra,
"-f",
fmt,
"-",
],
input=pcm,
capture_output=True,
timeout=ENCODE_SECONDS,
)
except subprocess.TimeoutExpired:
raise ProviderError(f"ffmpeg took longer than {ENCODE_SECONDS} s to encode the audio to send") from None
if out.returncode == 0 and out.stdout:
return out.stdout, name, ctype
raise ProviderError("ffmpeg couldn't encode the audio to send: " + out.stderr.decode(errors="replace")[-200:])
Expand Down
24 changes: 24 additions & 0 deletions fastapi_backend/tests/domain/test_speech.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import http.server
import json
import shutil
import sys
import threading
import urllib.parse

Expand Down Expand Up @@ -315,3 +317,25 @@ def test_words_become_segments():
segs = speech.group_words(w)
assert [len(json.loads(s["words"])) for s in segs] == [40, 5, 1] # long sentence breaks, then the pause
assert speech._short("en-US") == "en" and speech._short("english") == "en"


def test_stuck_converters_are_stopped(tmp_path, monkeypatch):
"""ffmpeg, antiword and pdftotext that never finish are given up on, so one bad file can't hold a worker."""
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
for name in ("ffmpeg", "antiword", "pdftotext"):
(bin_dir / name).write_text(f"#!/bin/sh\nexec {shutil.which('sleep')} 60\n")
(bin_dir / name).chmod(0o755)
monkeypatch.setenv("PATH", str(bin_dir)) # nothing else: no real catdoc or LibreOffice
for mod, name in ((ingest, "DECODE_SECONDS"), (ingest, "READ_SECONDS"), (speech, "ENCODE_SECONDS")):
monkeypatch.setattr(mod, name, 0.5)
with pytest.raises(RuntimeError, match="longer than"):
ingest.decode(tmp_path / "a.wav")
with pytest.raises(speech.ProviderError, match="longer than"):
speech.encode(np.zeros(1600, dtype=np.float32))
monkeypatch.setitem(sys.modules, "pypdf", None) # pdftotext is the fallback
with pytest.raises(SystemExit, match="longer than"):
ingest.read_pdf(tmp_path / "a.pdf")
(tmp_path / "a.doc").write_bytes(b"x")
with pytest.raises(SystemExit): # antiword gave up; no catdoc or LibreOffice here
ingest.read_doc(tmp_path / "a.doc")
Loading
0