8000 fix(transcribe): keep the language Whisper heard by adeelahmad · Pull Request #592 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changelog.d/whisper-lang.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- **Whisper keeps the language it heard.** With no language set in Settings › Transcription, recordings Whisper
transcribes now keep the language it detected, as other engines' do, for search filters and the text cleanup.
12 changes: 8 additions & 4 deletions fastapi_backend/app/domain/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,14 +351,18 @@ def __init__(self, cfg, mlx=False):
raise EngineMissing(f"{name} isn't installed: uv sync --extra {'mlx' if mlx else 'whisper'}") from e

def transcribe(self, audio, terms=()):
"""The segments; `terms` (vocabulary.py) go in as the prompt, the names Whisper should expect."""
"""The segments, in the language set in Settings or, with none set, the one Whisper heard (so the recording's
language is kept either way); `terms` (vocabulary.py) go in as the prompt, the names Whisper should expect."""
hint = {"initial_prompt": vocabulary.prompt(terms)} if terms else {}
if self.mlx:
segs = self.m.transcribe(audio, path_or_hf_repo=self.repo, word_timestamps=True, language=self.lang, **hint)["segments"]
got = self.m.transcribe(audio, path_or_hf_repo=self.repo, word_timestamps=True, language=self.lang, **hint)
heard, segs = got.get("language"), got["segments"]
rows = [(s["start"], s["end"], s["text"], [(w["word"], w["start"], w["end"]) for w in s.get("words") or []]) for s in segs]
else:
segs, _ = self.m.transcribe(audio, vad_filter=True, word_timestamps=True, language=self.lang, **hint)
segs, info = self.m.transcribe(audio, vad_filter=True, word_timestamps=True, language=self.lang, **hint)
heard = getattr(info, "language", None)
rows = [(s.start, s.end, s.text, [(w.word, w.start, w.end) for w in (s.words or [])]) for s in segs]
lang = self.lang or (str(heard).lower() if heard else None)
return [
{
"t0": int(a * 1000),
Expand All @@ -367,7 +371,7 @@ def transcribe(self, audio, terms=()):
"raw_text": None,
"emotion": None,
"event": None,
"lang": self.lang,
"lang": lang,
"words": json.dumps([[w.strip(), int(p * 1000), int(q * 1000)] for w, p, q in ws]),
}
for a, b, x, ws in rows
Expand Down
26 changes: 26 additions & 0 deletions fastapi_backend/tests/domain/test_speech.py
Original file line number Diff line number Diff line change
Expand Up @@ -617,3 +617,29 @@ def transcribe(self, audio, terms=()):
8000 monkeypatch.setattr(ingest, "decode", lambda p: clip())
ingest.transcribe_one(db, eff, rid, log=lambda *a: None, engine=Hearing())
assert Hearing.heard == words


def test_whisper_keeps_the_language_it_heard():
"""With no language set, the recording's language is the one Whisper detected; a language set in Settings wins."""
from types import SimpleNamespace as NS

from app.domain import ingest

class Faster:
def transcribe(self, audio, **kw):
seg = NS(start=0.0, end=1.5, text=" Bonjour à tous.", words=[NS(word=" Bonjour", start=0.0, end=0.6)])
return iter([seg]), NS(language="fr", language_probability=0.97)

class Mlx:
def transcribe(self, audio, **kw):
return {"language": "de", "segments": [{"start": 0.0, "end": 1.0, "text": " Guten Tag.", "words": []}]}

def engine(m, mlx, lang):
w = object.__new__(ingest.Whisper)
w.m, w.mlx, w.repo, w.lang, w.name = m, mlx, "repo", lang, "whisper"
return w

segs = engine(Faster(), False, None).transcribe(None)
assert ([s["lang"] for s in segs], ingest._language(segs)) == (["fr"], "fr")
assert [s["lang"] for s in engine(Mlx(), True, None).transcribe(None)] == ["de"]
assert [s["lang"] for s in engine(Faster(), False, "en").transcribe(None)] == ["en"] # the setting, as before
Loading
0