Repository navigation
Expand file tree
/
Copy pathlib.rs
More file actions
208 lines (184 loc) · 7.63 KB
/
Copy pathlib.rs
File metadata and controls
208 lines (184 loc) · 7.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
//! Speech-to-text for FilmCraft's Text panel ▸ Transcript.
//!
//! - [`Transcriber`]: the trait every speech recogniser implements. It takes mono 16 kHz samples
//! and returns a [`Transcript`] whose words carry media-time bounds (`Tick`s; one 16 kHz sample
//! is exactly 15 876 000 ticks).
//! - [`models`]: the catalogue of downloadable Whisper models (source URL pinned to a revision,
//! SHA-256, size, licence), where they live in the per-user data directory, and (feature
//! `download`) the verified downloader. Weights are **never** bundled or committed.
//! - [`whisper`] (feature `whisper`): Whisper inference in pure Rust on candle, with timestamp
//! decoding, language detection and word-level timestamps from cross-attention alignment (DTW).
//! - [`diarize`]: speaker labelling by clustering per-chunk MFCC statistics (classical, no model).
//! - [`mel`]: the log-mel front end shared by Whisper and diarization.
//! - [`vad`]: energy-based tightening of word bounds (keeps pauses out of words).
//!
//! See `docs/transcripts.md` for the user-facing behaviour and accuracy numbers.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
pub mod diarize;
pub mod mel;
pub mod models;
pub mod vad;
#[cfg(feature = "whisper")]
pub mod whisper;
use filmcraft_project::Transcript;
use filmcraft_time::{TICKS_PER_SECOND, Tick};
/// The sample rate every transcriber takes.
pub const SAMPLE_RATE: u32 = 16_000;
/// Ticks per 16 kHz sample (exact).
pub const TICKS_PER_SAMPLE: i64 = TICKS_PER_SECOND / SAMPLE_RATE as i64;
/// Media time of a 16 kHz sample index.
pub fn sample_tick(n: i64) -> Tick {
Tick(n * TICKS_PER_SAMPLE)
}
/// Media time of `s` seconds (rounded to the nearest 16 kHz sample).
pub fn seconds_tick(s: f64) -> Tick {
sample_tick((s * SAMPLE_RATE as f64).round() as i64)
}
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
pub enum SpeechError {
#[error("stopped")]
Cancelled,
#[error("the speech model `{0}` is not downloaded")]
NotInstalled(String),
#[error("unknown speech model `{0}`")]
UnknownModel(String),
#[error("speech-to-text is not available in this build ({0})")]
Unavailable(String),
#[error("model: {0}")]
Model(String),
#[error("download: {0}")]
Download(String),
#[error("{0}")]
Io(String),
}
impl From<std::io::Error> for SpeechError {
fn from(e: std::io::Error) -> Self {
SpeechError::Io(e.to_string())
}
}
/// Transcription options.
#[derive(Clone, Debug, PartialEq)]
pub struct Options {
/// ISO 639-1 language code; `None` = detect from the first 30 s.
pub language: Option<String>,
/// Label speakers ([`diarize`]); otherwise every word gets speaker 0.
pub diarize: bool,
/// Upper bound for the number of speakers found.
pub max_speakers: usize,
}
impl Default for Options {
fn default() -> Self {
Self { language: None, diarize: true, max_speakers: 6 }
}
}
/// Progress callback: `(fraction 0..1, status)`; return `false` to cancel.
pub type ProgressFn<'a> = &'a mut dyn FnMut(f32, &str) -> bool;
/// A speech recogniser.
pub trait Transcriber: Send + Sync {
/// Stable id of the model (`"whisper-base"`), stored in the transcript's `source`.
fn id(&self) -> String;
/// Transcribe mono 16 kHz `audio`. Word times are relative to the first sample.
fn transcribe(&self, audio: &[f32], opts: &Options, progress: ProgressFn) -> Result<Transcript, SpeechError>;
}
/// A transcriber that returns a fixed transcript (tests, demos and agents that bring their own
/// transcript); words beyond the audio's length are dropped.
pub struct FixedTranscriber {
pub transcript: Transcript,
pub id: String,
}
impl Transcriber for FixedTranscriber {
fn id(&self) -> String {
self.id.clone()
}
fn transcribe(&self, audio: &[f32], _: &Options, progress: ProgressFn) -> Result<Transcript, SpeechError> {
if !progress(0.5, "Transcribing") {
return Err(SpeechError::Cancelled);
}
let end = sample_tick(audio.len() as i64);
let mut t = self.transcript.clone();
t.words.retain(|w| w.start < end);
t.source = self.id.clone();
progress(1.0, "Done");
Ok(t)
}
}
/// Load the transcriber for catalogue model `id` from `models_dir` (feature `whisper`).
pub fn load(models_dir: &std::path::Path, id: &str) -> Result<std::sync::Arc<dyn Transcriber>, SpeechError> {
let m = models::find(id).ok_or_else(|| SpeechError::UnknownModel(id.into()))?;
if !models::installed(models_dir, m) {
return Err(SpeechError::NotInstalled(id.into()));
}
#[cfg(feature = "whisper")]
{
Ok(std::sync::Arc::new(whisper::Whisper::load(&models::model_dir(models_dir, m), m.id)?))
}
#[cfg(not(feature = "whisper"))]
{
Err(SpeechError::Unavailable("built without the `whisper` feature".into()))
}
}
/// Whether this build can run speech models.
pub const fn available() -> bool {
cfg!(feature = "whisper")
}
/// Mix a multi-channel buffer down to mono.
pub fn downmix(channels: &[Vec<f32>]) -> Vec<f32> {
let n = channels.iter().map(Vec::len).min().unwrap_or(0);
if channels.len() == 1 {
return channels[0][..n].to_vec();
}
let k = 1.0 / channels.len().max(1) as f32;
(0..n).map(|i| channels.iter().map(|c| c[i]).sum::<f32>() * k).collect()
}
/// Word error rate between a reference and a hypothesis (words compared case-insensitively
/// without punctuation, hyphenated words split): (substitutions + deletions + insertions) / reference words.
pub fn word_error_rate(reference: &str, hypothesis: &str) -> f64 {
let norm = |s: &str| -> Vec<String> {
s.split(|c: char| c.is_whitespace() || c == '-').map(filmcraft_project::transcript::normalize_word).filter(|w| !w.is_empty()).collect()
};
let (r, h) = (norm(reference), norm(hypothesis));
if r.is_empty() {
return if h.is_empty() { 0.0 } else { 1.0 };
}
let mut prev: Vec<usize> = (0..=h.len()).collect();
for i in 1..=r.len() {
let mut cur = vec![i; h.len() + 1];
for j in 1..=h.len() {
let sub = prev[j - 1] + usize::from(r[i - 1] != h[j - 1]);
cur[j] = sub.min(prev[j] + 1).min(cur[j - 1] + 1);
}
prev = cur;
}
prev[h.len()] as f64 / r.len() as f64
}
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_project::Word;
#[test]
fn sample_ticks_are_exact() {
assert_eq!(TICKS_PER_SAMPLE * SAMPLE_RATE as i64, TICKS_PER_SECOND);
assert_eq!(seconds_tick(1.5), Tick(TICKS_PER_SECOND * 3 / 2));
}
#[test]
fn wer_counts_edits() {
assert_eq!(word_error_rate("the cat sat", "The cat, sat."), 0.0);
assert!((word_error_rate("the cat sat on the mat", "the cat sat on mat") - 1.0 / 6.0).abs() < 1e-9);
assert!((word_error_rate("a b", "a x b y") - 1.0).abs() < 1e-9);
}
#[test]
fn fixed_transcriber_trims_and_cancels() {
let mut t = Transcript::default();
t.words.push(Word::new("hi", seconds_tick(0.1), seconds_tick(0.3)));
t.words.push(Word::new("late", seconds_tick(5.0), seconds_tick(5.5)));
let f = FixedTranscriber { transcript: t, id: "fixed".into() };
let out = f.transcribe(&vec![0.0; 16_000], &Options::default(), &mut |_, _| true).unwrap();
assert_eq!(out.words.len(), 1);
assert_eq!(out.source, "fixed");
assert_eq!(f.transcribe(&[], &Options::default(), &mut |_, _| false), Err(SpeechError::Cancelled));
}
#[test]
fn downmix_averages() {
assert_eq!(downmix(&[vec![1.0, 0.0], vec![0.0, 0.0, 9.0]]), vec![0.5, 0.0]);
}
}