Download model.py from viks66/mimi-endpointer: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/viks66/mimi-endpointer/resolve/main/model.py
- Command line
-
hf download hf://viks66/mimi-endpointer/model.py
-
curl -L -o model.py https://huggingface.co/viks66/mimi-endpointer/resolve/main/model.py
10.2 kB
| import torch | |
| import torch.nn as nn | |
| from huggingface_hub import hf_hub_download | |
| from moshi.models import loaders as moshi_loaders | |
| # Config (from ms_lstm_mimi-25hz-nq8_delay1f training run) | |
| MODEL_DEFAULTS = { | |
| "input_size": 512, # Mimi feat_size | |
| "hidden_size": 512, | |
| "num_layers": 2, | |
| "output_size": 5, | |
| "dropout": 0.1, | |
| "bidirectional": False, | |
| "project": 128, # projects 512 → 128 before LSTMs | |
| "num_project": 1, | |
| } | |
| AUDIO_DEFAULTS = { | |
| "sr": 24000, # Mimi native sample rate | |
| "num_quantizers": 8, | |
| "frame_rate_hz": 25.0, # after ×2 upsample from Mimi's native 12.5Hz | |
| } | |
| class MimiLSTM(nn.Module): | |
| """Two-stream LSTM over Mimi embeddings. | |
| Input x: (batch, 2, feat_size, T) — speaker_1 at index 0, speaker_2 at index 1. | |
| Output: (batch, T, output_size) | |
| """ | |
| def __init__(self, input_size, hidden_size, num_layers, output_size, | |
| dropout, bidirectional, project, **_): | |
| super().__init__() | |
| self.mel_embed = nn.Sequential( | |
| nn.Linear(input_size, project), | |
| nn.ReLU(), | |
| ) | |
| lstm_kwargs = dict( | |
| input_size=project, | |
| hidden_size=hidden_size, | |
| num_layers=num_layers, | |
| batch_first=True, | |
| dropout=dropout, | |
| bidirectional=bidirectional, | |
| ) | |
| self.model1 = nn.LSTM(**lstm_kwargs) | |
| self.model2 = nn.LSTM(**lstm_kwargs) | |
| self.linear = nn.Linear(hidden_size * 2, output_size) | |
| def init_hidden(self, batch_size, device): | |
| h = torch.zeros(self.model1.num_layers, batch_size, self.model1.hidden_size).to(device) | |
| c = torch.zeros(self.model1.num_layers, batch_size, self.model1.hidden_size).to(device) | |
| return h, c | |
| def infer(self, x): | |
| """Full-sequence inference. x: (1, 2, feat, T).""" | |
| x = x.permute(0, 1, 3, 2) # (1, 2, T, feat) | |
| x = self.mel_embed(x) # (1, 2, T, project) | |
| h, c = self.init_hidden(x.size(0), x.device) | |
| x1, _ = self.model1(x[:, 0], (h, c)) | |
| x2, _ = self.model2(x[:, 1], (h, c)) | |
| return self.linear(torch.cat([x1, x2], dim=-1)) # (1, T, output_size) | |
| def infer_ar_step(self, feat1, feat2, h1, c1, h2, c2): | |
| """Single-frame AR step. | |
| feat1, feat2: (1, feat_size) — one frame per speaker. | |
| Returns: logits (1, output_size), updated (h1, c1, h2, c2). | |
| """ | |
| f1 = self.mel_embed(feat1).unsqueeze(1) # (1, 1, project) | |
| f2 = self.mel_embed(feat2).unsqueeze(1) | |
| out1, (h1, c1) = self.model1(f1, (h1, c1)) # (1, 1, hidden) | |
| out2, (h2, c2) = self.model2(f2, (h2, c2)) | |
| logits = self.linear(torch.cat([out1, out2], dim=-1)).squeeze(1) # (1, output_size) | |
| return logits, h1, c1, h2, c2 | |
| def infer_ar(self, x): | |
| """Frame-by-frame AR inference. x: (1, 2, feat, T). | |
| Equivalent to infer() for a unidirectional LSTM but simulates real-time decoding. | |
| """ | |
| x = x.permute(0, 1, 3, 2) # (1, 2, T, feat) | |
| x = self.mel_embed(x) # (1, 2, T, project) | |
| T = x.size(2) | |
| h1, c1 = self.init_hidden(x.size(0), x.device) | |
| h2, c2 = self.init_hidden(x.size(0), x.device) | |
| outputs = [] | |
| for t in range(T): | |
| f1 = x[:, 0, t:t+1, :] # (1, 1, project) | |
| f2 = x[:, 1, t:t+1, :] | |
| out1, (h1, c1) = self.model1(f1, (h1, c1)) | |
| out2, (h2, c2) = self.model2(f2, (h2, c2)) | |
| outputs.append(self.linear(torch.cat([out1, out2], dim=-1))) | |
| return torch.cat(outputs, dim=1) # (1, T, output_size) | |
| class AudioFeatureExtractor: | |
| """Extracts Mimi embeddings from raw waveform using the moshi library.""" | |
| def __init__(self, sr, num_quantizers, device="cuda", **_): | |
| mimi_weight = hf_hub_download(moshi_loaders.DEFAULT_REPO, moshi_loaders.MIMI_NAME) | |
| self.mimi = moshi_loaders.get_mimi(mimi_weight, device=device, num_codebooks=num_quantizers) | |
| self.mimi.eval() | |
| self.sr = sr | |
| self.device = device | |
| def _to_tensor(self, wav): | |
| """1D numpy array → (1, 1, T) tensor on device.""" | |
| return torch.from_numpy(wav).float().to(self.device).unsqueeze(0).unsqueeze(0) | |
| def __call__(self, wav): | |
| """Full-sequence. wav: 1D numpy array. Returns (1, feat_size, T) at 25Hz.""" | |
| x = self._to_tensor(wav) | |
| emb = self.mimi.encode_to_latent(x, quantize=True) # (1, feat, T) at 12.5Hz | |
| return self.mimi.upsample(emb) # (1, feat, T) at 25Hz | |
| def stream(self, wav1, wav2): | |
| """Streaming both speakers batched together, one Mimi frame (1920 samples) at a time. | |
| Yields (1, feat_size, frames) for speaker1 and speaker2 per chunk. | |
| Single streaming context so KV cache is shared and there's no double-enter conflict. | |
| """ | |
| chunk_samples = self.mimi.frame_size # 1920 | |
| def pad(wav): | |
| t = torch.from_numpy(wav).float().to(self.device) | |
| r = len(t) % chunk_samples | |
| return torch.nn.functional.pad(t, (0, chunk_samples - r if r else 0)) | |
| w1, w2 = pad(wav1), pad(wav2) | |
| n_chunks = len(w1) // chunk_samples | |
| with self.mimi.streaming(batch_size=2): | |
| for i in range(n_chunks): | |
| s = i * chunk_samples | |
| chunk = torch.stack([w1[s:s+chunk_samples], w2[s:s+chunk_samples]]).unsqueeze(1) # (2, 1, 1920) | |
| emb = self.mimi.encode_to_latent(chunk, quantize=True) # (2, feat, 1) | |
| emb = self.mimi.upsample(emb) # (2, feat, 2) | |
| yield emb[0:1], emb[1:2] # (1, feat, 2) each | |
| def load_model(checkpoint_path, device="cpu"): | |
| model = MimiLSTM(**MODEL_DEFAULTS) | |
| ckpt = torch.load(checkpoint_path, map_location=device) | |
| model.load_state_dict(ckpt["model_state_dict"]) | |
| model.eval() | |
| return model.to(device) | |
| CLASS_NAMES = ["bos", "system_end", "user_end", "system", "user"] | |
| if __name__ == "__main__": | |
| import argparse | |
| import numpy as np | |
| import soundfile as sf | |
| import matplotlib.pyplot as plt | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--checkpoint", default=None) | |
| parser.add_argument("--sample_dir", required=True) | |
| parser.add_argument("--ar", action="store_true", help="Streaming Mimi (chunk-by-chunk) + AR LSTM step-by-step.") | |
| parser.add_argument("--out", default="eval_output.png") | |
| args = parser.parse_args() | |
| import torchaudio | |
| import numpy as np | |
| import soundfile as sf | |
| import matplotlib.pyplot as plt | |
| from pathlib import Path | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| if args.checkpoint: | |
| model = load_model(args.checkpoint, device=device) | |
| print(f"Loaded: {args.checkpoint}") | |
| else: | |
| model = MimiLSTM(**MODEL_DEFAULTS).to(device).eval() | |
| print("No checkpoint — using random weights.") | |
| extractor = AudioFeatureExtractor(**AUDIO_DEFAULTS, device=device) | |
| target_sr = AUDIO_DEFAULTS["sr"] | |
| d = Path(args.sample_dir) | |
| wav1, sr1 = sf.read(d / "speaker_1_audio.wav", dtype="float32") | |
| wav2, sr2 = sf.read(d / "speaker_2_audio.wav", dtype="float32") | |
| if sr1 != target_sr: | |
| wav1 = torchaudio.functional.resample(torch.from_numpy(wav1), sr1, target_sr).numpy() | |
| if sr2 != target_sr: | |
| wav2 = torchaudio.functional.resample(torch.from_numpy(wav2), sr2, target_sr).numpy() | |
| if args.ar: | |
| # streaming Mimi chunk-by-chunk + AR LSTM step-by-step | |
| print("Running streaming Mimi + AR LSTM...") | |
| from tqdm import tqdm | |
| h1, c1 = model.init_hidden(1, device) | |
| h2, c2 = model.init_hidden(1, device) | |
| all_logits = [] | |
| n_chunks = (len(wav1) + 1919) // 1920 | |
| with torch.no_grad(): | |
| for feat1_chunk, feat2_chunk in tqdm( | |
| extractor.stream(wav1, wav2), | |
| total=n_chunks, unit="chunk", desc="Streaming" | |
| ): | |
| feat1_chunk, feat2_chunk = feat1_chunk.to(device), feat2_chunk.to(device) | |
| for t in range(feat1_chunk.shape[-1]): | |
| logits, h1, c1, h2, c2 = model.infer_ar_step( | |
| feat1_chunk[:, :, t], feat2_chunk[:, :, t], h1, c1, h2, c2 | |
| ) | |
| all_logits.append(logits) | |
| out = torch.stack(all_logits, dim=1) # (1, T, output_size) | |
| else: | |
| feat1 = extractor(wav1) # (1, feat, T) | |
| feat2 = extractor(wav2) | |
| T = min(feat1.shape[-1], feat2.shape[-1]) | |
| x = torch.cat([feat1[:, :, :T], feat2[:, :, :T]], dim=0).unsqueeze(0) # (1, 2, feat, T) | |
| with torch.no_grad(): | |
| out = model.infer(x.to(device)) | |
| T = out.shape[1] | |
| print(f"Audio: {len(wav1)/target_sr:.1f}s, {T} frames @ {AUDIO_DEFAULTS['frame_rate_hz']} Hz") | |
| probs = torch.softmax(out[0], dim=-1).cpu().numpy() # (T, 5) | |
| frame_times = np.arange(T) / AUDIO_DEFAULTS["frame_rate_hz"] | |
| wav_times = np.arange(len(wav1)) / target_sr | |
| duration = len(wav1) / target_sr | |
| fig_width = max(28, int(duration * 0.2)) # ~0.2 inches per second | |
| fig, (ax_wav, ax_pred) = plt.subplots( | |
| 2, 1, figsize=(fig_width, 6), | |
| gridspec_kw={"hspace": 0.08, "height_ratios": [1, 1]}, | |
| ) | |
| ax_wav.plot(wav_times, wav1, linewidth=0.3, color="steelblue", alpha=0.7, label="Speaker 1") | |
| ax_wav.plot(wav_times, wav2, linewidth=0.3, color="darkorange", alpha=0.7, label="Speaker 2") | |
| ax_wav.set_ylabel("Amplitude") | |
| ax_wav.set_xlim(wav_times[0], wav_times[-1]) | |
| ax_wav.legend(loc="upper right", fontsize=8) | |
| ax_wav.set_xticklabels([]) | |
| for i, name in enumerate(CLASS_NAMES): | |
| ax_pred.plot(frame_times, probs[:, i], label=name, linewidth=1.0) | |
| ax_pred.set_ylabel("Softmax probability") | |
| ax_pred.set_xlabel("Time (s)") | |
| ax_pred.set_xlim(frame_times[0], frame_times[-1]) | |
| ax_pred.set_ylim(0, 1) | |
| ax_pred.legend(loc="upper right", fontsize=8) | |
| plt.savefig(args.out, dpi=150, bbox_inches="tight") | |
| print(f"Saved: {args.out}") | |