# Copyright (c) 2023 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import librosa
import torch
import numpy as np
def extract_mstft(
audio_ref,
audio_deg,
fs=None,
mid_freq=None,
high_freq=None,
method="cut",
version="pwg",
):
"""Compute Multi-Scale STFT Distance (mstft) between the predicted and the ground truth audio.
audio_ref: path to the ground truth audio.
audio_deg: path to the predicted audio.
fs: sampling rate.
med_freq: division frequency for mid frequency parts.
high_freq: division frequency for high frequency parts.
method: "dtw" will use dtw algorithm to align the length of the ground truth and predicted audio.
"cut" will cut both audios into a same length according to the one with the shorter length.
version: "pwg" will use the computational version provided by ParallelWaveGAN.
"encodec" will use the computational version provided by Encodec.
"""
# Load audio
if fs != None:
audio_ref, _ = librosa.load(audio_ref, sr=fs)
audio_deg, _ = librosa.load(audio_deg, sr=fs)
else:
audio_ref, fs = librosa.load(audio_ref)
audio_deg, fs = librosa.load(audio_deg)
# Automatically choose mid_freq and high_freq if they are not given
if mid_freq == None:
mid_freq = fs // 6
if high_freq == None:
high_freq = fs // 3
# Audio length alignment
if len(audio_ref) != len(audio_deg):
if method == "cut":
length = min(len(audio_ref), len(audio_deg))
audio_ref = audio_ref[:length]
audio_deg = audio_deg[:length]
elif method == "dtw":
_, wp = librosa.sequence.dtw(audio_ref, audio_deg, backtrack=True)
audio_ref_new = []
audio_deg_new = []
for i in range(wp.shape[0]):
ref_index = wp[i][0]
deg_index = wp[i][1]
audio_ref_new.append(audio_ref[ref_index])
audio_deg_new.append(audio_deg[deg_index])
audio_ref = np.array(audio_ref_new)
audio_deg = np.array(audio_deg_new)
assert len(audio_ref) == len(audio_deg)
# Define loss function
l1Loss = torch.nn.L1Loss(reduction="mean")
l2Loss = torch.nn.MSELoss(reduction="mean")
# Compute distance
if version == "encodec":
n_fft = 1024
mstft = 0
mstft_low = 0
mstft_mid = 0
mstft_high = 0
freq_resolution = fs / n_fft
mid_freq_index = 1 + int(np.floor(mid_freq / freq_resolution))
high_freq_index = 1 + int(np.floor(high_freq / freq_resolution))
for i in range(5, 11):
hop_length = 2**i // 4
win_length = 2**i
spec_ref = librosa.stft(
y=audio_ref, n_fft=n_fft, hop_length=hop_length, win_length=win_length
)
spec_deg = librosa.stft(
y=audio_deg, n_fft=n_fft, hop_length=hop_length, win_length=win_length
)
mag_ref = np.abs(spec_ref)
mag_deg = np.abs(spec_deg)
mag_ref = torch.from_numpy(mag_ref)
mag_deg = torch.from_numpy(mag_deg)
mstft += l1Loss(mag_ref, mag_deg) + l2Loss(mag_ref, mag_deg)
mag_ref_low = mag_ref[:mid_freq_index, :]
mag_deg_low = mag_deg[:mid_freq_index, :]
mstft_low += l1Loss(mag_ref_low, mag_deg_low) + l2Loss(
mag_ref_low, mag_deg_low
)
mag_ref_mid = mag_ref[mid_freq_index:high_freq_index, :]
mag_deg_mid = mag_deg[mid_freq_index:high_freq_index, :]
mstft_mid += l1Loss(mag_ref_mid, mag_deg_mid) + l2Loss(
mag_ref_mid, mag_deg_mid
)
mag_ref_high = mag_ref[high_freq_index:, :]
mag_deg_high = mag_deg[high_freq_index:, :]
mstft_high += l1Loss(mag_ref_high, mag_deg_high) + l2Loss(
mag_ref_high, mag_deg_high
)
mstft /= 6
mstft_low /= 6
mstft_mid /= 6
mstft_high /= 6
return mstft
elif version == "pwg":
fft_sizes = [1024, 2048, 512]
hop_sizes = [120, 240, 50]
win_sizes = [600, 1200, 240]
audio_ref = torch.from_numpy(audio_ref)
audio_deg = torch.from_numpy(audio_deg)
mstft_sc = 0
mstft_sc_low = 0
mstft_sc_mid = 0
mstft_sc_high = 0
mstft_mag = 0
mstft_mag_low = 0
mstft_mag_mid = 0
mstft_mag_high = 0
for n_fft, hop_length, win_length in zip(fft_sizes, hop_sizes, win_sizes):
spec_ref = torch.stft(
audio_ref, n_fft, hop_length, win_length, return_complex=False
)
spec_deg = torch.stft(
audio_deg, n_fft, hop_length, win_length, return_complex=False
)
real_ref = spec_ref[..., 0]
imag_ref = spec_ref[..., 1]
real_deg = spec_deg[..., 0]
imag_deg = spec_deg[..., 1]
mag_ref = torch.sqrt(
torch.clamp(real_ref**2 + imag_ref**2, min=1e-7)
).transpose(1, 0)
mag_deg = torch.sqrt(
torch.clamp(real_deg**2 + imag_deg**2, min=1e-7)
).transpose(1, 0)
sc_loss = torch.norm(mag_ref - mag_deg, p="fro") / torch.norm(
mag_ref, p="fro"
)
mag_loss = l1Loss(torch.log(mag_ref), torch.log(mag_deg))
mstft_sc += sc_loss
mstft_mag += mag_loss
freq_resolution = fs / n_fft
mid_freq_index = 1 + int(np.floor(mid_freq / freq_resolution))
high_freq_index = 1 + int(np.floor(high_freq / freq_resolution))
mag_ref_low = mag_ref[:, :mid_freq_index]
mag_deg_low = mag_deg[:, :mid_freq_index]
sc_loss_low = torch.norm(mag_ref_low - mag_deg_low, p="fro") / torch.norm(
mag_ref_low, p="fro"
)
mag_loss_low = l1Loss(torch.log(mag_ref_low), torch.log(mag_deg_low))
mstft_sc_low += sc_loss_low
mstft_mag_low += mag_loss_low
mag_ref_mid = mag_ref[:, mid_freq_index:high_freq_index]
mag_deg_mid = mag_deg[:, mid_freq_index:high_freq_index]
sc_loss_mid = torch.norm(mag_ref_mid - mag_deg_mid, p="fro") / torch.norm(
mag_ref_mid, p="fro"
)
mag_loss_mid = l1Loss(torch.log(mag_ref_mid), torch.log(mag_deg_mid))
mstft_sc_mid += sc_loss_mid
mstft_mag_mid += mag_loss_mid
mag_ref_high = mag_ref[:, high_freq_index:]
mag_deg_high = mag_deg[:, high_freq_index:]
sc_loss_high = torch.norm(
mag_ref_high - mag_deg_high, p="fro"
) / torch.norm(mag_ref_high, p="fro")
mag_loss_high = l1Loss(torch.log(mag_ref_high), torch.log(mag_deg_high))
mstft_sc_high += sc_loss_high
mstft_mag_high += mag_loss_high
# Normalize distances
mstft_sc /= len(fft_sizes)
mstft_sc_low /= len(fft_sizes)
mstft_sc_mid /= len(fft_sizes)
mstft_sc_high /= len(fft_sizes)
mstft_mag /= len(fft_sizes)
mstft_mag_low /= len(fft_sizes)
mstft_mag_mid /= len(fft_sizes)
mstft_mag_high /= len(fft_sizes)
# return (
# mstft_sc.numpy().tolist(),
# mstft_sc_low.numpy().tolist(),
# mstft_sc_mid.numpy().tolist(),
# mstft_sc_high.numpy().tolist(),
# mstft_mag.numpy().tolist(),
# mstft_mag_low.numpy().tolist(),
# mstft_mag_mid.numpy().tolist(),
# mstft_mag_high.numpy().tolist(),
# )
return mstft_sc.numpy().tolist() + mstft_mag.numpy().tolist()