Corrected issues for DUET code review
MODIFY
nussl/audio_signal.py
nussl/audio_signal.py
@@ -109,13 +109,15 @@ class AudioSignal(object):
# Plotting
##################################################
- def plot_time_domain(self):
- """
- Not implemented yet -- will raise and exception
- Returns:
+ # def plot_time_domain(self):
+ # """
+ # Not implemented yet -- will raise and exception
+ # Returns:
+ #
+ # """
+ # signal = self.audio_data
+ # for channel in num_channels
- """
- raise NotImplementedError('Not ready yet!')
_NAME_STEM = 'audio_signal'
@@ -231,8 +233,8 @@ class AudioSignal(object):
@property
def stft_data(self):
- """ (:obj:`np.ndarray`): Real-valued, uncompressed, time-domain representation of the audio.
- 2D numpy array with shape `(n_channels, n_samples)`.
+ """ (:obj:`np.ndarray`): Complex-valued, time-frequency representation of the audio.
+ 2D numpy array with shape `(n_frequency_bins, n_time_bins)`.
``None`` by default, this can be initialized at instantiation.
Usually, this is expected to be floats. Some functions will convert to floats if not already.
"""
@@ -1228,7 +1230,7 @@ class AudioSignal(object):
Args:
overwrite (bool, optional): If ``True`` this function will overwrite :attr:`audio_data`.
-
+ remove_channels (bool, optional): If ``True`` this function will remove the channel index axis.
Returns:
(:obj:`np.array`): Mono-ed version of :attr:`audio_data`.
@@ -1241,23 +1243,24 @@ class AudioSignal(object):
self.audio_data = mono
return mono
- def stft_to_mono(self, overwrite=False):
- """ Converts :attr:`stft_data` to mono by averaging every sample.
+ def stft_to_one_channel(self, overwrite=False):
+ """ Converts :attr:`stft_data` to a single channel by averaging every sample.
+ Shape: stft_data.shape will be (num_freq, num_time, 1) where the last axis is the channel number
Warning:
- If overwrite=True (default) this will overwrite any data in :attr:`audio_data`!
+ If overwrite=True (default) this will overwrite any data in :attr:`stft_data`!
Args:
overwrite (bool, optional): If ``True`` this function will overwrite :attr:`stft_data`.
Returns:
- (:obj:`np.array`): Mono-ed version of :attr:`stft_data`.
+ (:obj:`np.array`): Single channel version of :attr:`stft_data`.
"""
- mono_stft = np.mean(self.stft_data, axis=constants.CHAN_INDEX)
+ one_channel_stft = np.mean(self.stft_data, axis=constants.CHAN_INDEX)
if overwrite:
- self.stft_data = mono_stft
- return mono_stft
+ self.stft_data = one_channel_stft
+ return one_channel_stft
##################################################
# Operator overloading

MODIFY
nussl/separation/duet.py
nussl/separation/duet.py
@@ -19,7 +19,12 @@ class Duet(mask_separation_base.MaskSeparationBase):
"""Implements the Degenerate Unmixing Estimation Technique (DUET) algorithm.
The DUET algorithm was originally proposed by S.Rickard and F.Dietrich for DOA estimation
- and further developed for BSS and demixing by A.Jourjine, S.Rickard, and O. Yilmaz.
+ and further developed for BSS and demixing by A.Jourjine, S.Rickard, and O. Yilmaz. DUET extracts
+ sources using the symmetric attenuation and relative delay between two channels. The symmetric
+ attenuation is calculated from the ratio of the two channels' stft amplitudes, and the delay is the arrival delay
+ between the two sensors used to record the audio signal. These two values are clustered as peaks on a
+ histogram to determine where each source occurs. This implementation of DUET creates and returns Mask objects after
+ the run() function, which can then be applied to the original audio signal to extract each individual source.
References:
@@ -31,8 +36,8 @@ class Duet(mask_separation_base.MaskSeparationBase):
Parameters:
input_audio_signal (np.array): a 2-row Numpy matrix containing samples of the two-channel mixture.
num_sources (int): Number of sources to find.
- attenuation_min (int): Lower bound on attenuation value, used as minimum distance in utils.find_peak_indices.
- attenuation_max (int): Upper bound on attenuation, used for creating a histogram without outliers.
+ attenuation_min (int): Minimum distance in utils.find_peak_indices, change if not enough peaks are identified.
+ attenuation_max (int): Used for creating a histogram without outliers.
num_attenuation_bins (int): Number of bins for attenuation.
delay_min (int): Lower bound on delay, used as minimum distance in utils.find_peak_indices.
delay_max (int): Upper bound on delay, used for creating a histogram without outliers.
@@ -40,13 +45,14 @@ class Duet(mask_separation_base.MaskSeparationBase):
peak_threshold (float): Value in [0, 1] for peak picking.
attenuation_min_distance (int): Minimum distance between peaks wrt attenuation.
delay_min_distance (int): Minimum distance between peaks wrt delay.
- p (int): Weight the histogram with the symmetric attenuation estimator
+ p (int): Weight the histogram with the symmetric attenuation estimator.
q (int): Weight the histogram with the delay estimator
-
+ On page 8 of his paper, Rickard recommends p=1 and q=0 as a default starting point and p=.5, q=0 if one
+ source is more dominant.
Attributes:
stft_ch0 (np.array): A Numpy matrix containing the stft data of channel 0.
- stft_ch1 (np.array): A Numpy matrix containing the stft data of channel 0.
+ stft_ch1 (np.array): A Numpy matrix containing the stft data of channel 1.
frequency_matrix (np.array): A Numpy matrix containing the frequencies of analysis.
symmetric_atn (np.array): A Numpy matrix containing the symmetric attenuation between the two channels.
delay (np.array): A Numpy matrix containing the delay between the two channels.
@@ -69,10 +75,9 @@ class Duet(mask_separation_base.MaskSeparationBase):
def __init__(self, input_audio_signal, num_sources,
attenuation_min=-3, attenuation_max=3, num_attenuation_bins=50,
delay_min=-3, delay_max=3, num_delay_bins=50,
- peak_threshold=0.2, attenuation_min_distance=5, delay_min_distance=5, p=1, q=0,
- mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK, mask_threshold=0.5):
- super(Duet, self).__init__(input_audio_signal=input_audio_signal, mask_type=mask_type,
- mask_threshold=mask_threshold)
+ peak_threshold=0.2, attenuation_min_distance=5, delay_min_distance=5, p=1, q=0):
+ super(Duet, self).__init__(input_audio_signal=input_audio_signal,
+ mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
if self.audio_signal.num_channels != 2:
raise ValueError('Duet requires that the input_audio_signal has exactly 2 channels!')
@@ -142,6 +147,8 @@ class Duet(mask_separation_base.MaskSeparationBase):
i += 1
"""
+ self.masks =[]
+
if self.audio_signal.num_channels != 2: # double check this
raise ValueError('Cannot run Duet on audio signal without exactly 2 channels!')
@@ -188,10 +195,6 @@ class Duet(mask_separation_base.MaskSeparationBase):
stft_ch0 = self.audio_signal.get_stft_channel(0)
stft_ch1 = self.audio_signal.get_stft_channel(1)
- # remove dc component to avoid dividing by zero freq. in the delay estimation
- stft_ch0 += nussl.constants.EPSILON
- stft_ch1 += nussl.constants.EPSILON
-
# Compute the freq. matrix for later use in phase calculations
n_time_bins = len(self.audio_signal.time_bins_vector)
wmat = np.array(np.tile(np.mat(self.audio_signal.freq_vector).T, (1, n_time_bins))) * (2 * np.pi / sample_rate)
@@ -382,9 +385,11 @@ class Duet(mask_separation_base.MaskSeparationBase):
signals (List[AudioSignal]): List of AudioSignals extracted using DUET.
"""
+ if len(self.masks) == 0:
+ raise ValueError('Cannot make audio signals with no masks to apply!')
signals = []
for i in range(self.num_sources):
- # Apply masks to stft channels using equation provided by Rickard
+ # Apply masks to stft channels using equation 8.34 (pg. 11) provided by Rickard
stft_sources = ((self.stft_ch0 + self.atn_peak[i] * np.exp(1j * self.frequency_matrix * self.delay_peak[i])
* self.stft_ch1) / (1 + self.atn_peak[i] ** 2))
new_sig = self.audio_signal.make_copy_with_stft_data(stft_sources, verbose=False)
@@ -406,9 +411,10 @@ class Duet(mask_separation_base.MaskSeparationBase):
normalize (Optional[bool]): Flags whether the matrix should be normalized or not
"""
plt.close('all')
-
- aa = np.tile(self.attenuation_bins[1::], (self.num_delay_bins, 1)).T
- dd = np.tile(self.delay_bins[1::].T, (self.num_attenuation_bins, 1))
+ if self.attenuation_delay_histogram is None:
+ raise ValueError('Cannot plot with no histogram data!')
+ aa = np.tile(self.attenuation_bins[1:], (self.num_delay_bins, 1)).T
+ dd = np.tile(self.delay_bins[1:].T, (self.num_attenuation_bins, 1))
# save smoothed_hist (not normalized) as an option for plotting
smoothed_hist = self._smooth_matrix(self.attenuation_delay_histogram, np.array([3]))

MODIFY
tests/duet_reference/duet_benchmarks/benchmark_delay.npy.REMOVED.git-id
tests/duet_reference/duet_benchmarks/benchmark_delay.npy.REMOVED.git-id
-cb65959e4786d049fdd174d176be42c1d0f37262
\ No newline at end of file
+1493fba7a56219eb9361ab0b2daa39607ccbb64d
\ No newline at end of file

MODIFY
tests/duet_reference/duet_benchmarks/benchmark_final_signals.npy.REMOVED.git-id
tests/duet_reference/duet_benchmarks/benchmark_final_signals.npy.REMOVED.git-id
-9e0eae6d9663764f53b167799acec8f193f136bd
\ No newline at end of file
+9679e0e6b03d248e033187695c5b4517bb4f0a79
\ No newline at end of file

MODIFY
tests/duet_reference/duet_benchmarks/benchmark_stft_ch0.npy.REMOVED.git-id
tests/duet_reference/duet_benchmarks/benchmark_stft_ch0.npy.REMOVED.git-id
-f003de722b0b05fa5f3bd1c4e2a151c633330fda
\ No newline at end of file
+3462a19f9ecefe92aebecb786e5bc409af503a85
\ No newline at end of file

MODIFY
tests/duet_reference/duet_benchmarks/benchmark_stft_ch1.npy.REMOVED.git-id
tests/duet_reference/duet_benchmarks/benchmark_stft_ch1.npy.REMOVED.git-id
-1c194e8503fed55edb6753778e632e04abe28f4e
\ No newline at end of file
+8c97bdd8dd39c2a364bfb653c23c3713e29af927
\ No newline at end of file

MODIFY
tests/duet_reference/duet_benchmarks/benchmark_sym_atn.npy.REMOVED.git-id
tests/duet_reference/duet_benchmarks/benchmark_sym_atn.npy.REMOVED.git-id
-d8554310a7f3859c86e86cc8c62a1db5e4b4e099
\ No newline at end of file
+a5a852781704d287aaca51afcde7be132d1dc55d
\ No newline at end of file

MODIFY
tests/test_duet.py
tests/test_duet.py
@@ -31,6 +31,20 @@ class DuetUnitTests(unittest.TestCase):
benchmark_dict[key] = value
return benchmark_dict
+ def test_multiple_duet(self):
+ mask_path = os.path.join('duet_reference', 'duet_benchmarks', 'benchmark_masks.npy')
+ benchmark_mask = np.load(mask_path)
+ duet = nussl.Duet(self.signal, 3)
+ duet_masks = duet.run()
+ path_to_benchmark_file = os.path.join('..', 'Input', 'dev1_wdrums_inst_mix.wav')
+ duet.audio_signal = nussl.AudioSignal(path_to_benchmark_file)
+ duet_masks = duet.run()
+ path_to_benchmark_file = os.path.join('..', 'Input', 'dev1_female3_inst_mix.wav')
+ duet.audio_signal = nussl.AudioSignal(path_to_benchmark_file)
+ duet_masks = duet.run()
+ for i in range(len(duet_masks)):
+ assert np.all(benchmark_mask[i].mask == duet_masks[i].mask)
+
def test_duet_final_outputs(self):
#Test final outputs
mask_path = os.path.join('duet_reference', 'duet_benchmarks', 'benchmark_masks.npy')
@@ -188,7 +202,7 @@ def freeze_duet_values():
signal = nussl.AudioSignal(path)
duet = nussl.Duet(signal, 3)
- output_folder = os.path.abspath('duet_reference/duet_benchmarks')
+ output_folder = os.path.abspath('duet_reference/test')
duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)

Corrected issues for DUET code review #87.