Revised documentation for tests and nmf_mfcc.
MODIFY
nussl/separation/nmf_mfcc.py
nussl/separation/nmf_mfcc.py
@@ -17,44 +17,92 @@ import masks
class NMF_MFCC(mask_separation_base.MaskSeparationBase):
"""
Non Negative Matrix Factorization using K-Means Clustering on MFCC (NMF MFCC) is a source separation
- algorithm that runs Transformer NMF on the magnitude spectrogram of an input audio signal.
- It uses K means clustering to cluster the templates and activations returned by the NMF. The dot product of the
- clustered templates and activations results in a magnitude spectrogram only containing a separated source.
- This is used to create a Binary Mask object, and the whole process can be applied for each cluster to
- return a list of Audio Signal objects corresponding to each separated source.
+ algorithm that runs Transformer NMF on the magnitude spectrogram of an input audio signal. The templates matrix
+ is then converted to mel-space to reduce the dimensionality. The K means clustering then clusters the converted
+ templates and activations matrices. The dot product of the clustered templates and activations results in a
+ magnitude spectrogram only containing a separated source. This is used to create a Binary Mask object, and the
+ whole process can be applied for each cluster to return a list of Audio Signal objects corresponding to each
+ separated source.
References:
- Mel Frequency Cepstral Coefficients(MFCC): https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
- Volker Gnann, Martin Spiertz. "Source-filter based clustering for monaural blind source separation"
- October 2002: https://www.researchgate.net/profile/Martin_Spiertz/publication/228567105_Source-filter_based
- _clustering_for_monaural_blind_source_separation/links/0c960526b56dae0c69000000.pdf
- scikit-learn KMeans: http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
+ * `Mel Frequency Cepstral Coefficients (MFCC) <https://en.wikipedia.org/wiki/Mel-frequency_cepstrum>`_
+
+ * `scikit-learn KMeans <http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html>`_
+
+ * Spiertz, Martin, and Volker Gnann. "Source-filter based clustering for monaural blind source separation."
+ Proceedings of the 12th International Conference on Digital Audio Effects. 2009.
+
+ See Also:
+ :ref:`The NMF MFCC Demo Example <nmf_mfcc_demo>`
Parameters:
- input_audio_signal (np.array): An Audio Signal object of the input audio signal.
+ input_audio_signal (:class:`audio_signal.AudioSignal`): The :class:`audio_signal.AudioSignal` object that
+ NMF MFCC will be run on. This makes a copy of ``input_audio_signal``
num_sources (int): Number of sources to find.
- num_templates (int): Number of template vectors to used in NMF.
+ num_templates (int): Number of template vectors to used in NMF. Defaults to 50.
+ num_iterations (int): The number of iterations to go through in NMF. Defaults to 50.
+ random_seed (int): The seed to use in the numpy random generator in NMF and KMeans. See code examples below
+ for how this is used. Default uses no seed.
distance_measure (str): The type of distance measure to use in NMF - euclidean or divergence.
- num_iterations (int): The number of iterations to go through in NMF.
- random_seed (int): The seed to use in the numpy random generator
- kmeans_kwargs (dict): The kwargs for KMeans parameters.
- convert_to_mono (bool): Given a stereo signal, convert to mono.
- mask_type (object): A soft or binary mask object used for the source separation.
- mfcc_range (int/list/tuple): The range of mfcc to use for clustering.
- n_mfcc (int): The max number of mfccs to use.
+ Defaults to euclidean.
+ kmeans_kwargs (dict): The kwargs for KMeans parameters. Can be initialized with a dictionary of keys
+ corresponding to parameters in KMeans. See below for an example. Default is none.
+ convert_to_mono (bool): Given a stereo signal, convert to mono. Default set to False.
+ mask_type (str): A soft or binary mask object used for the source separation.
+ Defaults to Binary.
+ mfcc_range (int,list,tuple): The range of mfcc for clustering. See code examples below. Defaults to 1:14.
+ n_mfcc (int): The max number of mfccs to use. Defaults to 20.
Attributes:
- input_audio_signal (object): An Audio Signal object of the input audio signal.
- clusterer (object): A KMeans object for clustering the templates and activations.
- signal_stft (np.matrix): The stft data for the input audio signal.
- labeled_templates (np.array): A Numpy array containing the labeled templates columns
+ input_audio_signal (:class:`audio_signal.AudioSignal`): The :class:`audio_signal.AudioSignal` object that
+ NMF MFCC will be run on. This makes a copy of ``input_audio_signal``
+ clusterer (:obj:`KMeans`): A scikit-learn KMeans object for clustering the templates and activations.
+ signal_stft (:obj:`np.ndarray`): The stft data for the input audio signal.
+ labeled_templates (:obj:`list`): A Numpy array containing the labeled templates columns
from the templates matrix for a particular source.
- sources (np.array): A Numpy array containing the list of Audio Signal objects for each source.
- masks (np.array): A Numpy array containing the lists of Binary Mask objects for each channel.
+ sources (:obj:`list`): A list containing the lists of Audio Signal objects for each source.
+ result_masks (:obj:`list`): A list containing the lists of Binary Mask objects for each channel.
+
+ Initializing Example:
+
+ .. code-block:: python
+ :linenos:
+ # Initialize input signal
+ signal = nussl.AudioSignal(path_to_input_file='input_name.wav')
+
+ # Default initialization
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2)
+
+ # Random Seeding initialization
+ # Set NMF and KMeans seeds to 0 (no kmeans_kwarg initialized, so random_seed is used for random_state)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, random_seed=0)
+
+ # Use individual seeding
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, random_seed=0, kmeans_kwargs = {random_state: 1})
+
+ # Only set random_seed and not random_state
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, random_seed=0, kmeans_kwargs = {random_state: None})
+
+ # MFCC range initialization
+ # Set up the max of the MFCC range by only using an int
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=5)
+
+ # Set up the MFCC range by using a list [min, max]
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=[3, 15])
+
+ # Set up the MFCC range by using a tuple (min, max)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=(2, 14))
+
+ # KMeans initialization
+ # Initialize all KMeans arguments, see KMeans documentation for more detail.
+ kmeans_kwargs = { n_clusters: 8, init: 'k-means++', n_init: 10, max_iter: 300, tol: 1e-4,
+ precompute_distances: 'auto', verbose: 0, random_state: None, copy_x: True,
+ n_jobs: 1, algorithm: 'auto'}
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, kmeans_kwargs=kmeans_kwargs)
"""
- def __init__(self, input_audio_signal, num_sources, num_templates=50, distance_measure='euclidean',
- num_iterations=50, random_seed=None, kmeans_kwargs=None, convert_to_mono=False,
+ def __init__(self, input_audio_signal, num_sources, num_templates=50, num_iterations=50, random_seed=None,
+ distance_measure=transformer_nmf.TransformerNMF.EUCLIDEAN, kmeans_kwargs=None, convert_to_mono=False,
mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK, mfcc_range=(1, 14), n_mfcc=20):
super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal, mask_type=mask_type)
@@ -64,19 +112,15 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.num_iterations = num_iterations
self.random_seed = random_seed
self.kmeans_kwargs = kmeans_kwargs
- self.convert_to_mono = convert_to_mono
self.n_mfcc = n_mfcc
- self.mask_type = mask_type
self.signal_stft = None
- self.input_audio_signal = input_audio_signal
self.labeled_templates = None
self.sources = []
- self.masks = []
# Convert the stereo signal to mono if indicated
- if self.convert_to_mono:
- self.input_audio_signal.to_mono(overwrite=True, remove_channels=False)
+ if convert_to_mono:
+ self.audio_signal.to_mono(overwrite=True, remove_channels=False)
# Set the MFCC range
if isinstance(mfcc_range, int) and mfcc_range < n_mfcc:
@@ -84,7 +128,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
elif isinstance(mfcc_range, (tuple, list)) and len(mfcc_range) == 2:
self.mfcc_start, self.mfcc_end = mfcc_range[0], mfcc_range[1]
else:
- raise ValueError('mfcc_range is not set correctly! Must be a tuple or list with min and max, or int (max)')
+ raise ValueError('mfcc_range is not set correctly! Must be a tuple or list with (min, max), or int (max)')
# If kmeans_kwargs does not include the 'random_state', use the random_seed instead. Else, use the value
# provided in the dictionary. If kmeans_kwargs is None, use the random_seed.
@@ -101,11 +145,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
def run(self):
""" This function calls TransformerNMF on the magnitude spectrogram of each channel in the input audio signal.
The templates and activation matrices returned are clustered using K-Means clustering. These clusters are used
- to create mask objects for each source. Note: The masks are not returned in a particular order, but they are in
- the same order for each channel.
+ to create mask objects for each source. Note: The masks in self.result_masks are not returned in a particular
+ order corresponding to the sources, but they are in the same order for each channel.
Returns:
- self.masks (np.array): A list of binary mask objects that can be used to extract the sources.
+ result_masks (:obj:`list`): A list of :obj:`MaskBase`-derived objects for each source.
+ (to get a list of :obj:`AudioSignal`-derived objects run :func:`make_audio_signals`)
Example:
@@ -126,21 +171,18 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
"""
- self.masks = []
- self.input_audio_signal.stft_params = self.stft_params
- self.signal_stft = self.input_audio_signal.stft()
+ self.result_masks = []
+ self.audio_signal.stft_params = self.stft_params
- for i in range(self.input_audio_signal.num_channels):
- channel_stft = self.input_audio_signal.get_magnitude_spectrogram_channel(i)
+ for i in range(self.audio_signal.num_channels):
+ channel_stft = self.audio_signal.get_magnitude_spectrogram_channel(i)
# Set up NMF and run
nmf = transformer_nmf.TransformerNMF(input_matrix=channel_stft, num_components=self.num_templates,
- seed=self.random_seed)
+ seed=self.random_seed, should_do_epsilon=False,
+ max_num_iterations=self.num_iterations,
+ distance_measure=self.distance_measure)
- # Pass these to constructor (above)
- nmf.should_use_epsilon = False
- nmf.max_num_iterations = self.num_iterations
- nmf.distance_measure = self.distance_measure
channel_activation_matrix, channel_templates_matrix = nmf.transform()
# Cluster the templates matrix into Mel frequencies and retrieve labels
@@ -151,25 +193,24 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
# Extract sources from signal
channel_masks = self._extract_masks(channel_stft, channel_templates_matrix, channel_activation_matrix)
- self.masks.append(channel_masks)
+ self.result_masks.append(channel_masks)
- return self.masks
+ return self.result_masks
- def _extract_masks(self, signal_stft, templates_matrix, activation_matrix):
+ def _extract_masks(self, templates_matrix, activation_matrix):
""" Creates binary masks from clustered templates and activation matrices
Parameters:
- signal_stft (np.matrix): A 2D Numpy matrix containing the stft of the current channel
- templates_matrix (np.matrix): A 2D Numpy matrix containing the templates matrix after running NMF on
+ templates_matrix (np.ndarray): A 2D Numpy array containing the templates matrix after running NMF on
the current channel
- activation_matrix (np.matrix): A 2D Numpy matrix containing the activation matrix after running NMF on
+ activation_matrix (np.ndarray): A 2D Numpy array containing the activation matrix after running NMF on
the current channel
Returns:
- channel_mask_list (np.matrix): A list of Binary Mask objects corresponding to each source
+ channel_mask_list (list): A list of Binary Mask objects corresponding to each source
"""
- if signal_stft is None:
+ if self.audio_signal.stft_data is None:
raise ValueError('Cannot extract masks with no signal_stft data')
self.sources = []
@@ -188,6 +229,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
music_stft_max = np.maximum(mask_matrix, np.abs(signal_stft))
mask_matrix = np.divide(mask_matrix, music_stft_max)
mask = np.nan_to_num(mask_matrix)
+
if self.mask_type == self.BINARY_MASK:
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
@@ -203,10 +245,10 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.sources (np.array): An array of audio_signal objects containing each separated source
"""
self.sources = []
- for i in range(self.input_audio_signal.num_channels):
- channel_mask = self.masks[i]
+ for i in range(self.audio_signal.num_channels):
+ channel_mask = self.result_masks[i]
for j in range(self.num_sources):
- channel_stft = self.input_audio_signal.get_stft_channel(i)
+ channel_stft = self.audio_signal.get_stft_channel(i)
source = self.audio_signal.make_copy_with_stft_data(channel_stft, verbose=False)
source = source.apply_mask(channel_mask[j])
source.stft_params = self.stft_params

MODIFY
tests/freeze_duet_values.py
tests/freeze_duet_values.py
@@ -10,6 +10,9 @@ import scipy.io
import os
import warnings
+
+# Freezes essential values from DUET in its current implementation for benchmarking
+# See test_benchmark_duet() in test_duet.py for usage
def freeze_duet_values():
signal = nussl.AudioSignal(os.path.join('..', 'Input', 'dev1_female3_inst_mix.wav'))
duet = nussl.Duet(signal, 3)

MODIFY
tests/freeze_nmf_mfcc.py
tests/freeze_nmf_mfcc.py
@@ -5,7 +5,9 @@ import numpy as np
import nussl
import os
-#Freeze key NMF MFCC values to the benchmark folder
+
+# Freezes essential values from NMF MFCC in its current implementation for benchmarking
+# See test_benchmark_nmf_mfcc() in test_nmf_mfcc.py for usage
def main():
path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_benchmark_file)

MODIFY
tests/test_audio_signal.py
tests/test_audio_signal.py
@@ -327,12 +327,12 @@ class AudioSignalUnitTests(unittest.TestCase):
signal = nussl.AudioSignal(path_to_input_file=input_file_name)
signal.stft_params = signal.stft_params
signal_stft = signal.stft()
- assert (signal_stft.shape[2] == 2)
+ assert (signal_stft.shape[nussl.constants.STFT_CHAN_INDEX] == 2)
signal.to_mono(overwrite=True)
signal.stft_params = signal.stft_params
signal_stft = signal.stft()
- assert (signal_stft.shape[2] == 1)
+ assert (signal_stft.shape[nussl.constants.STFT_CHAN_INDEX]== 1)
def test_stft(self):
"""

MODIFY
tests/test_duet.py
tests/test_duet.py
@@ -191,6 +191,13 @@ class DuetUnitTests(unittest.TestCase):
final_signals = duet.make_audio_signals()
+ # Is the audio data the same?
+ assert all(np.array_equal(benchmark_final_signals[i].audio_data, final_signals[i].audio_data)
+ for i in range(len(final_signals)))
+
+ # Check to see if AudioSignal's API changed; do we need to refreeze?
+ assert all(benchmark_final_signals[i] == final_signals[i] for i in range(len(final_signals)))
+
assert np.all(benchmark_final_signals == final_signals)
# test_smooth_matrix

MODIFY
tests/test_nmf_mfcc.py
tests/test_nmf_mfcc.py
@@ -21,12 +21,14 @@ class NMFMFCCUnitTests(unittest.TestCase):
benchmark_dict[key] = value
return benchmark_dict
- def test_mfcc_initialization(self):
+ def test_nmf_mfcc_initialization(self):
+ # Test initializing the MFCC range
+
# Load input file
input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_input_file=input_file_name)
- # Set up NMMF MFCC using an int
+ # Set up the max of the MFCC range by only using an int
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=5, num_iterations=5)
assert nmf_mfcc.mfcc_start == 1
assert nmf_mfcc.mfcc_end == 5
@@ -34,12 +36,12 @@ class NMFMFCCUnitTests(unittest.TestCase):
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
- # Using a list
+ # Set up the MFCC range by using a list [min, max]
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=[3, 15], num_iterations=5)
assert nmf_mfcc.mfcc_start == 3
assert nmf_mfcc.mfcc_end == 15
- # Using a tuple
+ # Set up the MFCC range by using a tuple (min, max)
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, mfcc_range=(2, 14), num_iterations=5)
assert nmf_mfcc.mfcc_start == 2
assert nmf_mfcc.mfcc_end == 14
@@ -97,9 +99,6 @@ class NMFMFCCUnitTests(unittest.TestCase):
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
- for i, source in enumerate(sources):
- output_file_name = str(i) + '.wav'
- source.write_audio_to_file(output_file_name)
def test_piano_and_synth_stereo(self):
# Load input file
@@ -111,9 +110,6 @@ class NMFMFCCUnitTests(unittest.TestCase):
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
- for i, source in enumerate(sources):
- output_file_name = str(i) + '.wav'
- source.write_audio_to_file(output_file_name)
def test_piano_and_synth_stereo_to_mono(self):
# Load input file
@@ -126,9 +122,6 @@ class NMFMFCCUnitTests(unittest.TestCase):
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
- for i, source in enumerate(sources):
- output_file_name = str(i) + '.wav'
- source.write_audio_to_file(output_file_name)
def benchmark_nmf_mfcc(self):
# Load input file
@@ -137,16 +130,19 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150, random_seed=0)
+ num_iterations=100, random_seed=0)
# and run
nmf_mfcc.run()
- benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_labeled_templates.npy'))
- benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_masks.npy'))
+ benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks',
+ 'benchmark_labeled_templates.npy'))
+ benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_masks.npy'))
nmf_mfcc.make_audio_signals()
- benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_sources.npy'))
+ benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_sources.npy'))
assert np.all(benchmark_labeled_templates == nmf_mfcc.labeled_templates)
assert np.all(benchmark_masks == nmf_mfcc.masks)
- assert np.all(benchmark_sources == nmf_mfcc.sources)
+ for i in range(nmf_mfcc.num_sources):
+ assert np.all(benchmark_sources[i].audio_data == nmf_mfcc.sources[i].audio_data)
+ assert all(benchmark_sources[i] == nmf_mfcc.sources[i] for i in range(len(nmf_mfcc.sources)))

Revised documentation for tests and nmf_mfcc.