Added multi-channel functionality for nmf_mfcc, exposed random seeding, and created tests.
RENAME
input/piano_and_synth_arp_chord.wav.REMOVED.git-id
input/piano_and_synth_arp_chord_stereo.wav.REMOVED.git-id
MODIFY
nussl/separation/nmf_mfcc.py
nussl/separation/nmf_mfcc.py
@@ -15,7 +15,8 @@ import masks
class NMF_MFCC(mask_separation_base.MaskSeparationBase):
- def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure, num_iterations):
+ def __init__(self, input_audio_signal, num_sources, num_templates, distance_measure,
+ num_iterations, random_seed, mono):
super(NMF_MFCC, self).__init__(input_audio_signal=input_audio_signal,
mask_type=mask_separation_base.MaskSeparationBase.BINARY_MASK)
@@ -23,6 +24,9 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.num_templates = num_templates
self.distance_measure = distance_measure
self.num_iterations = num_iterations
+ self.random_seed = None
+ self.mono=False
+
self.input_audio_signal = input_audio_signal
self.audio_data = input_audio_signal.audio_data
self.clusterer = None
@@ -32,45 +36,58 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.labeled_templates = None
self.sources = None
self.masks = []
+
+ if self.mono:
+ self.input_audio_signal.audio_data.to_mono(overwrite=True)
+
self.clusterer = KMeans(n_clusters=self.num_sources)
self.input_audio_signal.stft_params = self.stft_params
self.signal_stft = self.input_audio_signal.stft()
def run(self):
- #TODO expose random seeding for NMF and KMeans
- templates_stft = self.signal_stft[:, 0:self.signal_stft.shape[1]]
- templates_stft = np.squeeze(templates_stft)
+ for i in range(self.audio_signal.num_channels):
+ channel_stft = self.signal_stft[:, :, i]
# Set up NMF and run
- nmf = transformer_nmf.TransformerNMF(np.abs(templates_stft), self.num_templates)
+ nmf = transformer_nmf.TransformerNMF(input_matrix=np.abs(channel_stft), num_components=self.num_templates,
+ seed=self.random_seed)
nmf.should_use_epsilon = False
nmf.max_num_iterations = self.num_iterations
nmf.distance_measure = self.distance_measure
- self.activation_matrix, self.templates_matrix = nmf.transform() # FREEZE!
+ channel_activation_matrix, channel_templates_matrix = nmf.transform()
# Cluster the templates matrix into Mel frequencies and retrieve labels
- cluster_templates = librosa.feature.mfcc(S=self.templates_matrix)[1:14]
+ cluster_templates = librosa.feature.mfcc(S=channel_templates_matrix)[1:14]
self.clusterer.fit_transform(cluster_templates.T)
- self.labeled_templates = self.clusterer.labels_ # FREEZE!
+ self.labeled_templates = self.clusterer.labels_
# Extract sources from signal
+ channel_masks = self.extract_masks(channel_stft, channel_templates_matrix, channel_activation_matrix)
+ self.masks.append(channel_masks)
+
+ return self.masks
+
+ def extract_masks(self, signal_stft, templates_matrix, activation_matrix):
+ if signal_stft is None:
+ raise ValueError('Cannot extract masks with no signal_stft data')
+
self.sources = []
+ channel_mask_list = []
for source_index in range(self.num_sources):
source_indices = np.where(self.labeled_templates == source_index)[0]
- templates_mask = np.copy(self.templates_matrix)
- activation_mask = np.copy(self.activation_matrix)
+ templates_mask = np.copy(templates_matrix)
+ activation_mask = np.copy(activation_matrix)
for i in range(templates_mask.shape[1]):
- templates_mask[:, i] = 0 if i in source_indices else self.templates_matrix[:, i]
- activation_mask[i, :] = 0 if i in source_indices else self.activation_matrix[i, :]
+ templates_mask[:, i] = 0 if i in source_indices else templates_matrix[:, i]
+ activation_mask[i, :] = 0 if i in source_indices else activation_matrix[i, :]
mask_matrix = templates_mask.dot(activation_mask)
- music_stft_max = np.maximum(mask_matrix, np.abs(np.squeeze(self.signal_stft)))
+ music_stft_max = np.maximum(mask_matrix, np.abs(signal_stft))
mask_matrix = np.divide(mask_matrix, music_stft_max)
mask = np.nan_to_num(mask_matrix)
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
- self.masks.append(mask_object)
-
- return self.masks # FREEZE!
+ channel_mask_list.append(mask_object)
+ return channel_mask_list
def make_audio_signals(self):
""" Applies each mask in self.masks and returns a list of audio_signal objects for each source.
@@ -78,10 +95,12 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
Returns:
self.sources (np.array): An array of audio_signal objects containing each separated source
"""
- for i in range(self.num_sources):
- source_audio_signal = self.audio_signal.make_copy_with_stft_data(np.squeeze(self.signal_stft), verbose=False)
- source_audio_signal = source_audio_signal.apply_mask(self.masks[i])
+ for i in range(self.audio_signal.num_channels):
+ channel_mask = self.masks[i]
+ for j in range(self.num_sources):
+ source_audio_signal = self.audio_signal.make_copy_with_stft_data((self.signal_stft[:,:,i]), verbose=False)
+ source_audio_signal = source_audio_signal.apply_mask(channel_mask[j])
source_audio_signal.stft_params = self.stft_params
source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
self.sources.append(source_audio_signal)
- return self.sources # FREEZE!
+ return self.sources

RENAME
nussl/separation/freeze_nmf_mfcc.py
tests/freeze_nmf_mfcc.py
@@ -6,11 +6,11 @@ import nussl
import os
#Freeze key NMF MFCC values to the benchmark folder
-def freeze_nmf_mfcc_values():
+def main():
path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
signal = nussl.AudioSignal(path_to_benchmark_file)
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=150, random_seed=0)
output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
nmf_mfcc.run()
@@ -23,4 +23,4 @@ def freeze_nmf_mfcc_values():
np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
if __name__ == '__main__':
- freeze_nmf_mfcc_values()
+ main()

ADD
tests/nmf_mfcc_reference/test/benchmark_masks.npy.REMOVED.git-id
+650d69d03451df290fda2fff1ba0b59e23b23a89
\ No newline at end of file

ADD
tests/nmf_mfcc_reference/test/benchmark_sources.npy.REMOVED.git-id
+14573679a559967da97103e306816f114cb3c1b4
\ No newline at end of file

MODIFY
tests/test_nmf_mfcc.py
tests/test_nmf_mfcc.py
@@ -33,6 +33,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
+
def load_benchmarks(self):
benchmark_dict = {}
directory = 'nmf_mfcc_reference/nmf_mfcc_benchmarks'
@@ -51,7 +52,22 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=10, random_seed=0, mono=False)
+ # and run
+ nmf_mfcc.run()
+ sources = nmf_mfcc.make_audio_signals()
+ for i, source in enumerate(sources):
+ output_file_name = str(i) + '.wav'
+ source.write_audio_to_file(output_file_name)
+
+ def test_piano_and_synth_stereo(self):
+ # Load input file
+ input_file_name = os.path.join('..', 'input', 'piano_and_synth_arp_chord_stereo.wav')
+ signal = nussl.AudioSignal(path_to_input_file=input_file_name)
+
+ # Set up NMMF MFCC
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=20, random_seed=0, mono=False)
# and run
nmf_mfcc.run()
sources = nmf_mfcc.make_audio_signals()
@@ -66,17 +82,17 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
- num_iterations=150)
+ num_iterations=150, random_seed=0)
# and run
nmf_mfcc.run()
- benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_activation_matrix.npy'))
- benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_templates_matrix.npy'))
- benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_labeled_templates.npy'))
- benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_masks.npy'))
+ benchmark_activation_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_activation_matrix.npy'))
+ benchmark_templates_matrix = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_templates_matrix.npy'))
+ benchmark_labeled_templates = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_labeled_templates.npy'))
+ benchmark_masks = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_masks.npy'))
nmf_mfcc.make_audio_signals()
- benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'nmf_mfcc_benchmarks', 'benchmark_sources.npy'))
+ benchmark_sources = np.load(os.path.join('nmf_mfcc_reference', 'test', 'benchmark_sources.npy'))
assert np.all(benchmark_activation_matrix == nmf_mfcc.activation_matrix)
assert np.all(benchmark_templates_matrix == nmf_mfcc.templates_matrix)
@@ -84,3 +100,4 @@ class NMFMFCCUnitTests(unittest.TestCase):
assert np.all(benchmark_masks == nmf_mfcc.masks)
assert np.all(benchmark_sources == nmf_mfcc.sources)
+

Added multi-channel functionality for nmf_mfcc, exposed random seeding, and created tests.