Removed second NMF to optimize running time and added benchmarks for algorithm with 2 NMFs.
MODIFY
nussl/separation/nmf_mfcc.py
nussl/separation/nmf_mfcc.py
@@ -37,6 +37,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
self.signal_stft = self.input_audio_signal.stft()
def run(self):
+ #TODO expose random seeding for NMF and KMeans
templates_stft = self.signal_stft[:, 0:self.signal_stft.shape[1]]
templates_stft = np.squeeze(templates_stft)
@@ -45,53 +46,64 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
nmf.should_use_epsilon = False
nmf.max_num_iterations = self.num_iterations
nmf.distance_measure = self.distance_measure
- self.activation_matrix, self.templates_matrix = nmf.transform()
+ self.activation_matrix, self.templates_matrix = nmf.transform() # FREEZE!
- # Cluster the templates matrix into Mel frequencies
+ # Cluster the templates matrix into Mel frequencies and retrieve labels
cluster_templates = librosa.feature.mfcc(S=self.templates_matrix)[1:14]
self.clusterer.fit_transform(cluster_templates.T)
-
- # Retrieve cluster labels for the templates
- self.labeled_templates = self.clusterer.labels_
- self.sources = []
+ self.labeled_templates = self.clusterer.labels_ # FREEZE!
# Extract sources from signal
+ self.sources = []
for source_index in range(self.num_sources):
source_indices = np.where(self.labeled_templates == source_index)[0]
- self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
- signal_stft=self.signal_stft)
- return self.masks
-
- def extract_masks(self, clustered_templates, signal_stft):
- """ Extracts the binary mask objects for each source.
- Parameters:
- clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
- source being extracted
- signal_stft (np.matrix): The stft of the input audio signal
- """
- new_templates_stft = np.abs(np.squeeze(signal_stft))
-
- # Set up and run NMF using each clustered template
- reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
- num_components=clustered_templates.shape[1], should_update_template=False)
- reconstruct_nmf.should_use_epsilon = False
- reconstruct_nmf.max_num_iterations = self.num_iterations
- reconstruct_nmf.distance_measure = self.distance_measure
- activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
-
- # Reconstruct the signal
- reconstructed_signal = new_templates_matrix.dot(activation_matrix)
-
- # Mask the input signal
- music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
- mask = np.divide(reconstructed_signal, music_stft_max)
- mask = np.nan_to_num(mask)
-
- # Create the binary mask
+ templates_mask = np.copy(self.templates_matrix)
+ activation_mask = np.copy(self.activation_matrix)
+ for i in range(templates_mask.shape[1]):
+ templates_mask[:, i] = 0 if i in source_indices else self.templates_matrix[:, i]
+ activation_mask[i, :] = 0 if i in source_indices else self.activation_matrix[i, :]
+ mask_matrix = templates_mask.dot(activation_mask)
+ music_stft_max = np.maximum(mask_matrix, np.abs(np.squeeze(self.signal_stft)))
+ mask_matrix = np.divide(mask_matrix, music_stft_max)
+ mask = np.nan_to_num(mask_matrix)
mask = np.round(mask)
mask_object = masks.BinaryMask(np.array(mask))
self.masks.append(mask_object)
+ # self.extract_masks(clustered_templates=self.templates_matrix[:, source_indices],
+ # signal_stft=self.signal_stft)
+ return self.masks # FREEZE!
+
+ # def extract_masks(self, clustered_templates, signal_stft):
+ # """ Extracts the binary mask objects for each source.
+ # Parameters:
+ # clustered_templates (np.matrix): The columns from self.templates_matrix that correspond to the current
+ # source being extracted
+ # signal_stft (np.matrix): The stft of the input audio signal
+ # """
+ # new_templates_stft = np.abs(np.squeeze(signal_stft)) # TODO: multichannel
+ #
+ # # Set up and run NMF using each clustered template
+ # reconstruct_nmf = transformer_nmf.TransformerNMF(input_matrix=new_templates_stft, templates=clustered_templates,
+ # num_components=clustered_templates.shape[1], should_update_template=False)
+ # reconstruct_nmf.should_use_epsilon = False
+ # reconstruct_nmf.max_num_iterations = self.num_iterations
+ # reconstruct_nmf.distance_measure = self.distance_measure
+ # activation_matrix, new_templates_matrix = reconstruct_nmf.transform()
+ #
+ # # Reconstruct the signal
+ # reconstructed_signal = new_templates_matrix.dot(activation_matrix)
+ #
+ # # Mask the input signal
+ # music_stft_max = np.maximum(reconstructed_signal, np.abs(new_templates_stft))
+ # mask = np.divide(reconstructed_signal, music_stft_max)
+ # mask = np.nan_to_num(mask)
+ #
+ # # Create the binary mask
+ # mask = np.round(mask)
+ # mask_object = masks.BinaryMask(np.array(mask))
+ # self.masks.append(mask_object)
+
def make_audio_signals(self):
""" Applies each mask in self.masks and returns a list of audio_signal objects for each source.
@@ -104,7 +116,7 @@ class NMF_MFCC(mask_separation_base.MaskSeparationBase):
source_audio_signal.stft_params = self.stft_params
source_audio_signal.istft(overwrite=True, truncate_to_length=self.audio_signal.signal_length)
self.sources.append(source_audio_signal)
- return self.sources
+ return self.sources # FREEZE!
# # template - extracted template, residual - everything that's leftover.
# template = np.multiply(np.squeeze(signal_stft), mask)

ADD
tests/nmf_mfcc_reference/nmf_mfcc_benchmarks/benchmark_masks.npy.REMOVED.git-id
+5ee5d25fb0e0d4e73bd13e7a76e10ea77cc60ce9
\ No newline at end of file

ADD
tests/nmf_mfcc_reference/nmf_mfcc_benchmarks/benchmark_sources.npy.REMOVED.git-id
+e80ca319a52501f5b340d3d34821120f19a790d5
\ No newline at end of file

MODIFY
tests/test_duet.py
tests/test_duet.py
@@ -202,7 +202,7 @@ def freeze_duet_values():
signal = nussl.AudioSignal(path)
duet = nussl.Duet(signal, 3)
- output_folder = os.path.abspath('duet_reference/test')
+ output_folder = os.path.abspath('duet_reference/duet_benchmarks')
duet.stft_ch0, duet.stft_ch1, duet.frequency_matrix = duet._compute_spectrogram(duet.sample_rate)
np.save(os.path.join(output_folder, "benchmark_stft_ch0"), duet.stft_ch0)

MODIFY
tests/test_nmf_mfcc.py
tests/test_nmf_mfcc.py
@@ -10,6 +10,9 @@ import os
class NMFMFCCUnitTests(unittest.TestCase):
+ # Update this if the benchmark file changes and rerun freeze_duet_values() (below)
+ path_to_benchmark_file = os.path.join('..', 'Input', 'piano_and_synth_arp_chord_mono.wav')
+
def test_2_sin_waves(self):
sr = nussl.constants.DEFAULT_SAMPLE_RATE
dur = 3 # seconds
@@ -22,7 +25,7 @@ class NMFMFCCUnitTests(unittest.TestCase):
# Set up NMMF MFCC
nmf_mfcc = nussl.NMF_MFCC(test_audio, num_sources=2, num_templates=2, distance_measure="euclidean",
- num_iterations=50)
+ num_iterations=5)
# and run
nmf_mfcc.run()
@@ -46,3 +49,22 @@ class NMFMFCCUnitTests(unittest.TestCase):
for i, source in enumerate(sources):
output_file_name = str(i) + '.wav'
source.write_audio_to_file(output_file_name)
+
+def freeze_nmf_mfcc_values():
+ path = NMFMFCCUnitTests.path_to_benchmark_file
+
+ signal = nussl.AudioSignal(path)
+ nmf_mfcc = nussl.NMF_MFCC(signal, num_sources=2, num_templates=6, distance_measure="euclidean",
+ num_iterations=150)
+
+ output_folder = os.path.abspath('nmf_mfcc_reference/nmf_mfcc_benchmarks')
+
+ nmf_mfcc.run()
+ np.save(os.path.join(output_folder, "benchmark_activation_matrix"), nmf_mfcc.activation_matrix)
+ np.save(os.path.join(output_folder, "benchmark_templates_matrix"), nmf_mfcc.templates_matrix)
+ np.save(os.path.join(output_folder, "benchmark_labeled_templates"), nmf_mfcc.labeled_templates)
+ np.save(os.path.join(output_folder, "benchmark_masks"), nmf_mfcc.masks)
+ nmf_mfcc.make_audio_signals()
+ np.save(os.path.join(output_folder, "benchmark_sources"), nmf_mfcc.sources)
+
+freeze_nmf_mfcc_values()
\ No newline at end of file

Removed second NMF to optimize running time and added benchmarks for algorithm with 2 NMFs.