From 609fe418245379b6089df0e62181c91bc96ee012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian-Robert=20St=C3=B6ter?= Date: Tue, 24 Apr 2018 15:02:32 +0200 Subject: [PATCH] add paper plots --- aggregate.py | 3 +- sisec-2018-paper-figures/boxplot.py | 102 +++++++++++++ sisec-2018-paper-figures/heatmap_acc.py | 90 ++++++++++++ sisec-2018-paper-figures/heatmap_vocal.py | 90 ++++++++++++ sisec-2018-paper-figures/stats.py | 115 +++++++++++++++ sisec-2018-paper-figures/track_plot.py | 153 +++++++++++++++++++ sisec-2018-paper-figures/v3v4.py | 170 ++++++++++++++++++++++ 7 files changed, 722 insertions(+), 1 deletion(-) create mode 100644 sisec-2018-paper-figures/boxplot.py create mode 100644 sisec-2018-paper-figures/heatmap_acc.py create mode 100644 sisec-2018-paper-figures/heatmap_vocal.py create mode 100644 sisec-2018-paper-figures/stats.py create mode 100644 sisec-2018-paper-figures/track_plot.py create mode 100644 sisec-2018-paper-figures/v3v4.py diff --git a/aggregate.py b/aggregate.py index 9c0cf55..84c4082 100644 --- a/aggregate.py +++ b/aggregate.py @@ -27,6 +27,7 @@ def museval2df(json_path): value_vars=['SDR', 'SAR', 'ISR', 'SIR'] ) df['track'] = json_path.stem + df = df.rename(index=str, columns={"name": "target"}) return df @@ -39,7 +40,7 @@ def aggregate(input_dirs, output_path=None): json_paths = p.glob('**/*.json') for json_path in json_paths: df = museval2df(json_path) - df['estimate'] = p.stem + df['method'] = p.stem data.append(df) df = pd.concat(data, ignore_index=True) diff --git a/sisec-2018-paper-figures/boxplot.py b/sisec-2018-paper-figures/boxplot.py new file mode 100644 index 0000000..2f89444 --- /dev/null +++ b/sisec-2018-paper-figures/boxplot.py @@ -0,0 +1,102 @@ +import pandas as pd +import seaborn as sns +import matplotlib.pyplot as plt +import matplotlib as mpl +import math +import numpy as np + +sns.set() +sns.set_context("notebook") + +metrics = ['SDR', 'SIR', 'SAR', 'ISR'] +targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other'] +selected_targets = ['vocals', 'accompaniment'] +oracles = [ + 'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF', 'IMSK' +] + +# Convert to Pandas Dataframes +df = pd.read_pickle("sisec18_mus.pandas") +df['oracle'] = df.method.isin(oracles) +# df = df[df.target.isin(selected_targets)].dropna() + +# aggregate methods by mean using median by track +df = df.groupby( + ['method', 'track', 'target', 'metric'] +).median().reset_index() + +# Get sorting keys (sorted by median of SDR:vocals) +df_sort_by = df[ + (df.metric == "SDR") & + (df.target == "vocals") +] + +methods_by_sdr = df_sort_by.score.groupby( + df_sort_by.method +).median().sort_values().index.tolist() + + +plt.rc('text', usetex=True) +plt.rc('font', family='serif') + +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['text.latex.unicode'] = 'True' + +sns.set() +sns.set_context("paper") + +params = { + 'backend': 'ps', + 'axes.labelsize': 18, + 'font.size': 15, + 'legend.fontsize': 16, + 'xtick.labelsize': 13, + 'ytick.labelsize': 15, + 'text.usetex': True, + 'font.family': 'serif', + 'font.serif': 'ptmrr8re', +} + +sns.set_style("darkgrid", { + 'pgf.texsystem': 'xelatex', # pdflatex, xelatex, lualatex + "axes.facecolor": "0.925", + 'text.usetex': True, + 'font.family': 'serif', + 'axes.labelsize': 14, + 'font.size': 14, + 'legend.fontsize': 15, + 'xtick.labelsize': 15, + 'ytick.labelsize': 17, + 'font.serif': [], +}) +plt.rcParams.update(params) + +g = sns.FacetGrid( + df, + row="target", + col="metric", + row_order=targets, + col_order=metrics, + size=6, + sharex=False, + aspect=0.7 +) +g = (g.map( + sns.boxplot, + "score", + "method", + "oracle", + orient='h', + order=methods_by_sdr[::-1], + hue_order=[True, False], + showfliers=False, + notch=True +)) + +g.fig.tight_layout() +plt.subplots_adjust(hspace=0.2, wspace=0.1) +g.fig.savefig( + "boxplot.pdf", + bbox_inches='tight', + dpi=300 +) diff --git a/sisec-2018-paper-figures/heatmap_acc.py b/sisec-2018-paper-figures/heatmap_acc.py new file mode 100644 index 0000000..df3c684 --- /dev/null +++ b/sisec-2018-paper-figures/heatmap_acc.py @@ -0,0 +1,90 @@ +import pandas as pd +import seaborn as sns +import matplotlib.pyplot as plt +import matplotlib as mpl +import math +import numpy as np +from matplotlib import gridspec + + +sns.set() +sns.set_context("notebook") + +metrics = ['SDR', 'SIR', 'SAR', 'ISR'] +targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other'] +selected_targets = ['vocals', 'accompaniment'] +oracles = [ + 'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF' +] + +df = pd.read_pickle("sisec18_mus.pandas") +df['oracle'] = df.method.isin(oracles) + +# aggregate methods by mean using median by track +df = df.groupby( + ['method', 'track', 'target', 'metric'] +).median().reset_index() + +plt.rc('text', usetex=True) +plt.rc('font', family='serif') + +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['text.latex.unicode'] = 'True' + +sns.set() +sns.set_context("paper") + +params = { + 'backend': 'ps', + 'axes.labelsize': 18, + 'font.size': 15, + 'legend.fontsize': 16, + 'xtick.labelsize': 13, + 'ytick.labelsize': 15, + 'text.usetex': False, + 'font.family': 'serif', + 'font.serif': 'ptmrr8re', + 'text.latex.unicode': False +} + +sns.set_style("darkgrid", { + 'pgf.texsystem': 'xelatex', # pdflatex, xelatex, lualatex + "axes.facecolor": "0.925", + 'text.usetex': False, + 'font.family': 'serif', + 'axes.labelsize': 14, + 'font.size': 14, + 'legend.fontsize': 15, + 'xtick.labelsize': 15, + 'ytick.labelsize': 17, + 'font.serif': [], + 'text.latex.unicode': False +}) +plt.rcParams.update(params) + +target = 'accompaniment' +f, ax = plt.subplots(1, 1, figsize=(16, 10)) + +df_target = df[(df.target == target) & (df.metric == 'SDR')] + +targets_by_score = df_target.score.groupby( + df_target.method +).median().sort_values().index.tolist() + +tracks_by_score = df_target.score.groupby( + df_target.track +).median().sort_values().index.tolist() + +pivoted = pd.pivot_table(df_target, values='score', index='method', columns='track') + +pivoted = pivoted.reindex(index=targets_by_score[::-1], columns=tracks_by_score[::-1]) +sns.heatmap( + pivoted, square=True, ax=ax, cmap='viridis', vmin=np.percentile(pivoted, 10), vmax=np.percentile(pivoted, 90) +) +for label in ax.get_yticklabels(): + label.set_rotation(0) +f.savefig( + "heatmap_acc.pdf", + bbox_inches='tight', + dpi=300 +) diff --git a/sisec-2018-paper-figures/heatmap_vocal.py b/sisec-2018-paper-figures/heatmap_vocal.py new file mode 100644 index 0000000..c67fe13 --- /dev/null +++ b/sisec-2018-paper-figures/heatmap_vocal.py @@ -0,0 +1,90 @@ +import pandas as pd +import seaborn as sns +import matplotlib.pyplot as plt +import matplotlib as mpl +import math +import numpy as np +from matplotlib import gridspec + + +sns.set() +sns.set_context("notebook") + +metrics = ['SDR', 'SIR', 'SAR', 'ISR'] +targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other'] +selected_targets = ['vocals', 'accompaniment'] +oracles = [ + 'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF' +] + +df = pd.read_pickle("sisec18_mus.pandas") +df['oracle'] = df.method.isin(oracles) + +# aggregate methods by mean using median by track +df = df.groupby( + ['method', 'track', 'target', 'metric'] +).median().reset_index() + +plt.rc('text', usetex=True) +plt.rc('font', family='serif') + +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['text.latex.unicode'] = 'True' + +sns.set() +sns.set_context("paper") + +params = { + 'backend': 'ps', + 'axes.labelsize': 18, + 'font.size': 15, + 'legend.fontsize': 16, + 'xtick.labelsize': 13, + 'ytick.labelsize': 15, + 'text.usetex': False, + 'font.family': 'serif', + 'font.serif': 'ptmrr8re', + 'text.latex.unicode': False +} + +sns.set_style("darkgrid", { + 'pgf.texsystem': 'xelatex', # pdflatex, xelatex, lualatex + "axes.facecolor": "0.925", + 'text.usetex': False, + 'font.family': 'serif', + 'axes.labelsize': 14, + 'font.size': 14, + 'legend.fontsize': 15, + 'xtick.labelsize': 15, + 'ytick.labelsize': 17, + 'font.serif': [], + 'text.latex.unicode': False +}) +plt.rcParams.update(params) + +target = 'vocals' +f, ax = plt.subplots(1, 1, figsize=(16, 10*(31/27))) + +df_target = df[(df.target == target) & (df.metric == 'SDR')] + +targets_by_score = df_target.score.groupby( + df_target.method +).median().sort_values().index.tolist() + +tracks_by_score = df_target.score.groupby( + df_target.track +).median().sort_values().index.tolist() + +pivoted = pd.pivot_table(df_target, values='score', index='method', columns='track') + +pivoted = pivoted.reindex(index=targets_by_score[::-1], columns=tracks_by_score[::-1]) +sns.heatmap( + pivoted, square=True, ax=ax, cmap='viridis', vmin=np.percentile(pivoted, 10), vmax=np.percentile(pivoted, 90) +) +for label in ax.get_yticklabels(): + label.set_rotation(0) +f.savefig( + "heatmap_vocals.pdf", + bbox_inches='tight', + dpi=300 +) diff --git a/sisec-2018-paper-figures/stats.py b/sisec-2018-paper-figures/stats.py new file mode 100644 index 0000000..0715c1e --- /dev/null +++ b/sisec-2018-paper-figures/stats.py @@ -0,0 +1,115 @@ +import pandas as pd +import seaborn as sns +import matplotlib.pyplot as plt +import matplotlib as mpl +import math +import numpy as np +from matplotlib import gridspec +import scikit_posthocs as sp + + +sns.set() +sns.set_context("notebook") + +metrics = ['SDR', 'SIR', 'SAR', 'ISR'] +targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other'] +selected_targets = ['vocals', 'accompaniment'] +oracles = [ + 'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF' +] + +df = pd.read_pickle("sisec18_mus.pandas") +df['oracle'] = df.method.isin(oracles) + +# aggregate methods by mean using median by track +df = df.groupby( + ['method', 'track', 'target', 'metric'] +).median().reset_index() + +plt.rc('text', usetex=True) +plt.rc('font', family='serif') + +mpl.rcParams['font.family'] = 'serif' +mpl.rcParams['text.latex.unicode'] = 'True' + +sns.set() +sns.set_context("paper") + +params = { + 'backend': 'ps', + 'axes.labelsize': 18, + 'font.size': 15, + 'legend.fontsize': 16, + 'xtick.labelsize': 13, + 'ytick.labelsize': 15, + 'text.usetex': True, + 'font.family': 'serif', + 'font.serif': 'ptmrr8re', + 'text.latex.unicode': True +} + +sns.set_style("darkgrid", { + 'pgf.texsystem': 'xelatex', # pdflatex, xelatex, lualatex + "axes.facecolor": "0.925", + 'text.usetex': True, + 'font.family': 'serif', + 'axes.labelsize': 14, + 'font.size': 14, + 'legend.fontsize': 15, + 'xtick.labelsize': 15, + 'ytick.labelsize': 17, + 'font.serif': [], + 'text.latex.unicode': True +}) +plt.rcParams.update(params) + +f = plt.figure(figsize=(22, 20)) +# resort them by median SDR +# Get sorting keys (sorted by median of SDR:vocals score) +df_voc = df[(df.target == 'vocals') & (df.metric == "SDR")] +df_acc = df[(df.target == 'accompaniment') & (df.metric == "SDR")] + +targets_by_voc_sdr = df_voc.score.groupby( + df_voc.method +).median().sort_values().index.tolist() + +targets_by_acc_sdr = df_acc.score.groupby( + df_acc.method +).median().sort_values().index.tolist() + +targets_by_voc_sdr_acc = [x for x in targets_by_voc_sdr if x in targets_by_acc_sdr] + +# get the two sortings +df_voc['method'] = df_voc['method'].astype('category', categories=targets_by_voc_sdr, ordered=True) +df_acc['method'] = df_acc['method'].astype('category', categories=targets_by_acc_sdr, ordered=True) + +# prepare the pairwise plots +pc_voc = sp.posthoc_conover(df_voc, val_col='score', group_col='method') +pc_acc = sp.posthoc_conover(df_acc, val_col='score', group_col='method') + +f = plt.figure(figsize=(10, 10)) +# Format: diagonal, non-significant, p<0.001, p<0.01, p<0.05 +cmap = ['1', '#ff2626', '#ffffff', '#fcbdbd', '#ff7272'] +heatmap_args = {'cmap': cmap, 'linewidths': 0.25, 'linecolor': '0.5', 'clip_on': False, 'square': True, 'cbar_ax_bbox': [0.90, 0.35, 0.04, 0.3]} +sp.sign_plot(pc_voc, **heatmap_args) + +f.tight_layout() +f.savefig( + "pairwise_vocals.pdf", + bbox_inches='tight', + dpi=300 +) + +f = plt.figure(figsize=(8.709677419, 8.709677419)) + +# Format: diagonal, non-significant, p<0.001, p<0.01, p<0.05 +cmap = ['1', '#ff2626', '#ffffff', '#fcbdbd', '#ff7272'] +heatmap_args = {'cmap': cmap, 'linewidths': 0.25, 'linecolor': '0.5', 'clip_on': False, 'square': True, 'cbar_ax_bbox': [0.90, 0.35, 0.04, 0.3]} +sp.sign_plot(pc_acc, **heatmap_args) + +f.tight_layout() +f.savefig( + "pairwise_acc.pdf", + bbox_inches='tight', + dpi=300 +) diff --git a/sisec-2018-paper-figures/track_plot.py b/sisec-2018-paper-figures/track_plot.py new file mode 100644 index 0000000..74322c8 --- /dev/null +++ b/sisec-2018-paper-figures/track_plot.py @@ -0,0 +1,153 @@ +import os +import glob +import yaml +import pandas as pd +import json +import scipy.io +import numpy as np +import difflib +import csv +import argparse +from pathlib import Path +from pandas.io.json import json_normalize +import math +import seaborn as sns +import matplotlib.pyplot as plt +from matplotlib.transforms import BlendedGenericTransform +import matplotlib as mpl +from matplotlib import gridspec +import soundfile as sf + + +class Framing: + """helper iterator class to do overlapped windowing""" + def __init__(self, window, hop, length): + self.current = 0 + self.window = window + self.hop = hop + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.current >= self.nwin: + raise StopIteration + else: + start = self.current * self.hop + if np.isnan(start) or np.isinf(start): + start = 0 + stop = min(self.current * self.hop + self.window, self.length) + if np.isnan(stop) or np.isinf(stop): + stop = self.length + result = slice(start, stop) + self.current += 1 + return result + + @property + def nwin(self): + if self.window < self.length: + return int( + np.floor((self.length - self.window + self.hop) / self.hop) + ) + else: + return 1 + + next = __next__ + + +def discrete_cmap(N, base_cmap=None): + """Create an N-bin discrete colormap from the specified input map""" + + # Note that if base_cmap is a string or None, you can simply do + # return plt.cm.get_cmap(base_cmap, N) + # The following works for string, None, or a colormap instance: + + base = plt.cm.get_cmap(base_cmap) + color_list = base(np.linspace(0, 1, N)) + cmap_name = base.name + str(N) + color_tuples = map(tuple, color_list.tolist()) + return base.from_list(cmap_name, color_list, N), color_tuples + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Aggregate Folder') + parser.add_argument( + 'submission_dirs', + help='method folder', + nargs='+', + type=str + ) + + parser.add_argument( + '--track_name', + help='method folder', + type=str, + default="AM Contra - Heart Peripheral" + ) + + args = parser.parse_args() + + oracles = [ + 'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF', 'IMSK' + ] + + data = [] + color_list, color_tuples = discrete_cmap( + len(args.submission_dirs) + 2, + 'cubehelix_r' + ) + fig, ax1 = plt.subplots() + scores = [] + data = [] + for k, path in enumerate(args.submission_dirs): + p = Path(path) + is_oracle = p.stem in oracles + p = p / 'test' / (args.track_name + '.json') + with open(p) as json_file: + json_string = json.loads(json_file.read()) + try: + scores = list( + filter( + lambda target: target['name'] == 'vocals', + json_string['targets'] + ) + ) + frames = scores[0]['frames'] + except IndexError: + continue + + score = np.array([s['metrics']['SDR'] for s in frames]) + t = np.array([s['time'] for s in frames]) + if is_oracle: + ax1.plot(t, score, lw=2, color='red', alpha=0.4) + else: + data.append(score) + ax1.plot(t, score, lw=2, color='gray', alpha=0.4) + + import musdb + mus = musdb.DB() + track = mus.load_mus_tracks(tracknames=[p.stem])[0] + (nsampl, nchan) = track.audio.shape + framer = Framing(track.rate, track.rate, nsampl) + nwin = framer.nwin + audio = track.targets['vocals'].audio + energy = [ + np.sqrt(np.mean(audio[win, :]**2)) + for win in framer + ] + ax2 = ax1.twinx() + # ax2.step(range(nwin), energy, alpha=0.5, color='red') + ax2.plot(range(nwin), energy, alpha=0.5, color='blue') + # ax2.set_ylim([0.1, .3]) + D = np.array(data) + U = np.argmax(np.nanmean(D, axis=1)) + print("best method:", np.max(np.nanmean(D, axis=1))) + print("upper_bound:", np.mean(np.nanmax(D, axis=0))) + L = np.argmin(np.nanmean(D, axis=1)) + lower_bound = np.nanmin(D, axis=0) + upper_bound = np.nanmax(D, axis=0) + # ax1.plot(t, D[U], lw=2, label='Best Method', color='green') + ax1.fill_between(t, lower_bound, upper_bound, facecolor='gray', alpha=0.1) + ax1.legend() + plt.show() diff --git a/sisec-2018-paper-figures/v3v4.py b/sisec-2018-paper-figures/v3v4.py new file mode 100644 index 0000000..bc68c04 --- /dev/null +++ b/sisec-2018-paper-figures/v3v4.py @@ -0,0 +1,170 @@ +import json +import numpy as np +import argparse +from pathlib import Path +import math +import seaborn as sns +import matplotlib.pyplot as plt +import matplotlib as mpl +import musdb + + +class Framing: + """helper iterator class to do overlapped windowing""" + def __init__(self, window, hop, length): + self.current = 0 + self.window = window + self.hop = hop + self.length = length + + def __iter__(self): + return self + + def __next__(self): + if self.current >= self.nwin: + raise StopIteration + else: + start = self.current * self.hop + if np.isnan(start) or np.isinf(start): + start = 0 + stop = min(self.current * self.hop + self.window, self.length) + if np.isnan(stop) or np.isinf(stop): + stop = self.length + result = slice(start, stop) + self.current += 1 + return result + + @property + def nwin(self): + if self.window < self.length: + return int( + np.floor((self.length - self.window + self.hop) / self.hop) + ) + else: + return 1 + + next = __next__ + + +def discrete_cmap(N, base_cmap=None): + """Create an N-bin discrete colormap from the specified input map""" + + # Note that if base_cmap is a string or None, you can simply do + # return plt.cm.get_cmap(base_cmap, N) + # The following works for string, None, or a colormap instance: + + base = plt.cm.get_cmap(base_cmap) + color_list = base(np.linspace(0, 1, N)) + cmap_name = base.name + str(N) + color_tuples = map(tuple, color_list.tolist()) + return base.from_list(cmap_name, color_list, N), color_tuples + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Aggregate Folder') + parser.add_argument( + 'submission_dirs', + help='directories of submissions', + nargs='+', + type=str + ) + + plt.rc('text', usetex=True) + plt.rc('font', family='serif') + + mpl.rcParams['font.family'] = 'serif' + mpl.rcParams['text.latex.unicode'] = 'True' + + sns.set() + sns.set_context("paper") + + # Get this from LaTeX using \showthe\columnwidth + fig_width_pt = 244.6937 + # Convert pt to inch + inches_per_pt = 1.0 / 72.27 + # Aesthetic ratio + golden_mean = (math.sqrt(5) - 1.0) / 2.0 + # width in inches + fig_width = fig_width_pt * inches_per_pt + # height in inches + fig_height = fig_width * golden_mean + fig_size = np.array([fig_width*2, fig_height*2]) + + params = { + 'backend': 'ps', + 'axes.labelsize': 18, + 'font.size': 15, + 'legend.fontsize': 16, + 'xtick.labelsize': 13, + 'ytick.labelsize': 15, + 'text.usetex': True, + 'font.family': 'serif', + 'font.serif': 'ptmrr8re', + 'figure.figsize': fig_size + } + + sns.set_style("darkgrid", { + "axes.facecolor": "0.925", + 'text.usetex': True, + 'font.family': 'serif', + 'axes.labelsize': 14, + 'font.size': 14, + 'legend.fontsize': 15, + 'xtick.labelsize': 15, + 'ytick.labelsize': 17, + 'font.serif': 'ptmrr8re', + }) + plt.rcParams.update(params) + args = parser.parse_args() + data = [] + mus = musdb.DB() + v3 = [] + v4 = [] + plot_target = 'vocals' + for path in args.submission_dirs: + p = Path(path) + if p.exists(): + json_paths = p.glob('**/*.json') + for json_path in json_paths: + with open(json_path) as json_file: + print(json_path.stem) + json_string = json.loads(json_file.read()) + track = mus.load_mus_tracks(tracknames=[json_path.stem])[0] + (nsampl, nchan) = track.targets[plot_target].audio.shape + framer = Framing(track.rate, track.rate, nsampl) + nwin = framer.nwin + audio = track.targets[plot_target].audio + energy = [ + np.sqrt(np.mean(audio[win, :]**2)) + for win in framer + ] + + vocal_scores = list( + filter( + lambda target: target['name'] == plot_target, + json_string['targets'] + ) + ) + frames = vocal_scores[0]['frames'] + for s, e in zip(frames, energy): + if "v3" in str(json_path): + v3.append([e, s['metrics']['SIR']]) + else: + v4.append([e, s['metrics']['SIR']]) + + fig, ax1 = plt.subplots(figsize=fig_size) + v3 = np.array(v3) + v4 = np.array(v4) + ax1.scatter(v3[:, 0], v3[:, 1], label="v3", s=3, alpha=0.5) + ax1.scatter(v4[:, 0], v4[:, 1], label="v4", s=3, alpha=0.5) + ax1.legend() + ax1.set_xlabel('Framewise RMS (%s)' % plot_target.capitalize()) + ax1.set_ylabel('Framewise SIR in dB (%s)' % plot_target.capitalize()) + ax1.set_ylim([-50, 100]) + ax1.set_xlim([0, 0.175]) + fig.set_tight_layout(True) + fig.savefig( + "timeplot_sir_%s.pdf" % plot_target.capitalize(), + bbox_inches='tight', + dpi=300 + )