add paper plots

aggregate.py

@@ -27,6 +27,7 @@ def museval2df(json_path):
             value_vars=['SDR', 'SAR', 'ISR', 'SIR']
         )
         df['track'] = json_path.stem
+        df = df.rename(index=str, columns={"name": "target"})
         return df
 
 
@@ -39,7 +40,7 @@ def aggregate(input_dirs, output_path=None):
             json_paths = p.glob('**/*.json')
             for json_path in json_paths:
                 df = museval2df(json_path)
-                df['estimate'] = p.stem
+                df['method'] = p.stem
                 data.append(df)
 
     df = pd.concat(data, ignore_index=True)

sisec-2018-paper-figures/boxplot.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import math
+import numpy as np
+
+sns.set()
+sns.set_context("notebook")
+
+metrics = ['SDR', 'SIR', 'SAR', 'ISR']
+targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other']
+selected_targets = ['vocals', 'accompaniment']
+oracles = [
+    'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF', 'IMSK'
+]
+
+# Convert to Pandas Dataframes
+df = pd.read_pickle("sisec18_mus.pandas")
+df['oracle'] = df.method.isin(oracles)
+# df = df[df.target.isin(selected_targets)].dropna()
+
+# aggregate methods by mean using median by track
+df = df.groupby(
+    ['method', 'track', 'target', 'metric']
+).median().reset_index()
+
+# Get sorting keys (sorted by median of SDR:vocals)
+df_sort_by = df[
+    (df.metric == "SDR") &
+    (df.target == "vocals")
+]
+
+methods_by_sdr = df_sort_by.score.groupby(
+    df_sort_by.method
+).median().sort_values().index.tolist()
+
+
+plt.rc('text', usetex=True)
+plt.rc('font', family='serif')
+
+mpl.rcParams['font.family'] = 'serif'
+mpl.rcParams['text.latex.unicode'] = 'True'
+
+sns.set()
+sns.set_context("paper")
+
+params = {
+    'backend': 'ps',
+    'axes.labelsize': 18,
+    'font.size': 15,
+    'legend.fontsize': 16,
+    'xtick.labelsize': 13,
+    'ytick.labelsize': 15,
+    'text.usetex': True,
+    'font.family': 'serif',
+    'font.serif': 'ptmrr8re',
+}
+
+sns.set_style("darkgrid", {
+    'pgf.texsystem': 'xelatex',  # pdflatex, xelatex, lualatex
+    "axes.facecolor": "0.925",
+    'text.usetex': True,
+    'font.family': 'serif',
+    'axes.labelsize': 14,
+    'font.size': 14,
+    'legend.fontsize': 15,
+    'xtick.labelsize': 15,
+    'ytick.labelsize': 17,
+    'font.serif': [],
+})
+plt.rcParams.update(params)
+
+g = sns.FacetGrid(
+    df,
+    row="target",
+    col="metric",
+    row_order=targets,
+    col_order=metrics,
+    size=6,
+    sharex=False,
+    aspect=0.7
+)
+g = (g.map(
+    sns.boxplot,
+    "score",
+    "method",
+    "oracle",
+    orient='h',
+    order=methods_by_sdr[::-1],
+    hue_order=[True, False],
+    showfliers=False,
+    notch=True
+))
+
+g.fig.tight_layout()
+plt.subplots_adjust(hspace=0.2, wspace=0.1)
+g.fig.savefig(
+    "boxplot.pdf",
+    bbox_inches='tight',
+    dpi=300
+)

sisec-2018-paper-figures/heatmap_acc.py

@@ -0,0 +1,90 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import math
+import numpy as np
+from matplotlib import gridspec
+
+
+sns.set()
+sns.set_context("notebook")
+
+metrics = ['SDR', 'SIR', 'SAR', 'ISR']
+targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other']
+selected_targets = ['vocals', 'accompaniment']
+oracles = [
+    'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF'
+]
+
+df = pd.read_pickle("sisec18_mus.pandas")
+df['oracle'] = df.method.isin(oracles)
+
+# aggregate methods by mean using median by track
+df = df.groupby(
+    ['method', 'track', 'target', 'metric']
+).median().reset_index()
+
+plt.rc('text', usetex=True)
+plt.rc('font', family='serif')
+
+mpl.rcParams['font.family'] = 'serif'
+mpl.rcParams['text.latex.unicode'] = 'True'
+
+sns.set()
+sns.set_context("paper")
+
+params = {
+    'backend': 'ps',
+    'axes.labelsize': 18,
+    'font.size': 15,
+    'legend.fontsize': 16,
+    'xtick.labelsize': 13,
+    'ytick.labelsize': 15,
+    'text.usetex': False,
+    'font.family': 'serif',
+    'font.serif': 'ptmrr8re',
+    'text.latex.unicode': False
+}
+
+sns.set_style("darkgrid", {
+    'pgf.texsystem': 'xelatex',  # pdflatex, xelatex, lualatex
+    "axes.facecolor": "0.925",
+    'text.usetex': False,
+    'font.family': 'serif',
+    'axes.labelsize': 14,
+    'font.size': 14,
+    'legend.fontsize': 15,
+    'xtick.labelsize': 15,
+    'ytick.labelsize': 17,
+    'font.serif': [],
+    'text.latex.unicode': False
+})
+plt.rcParams.update(params)
+
+target = 'accompaniment'
+f, ax = plt.subplots(1, 1, figsize=(16, 10))
+
+df_target = df[(df.target == target) & (df.metric == 'SDR')]
+
+targets_by_score = df_target.score.groupby(
+    df_target.method
+).median().sort_values().index.tolist()
+
+tracks_by_score = df_target.score.groupby(
+    df_target.track
+).median().sort_values().index.tolist()
+
+pivoted = pd.pivot_table(df_target, values='score', index='method', columns='track')
+
+pivoted = pivoted.reindex(index=targets_by_score[::-1], columns=tracks_by_score[::-1])
+sns.heatmap(
+    pivoted, square=True, ax=ax, cmap='viridis', vmin=np.percentile(pivoted, 10), vmax=np.percentile(pivoted, 90)
+)
+for label in ax.get_yticklabels():
+    label.set_rotation(0)
+f.savefig(
+    "heatmap_acc.pdf",
+    bbox_inches='tight',
+    dpi=300
+)

sisec-2018-paper-figures/heatmap_vocal.py

@@ -0,0 +1,90 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import math
+import numpy as np
+from matplotlib import gridspec
+
+
+sns.set()
+sns.set_context("notebook")
+
+metrics = ['SDR', 'SIR', 'SAR', 'ISR']
+targets = ['vocals', 'accompaniment', 'drums', 'bass', 'other']
+selected_targets = ['vocals', 'accompaniment']
+oracles = [
+    'IBM1', 'IBM2', 'IRM1', 'IRM2', 'MWF'
+]
+
+df = pd.read_pickle("sisec18_mus.pandas")
+df['oracle'] = df.method.isin(oracles)
+
+# aggregate methods by mean using median by track
+df = df.groupby(
+    ['method', 'track', 'target', 'metric']
+).median().reset_index()
+
+plt.rc('text', usetex=True)
+plt.rc('font', family='serif')
+
+mpl.rcParams['font.family'] = 'serif'
+mpl.rcParams['text.latex.unicode'] = 'True'
+
+sns.set()
+sns.set_context("paper")
+
+params = {
+    'backend': 'ps',
+    'axes.labelsize': 18,
+    'font.size': 15,
+    'legend.fontsize': 16,
+    'xtick.labelsize': 13,
+    'ytick.labelsize': 15,
+    'text.usetex': False,
+    'font.family': 'serif',
+    'font.serif': 'ptmrr8re',
+    'text.latex.unicode': False
+}
+
+sns.set_style("darkgrid", {
+    'pgf.texsystem': 'xelatex',  # pdflatex, xelatex, lualatex
+    "axes.facecolor": "0.925",
+    'text.usetex': False,
+    'font.family': 'serif',
+    'axes.labelsize': 14,
+    'font.size': 14,
+    'legend.fontsize': 15,
+    'xtick.labelsize': 15,
+    'ytick.labelsize': 17,
+    'font.serif': [],
+    'text.latex.unicode': False
+})
+plt.rcParams.update(params)
+
+target = 'vocals'
+f, ax = plt.subplots(1, 1, figsize=(16, 10*(31/27)))
+
+df_target = df[(df.target == target) & (df.metric == 'SDR')]
+
+targets_by_score = df_target.score.groupby(
+    df_target.method
+).median().sort_values().index.tolist()
+
+tracks_by_score = df_target.score.groupby(
+    df_target.track
+).median().sort_values().index.tolist()
+
+pivoted = pd.pivot_table(df_target, values='score', index='method', columns='track')
+
+pivoted = pivoted.reindex(index=targets_by_score[::-1], columns=tracks_by_score[::-1])
+sns.heatmap(
+    pivoted, square=True, ax=ax, cmap='viridis', vmin=np.percentile(pivoted, 10), vmax=np.percentile(pivoted, 90)
+)
+for label in ax.get_yticklabels():
+    label.set_rotation(0)
+f.savefig(
+    "heatmap_vocals.pdf",
+    bbox_inches='tight',
+    dpi=300
+)