Exps_on_subset_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Exps on subset 1.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "Tse4tfn2Ue4I",
        "xfY6ukIWgYzH",
        "xk81V9u_I5RV"
      ],
      "authorship_tag": "ABX9TyOP3IS0SyJNV0P7HjGcNtyS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jwang44/Plucking-Style-Detection/blob/main/Exps_on_subset_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jIRN3f8QLeY0",
        "outputId": "8b22ad88-e389-43ad-ae4e-c469dd43016f"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tse4tfn2Ue4I"
      },
      "source": [
        "## Preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQgdVcR-jj7X"
      },
      "source": [
        "!pip install essentia"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJbPMOX3jj7Z"
      },
      "source": [
        "import numpy as np\n",
        "import pickle\n",
        "import glob, os\n",
        "\n",
        "from collections import Counter\n",
        "from statistics import stdev\n",
        "\n",
        "import essentia\n",
        "from essentia.standard import *"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xfY6ukIWgYzH"
      },
      "source": [
        "## String number estimation using Dataset1\n",
        "Use Dataset1 (non-chord portion). Monophonic string number estimation.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EaoDs9JhCL7p"
      },
      "source": [
        "#### Feature extraction on single note recordings and save features and labels to pickle"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "frxKNiCSmKB4"
      },
      "source": [
        "w = Windowing(type = 'hann')\n",
        "spectrum = Spectrum()\n",
        "mfcc = MFCC()\n",
        "spectral_peaks = SpectralPeaks()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_TlDGgdga3n"
      },
      "source": [
        "# iterate over all files in dataset1 (non-chord portion)\n",
        "AUDIO_DIR = '/content/drive/MyDrive/IDMT-SMT-GUITAR_V2/dataset1/*[!hords]/audio/*.wav'\n",
        "\n",
        "# parse dataset1\n",
        "i = 0\n",
        "data = []\n",
        "string_labels = []\n",
        "pitch_labels = []\n",
        "fret_labels = []\n",
        "\n",
        "for audio_file in glob.iglob(AUDIO_DIR):\n",
        "  # G53-40100-1111-00001.wav\n",
        "  label = os.path.basename(audio_file).split('-')[1]  # 40100\n",
        "  pitch_label = label[0:2] # 40\n",
        "  string_label = label[2] # 1\n",
        "  fret_label = label[3:] # 00\n",
        "\n",
        "  audio = MonoLoader(filename=audio_file)()\n",
        "  pool = essentia.Pool()\n",
        "  for frame in FrameGenerator(audio, frameSize=1024, hopSize=512, startFromZero=True):\n",
        "      _, mfcc_coeffs = mfcc(spectrum(w(frame)))\n",
        "      # spectral_peaks(spectrum(w(frame)))\n",
        "      pool.add('mfcc', mfcc_coeffs)\n",
        "\n",
        "  # compute statistics of the frame-level features\n",
        "  aggr_pool = PoolAggregator(defaultStats = ['min', 'max', 'median', 'mean', 'var', 'skew', 'kurt'])(pool)\n",
        "\n",
        "  feature = np.array([])\n",
        "  for key in aggr_pool.descriptorNames():\n",
        "    feature = np.concatenate((feature, aggr_pool[key]))\n",
        "  \n",
        "  data.append(feature)\n",
        "  string_labels.append(string_label)\n",
        "  pitch_labels.append(pitch_label)\n",
        "  fret_labels.append(fret_label)\n",
        "\n",
        "  i+=1\n",
        "  if i%100==0:\n",
        "    print(i)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cBSK4ZCTm1k9",
        "outputId": "6b9f70b8-55f5-4d5f-82cf-6621d0a74507"
      },
      "source": [
        "print(len(data))\n",
        "print(len(string_labels))\n",
        "print(len(pitch_labels))\n",
        "print(len(fret_labels))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "312\n",
            "312\n",
            "312\n",
            "312\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G3TTcTX5m1k-",
        "outputId": "cd38160f-b694-44d0-efa4-a20657bbcb05"
      },
      "source": [
        "print(Counter(string_labels))\n",
        "print(Counter(pitch_labels))\n",
        "print(Counter(fret_labels))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counter({'1': 52, '2': 52, '5': 52, '6': 52, '4': 52, '3': 52})\n",
            "Counter({'51': 12, '50': 12, '52': 12, '64': 12, '67': 12, '56': 12, '55': 12, '61': 12, '66': 12, '60': 12, '65': 12, '59': 12, '57': 12, '62': 12, '47': 8, '49': 8, '48': 8, '46': 8, '45': 8, '53': 8, '63': 8, '54': 8, '68': 8, '69': 8, '71': 8, '70': 8, '58': 8, '42': 4, '40': 4, '41': 4, '43': 4, '44': 4, '72': 4, '74': 4, '73': 4, '75': 4, '76': 4})\n",
            "Counter({'07': 24, '11': 24, '02': 24, '09': 24, '10': 24, '08': 24, '01': 24, '05': 24, '00': 24, '06': 24, '12': 24, '03': 24, '04': 24})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p0gbWqkWm1k_"
      },
      "source": [
        "with open('/content/drive/MyDrive/dataset1_pkl/MFCC.pkl', 'wb') as file:\n",
        "    pickle.dump(data, file)\n",
        "with open('/content/drive/MyDrive/dataset1_pkl/STR_LABEL.pkl', 'wb') as file:\n",
        "    pickle.dump(string_labels, file)\n",
        "with open('/content/drive/MyDrive/dataset1_pkl/PITCH_LABEL.pkl', 'wb') as file:\n",
        "    pickle.dump(pitch_labels, file)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ogTbM97tyHu"
      },
      "source": [
        "#### Load features and labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEmZpwSUtyH0"
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
        "from sklearn.svm import SVC, LinearSVC\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.model_selection import KFold, train_test_split, GridSearchCV\n",
        "from sklearn.utils import shuffle\n",
        "from statistics import stdev"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4X8LdJsmtyHy"
      },
      "source": [
        "with open('/content/drive/MyDrive/dataset1_pkl/MFCC.pkl', 'rb') as file:\n",
        "    data = pickle.load(file)\n",
        "with open('/content/drive/MyDrive/dataset1_pkl/STR_LABEL.pkl', 'rb') as file:\n",
        "    string_labels = pickle.load(file)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1shbOg90tyHz"
      },
      "source": [
        "data = np.vstack(data)\n",
        "le = LabelEncoder()\n",
        "string_labels = le.fit_transform(string_labels) # convert category from string to numerical"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TNlaUzALtyH0"
      },
      "source": [
        "#### Train monophonic string number classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qIBUoA4A4oV6"
      },
      "source": [
        "# split into train and test\n",
        "X = data\n",
        "y = string_labels\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42)"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUapiVOZ8Ohh"
      },
      "source": [
        "model = make_pipeline(StandardScaler(), SVC())\n",
        "parameters = {\n",
        "    'svc__kernel': ['linear', 'poly', 'rbf', 'sigmoid'], \n",
        "    'svc__C':[0.2, 1, 5], \n",
        "    'svc__gamma': ['scale', 'auto']\n",
        "    }\n",
        "clf = GridSearchCV(model, parameters, n_jobs=-1, refit=True, cv=10, return_train_score=True)\n",
        "clf.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KMs29PIaAh4m"
      },
      "source": [
        "clf.cv_results_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NS6T1lSwAIWI",
        "outputId": "49372f60-7d1e-43b4-a13f-1bd1214db352"
      },
      "source": [
        "clf.score(X_test, y_test)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9365079365079365"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7gz5i49E_Px5",
        "outputId": "e30b60d5-38e2-404b-855b-ee3b52306621"
      },
      "source": [
        "clf.best_params_"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'svc__C': 5, 'svc__gamma': 'scale', 'svc__kernel': 'rbf'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fsjB2pJ8tyH4",
        "outputId": "7d235078-f3d2-4610-bba3-29d832d2bf9a"
      },
      "source": [
        "# Sanity check use normal Kfold\n",
        "X = X_train\n",
        "y = y_train\n",
        "model = make_pipeline(StandardScaler(), SVC(C=5, gamma='scale'))\n",
        "kf = KFold(n_splits=10, shuffle=True, random_state=42)\n",
        "train_accus = []\n",
        "val_accus = []\n",
        "for train_index, val_index in kf.split(X):\n",
        "    X_train, X_val = X[train_index], X[val_index]\n",
        "    y_train, y_val = y[train_index], y[val_index]\n",
        "    model.fit(X_train, y_train)\n",
        "    train_accus.append(model.score(X_train, y_train))\n",
        "    val_accus.append(model.score(X_val, y_val))\n",
        "train_accus = np.array(train_accus)\n",
        "val_accus = np.array(val_accus)\n",
        "print(\"-------------Monophonic String Number Classification---------------\")\n",
        "print(\"train accu: \", train_accus.mean(), \"Std dev: \", stdev(train_accus))\n",
        "print(\"val accu: \", val_accus.mean(), \"Std dev: \", stdev(val_accus))\n",
        "print(\"test accu: \", model.score(X_test, y_test))"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-------------Monophonic String Number Classification---------------\n",
            "train accu:  1.0 Std dev:  0.0\n",
            "val accu:  0.9756666666666666 Std dev:  0.02871958939147313\n",
            "test accu:  0.9365079365079365\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xk81V9u_I5RV"
      },
      "source": [
        "## Monophonic Pitch Estimation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KXFxXahtLAxV"
      },
      "source": [
        "#### Estimate pitch on dataset1 single notes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfU7h-TOJaID"
      },
      "source": [
        "with open('/content/drive/MyDrive/dataset1_pkl/PITCH_LABEL.pkl', 'rb') as file:\n",
        "    pitch_labels = pickle.load(file)"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xq2aWfuILFRl",
        "outputId": "0ff21f81-2bb8-46a4-a9e2-b4b8480a0d4a"
      },
      "source": [
        "AUDIO_DIR = '/content/drive/MyDrive/IDMT-SMT-GUITAR_V2/dataset1/*[!hords]/audio/*.wav'\n",
        "from essentia.standard import MultiPitchKlapuri, PitchMelodia, PitchYin\n",
        "i=0\n",
        "correct = 0\n",
        "\n",
        "for audio_file in glob.iglob(AUDIO_DIR):\n",
        "  # pitch label in str\n",
        "  pitch_label = int(pitch_labels[i])\n",
        "  audio = MonoLoader(filename=audio_file)()\n",
        "  mp = MultiPitchKlapuri()\n",
        "  # pitchyin = PitchYin()\n",
        "  freqs = mp(audio) # this is a list of arrays\n",
        "\n",
        "  notes = []\n",
        "  for freq_array in freqs:\n",
        "    note_array = np.round(12*np.log2(freq_array/440)+69)\n",
        "    note_tuple = tuple(note_array)\n",
        "    notes.append(note_tuple)\n",
        "\n",
        "  pitch_pred = max(set(notes), key=notes.count)\n",
        "  if pitch_label == pitch_pred[0]:\n",
        "    correct+=1\n",
        "\n",
        "  i+=1\n",
        "  if i%50==0:\n",
        "    print(i)\n",
        "\n",
        "print(\"Pitch Estimation accuracy: \", correct/i)\n"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "50\n",
            "100\n",
            "150\n",
            "200\n",
            "250\n",
            "300\n",
            "Pitch Estimation accuracy:  0.9935897435897436\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}