mir_eval_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:\n",
      "    PyTorch 2.1.0+cu121 with CUDA 1201 (you have 2.1.0+cu118)\n",
      "    Python  3.8.18 (you have 3.8.10)\n",
      "  Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)\n",
      "  Memory-efficient attention, SwiGLU, sparse and more won't be available.\n",
      "  Set XFORMERS_MORE_DETAILS=1 for more details\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jongmin/.local/share/virtualenvs/audiocraft-ssbSDm-j/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from audiocraft.data.midi_dataset import EnCodecTokenMIDIDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths_loaded = torch.load('/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/maestro-v3.0.0_split.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 4/137 [00:00<00:05, 22.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_12_01_2004_01-05_ORIG_MID--AUDIO_12_R1_2004_03_Track03_wav--1.midi\n",
      "39 38\n",
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2004/MIDI-Unprocessed_SMF_16_R1_2004_01-08_ORIG_MID--AUDIO_16_R1_2004_03_Track03_wav.midi\n",
      "12 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 13/137 [00:00<00:05, 21.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2004/MIDI-Unprocessed_XP_11_R1_2004_03-04_ORIG_MID--AUDIO_11_R1_2004_04_Track04_wav.midi\n",
      "20 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 48/137 [00:02<00:03, 28.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2008/MIDI-Unprocessed_15_R2_2008_01-04_ORIG_MID--AUDIO_15_R2_2008_wav--4.midi\n",
      "22 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 75/137 [00:03<00:02, 28.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2011/MIDI-Unprocessed_16_R1_2011_MID--AUDIO_R1-D6_15_Track15_wav.midi\n",
      "13 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 99/137 [00:04<00:01, 22.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2015/MIDI-Unprocessed_R1_D1-1-8_mid--AUDIO-from_mp3_01_R1_2015_wav--2.midi\n",
      "11 10\n",
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2015/MIDI-Unprocessed_R1_D1-1-8_mid--AUDIO-from_mp3_03_R1_2015_wav--4.midi\n",
      "19 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 137/137 [00:05<00:00, 24.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length mismatch! Popping -2 index\n",
      "/home/jongmin/userdata/MAESTRO/maestro-v3.0.0/2018/MIDI-Unprocessed_Recital9-11_MID--AUDIO_09_R1_2018_wav--4.midi\n",
      "10 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "dataset = EnCodecTokenMIDIDataset(paths_loaded['validation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "137"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "dataloader = DataLoader(dataset, batch_size=8, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from audiocraft.modules.semcodec import SemCodecMidiDecoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SemCodecMidiDecoder().to(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "input, target_pr = next(iter(dataloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = model(input.to(\"cuda\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 2, 88, 1500])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "qq = extract_notes(out[0,0].T,out[0,1].T,None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 1,  3,  7, ..., 77, 80, 82]),\n",
       " array([[   0,    3],\n",
       "        [   0,    1],\n",
       "        [   0,    4],\n",
       "        ...,\n",
       "        [1499, 1500],\n",
       "        [1499, 1500],\n",
       "        [1499, 1500]]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "\n",
    "def extract_notes(onsets, frames, velocity, onset_threshold=0.5, frame_threshold=0.5):\n",
    "    \"\"\"\n",
    "    Finds the note timings based on the onsets and frames information\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    onsets: torch.FloatTensor, shape = [frames, bins]\n",
    "    frames: torch.FloatTensor, shape = [frames, bins]\n",
    "    velocity: torch.FloatTensor, shape = [frames, bins]\n",
    "    onset_threshold: float\n",
    "    frame_threshold: float\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    pitches: np.ndarray of bin_indices\n",
    "    intervals: np.ndarray of rows containing (onset_index, offset_index)\n",
    "    velocities: np.ndarray of velocity values\n",
    "    \"\"\"\n",
    "    onsets = (onsets > onset_threshold).cpu().to(torch.uint8)\n",
    "    frames = (frames > frame_threshold).cpu().to(torch.uint8)\n",
    "    onset_diff = torch.cat([onsets[:1, :], onsets[1:, :] - onsets[:-1, :]], dim=0) == 1\n",
    "\n",
    "    pitches = []\n",
    "    intervals = []\n",
    "    # velocities = []\n",
    "\n",
    "    for nonzero in onset_diff.nonzero():\n",
    "        frame = nonzero[0].item()\n",
    "        pitch = nonzero[1].item()\n",
    "\n",
    "        onset = frame\n",
    "        offset = frame\n",
    "        # velocity_samples = []\n",
    "\n",
    "        while onsets[offset, pitch].item() or frames[offset, pitch].item():\n",
    "            # if onsets[offset, pitch].item():\n",
    "                # velocity_samples.append(velocity[offset, pitch].item())\n",
    "            offset += 1\n",
    "            if offset == onsets.shape[0]:\n",
    "                break\n",
    "\n",
    "        if offset > onset:\n",
    "            pitches.append(pitch)\n",
    "            intervals.append([onset, offset])\n",
    "            # velocities.append(np.mean(velocity_samples) if len(velocity_samples) > 0 else 0)\n",
    "\n",
    "    return np.array(pitches), np.array(intervals) #, np.array(velocities)\n",
    "\n",
    "\n",
    "def notes_to_frames(pitches, intervals, shape):\n",
    "    \"\"\"\n",
    "    Takes lists specifying notes sequences and return\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    pitches: list of pitch bin indices\n",
    "    intervals: list of [onset, offset] ranges of bin indices\n",
    "    shape: the shape of the original piano roll, [n_frames, n_bins]\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    time: np.ndarray containing the frame indices\n",
    "    freqs: list of np.ndarray, each containing the frequency bin indices\n",
    "    \"\"\"\n",
    "    roll = np.zeros(tuple(shape))\n",
    "    for pitch, (onset, offset) in zip(pitches, intervals):\n",
    "        roll[onset:offset, pitch] = 1\n",
    "\n",
    "    time = np.arange(roll.shape[0])\n",
    "    freqs = [roll[t, :].nonzero()[0] for t in time]\n",
    "    return time, freqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from collections import defaultdict\n",
    "\n",
    "import numpy as np\n",
    "from mir_eval.multipitch import evaluate as evaluate_frames\n",
    "from mir_eval.transcription import precision_recall_f1_overlap as evaluate_notes\n",
    "from mir_eval.transcription_velocity import precision_recall_f1_overlap as evaluate_notes_with_velocity\n",
    "from mir_eval.util import midi_to_hz\n",
    "from scipy.stats import hmean\n",
    "from tqdm import tqdm\n",
    "\n",
    "SAMPLE_RATE = 32000\n",
    "HOP_LENGTH = 640\n",
    "MIN_MIDI = 21\n",
    "MAX_MIDI = 108\n",
    "\n",
    "eps = sys.float_info.epsilon\n",
    "\n",
    "\n",
    "def evaluate(pred, label, onset_threshold=0.5, frame_threshold=0.5):\n",
    "    metrics = defaultdict(list)\n",
    "\n",
    "    pred = torch.nn.functional.relu(pred)\n",
    "\n",
    "    p_ref, i_ref = extract_notes(label[0].T, label[1].T, None)\n",
    "    p_est, i_est = extract_notes(pred[0].T, pred[1].T, None, onset_threshold, frame_threshold)\n",
    "\n",
    "    t_ref, f_ref = notes_to_frames(p_ref, i_ref, label[1].shape)\n",
    "    t_est, f_est = notes_to_frames(p_est, i_est, pred[1].shape)\n",
    "\n",
    "    scaling = HOP_LENGTH / SAMPLE_RATE\n",
    "\n",
    "    i_ref = (i_ref * scaling).reshape(-1, 2)\n",
    "    p_ref = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_ref])\n",
    "    i_est = (i_est * scaling).reshape(-1, 2)\n",
    "    p_est = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_est])\n",
    "\n",
    "    t_ref = t_ref.astype(np.float64) * scaling\n",
    "    f_ref = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_ref]\n",
    "    t_est = t_est.astype(np.float64) * scaling\n",
    "    f_est = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_est]\n",
    "\n",
    "    p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est, offset_ratio=None)\n",
    "    metrics['metric/note/precision'].append(p)\n",
    "    metrics['metric/note/recall'].append(r)\n",
    "    metrics['metric/note/f1'].append(f)\n",
    "    metrics['metric/note/overlap'].append(o)\n",
    "\n",
    "    p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est)\n",
    "    metrics['metric/note-with-offsets/precision'].append(p)\n",
    "    metrics['metric/note-with-offsets/recall'].append(r)\n",
    "    metrics['metric/note-with-offsets/f1'].append(f)\n",
    "    metrics['metric/note-with-offsets/overlap'].append(o)\n",
    "\n",
    "    frame_metrics = evaluate_frames(t_ref, f_ref, t_est, f_est)\n",
    "    metrics['metric/frame/f1'].append(hmean([frame_metrics['Precision'] + eps, frame_metrics['Recall'] + eps]) - eps)\n",
    "\n",
    "    for key, loss in frame_metrics.items():\n",
    "        metrics['metric/frame/' + key.lower().replace(' ', '_')].append(loss)\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([88, 1500])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out[0,0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(list,\n",
       "            {'metric/note/precision': [0.0035287541052337365],\n",
       "             'metric/note/recall': [0.8706896551724138],\n",
       "             'metric/note/f1': [0.007029020808685365],\n",
       "             'metric/note/overlap': [0.13458018355039242],\n",
       "             'metric/note-with-offsets/precision': [6.987631891551953e-05],\n",
       "             'metric/note-with-offsets/recall': [0.017241379310344827],\n",
       "             'metric/note-with-offsets/f1': [0.00013918853086505672],\n",
       "             'metric/note-with-offsets/overlap': [0.8754480286738355],\n",
       "             'metric/frame/f1': [0.10490753911806561],\n",
       "             'metric/frame/precision': [0.05635148042024833],\n",
       "             'metric/frame/recall': [0.7583547557840618],\n",
       "             'metric/frame/accuracy': [0.05535747795083505],\n",
       "             'metric/frame/substitution_error': [0.2416452442159383],\n",
       "             'metric/frame/miss_error': [0.0],\n",
       "             'metric/frame/false_alarm_error': [12.45758354755784],\n",
       "             'metric/frame/total_error': [12.699228791773779],\n",
       "             'metric/frame/chroma_precision': [0.07373447946513849],\n",
       "             'metric/frame/chroma_recall': [0.9922879177377892],\n",
       "             'metric/frame/chroma_accuracy': [0.07369224894998092],\n",
       "             'metric/frame/chroma_substitution_error': [0.007712082262210797],\n",
       "             'metric/frame/chroma_miss_error': [0.0],\n",
       "             'metric/frame/chroma_false_alarm_error': [12.45758354755784],\n",
       "             'metric/frame/chroma_total_error': [12.465295629820051]})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 7\n",
    "a = evaluate(out[i], target_pr[i])\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(list,\n",
       "            {'metric/note/precision': [0.003123635508682363],\n",
       "             'metric/note/recall': [0.8454545454545455],\n",
       "             'metric/note/f1': [0.00622427467121775],\n",
       "             'metric/note/overlap': [0.1902006336988417],\n",
       "             'metric/note-with-offsets/precision': [0.0004366372216437712],\n",
       "             'metric/note-with-offsets/recall': [0.11818181818181818],\n",
       "             'metric/note-with-offsets/f1': [0.0008700599002777498],\n",
       "             'metric/note-with-offsets/overlap': [0.6670681882220355],\n",
       "             'metric/frame/f1': [0.05838876570583906],\n",
       "             'metric/frame/precision': [0.030291411042944784],\n",
       "             'metric/frame/recall': [0.8061224489795918],\n",
       "             'metric/frame/accuracy': [0.030072325846973735],\n",
       "             'metric/frame/substitution_error': [0.19387755102040816],\n",
       "             'metric/frame/miss_error': [0.0],\n",
       "             'metric/frame/false_alarm_error': [25.612244897959183],\n",
       "             'metric/frame/total_error': [25.806122448979593],\n",
       "             'metric/frame/chroma_precision': [0.037576687116564415],\n",
       "             'metric/frame/chroma_recall': [1.0],\n",
       "             'metric/frame/chroma_accuracy': [0.037576687116564415],\n",
       "             'metric/frame/chroma_substitution_error': [0.0],\n",
       "             'metric/frame/chroma_miss_error': [0.0],\n",
       "             'metric/frame/chroma_false_alarm_error': [25.612244897959183],\n",
       "             'metric/frame/chroma_total_error': [25.612244897959183]})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "audiocraft-ssbSDm-j",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}