Animate_Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T14:29:00.668345Z",
     "start_time": "2021-01-27T14:28:55.952395Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "# Import classifier models\n",
    "from sklearn.linear_model import LogisticRegression # Logistic Regression\n",
    "from sklearn.naive_bayes import GaussianNB # Naïve Bayes\n",
    "from sklearn.linear_model import SGDClassifier # Stochastic Gradient Descent\n",
    "from sklearn.neighbors import KNeighborsClassifier # K-Nearest Neighbors\n",
    "from sklearn.tree import DecisionTreeClassifier # Decision Tree\n",
    "from sklearn.ensemble import RandomForestClassifier # Random Forest\n",
    "from sklearn.svm import SVC # Support Vector Classifier\n",
    "from sklearn.neural_network import MLPClassifier # Neural Network\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import pickle\n",
    "import imageio  # for animation\n",
    "try:\n",
    "    os.mkdir('./figures')\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T14:29:00.692197Z",
     "start_time": "2021-01-27T14:29:00.673287Z"
    }
   },
   "outputs": [],
   "source": [
    "def make_data(n_samples,noise,shift):\n",
    "    data,label = make_moons(n_samples=n_samples,noise=noise)\n",
    "    x = data[:,0]\n",
    "    y = data[:,1]\n",
    "\n",
    "    df = pd.DataFrame(columns=['x','y','label'])\n",
    "    df['x'] = data[:,0]\n",
    "    df['y'] = data[:,1]\n",
    "    df['label'] = label\n",
    "    \n",
    "    df['x'] = np.where(df['label'] == 0,df['x'],df['x'] + shift)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T14:29:00.959193Z",
     "start_time": "2021-01-27T14:29:00.696253Z"
    }
   },
   "outputs": [],
   "source": [
    "df = make_data(n_samples=1000,noise=0.1,shift=0.0)\n",
    "plt.scatter(df['x'],df['y'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "lr, knn, dt, svm, nb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T16:31:32.920509Z",
     "start_time": "2021-01-28T16:31:32.906510Z"
    }
   },
   "outputs": [],
   "source": [
    "nb=GaussianNB()\n",
    "lr=LogisticRegression()\n",
    "sgd=SGDClassifier(max_iter=2000)\n",
    "knn=KNeighborsClassifier()\n",
    "dt=DecisionTreeClassifier()\n",
    "rfm=RandomForestClassifier()\n",
    "svm=SVC()\n",
    "nn=MLPClassifier(max_iter=2000)\n",
    "ab=AdaBoostClassifier()\n",
    "qda=QuadraticDiscriminantAnalysis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T17:01:13.227792Z",
     "start_time": "2021-01-28T17:01:06.057474Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "clsfrs = [[nb,'Naive Bayes'],\n",
    "          [dt,'Decision Tree'],\n",
    "          [knn,'K Nearest Neighbors'],\n",
    "          [svm,'Support Vector Machine'],\n",
    "          [lr,'Logistic Regression'],\n",
    "          [sgd,'Stochastic Gradient Descent'],\n",
    "          [rfm,'Random Forest Classifier'],\n",
    "          [nn,'Neural Network'],\n",
    "          [ab,'AdaBoost'],\n",
    "          [qda,'Quadratic Discriminant Analysis']\n",
    "         ]\n",
    "\n",
    "for j, (clf, name) in enumerate(clsfrs):\n",
    "    figpath = './figures/Sup_'+name\n",
    "    try:\n",
    "        os.mkdir(figpath)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "colors = np.array(['orange','blue'])\n",
    "xlim = [-2,5];   x_min,x_max = xlim\n",
    "ylim = [-1,1.5]; y_min,y_max = ylim\n",
    "step = 0.02\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max+step, step), \\\n",
    "                     np.arange(y_min, y_max+step, step))\n",
    "cm = plt.cm.RdBu\n",
    "\n",
    "for i,s in enumerate(np.linspace(2,0,80)):\n",
    "    df = make_data(n_samples=2000,noise=0.1,shift=s)\n",
    "    Xtrain, Xtest, ytrain, ytest = train_test_split(df[['x','y']],\n",
    "                                                    df['label'],\n",
    "                                                    test_size = 0.2)\n",
    "    for j, (clf, name) in enumerate(clsfrs):\n",
    "        figname = './figures/Sup_'+name+'/'+str(10+i)+'.png'\n",
    "\n",
    "        # fit\n",
    "        t0 = time.time()\n",
    "        clf.fit(Xtrain,ytrain)\n",
    "        tfit = time.time()-t0\n",
    "        # predict\n",
    "        t0 = time.time()\n",
    "        yp = clf.predict(Xtest)\n",
    "        tpred = time.time()-t0\n",
    "        # size\n",
    "        pickle.dump(clf,open('size.pkl','wb'))\n",
    "        kb = int(os.path.getsize('size.pkl'))\n",
    "        os.remove('size.pkl')\n",
    "        \n",
    "        # Mismatches red\n",
    "        match = np.where(yp!=ytest.values,0,1)\n",
    "        \n",
    "        accuracy = accuracy_score(yp,ytest)\n",
    "        \n",
    "        plt.figure(0,figsize=(4,4))\n",
    "        plt_data = pd.DataFrame(columns=['x','y','color','marker'])\n",
    "        plt_data['x'] = Xtest['x']\n",
    "        plt_data['y'] = Xtest['y']\n",
    "        plt_data['color'] = yp\n",
    "        plt_data['marker'] = match\n",
    "\n",
    "        # contour plot\n",
    "        if hasattr(clf, \"decision_function\"):\n",
    "            z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\n",
    "        else:\n",
    "            z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n",
    "        z = z.reshape(xx.shape)\n",
    "        plt.contourf(xx, yy, z, cmap=cm, alpha=0.7)        \n",
    "        \n",
    "        plt.scatter(plt_data[plt_data['marker']==1]['x'],\n",
    "                    plt_data[plt_data['marker']==1]['y'],\n",
    "                    color = colors[plt_data[plt_data['marker']==1]['color']],\n",
    "                    marker = '.'\n",
    "                   )\n",
    "        \n",
    "        plt.scatter(plt_data[plt_data['marker']==0]['x'],\n",
    "                    plt_data[plt_data['marker']==0]['y'],\n",
    "                    color = colors[plt_data[plt_data['marker']==0]['color']],\n",
    "                    edgecolors='k',\n",
    "                    marker = 'x'\n",
    "                   )\n",
    "        \n",
    "        plt.text(1.8,1.35,'Train: %.4f s'%(tfit))\n",
    "        plt.text(1.8,1.20,'Predict: %.4f s'%(tpred))\n",
    "        if kb<1024:\n",
    "            plt.text(1.8,1.05,'Size: %.0f Bytes'%(kb))\n",
    "        else:\n",
    "            plt.text(1.8,1.05,'Size: %.0f kB'%(kb/1024.0))            \n",
    "        plt.text(1.8,0.90,'Accuracy: %.0f%%'%(accuracy*100))\n",
    "        plt.xlim([-2,5])\n",
    "        plt.ylim([-1,1.5])\n",
    "        plt.title(name)\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(figname,dpi=300)\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 80\n",
    "for j, (clf, name) in enumerate(clsfrs):\n",
    "    images = []\n",
    "    for i in range(n):\n",
    "        filename = './figures/Sup_'+name+'/'+str(10+i)+'.png'\n",
    "        images.append(imageio.imread(filename))\n",
    "    if images!=[]:\n",
    "        imageio.mimsave('./figures/Sup_'+name+'.mp4', images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T16:59:21.569537Z",
     "start_time": "2021-01-28T16:59:21.554539Z"
    }
   },
   "outputs": [],
   "source": [
    "# Unsupervised\n",
    "from sklearn.cluster import KMeans, DBSCAN, SpectralClustering\n",
    "from sklearn.mixture import GaussianMixture\n",
    "\n",
    "km = KMeans(n_clusters=2)\n",
    "gm = GaussianMixture(n_components=2)\n",
    "dbs = DBSCAN(eps=0.15)\n",
    "sc = SpectralClustering(n_clusters=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-28T17:00:01.445404Z",
     "start_time": "2021-01-28T16:59:57.317759Z"
    }
   },
   "outputs": [],
   "source": [
    "clsfrs = [[km,'K Means'],\n",
    "          [gm,'Gaussian Mixture Model'],\n",
    "          [sc,'Spectral Clustering'],\n",
    "          [dbs,'DBSCAN']\n",
    "         ]\n",
    "\n",
    "colors = np.array(['orange','blue','green','purple'])\n",
    "markers = np.array(['x','.'])\n",
    "\n",
    "for j, (clf, name) in enumerate(clsfrs):\n",
    "    figpath = './figures/Unsup_'+name\n",
    "    try:\n",
    "        os.mkdir(figpath)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "colors = np.array(['orange','blue','green','purple'])\n",
    "xlim = [-2,5];   x_min,x_max = xlim\n",
    "ylim = [-1,1.5]; y_min,y_max = ylim\n",
    "step = 0.02\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max+step, step), \\\n",
    "                     np.arange(y_min, y_max+step, step))\n",
    "cm = plt.cm.RdBu\n",
    "\n",
    "for i,s in enumerate(np.linspace(2,0,80)):\n",
    "    df = make_data(n_samples=2000,noise=0.1,shift=s)\n",
    "    Xtrain, Xtest, ytrain, ytest = train_test_split(df[['x','y']],\n",
    "                                                    df['label'],\n",
    "                                                    test_size = 0.2)\n",
    "    for j, (clf, name) in enumerate(clsfrs):\n",
    "        figname = './figures/Unsup_'+name+'/'+str(10+i)+'.png'\n",
    "\n",
    "        # fit\n",
    "        t0 = time.time()\n",
    "        clf.fit(Xtrain,ytrain)\n",
    "        tfit = time.time()-t0\n",
    "        # predict\n",
    "        t0 = time.time()\n",
    "        if hasattr(clf,'predict'):\n",
    "            yp = clf.predict(Xtest)\n",
    "        else:\n",
    "            yp = clf.fit_predict(Xtest)\n",
    "        tpred = time.time()-t0\n",
    "        # size\n",
    "        pickle.dump(clf,open('size.pkl','wb'))\n",
    "        kb = int(os.path.getsize('size.pkl'))\n",
    "        os.remove('size.pkl')\n",
    "                    \n",
    "        # Mismatches red\n",
    "        match = np.where(yp!=ytest.values,0,1)\n",
    "        \n",
    "        accuracy = accuracy_score(yp,ytest)\n",
    "        \n",
    "        plt.figure(0,figsize=(4,4))\n",
    "        plt_data = pd.DataFrame(columns=['x','y','color','marker'])\n",
    "        plt_data['x'] = Xtest['x']\n",
    "        plt_data['y'] = Xtest['y']\n",
    "        plt_data['color'] = yp\n",
    "        plt_data['marker'] = match\n",
    "\n",
    "        # contour plot\n",
    "        skip = False\n",
    "        if hasattr(clf, \"decision_function\"):\n",
    "            z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\n",
    "        elif hasattr(clf, \"predict_proba\"):\n",
    "            z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n",
    "        elif hasattr(clf, \"predict\"):\n",
    "            z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "        else:\n",
    "            skip = True # for Spectral Clustering\n",
    "            #z = clf.fit_predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "\n",
    "        # skip contour plot for certain methods\n",
    "        if not skip:\n",
    "            z = z.reshape(xx.shape)\n",
    "            plt.contourf(xx, yy, z, cmap=cm, alpha=0.7)        \n",
    "        \n",
    "        try:\n",
    "            clr = colors[min(3,plt_data[plt_data['marker']==1]['color'])],\n",
    "        except:\n",
    "            clr = 'orange'\n",
    "        plt.scatter(plt_data[plt_data['marker']==1]['x'],\n",
    "                    plt_data[plt_data['marker']==1]['y'],\n",
    "                    color = clr,\n",
    "                    marker = '.'\n",
    "                   )\n",
    "        \n",
    "        try:\n",
    "            clr = colors[min(3,plt_data[plt_data['marker']==0]['color'])],\n",
    "        except:\n",
    "            clr = 'orange'\n",
    "        plt.scatter(plt_data[plt_data['marker']==0]['x'],\n",
    "                    plt_data[plt_data['marker']==0]['y'],\n",
    "                    color = clr,\n",
    "                    edgecolors='k',\n",
    "                    marker = 'x'\n",
    "                   )\n",
    "        \n",
    "        plt.text(1.8,1.35,'Train: %.4f s'%(tfit))\n",
    "        plt.text(1.8,1.20,'Predict: %.4f s'%(tpred))\n",
    "        if kb<1024:\n",
    "            plt.text(1.8,1.05,'Size: %.0f Bytes'%(kb))\n",
    "        else:\n",
    "            plt.text(1.8,1.05,'Size: %.0f kB'%(kb/1024.0))            \n",
    "        plt.text(1.8,0.90,'Accuracy: %.0f%%'%(accuracy*100))\n",
    "        plt.xlim([-2,5])\n",
    "        plt.ylim([-1,1.5])\n",
    "        plt.title(name)\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(figname,dpi=300)\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 80\n",
    "for j, (clf, name) in enumerate(clsfrs):\n",
    "    images = []\n",
    "    for i in range(n):\n",
    "        filename = './figures/Unsup_'+name+'/'+str(10+i)+'.png'\n",
    "        images.append(imageio.imread(filename))\n",
    "    if images!=[]:\n",
    "        imageio.mimsave('./figures/Unsup_'+name+'.mp4', images)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}