1_08_FeatureEngineering6_Vectorization_By_Gensim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Gensim to vectorize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import random\n",
    "import numexpr as ne\n",
    "import mskcc_functions as ski\n",
    "import scipy.stats as scs\n",
    "import feature_engineering as fe\n",
    "import xgboost as xgb\n",
    "\n",
    "from pprint import pprint\n",
    "from matplotlib  import cm\n",
    "from collections import Counter\n",
    "from importlib import reload\n",
    "from gensim import corpora, matutils, models, similarities\n",
    "from nltk import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.preprocessing import scale, normalize, robust_scale\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "sns.set_context(\"paper\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Matrix1 Shape Before:  (3321, 22190)\n",
      "Feature Matrix1 Shape After:  (3321, 22156)\n"
     ]
    }
   ],
   "source": [
    "class_train = pd.read_csv('./data/training_variants')\n",
    "text_train = pd.read_csv(\"./data/training_text\", sep=\"\\|\\|\", engine='python',\n",
    "                         header=None, skiprows=1, names=[\"ID\",\"Text\"])\n",
    "\n",
    "feature_matrix1 = pd.read_csv('./data/feature_matrix_pass1.csv')\n",
    "print(\"Feature Matrix1 Shape Before: \", feature_matrix1.shape)\n",
    "\n",
    "total_frequency = feature_matrix1.sum(axis=0)\n",
    "zero_frequency_index = list(np.argwhere(total_frequency == 0).ravel())\n",
    "nonzero_frequency_index = [ind for ind in range(feature_matrix1.shape[1]) if ind not in zero_frequency_index]\n",
    "feature_matrix1 = feature_matrix1.iloc[:, nonzero_frequency_index]\n",
    "print(\"Feature Matrix1 Shape After: \", feature_matrix1.shape)\n",
    "\n",
    "feature_words = list(feature_matrix1.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "additional_removal = ['of', 'out', 'one', 'the', 'and', 'from', 'that', 'this', 'these', 'no',\n",
    "                     'those', 'he', 'she', 'me', 'you', 'her', 'by','we', 'is', 'to', 'at', \n",
    "                      'into', 'onto', 'on', 'out', 'can', 'should', 'must', 'would', 'will',\n",
    "                      'as', 'between', 'within','among','which', 'where', 'when','how','mm',\n",
    "                     'na', 'many', 'much', 'min', 'sec', 'hr', 'go', 'via']\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of words before removal: 22156\n",
      "# of words after removal: 22089\n"
     ]
    }
   ],
   "source": [
    "print('# of words before removal: %d' % len(feature_words))\n",
    "feature_words = [word for word in feature_words \\\n",
    "                 if word not in stop_words \\\n",
    "                 if word not in additional_removal]\n",
    "print('# of words after removal: %d' % len(feature_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# redundancy from plural forms to be removed and taken care of.\n",
    "# This requires some fiddling since the word still has to be found\n",
    "# feature_words = [re.sub(r's$', '', word) for word in feature_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 289 µs, sys: 0 ns, total: 289 µs\n",
      "Wall time: 295 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "documents = []\n",
    "for i in range(10):\n",
    "    document = text_train['Text'].iloc[i] + ''\n",
    "    documents.append(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.4 s, sys: 1.08 ms, total: 13.4 s\n",
      "Wall time: 13.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "textome = [[word for word in document.lower().split() if word in feature_words] for document in documents]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This would take too much time (>1hr) to process, so use subset of the feature_words space as below"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>With narrower feature space</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "whole_text = ''\n",
    "for i in range(len(text_train)):\n",
    "    text = text_train.loc[i, 'Text'] + ''\n",
    "    whole_text += text\n",
    "\n",
    "# Remove irrelevant characters by replacing them with whitespace\n",
    "whole_text_white = fe.replace_with_whitespace(whole_text, hyphens='on')\n",
    "\n",
    "# Tokenize the whole_text_white\n",
    "tokens_white = word_tokenize(whole_text_white)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 51s, sys: 999 µs, total: 1min 51s\n",
      "Wall time: 1min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Get race words\n",
    "race_words = set(fe.get_race_words(tokens_white))\n",
    "\n",
    "#Get drug words\n",
    "drug_words = set(fe.get_drug_words(tokens_white))\n",
    "\n",
    "#Get tissue type words (organ, tumor type, and cell type), combine the#\n",
    "tissue_type_words = set(fe.get_tissue_type_words(tokens_white))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of race words: 10\n",
      "# of drug words: 202\n",
      "# of tissue type words: 467\n"
     ]
    }
   ],
   "source": [
    "# Get number of words obtained from each\n",
    "print('# of race words: %d' % len(race_words))\n",
    "print('# of drug words: %d' % len(drug_words))\n",
    "print('# of tissue type words: %d' % len(tissue_type_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "662\n"
     ]
    }
   ],
   "source": [
    "feature_words2 = list(race_words) + list(drug_words) + list(tissue_type_words)\n",
    "feature_words2 = set(feature_words2)\n",
    "\n",
    "feature_words2 = [re.sub(r'^[\\W0-9]+', '', word) for word in feature_words2]\n",
    "feature_words2 = [word for word in feature_words2 if word not in additional_removal]\n",
    "feature_words2 = sorted(list(set(feature_words2)))\n",
    "print(len(feature_words2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save the 1st pass feature names\n",
    "pd.Series(feature_words2).to_csv('./data/features_pass1_subset1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mskcc_dictionary1 = corpora.Dictionary([feature_words2])\n",
    "mskcc_dictionary1.save('/tmp/mskcc_dictionary1.dict')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'aadenocarcinoma': 0, 'abagovomab': 1, 'acanthoma': 2, 'acthoma': 3, 'adamantinoma': 4, 'adenenocarcinoma': 5, 'adenoacanthoma': 6, 'adenocanthoma': 7, 'adenocarcinoma': 8, 'adenocarcioma': 9, 'adenocarinoma': 10, 'adenocrcinoma': 11, 'adenolymphoma': 12, 'adenoma': 13, 'adenoma–carcinoma': 14, 'adenomyoepithelialoma': 15, 'adenomyoepithelioma': 16, 'adenomyoepithioma': 17, 'adeocarcinoma': 18, 'adipocyte': 19, 'administrationcrizotinib': 20, 'adnocarcinoma': 21, 'adrenal': 22, 'afatanib': 23, 'afatinib': 24, 'african': 25, 'agerminoma': 26, 'alectinib': 27, 'alemtuzumab': 28, 'alk+lymphoma': 29, 'alkoma': 30, 'altiratinib': 31, 'ampullarycarcinoma': 32, 'amuvatinib': 33, 'andlymphocyte': 34, 'angiofibroma': 35, 'angioleiomyoma': 36, 'angioma': 37, 'angiomyofibroblastoma': 38, 'angiomyolipoma': 39, 'angiomyxoma': 40, 'angiosarcoma': 41, 'anoma': 42, 'antihepatocyte': 43, 'antimelanoma': 44, 'antimyeloma': 45, 'anti–glioblastoma': 46, 'arcinoma': 47, 'arms—erlotinib': 48, 'aroma': 49, 'asian': 50, 'aspergilloma': 51, 'astocytoma': 52, 'astroctyoma': 53, 'astrocyte': 54, 'astrocytoma': 55, 'astro\\xadcytoma': 56, 'auto`noma': 57, 'autã³noma': 58, 'autònoma': 59, 'axitinib': 60, 'bevacizumab': 61, 'black': 62, 'bladder': 63, 'blastoma': 64, 'bone': 65, 'bortezomib': 66, 'bosutinib': 67, 'brain': 68, 'breast': 69, 'brentuximab': 70, 'brigatinib': 71, 'brivanib': 72, 'broblast': 73, 'brosarcoma': 74, 'bsarcoma': 75, 'c491dupnilotinib': 76, 'cabozantinib': 77, 'cabozatinib': 78, 'cancer—adenocarcinoma': 79, 'canertinib': 80, 'capmatinib': 81, 'carcinoma': 82, 'carcinomaadenocarcinomacolonadenocarcinomaadenomaadenoma': 83, 'carcinosarcoma': 84, 'cardiac': 85, 'cardiomiocyte': 86, 'cardiomyocyte': 87, 'carfilzomib': 88, 'casesadenocarcinoma': 89, 'caucasian': 90, 'ccarcinoma': 91, 'cediranib': 92, 'celllinesn161carcinoma': 93, 'celllymphoma': 94, 'cellswasimatinib': 95, 'centroblast': 96, 'centrocyte': 97, 'cerdulatinib': 98, 'ceritinib': 99, 'certinib': 100, 'cervical': 101, 'cervix': 102, 'cetuximab': 103, 'cholangioacrinoma': 104, 'cholangiocarcinoma': 105, 'cholangiocarincoma': 106, 'cholangiocarinoma': 107, 'cholangiocyte': 108, 'cholangiosarcoma': 109, 'cholesteatoma': 110, 'chondroblast': 111, 'chondrocyte': 112, 'chondroma': 113, 'chondrosarcoma': 114, 'chordoma': 115, 'choriocarcinoma': 116, 'chorioepithelioma': 117, 'chroma': 118, 'chromocytoma': 119, 'cinoma': 120, 'cobimetinib': 121, 'colagenoma': 122, 'colangiocarcinoma': 123, 'collagenoma': 124, 'coloboma': 125, 'colon': 126, 'colonocyte': 127, 'coma': 128, 'conditionsfibroblast': 129, 'craniopharyngioma': 130, 'crenolanib': 131, 'creonolanib': 132, 'crioztinib': 133, 'critoztinib': 134, 'crizotiinib': 135, 'crizotinib': 136, 'crizotonib': 137, 'culturedmelanoma': 138, 'cylindroma': 139, 'cyneuroblastoma': 140, 'cystadenocarcinoma': 141, 'cystadenoma': 142, 'cysticleiomyoma': 143, 'cysticmesothelioma': 144, 'cytotrophoblast': 145, 'dabrafenib': 146, 'dabrafenib–trametinib': 147, 'dacomitinib': 148, 'dacomitnib': 149, 'dasatanib': 150, 'dasatinib': 151, 'dasitinib': 152, 'depocyte': 153, 'dermatofibroma': 154, 'dermatofibrosarcoma': 155, 'desminib': 156, 'dictyoma': 157, 'differentfibroblast': 158, 'differentiatedadenocarcinoma': 159, 'discussionosteosarcoma': 160, 'disease–neuroblastoma': 161, 'disgerminoma': 162, 'doma': 163, 'dovitinib': 164, 'duligotuzumab': 165, 'dysgermimoma': 166, 'dysgerminoma': 167, 'ectomesenchymoma': 168, 'egfr–afatinib': 169, 'egfr—cetuximab': 170, 'eitherstroma': 171, 'elotinib': 172, 'emangioendothelioma': 173, 'emangioma': 174, 'enchondroma': 175, 'enteroblast': 176, 'enterocyte': 177, 'entrecitnib': 178, 'entrectinib': 179, 'ependemoma': 180, 'ependymoblastoma': 181, 'ependymoma': 182, 'epiblast': 183, 'epitheliocyte': 184, 'epithelioma': 185, 'erlotinib': 186, 'erolotinib': 187, 'ertumaxomab': 188, 'erythroblast': 189, 'erythrocyte': 190, 'erythromegakaryocyte': 191, 'esthesioneuroblastoma': 192, 'estroma': 193, 'fedratinib': 194, 'fenib': 195, 'fibroadenoma': 196, 'fibroblast': 197, 'fibroblastmyofibroblast': 198, 'fibroblastoma': 199, 'fibrolipoma': 200, 'fibroma': 201, 'fibrosarcoma': 202, 'fibroxanthoma': 203, 'fibroxantoma': 204, 'ficlatuzumab': 205, 'figitumumab': 206, 'foimelanoma': 207, 'foretinib': 208, 'fostamatinib': 209, 'fractionsfibroblast': 210, 'froma': 211, 'g1202rcrizotinib': 212, 'gallbladder': 213, 'gangliocytoma': 214, 'ganglioglioma': 215, 'ganglioma': 216, 'ganglioneuroblastoma': 217, 'ganglioneuroma': 218, 'gastric': 219, 'gastriccarcinoma': 220, 'gastrinoma': 221, 'gefinib': 222, 'gefitanib': 223, 'gefitinib': 224, 'gefitnib': 225, 'geftinib': 226, 'gemtuzumab': 227, 'gentuzumab': 228, 'germinoma': 229, 'gewtinib': 230, 'ghoma': 231, 'giloblastoma': 232, 'gist–regorafenib': 233, 'gland': 234, 'glandcarcinoma': 235, 'glaucoma': 236, 'glesatinib': 237, 'gliobastoma': 238, 'glioblastoma': 239, 'glioblostoma': 240, 'glioma': 241, 'gliosarcoma': 242, 'glomangiopericytoma': 243, 'glucagonoma': 244, 'gonadoblastoma': 245, 'gonocyte': 246, 'granulocyte': 247, 'granulocytemonocyte': 248, 'granuloma': 249, 'gynandoblastoma': 250, 'gynandroblastoma': 251, 'haemangioblast': 252, 'haemangioblastoma': 253, 'haemangioma': 254, 'haemangiopericytoma': 255, 'haemangiosarcoma': 256, 'haematoma': 257, 'hamartoma': 258, 'harmatoma': 259, 'heamangioblastoma': 260, 'heart': 261, 'hemangioblastoma': 262, 'hemangioendothelioma': 263, 'hemangioma': 264, 'hemangiopericytoma': 265, 'hemangiosarcoma': 266, 'hematoma': 267, 'hemocyte': 268, 'hepatoblast': 269, 'hepatoblastoma': 270, 'hepatocarcinoma': 271, 'hepatocyte': 272, 'hepatoma': 273, 'her2–afatinib': 274, 'hidradenoma': 275, 'hispanic': 276, 'histiocyte': 277, 'histiocytoma': 278, 'histocytoma': 279, 'humanpapilloma': 280, 'hybridoma': 281, 'hygroma': 282, 'ibrutinib': 283, 'igblast': 284, 'imatimib': 285, 'imatinib': 286, 'imelanoma': 287, 'imitinib': 288, 'immunoblast': 289, 'immunocyte': 290, 'immunocytoma': 291, 'incidentaloma': 292, 'incucyte': 293, 'incyte': 294, 'indian': 295, 'inhibition—sorafenib': 296, 'inmelanoma': 297, 'innocyte': 298, 'insulinoma': 299, 'intestinal': 300, 'intestine': 301, 'ipilimumab': 302, 'keratinocyte': 303, 'kerationcyte': 304, 'keratoacanthoma': 305, 'keratoadenoma': 306, 'keratocanthoma': 307, 'kidney': 308, 'kitimatinib': 309, 'kit–imatinib': 310, 'kocyte': 311, 'koma': 312, 'lapatinib': 313, 'laptinib': 314, 'latino': 315, 'leimyosarcoma': 316, 'leioblastoma': 317, 'leiomyoblastoma': 318, 'leiomyoma': 319, 'leiomyomsarcoma': 320, 'leiomyosarcoma': 321, 'lenvatinib': 322, 'lestaurtinib': 323, 'leucocyte': 324, 'leukemialymphoma': 325, 'leukemia–lymphoma': 326, 'leukocyte': 327, 'levantinib': 328, 'lexatumumab': 329, 'linsitinib': 330, 'lipoblast': 331, 'lipoblastoma': 332, 'lipoma': 333, 'liposarcoma': 334, 'liver': 335, 'lonafarnib': 336, 'loopweremutatedinimatinib': 337, 'lorlatinib': 338, 'lucitanib': 339, 'lumretuzumab': 340, 'lung': 341, 'luteoma': 342, 'lymphangioma': 343, 'lymphoblast': 344, 'lymphocyte': 345, 'lymphoepitelioma': 346, 'lymphoepithelioma': 347, 'lymphoma': 348, 'lymphosarcoma': 349, 'macroadenoma': 350, 'macroprolactinoma': 351, 'margetuximab': 352, 'masatinib': 353, 'masitinib': 354, 'mastocytoma': 355, 'medullablastoma': 356, 'medullobastoma': 357, 'medulloblastoma': 358, 'medulloepitheioma': 359, 'medulloepithelioma': 360, 'medulloepthelioma': 361, 'meduloblastoma': 362, 'megablast': 363, 'megakarocyte': 364, 'megakaryocyte': 365, 'melanoblast': 366, 'melanocyte': 367, 'melanocytoma': 368, 'melanoma': 369, 'melanomaimatinib': 370, 'melanoma—ipilimumab': 371, 'mela¬noma': 372, 'melonoma': 373, 'meningioma': 374, 'meninigioma': 375, 'mepolizumab': 376, 'merestinib': 377, 'mesenchymoma': 378, 'mesothelioma': 379, 'metamyelocyte': 380, 'metastaticmelanoma': 381, 'methodsosteosarcoma': 382, 'methodsreagentscrizotinib': 383, 'meyloma': 384, 'mgimatinib': 385, 'microadenoma': 386, 'microcarcinoma': 387, 'micromegakaryocyte': 388, 'minigemistocyte': 389, 'mixosarcoma': 390, 'monoblast': 391, 'monocyte': 392, 'motesanib': 393, 'mucosalmelanoma': 394, 'murinefibroblast': 395, 'mutantcabozantinib': 396, 'mutantfibroblast': 397, 'myeloblast': 398, 'myelocyte': 399, 'myeloma': 400, 'myelomonoblast': 401, 'myelosarcoma': 402, 'myoblast': 403, 'myocyte': 404, 'myoepithelioma': 405, 'myofibroblast': 406, 'myofibroma': 407, 'myoma': 408, 'myopericytoma': 409, 'myxofibrosarcoma': 410, 'myxoma': 411, 'native american': 412, 'nentedanib': 413, 'nephroblastoma': 414, 'nephroma': 415, 'neratinib': 416, 'neratinib+trastuzumab': 417, 'neruroblastoma': 418, 'neurblastoma': 419, 'neurilemmoma': 420, 'neurilemoma': 421, 'neurinoma': 422, 'neuroblast': 423, 'neuroblastoma': 424, 'neuroblatoma': 425, 'neurocyte': 426, 'neurocytoma': 427, 'neuroepithelioma': 428, 'neurofibroma': 429, 'neurofibrosarcoma': 430, 'neuroma': 431, 'nevocyte': 432, 'nevomelanocyte': 433, 'nilotinib': 434, 'nimotuzumab': 435, 'nintedanib': 436, 'noma': 437, 'nonadenocarcinoma': 438, 'nonastrocytoma': 439, 'noncarcinoma': 440, 'nonerlotinib': 441, 'nonkeratinocyte': 442, 'nonlymphocyte': 443, 'nonlymphoma': 444, 'nonmelanoma': 445, 'nonseminoma': 446, 'non–panitumumab': 447, 'nulloligodendroglioma': 448, 'ochromocytoma': 449, 'odontoblast': 450, 'odontoma': 451, 'ofimatinib': 452, 'ofmelanoma': 453, 'ofrhabdomyosarcoma': 454, 'oklahoma': 455, 'oligoastroctyoma': 456, 'oligoastrocytoma': 457, 'oligodendrocyte': 458, 'oligodendroglioma': 459, 'oligodendroma': 460, 'oligodrendrocyte': 461, 'onartuzumab': 462, 'oncocyte': 463, 'oncocytoma': 464, 'oocyte': 465, 'orantinib': 466, 'osetosarcoma': 467, 'osimertinib': 468, 'osteoblast': 469, 'osteoblastoma': 470, 'osteocyte': 471, 'osteoma': 472, 'osteosarcoma': 473, 'osteoscarcoma': 474, 'ovarian': 475, 'ovariancarcinoma': 476, 'ovary': 477, 'p53y220sfibroblast': 478, 'pacific islander': 479, 'pancreas': 480, 'pancreatic': 481, 'pancreatoblastoma': 482, 'panitumumab': 483, 'papilloma': 484, 'paraganglioma': 485, 'parathyroid': 486, 'patritumab': 487, 'pazopanib': 488, 'pecoma': 489, 'pelitinib': 490, 'pembrolizumab': 491, 'pericyte': 492, 'pericytoma': 493, 'perosteosarcoma': 494, 'pertuzumab': 495, 'phaeochomocytoma': 496, 'phaeochromocytoma': 497, 'phagocyte': 498, 'phakoma': 499, 'pharmacokineticsimatinib': 500, 'pheochomocytoma': 501, 'pheochromocytoma': 502, 'pheochromocytoma–paraganglioma': 503, 'pheocromocytoma': 504, 'phocyte': 505, 'pilomatricoma': 506, 'pineoblastoma': 507, 'pituitary': 508, 'plasmablast': 509, 'plasmacytoma': 510, 'plasmocyte': 511, 'pleuramesothelioma': 512, 'pmab': 513, 'pneumocyte': 514, 'pneumonocyte': 515, 'podocyte': 516, 'poikilocyte': 517, 'polyoma': 518, 'ponatinib': 519, 'postimatinib': 520, 'post–crizotinib': 521, 'post–erlotinib': 522, 'preadipocyte': 523, 'precrizotinib': 524, 'preimatinib': 525, 'preosteoblast': 526, 'pre–bevacizumab': 527, 'pre–crizotinib': 528, 'proerythroblast': 529, 'prolactinoma': 530, 'prolymphocyte': 531, 'promonocyte': 532, 'promyelocyte': 533, 'prostate': 534, 'psammoma': 535, 'pseudopsammoma': 536, 'quizartinib': 537, 'ramicirumab': 538, 'ramucirumab': 539, 'rebinoblastoma': 540, 'rectal': 541, 'rectum': 542, 'refametinib': 543, 'regorafenib': 544, 'renal': 545, 'reslizumab': 546, 'resultedinmoderateimatinib': 547, 'resultsosteosarcoma': 548, 'reticulocyte': 549, 'retinoblasoma': 550, 'retinoblast': 551, 'retinoblastoma': 552, 'retinoma': 553, 'rhabdoleiomyoma': 554, 'rhabdomyoblast': 555, 'rhabdomyoma': 556, 'rhabdomyosarcoma': 557, 'rhabomyosarcoma': 558, 'rilotumumab': 559, 'rituximab': 560, 'rofibroma': 561, 'roma': 562, 'ros1–crizotinib': 563, 'roussarcoma': 564, 'rutuximab': 565, 'ruxolitinib': 566, 'ruxotilinib': 567, 'saracatinib': 568, 'sarcoma': 569, 'schistosoma': 570, 'schwannoma': 571, 'scwannoma': 572, 'sebocyte': 573, 'selumetinib': 574, 'seluteminib': 575, 'seminoma': 576, 'seribantumab': 577, 'sideroblast': 578, 'siderocyte': 579, 'siltuximab': 580, 'sitravatinib': 581, 'skeletal': 582, 'skin': 583, 'skincarcinoma': 584, 'soma': 585, 'somatostatinoma': 586, 'somatostinoma': 587, 'sorafanib': 588, 'sorafenib': 589, 'sorafenib+sunitinib': 590, 'sorafinib': 591, 'spermatocyte': 592, 'spinal': 593, 'spleen': 594, 'spleenocyte': 595, 'splenic': 596, 'splenocyte': 597, 'spongiotrophoblast': 598, 'sporadicmelanoma': 599, 'stoma': 600, 'stomach': 601, 'stroma': 602, 'subependymoma': 603, 'sulfatinib': 604, 'sunitinib': 605, 'suntinib': 606, 'synoviocyte': 607, 'syringoma': 608, 'tandutinib': 609, 'tasocitinib': 610, 'tepotinib': 611, 'teratocarcinoma': 612, 'teratoma': 613, 'thecoma': 614, 'thesefibroblast': 615, 'thoma': 616, 'thrombocyte': 617, 'thymocyte': 618, 'thymoma': 619, 'thyrocyte': 620, 'thyroid': 621, 'tinib': 622, 'tipifarnib': 623, 'tivantinib': 624, 'tivatinib': 625, 'tkis—nilotinib': 626, 'tmab': 627, 'tofacitinib': 628, 'toimatinib': 629, 'toma': 630, 'trametimib': 631, 'trametinib': 632, 'transtuzumab': 633, 'trastuzamab': 634, 'trastuzumab': 635, 'trastuzumab+lapatinib': 636, 'trastuzumab+neratinib': 637, 'tremelimumab': 638, 'trichilemmoma': 639, 'trichoepithelioma': 640, 'trophoblast': 641, 'trypanosoma': 642, 'tumor–stroma': 643, 'tumour–stroma': 644, 'underimatinib': 645, 'unitgranulocyte': 646, 'vandetanib': 647, 'vantetanib': 648, 'vatalanib': 649, 'vemurafenib': 650, 'vemurafinib': 651, 'vemuraftenib': 652, 'vemurfenib': 653, 'white': 654, 'withdasatinib': 655, 'withimatinib': 656, 'xanthoastrocytoma': 657, 'xanthogranuloma': 658, 'zomib': 659, 'zumab': 660, 'ﬁbroblast': 661}\n"
     ]
    }
   ],
   "source": [
    "print(mskcc_dictionary1.token2id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start over with saved files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_words2= pd.read_csv('./data/features_pass1_subset1.csv', header=None)\n",
    "feature_words2 = list(feature_words2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict1 = corpora.Dictionary([feature_words2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3321\n",
      "3321\n",
      "CPU times: user 1min 55s, sys: 489 ms, total: 1min 55s\n",
      "Wall time: 1min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "documents = []\n",
    "tokenized_documents = []\n",
    "for i in range(len(text_train)):\n",
    "    doc= text_train['Text'].iloc[i]\n",
    "    tokenized_doc = word_tokenize(fe.replace_with_whitespace(doc, hyphens='on'))\n",
    "    \n",
    "    documents.append(doc)\n",
    "    tokenized_documents.append(tokenized_doc)\n",
    "\n",
    "print(len(documents))     # double checking length\n",
    "print(len(tokenized_documents))     # double checking length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1e+03 ns, sys: 0 ns, total: 1e+03 ns\n",
      "Wall time: 3.1 µs\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "corpus = [dict1.doc2bow(token) for token in tokenized_documents]\n",
    "tfidf = models.TfidfModel(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(68, 0.10180953731779364), (69, 0.28417097593268426), (102, 0.30423889726812786), (125, 0.5501255110547993), (126, 0.08571365011797719), (261, 0.18349439254579827), (308, 0.21121164590425284), (414, 0.5200431794352696), (545, 0.3970012597830717)]\n",
      "[(8, 0.13839522847728142), (69, 0.03125203608407436), (82, 0.08968356109962472), (126, 0.028279377345904216), (308, 0.06968474480032831), (335, 0.0363244362091049), (341, 0.9802441363339809), (348, 0.04208912438416052), (369, 0.029899754762902034), (534, 0.03616243622163127)]\n",
      "[(8, 0.13839522847728142), (69, 0.03125203608407436), (82, 0.08968356109962472), (126, 0.028279377345904216), (308, 0.06968474480032831), (335, 0.0363244362091049), (341, 0.9802441363339809), (348, 0.04208912438416052), (369, 0.029899754762902034), (534, 0.03616243622163127)]\n",
      "[(65, 0.4018172964218958), (327, 0.9026776097619474), (348, 0.12554575775806914), (369, 0.08918663487097019)]\n",
      "[(101, 0.6071315660697193), (308, 0.1844343787916951), (341, 0.2937066618238871), (348, 0.44558857362092025), (569, 0.5590728407463504)]\n"
     ]
    }
   ],
   "source": [
    "corpus_tfidf = tfidf[corpus]\n",
    "for doc in corpus_tfidf[:5]:\n",
    "    print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_matrix = matutils.corpus2dense(corpus_tfidf, num_terms=len(feature_words2), num_docs=len(corpus_tfidf)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(feature_matrix).to_csv('feature_matrix_pass1_subset1.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Try RFC</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Data and extract gene and mutation type info\n",
    "# Get Gene feature from 'train_variants' data\n",
    "X_gene = np.array(class_train.Gene)\n",
    "X_gene_int = LabelEncoder().fit_transform(X_gene.ravel()).reshape(-1, 1)\n",
    "X_gene_bin = OneHotEncoder().fit_transform(X_gene_int).toarray()\n",
    "gene_table = pd.DataFrame(X_gene_bin)\n",
    "\n",
    "# Get Mutation Type from 'train_variants' data\n",
    "mut_type = ski.convert_mutation_type(class_train)\n",
    "X_mtype = np.array(mut_type['mutation_type'])\n",
    "X_mtype_int = LabelEncoder().fit_transform(X_mtype.ravel()).reshape(-1, 1)\n",
    "X_mtype_bin = OneHotEncoder().fit_transform(X_mtype_int).toarray()\n",
    "mtype_table = pd.DataFrame(X_mtype_bin)\n",
    "\n",
    "feat_mat = pd.DataFrame(feature_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = pd.concat([gene_table, mtype_table, feat_mat], axis=1)\n",
    "X = np.array(features).astype(float)\n",
    "y = np.array(class_train.Class).astype(int).ravel()\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.681362725451\n",
      "1.35105150702\n",
      "CPU times: user 1.21 s, sys: 16.1 ms, total: 1.23 s\n",
      "Wall time: 569 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Run RFC on the data\n",
    "rfc = RandomForestClassifier(n_estimators=100, max_depth=100, n_jobs=4)\n",
    "rfc.fit(X_train, y_train)\n",
    "y_pred = rfc.predict(X_test)\n",
    "y_proba = rfc.predict_proba(X_test)\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(log_loss(y_test, y_proba))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>XGBoost</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Play with max_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== max_depth: 10 ==========\n",
      "[0]\ttrain-mlogloss:2.01589\tvalid-mlogloss:2.04873\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[123]\ttrain-mlogloss:0.411008\tvalid-mlogloss:0.944508\n",
      "\n",
      "========== max_depth: 12 ==========\n",
      "[0]\ttrain-mlogloss:2.00688\tvalid-mlogloss:2.04385\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[113]\ttrain-mlogloss:0.366464\tvalid-mlogloss:0.947517\n",
      "\n",
      "========== max_depth: 15 ==========\n",
      "[0]\ttrain-mlogloss:1.99741\tvalid-mlogloss:2.03932\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[79]\ttrain-mlogloss:0.365388\tvalid-mlogloss:0.963744\n",
      "\n",
      "========== max_depth: 18 ==========\n",
      "[0]\ttrain-mlogloss:1.99249\tvalid-mlogloss:2.03599\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[61]\ttrain-mlogloss:0.369109\tvalid-mlogloss:0.962098\n",
      "\n",
      "========== max_depth: 20 ==========\n",
      "[0]\ttrain-mlogloss:1.98998\tvalid-mlogloss:2.0353\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[66]\ttrain-mlogloss:0.332457\tvalid-mlogloss:0.966193\n",
      "\n",
      "CPU times: user 24min 51s, sys: 229 ms, total: 24min 52s\n",
      "Wall time: 6min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Adjust the class labels so it starts with 0\n",
    "y_adj = y-1\n",
    "\n",
    "# Split into test and train\n",
    "x1, x2, y1, y2 = train_test_split(X, y_adj, test_size=0.15)\n",
    "\n",
    "depths =[10, 12, 15, 18, 20]\n",
    "for i in range(5):\n",
    "    print('========== max_depth: %d ==========' % depths[i])\n",
    "    \n",
    "    # Set up parameters for xgboost\n",
    "    param = params = {\n",
    "            'eta': 0.1,\n",
    "            'max_depth': depths[i],\n",
    "            'objective': 'multi:softprob',\n",
    "            'eval_metric': 'mlogloss',\n",
    "            'num_class': 9,\n",
    "            'seed': 1,\n",
    "            'silent': True,\n",
    "            'nthread' :4\n",
    "            }\n",
    "\n",
    "    watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\n",
    "\n",
    "    model = xgb.train(params, xgb.DMatrix(x1, y1), 500,  watchlist, verbose_eval=250, early_stopping_rounds=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Play with min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== min_child_weight: 1 ==========\n",
      "[0]\ttrain-mlogloss:2.00035\tvalid-mlogloss:2.02778\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[55]\ttrain-mlogloss:0.43163\tvalid-mlogloss:1.01864\n",
      "\n",
      "========== min_child_weight: 2 ==========\n",
      "[0]\ttrain-mlogloss:2.01333\tvalid-mlogloss:2.03051\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[60]\ttrain-mlogloss:0.462865\tvalid-mlogloss:1.01382\n",
      "\n",
      "========== min_child_weight: 5 ==========\n",
      "[0]\ttrain-mlogloss:2.04257\tvalid-mlogloss:2.05592\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[69]\ttrain-mlogloss:0.52694\tvalid-mlogloss:1.03538\n",
      "\n",
      "CPU times: user 13min, sys: 92.6 ms, total: 13min\n",
      "Wall time: 3min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Adjust the class labels so it starts with 0\n",
    "y_adj = y-1\n",
    "\n",
    "# Split into test and train\n",
    "x1, x2, y1, y2 = train_test_split(X, y_adj, test_size=0.15)\n",
    "\n",
    "min_child_weights =[1, 2, 5]\n",
    "for i in range(len(min_child_weights)):\n",
    "    print('========== min_child_weight: %d ==========' % min_child_weights[i])\n",
    "    \n",
    "    # Set up parameters for xgboost\n",
    "    param = params = {\n",
    "            'eta': 0.1,\n",
    "            'max_depth': 15,\n",
    "            'min_child_weight': min_child_weights[i],\n",
    "            'objective': 'multi:softprob',\n",
    "            'eval_metric': 'mlogloss',\n",
    "            'num_class': 9,\n",
    "            'seed': 1,\n",
    "            'silent': True,\n",
    "            'nthread' :4\n",
    "            }\n",
    "\n",
    "    watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\n",
    "\n",
    "    model = xgb.train(params, xgb.DMatrix(x1, y1), 500,  watchlist, verbose_eval=250, early_stopping_rounds=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try different gammas, also make max_delta_step=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== gamma: 0.600000 ==========\n",
      "[0]\ttrain-mlogloss:2.09678\tvalid-mlogloss:2.11528\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[98]\ttrain-mlogloss:0.405016\tvalid-mlogloss:1.04103\n",
      "\n",
      "========== gamma: 0.800000 ==========\n",
      "[0]\ttrain-mlogloss:2.09735\tvalid-mlogloss:2.11554\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[145]\ttrain-mlogloss:0.454069\tvalid-mlogloss:1.0462\n",
      "\n",
      "========== gamma: 1.000000 ==========\n",
      "[0]\ttrain-mlogloss:2.09744\tvalid-mlogloss:2.11533\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "[250]\ttrain-mlogloss:0.489357\tvalid-mlogloss:1.06041\n",
      "Stopping. Best iteration:\n",
      "[229]\ttrain-mlogloss:0.489357\tvalid-mlogloss:1.06041\n",
      "\n",
      "CPU times: user 21min 33s, sys: 213 ms, total: 21min 34s\n",
      "Wall time: 5min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Adjust the class labels so it starts with 0\n",
    "y_adj = y-1\n",
    "\n",
    "# Split into test and train\n",
    "x1, x2, y1, y2 = train_test_split(X, y_adj, test_size=0.15)\n",
    "\n",
    "gammas =[0.6, 0.8, 1]\n",
    "for i in range(len(gammas)):\n",
    "    print('========== gamma: %f ==========' % gammas[i])\n",
    "    \n",
    "    # Set up parameters for xgboost\n",
    "    param = params = {\n",
    "            'eta': 0.1,\n",
    "            'gamma': gammas[i],\n",
    "            'max_delta_step': 1,\n",
    "            'max_depth': 15,\n",
    "            'min_child_weight': 1,\n",
    "            'objective': 'multi:softprob',\n",
    "            'eval_metric': 'mlogloss',\n",
    "            'num_class': 9,\n",
    "            'seed': 1,\n",
    "            'silent': True,\n",
    "            'nthread' :4\n",
    "            }\n",
    "\n",
    "    watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\n",
    "\n",
    "    model = xgb.train(params, xgb.DMatrix(x1, y1), 500,  watchlist, verbose_eval=250, early_stopping_rounds=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cross Validate with multiple rounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========== Iteration 0/5 ==========\n",
      "[0]\ttrain-mlogloss:2.09675\tvalid-mlogloss:2.11377\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[259]\ttrain-mlogloss:0.43171\tvalid-mlogloss:0.990014\n",
      "\n",
      "0.990014515522\n",
      "========== Iteration 1/5 ==========\n",
      "[0]\ttrain-mlogloss:2.09706\tvalid-mlogloss:2.11376\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[93]\ttrain-mlogloss:0.440223\tvalid-mlogloss:1.03176\n",
      "\n",
      "1.0317560281\n",
      "========== Iteration 2/5 ==========\n",
      "[0]\ttrain-mlogloss:2.09615\tvalid-mlogloss:2.11687\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[91]\ttrain-mlogloss:0.457849\tvalid-mlogloss:1.05146\n",
      "\n",
      "1.05145463818\n",
      "========== Iteration 3/5 ==========\n",
      "[0]\ttrain-mlogloss:2.09825\tvalid-mlogloss:2.11671\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[98]\ttrain-mlogloss:0.440953\tvalid-mlogloss:1.03102\n",
      "\n",
      "1.03102123901\n",
      "========== Iteration 4/5 ==========\n",
      "[0]\ttrain-mlogloss:2.09651\tvalid-mlogloss:2.11246\n",
      "Multiple eval metrics have been passed: 'valid-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until valid-mlogloss hasn't improved in 100 rounds.\n",
      "Stopping. Best iteration:\n",
      "[115]\ttrain-mlogloss:0.447289\tvalid-mlogloss:0.996285\n",
      "\n",
      "0.996284923955\n",
      "Average mlogloss: 1.020106\n",
      "CPU times: user 31min 56s, sys: 272 ms, total: 31min 56s\n",
      "Wall time: 8min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Adjust the class labels so it starts with 0\n",
    "y_adj = y-1\n",
    "\n",
    "scores = []\n",
    "\n",
    "iteration = 5\n",
    "for i in range(iteration):\n",
    "    print('========== Iteration %d/%d ==========' % (i, iteration))\n",
    "    \n",
    "    # Split into test and train randomly, every iteration\n",
    "    x1, x2, y1, y2 = train_test_split(X, y_adj, test_size=0.15)\n",
    "    \n",
    "    # Set up parameters for xgboost\n",
    "    param = params = {\n",
    "            'eta': 0.1,\n",
    "            'gamma': 0.8,\n",
    "            'max_delta_step': 1,\n",
    "            'max_depth': 15,\n",
    "            'min_child_weight': 1,\n",
    "            'objective': 'multi:softprob',\n",
    "            'eval_metric': 'mlogloss',\n",
    "            'num_class': 9,\n",
    "            'seed': 1,\n",
    "            'silent': True,\n",
    "            'nthread' :4\n",
    "            }\n",
    "\n",
    "    watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'valid')]\n",
    "\n",
    "    model = xgb.train(params, xgb.DMatrix(x1, y1), 500,  watchlist, \n",
    "                      verbose_eval=500, early_stopping_rounds=100)\n",
    "\n",
    "    score = log_loss(y2, model.predict(xgb.DMatrix(x2), \n",
    "                     ntree_limit=model.best_ntree_limit), \n",
    "                     labels = list(range(9)))\n",
    "    scores.append(score)\n",
    "    print(score)\n",
    "          \n",
    "print(\"Average mlogloss: %f\" % np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}