Ludwig.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ludwig.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOMzzeXx6IrOVDQ3KRcsRYk",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/paiml/practical-mlops-book/blob/main/Ludwig.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCkVd-evwGdR"
      },
      "source": [
        "# Ludwig AutoML with Text Categorization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5szU_JcSwu_A"
      },
      "source": [
        "Uses data set from here: [http://boston.lti.cs.cmu.edu/classes/95-865-K/HW/HW2/](http://boston.lti.cs.cmu.edu/classes/95-865-K/HW/HW2/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ubMkNKHUv41s"
      },
      "source": [
        "## Ingest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z5OztwvJ5ds_",
        "outputId": "c3a48e8c-af43-46ce-fbd3-fb22114b56ab",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install -q ludwig"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 235kB 5.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 9.5MB 7.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 706kB 29.2MB/s \n",
            "\u001b[?25h  Building wheel for ludwig (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "25UIDqQbv2Xb",
        "outputId": "9e8000ab-c16a-4c89-e2b2-f214edec8436"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/paiml/practical-mlops-book/main/chap05/config.yaml\n",
        "!wget https://raw.githubusercontent.com/paiml/practical-mlops-book/main/chap05/reuters-allcats.csv\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-01-25 14:35:26--  https://raw.githubusercontent.com/paiml/practical-mlops-book/main/chap05/config.yaml\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 176 [text/plain]\n",
            "Saving to: ‘config.yaml’\n",
            "\n",
            "\rconfig.yaml           0%[                    ]       0  --.-KB/s               \rconfig.yaml         100%[===================>]     176  --.-KB/s    in 0s      \n",
            "\n",
            "2021-01-25 14:35:27 (7.99 MB/s) - ‘config.yaml’ saved [176/176]\n",
            "\n",
            "--2021-01-25 14:35:27--  https://raw.githubusercontent.com/paiml/practical-mlops-book/main/chap05/reuters-allcats.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3381488 (3.2M) [text/plain]\n",
            "Saving to: ‘reuters-allcats.csv’\n",
            "\n",
            "reuters-allcats.csv 100%[===================>]   3.22M  --.-KB/s    in 0.08s   \n",
            "\n",
            "2021-01-25 14:35:27 (38.0 MB/s) - ‘reuters-allcats.csv’ saved [3381488/3381488]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVs8bHF35Bn1"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4haCp4x55DBP",
        "outputId": "c80f760c-2f80-48c0-c04d-fdf95b5f05c8"
      },
      "source": [
        "!ludwig experiment \\\n",
        "  --dataset reuters-allcats.csv \\\n",
        "  --config_file config.yaml"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-01-25 14:35:35.946551: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
            "/usr/local/lib/python3.6/dist-packages/typeguard/__init__.py:804: UserWarning: no type annotations present -- not typechecking tensorflow_addons.layers.max_unpooling_2d.MaxUnpooling2D.__init__\n",
            "  warn('no type annotations present -- not typechecking {}'.format(function_name(func)))\n",
            "███████████████████████\n",
            "█ █ █ █  ▜█ █ █ █ █   █\n",
            "█ █ █ █ █ █ █ █ █ █ ███\n",
            "█ █   █ █ █ █ █ █ █ ▌ █\n",
            "█ █████ █ █ █ █ █ █ █ █\n",
            "█     █  ▟█     █ █   █\n",
            "███████████████████████\n",
            "ludwig v0.3.2 - Experiment\n",
            "\n",
            "2021-01-25 14:35:37.626504: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
            "2021-01-25 14:35:37.652352: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
            "2021-01-25 14:35:37.713898: E tensorflow/stream_executor/cuda/cuda_driver.cc:328] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "2021-01-25 14:35:37.713957: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (8cdcad013cef): /proc/driver/nvidia/version does not exist\n",
            "Experiment name: experiment\n",
            "Model name: run\n",
            "Output directory: results/experiment_run\n",
            "\n",
            "\n",
            "ludwig_version: '0.3.2'\n",
            "command: ('/usr/local/bin/ludwig experiment --dataset reuters-allcats.csv --config_file '\n",
            " 'config.yaml')\n",
            "random_seed: 42\n",
            "dataset: 'reuters-allcats.csv'\n",
            "data_format: 'csv'\n",
            "config: {   'combiner': {'type': 'concat'},\n",
            "    'input_features': [   {   'column': 'text',\n",
            "                              'encoder': 'parallel_cnn',\n",
            "                              'level': 'word',\n",
            "                              'name': 'text',\n",
            "                              'proc_column': 'text_mZFLky',\n",
            "                              'tied': None,\n",
            "                              'type': 'text'}],\n",
            "    'output_features': [   {   'column': 'class',\n",
            "                               'dependencies': [],\n",
            "                               'loss': {   'class_similarities_temperature': 0,\n",
            "                                           'class_weights': 1,\n",
            "                                           'confidence_penalty': 0,\n",
            "                                           'labels_smoothing': 0,\n",
            "                                           'robust_lambda': 0,\n",
            "                                           'type': 'softmax_cross_entropy',\n",
            "                                           'weight': 1},\n",
            "                               'name': 'class',\n",
            "                               'proc_column': 'class_mZFLky',\n",
            "                               'reduce_dependencies': 'sum',\n",
            "                               'reduce_input': 'sum',\n",
            "                               'top_k': 3,\n",
            "                               'type': 'category'}],\n",
            "    'preprocessing': {   'audio': {   'audio_feature': {'type': 'raw'},\n",
            "                                      'audio_file_length_limit_in_s': 7.5,\n",
            "                                      'in_memory': True,\n",
            "                                      'missing_value_strategy': 'backfill',\n",
            "                                      'norm': None,\n",
            "                                      'padding_value': 0},\n",
            "                         'bag': {   'fill_value': '<UNK>',\n",
            "                                    'lowercase': False,\n",
            "                                    'missing_value_strategy': 'fill_with_const',\n",
            "                                    'most_common': 10000,\n",
            "                                    'tokenizer': 'space'},\n",
            "                         'binary': {   'fill_value': 0,\n",
            "                                       'missing_value_strategy': 'fill_with_const'},\n",
            "                         'category': {   'fill_value': '<UNK>',\n",
            "                                         'lowercase': False,\n",
            "                                         'missing_value_strategy': 'fill_with_const',\n",
            "                                         'most_common': 10000},\n",
            "                         'date': {   'datetime_format': None,\n",
            "                                     'fill_value': '',\n",
            "                                     'missing_value_strategy': 'fill_with_const'},\n",
            "                         'force_split': False,\n",
            "                         'h3': {   'fill_value': 576495936675512319,\n",
            "                                   'missing_value_strategy': 'fill_with_const'},\n",
            "                         'image': {   'in_memory': True,\n",
            "                                      'missing_value_strategy': 'backfill',\n",
            "                                      'num_processes': 1,\n",
            "                                      'resize_method': 'interpolate',\n",
            "                                      'scaling': 'pixel_normalization'},\n",
            "                         'numerical': {   'fill_value': 0,\n",
            "                                          'missing_value_strategy': 'fill_with_const',\n",
            "                                          'normalization': None},\n",
            "                         'sequence': {   'fill_value': '<UNK>',\n",
            "                                         'lowercase': False,\n",
            "                                         'missing_value_strategy': 'fill_with_const',\n",
            "                                         'most_common': 20000,\n",
            "                                         'padding': 'right',\n",
            "                                         'padding_symbol': '<PAD>',\n",
            "                                         'sequence_length_limit': 256,\n",
            "                                         'tokenizer': 'space',\n",
            "                                         'unknown_symbol': '<UNK>',\n",
            "                                         'vocab_file': None},\n",
            "                         'set': {   'fill_value': '<UNK>',\n",
            "                                    'lowercase': False,\n",
            "                                    'missing_value_strategy': 'fill_with_const',\n",
            "                                    'most_common': 10000,\n",
            "                                    'tokenizer': 'space'},\n",
            "                         'split_probabilities': (0.7, 0.1, 0.2),\n",
            "                         'stratify': None,\n",
            "                         'text': {   'char_most_common': 70,\n",
            "                                     'char_sequence_length_limit': 1024,\n",
            "                                     'char_tokenizer': 'characters',\n",
            "                                     'char_vocab_file': None,\n",
            "                                     'fill_value': '<UNK>',\n",
            "                                     'lowercase': True,\n",
            "                                     'missing_value_strategy': 'fill_with_const',\n",
            "                                     'padding': 'right',\n",
            "                                     'padding_symbol': '<PAD>',\n",
            "                                     'pretrained_model_name_or_path': None,\n",
            "                                     'unknown_symbol': '<UNK>',\n",
            "                                     'word_most_common': 20000,\n",
            "                                     'word_sequence_length_limit': 256,\n",
            "                                     'word_tokenizer': 'space_punct',\n",
            "                                     'word_vocab_file': None},\n",
            "                         'timeseries': {   'fill_value': '',\n",
            "                                           'missing_value_strategy': 'fill_with_const',\n",
            "                                           'padding': 'right',\n",
            "                                           'padding_value': 0,\n",
            "                                           'timeseries_length_limit': 256,\n",
            "                                           'tokenizer': 'space'},\n",
            "                         'vector': {   'fill_value': '',\n",
            "                                       'missing_value_strategy': 'fill_with_const'}},\n",
            "    'training': {   'batch_size': 128,\n",
            "                    'bucketing_field': None,\n",
            "                    'decay': False,\n",
            "                    'decay_rate': 0.96,\n",
            "                    'decay_steps': 10000,\n",
            "                    'early_stop': 5,\n",
            "                    'epochs': 100,\n",
            "                    'eval_batch_size': 0,\n",
            "                    'gradient_clipping': None,\n",
            "                    'increase_batch_size_on_plateau': 0,\n",
            "                    'increase_batch_size_on_plateau_max': 512,\n",
            "                    'increase_batch_size_on_plateau_patience': 5,\n",
            "                    'increase_batch_size_on_plateau_rate': 2,\n",
            "                    'learning_rate': 0.001,\n",
            "                    'learning_rate_warmup_epochs': 1,\n",
            "                    'optimizer': {   'beta_1': 0.9,\n",
            "                                     'beta_2': 0.999,\n",
            "                                     'epsilon': 1e-08,\n",
            "                                     'type': 'adam'},\n",
            "                    'reduce_learning_rate_on_plateau': 0,\n",
            "                    'reduce_learning_rate_on_plateau_patience': 5,\n",
            "                    'reduce_learning_rate_on_plateau_rate': 0.5,\n",
            "                    'regularization_lambda': 0,\n",
            "                    'regularizer': 'l2',\n",
            "                    'staircase': False,\n",
            "                    'validation_field': 'combined',\n",
            "                    'validation_metric': 'loss'}}\n",
            "tf_version: '2.4.0'\n",
            "\n",
            "\n",
            "Using full raw csv, no hdf5 and json file with the same name have been found\n",
            "Building dataset (it may take a while)\n",
            "Writing preprocessed dataset cache\n",
            "Writing train set metadata\n",
            "Training set: 2868\n",
            "Validation set: 389\n",
            "Test set: 822\n",
            "2021-01-25 14:35:39.932229: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX512F\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2021-01-25 14:35:39.932413: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
            "\n",
            "╒══════════╕\n",
            "│ TRAINING │\n",
            "╘══════════╛\n",
            "\n",
            "\n",
            "Epoch   1\n",
            "Training:   0% 0/23 [00:00<?, ?it/s]2021-01-25 14:35:42.964572: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n",
            "2021-01-25 14:35:42.980281: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999995000 Hz\n",
            "Training: 100% 23/23 [01:01<00:00,  2.67s/it]\n",
            "Evaluation train: 100% 23/23 [00:14<00:00,  1.60it/s]\n",
            "Evaluation vali : 100% 4/4 [00:02<00:00,  1.96it/s]\n",
            "Evaluation test : 100% 7/7 [00:04<00:00,  1.72it/s]\n",
            "Took 1m 21.9641s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.9258 │     0.7148 │      0.9826 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.9134 │     0.6992 │      0.9692 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.9420 │     0.7311 │      0.9781 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.9376 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.9353 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 1.0277 │\n",
            "╘════════════╧════════╛\n",
            "Validation loss on combined improved, model saved\n",
            "\n",
            "\n",
            "Epoch   2\n",
            "Training: 100% 23/23 [00:57<00:00,  2.51s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.72it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.19it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.83it/s]\n",
            "Took 1m 16.7901s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.2788 │     0.9355 │      0.9857 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.3220 │     0.8869 │      0.9846 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.3516 │     0.8966 │      0.9830 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.2821 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.3514 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.4247 │\n",
            "╘════════════╧════════╛\n",
            "Validation loss on combined improved, model saved\n",
            "\n",
            "\n",
            "Epoch   3\n",
            "Training: 100% 23/23 [00:57<00:00,  2.50s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.72it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.21it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.83it/s]\n",
            "Took 1m 16.5517s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0985 │     0.9749 │      0.9857 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.1960 │     0.9306 │      0.9846 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2298 │     0.9355 │      0.9854 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.1016 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.2253 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2648 │\n",
            "╘════════════╧════════╛\n",
            "Validation loss on combined improved, model saved\n",
            "\n",
            "\n",
            "Epoch   4\n",
            "Training: 100% 23/23 [00:57<00:00,  2.50s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.72it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.21it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.83it/s]\n",
            "Took 1m 16.5802s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0598 │     0.9812 │      0.9941 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.2486 │     0.9203 │      0.9717 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2761 │     0.9246 │      0.9842 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.0658 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.3045 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2965 │\n",
            "╘════════════╧════════╛\n",
            "Last improvement of combined validation loss happened 1 epoch ago\n",
            "\n",
            "\n",
            "Epoch   5\n",
            "Training: 100% 23/23 [00:57<00:00,  2.50s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.71it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.20it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.81it/s]\n",
            "Took 1m 16.6541s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0425 │     0.9847 │      0.9944 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.2227 │     0.9306 │      0.9743 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2465 │     0.9416 │      0.9805 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.0463 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.2778 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2499 │\n",
            "╘════════════╧════════╛\n",
            "Last improvement of combined validation loss happened 2 epochs ago\n",
            "\n",
            "\n",
            "Epoch   6\n",
            "Training: 100% 23/23 [00:57<00:00,  2.50s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.71it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.21it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.81it/s]\n",
            "Took 1m 16.7602s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0288 │     0.9906 │      0.9965 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.2748 │     0.9100 │      0.9614 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2827 │     0.9282 │      0.9659 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.0323 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.3572 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2811 │\n",
            "╘════════════╧════════╛\n",
            "Last improvement of combined validation loss happened 3 epochs ago\n",
            "\n",
            "\n",
            "Epoch   7\n",
            "Training: 100% 23/23 [00:57<00:00,  2.51s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.70it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.19it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.82it/s]\n",
            "Took 1m 16.8417s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0248 │     0.9878 │      0.9972 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.2584 │     0.9280 │      0.9666 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2730 │     0.9270 │      0.9732 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.0281 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.3373 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2725 │\n",
            "╘════════════╧════════╛\n",
            "Last improvement of combined validation loss happened 4 epochs ago\n",
            "\n",
            "\n",
            "Epoch   8\n",
            "Training: 100% 23/23 [00:57<00:00,  2.50s/it]\n",
            "Evaluation train: 100% 23/23 [00:13<00:00,  1.72it/s]\n",
            "Evaluation vali : 100% 4/4 [00:01<00:00,  2.19it/s]\n",
            "Evaluation test : 100% 7/7 [00:03<00:00,  1.83it/s]\n",
            "Took 1m 16.5654s\n",
            "╒═════════╤════════╤════════════╤═════════════╕\n",
            "│ class   │   loss │   accuracy │   hits_at_k │\n",
            "╞═════════╪════════╪════════════╪═════════════╡\n",
            "│ train   │ 0.0227 │     0.9927 │      0.9972 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ vali    │ 0.2478 │     0.9357 │      0.9666 │\n",
            "├─────────┼────────┼────────────┼─────────────┤\n",
            "│ test    │ 0.2783 │     0.9319 │      0.9659 │\n",
            "╘═════════╧════════╧════════════╧═════════════╛\n",
            "╒════════════╤════════╕\n",
            "│ combined   │   loss │\n",
            "╞════════════╪════════╡\n",
            "│ train      │ 0.0266 │\n",
            "├────────────┼────────┤\n",
            "│ vali       │ 0.3569 │\n",
            "├────────────┼────────┤\n",
            "│ test       │ 0.2758 │\n",
            "╘════════════╧════════╛\n",
            "Last improvement of combined validation loss happened 5 epochs ago\n",
            "\n",
            "EARLY STOPPING due to lack of validation improvement, it has been 5 epochs since last validation improvement\n",
            "\n",
            "Best validation model epoch: 3\n",
            "Best validation model loss on validation set combined: 0.22534877061843872\n",
            "Best validation model loss on test set combined: 0.26480552554130554\n",
            "\n",
            "Finished: experiment_run\n",
            "Saved to: results/experiment_run\n",
            "Evaluation: 100% 7/7 [00:03<00:00,  1.82it/s]\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "\n",
            "===== class =====\n",
            "accuracy: 0.9355231\n",
            "hits_at_k: 0.98540145\n",
            "loss: 0.22983195\n",
            "overall_stats: { 'avg_f1_score_macro': 0.4707728205334513,\n",
            "  'avg_f1_score_micro': 0.9355231143552312,\n",
            "  'avg_f1_score_weighted': 0.9252548584390955,\n",
            "  'avg_precision_macro': 0.46514839380709594,\n",
            "  'avg_precision_micro': 0.9355231143552312,\n",
            "  'avg_precision_weighted': 0.9355231143552312,\n",
            "  'avg_recall_macro': 0.47655637160159775,\n",
            "  'avg_recall_micro': 0.9355231143552312,\n",
            "  'avg_recall_weighted': 0.9355231143552312,\n",
            "  'kappa_score': 0.897569852629995,\n",
            "  'token_accuracy': 0.9355231143552312}\n",
            "per_class_stats: {<UNK>: {   'accuracy': 1.0,\n",
            "    'f1_score': 0,\n",
            "    'fall_out': 0.0,\n",
            "    'false_discovery_rate': 1.0,\n",
            "    'false_negative_rate': 1.0,\n",
            "    'false_negatives': 0,\n",
            "    'false_omission_rate': 0.0,\n",
            "    'false_positive_rate': 0.0,\n",
            "    'false_positives': 0,\n",
            "    'hit_rate': 0,\n",
            "    'informedness': 0.0,\n",
            "    'markedness': 0.0,\n",
            "    'matthews_correlation_coefficient': 0,\n",
            "    'miss_rate': 1.0,\n",
            "    'negative_predictive_value': 1.0,\n",
            "    'positive_predictive_value': 0,\n",
            "    'precision': 0,\n",
            "    'recall': 0,\n",
            "    'sensitivity': 0,\n",
            "    'specificity': 1.0,\n",
            "    'true_negative_rate': 1.0,\n",
            "    'true_negatives': 822,\n",
            "    'true_positive_rate': 0,\n",
            "    'true_positives': 0},\n",
            "  Neg-: {   'accuracy': 0.9440389294403893,\n",
            "    'f1_score': 0.9427860696517413,\n",
            "    'fall_out': 0.06367924528301883,\n",
            "    'false_discovery_rate': 0.06650246305418717,\n",
            "    'false_negative_rate': 0.047738693467336724,\n",
            "    'false_negatives': 19,\n",
            "    'false_omission_rate': 0.04567307692307687,\n",
            "    'false_positive_rate': 0.06367924528301883,\n",
            "    'false_positives': 27,\n",
            "    'hit_rate': 0.9522613065326633,\n",
            "    'informedness': 0.8885820612496445,\n",
            "    'markedness': 0.8878244600227361,\n",
            "    'matthews_correlation_coefficient': 0.8882031798608105,\n",
            "    'miss_rate': 0.047738693467336724,\n",
            "    'negative_predictive_value': 0.9543269230769231,\n",
            "    'positive_predictive_value': 0.9334975369458128,\n",
            "    'precision': 0.9334975369458128,\n",
            "    'recall': 0.9522613065326633,\n",
            "    'sensitivity': 0.9522613065326633,\n",
            "    'specificity': 0.9363207547169812,\n",
            "    'true_negative_rate': 0.9363207547169812,\n",
            "    'true_negatives': 397,\n",
            "    'true_positive_rate': 0.9522613065326633,\n",
            "    'true_positives': 379},\n",
            "  Pos-earn: {   'accuracy': 0.9854014598540146,\n",
            "    'f1_score': 0.976190476190476,\n",
            "    'fall_out': 0.013986013986013957,\n",
            "    'false_discovery_rate': 0.03149606299212604,\n",
            "    'false_negative_rate': 0.016000000000000014,\n",
            "    'false_negatives': 4,\n",
            "    'false_omission_rate': 0.007042253521126751,\n",
            "    'false_positive_rate': 0.013986013986013957,\n",
            "    'false_positives': 8,\n",
            "    'hit_rate': 0.984,\n",
            "    'informedness': 0.970013986013986,\n",
            "    'markedness': 0.9614616834867471,\n",
            "    'matthews_correlation_coefficient': 0.9657283676058692,\n",
            "    'miss_rate': 0.016000000000000014,\n",
            "    'negative_predictive_value': 0.9929577464788732,\n",
            "    'positive_predictive_value': 0.968503937007874,\n",
            "    'precision': 0.968503937007874,\n",
            "    'recall': 0.984,\n",
            "    'sensitivity': 0.984,\n",
            "    'specificity': 0.986013986013986,\n",
            "    'true_negative_rate': 0.986013986013986,\n",
            "    'true_negatives': 564,\n",
            "    'true_positive_rate': 0.984,\n",
            "    'true_positives': 246},\n",
            "  Pos-acq: {   'accuracy': 0.9635036496350365,\n",
            "    'f1_score': 0.9056603773584906,\n",
            "    'fall_out': 0.027027027027026973,\n",
            "    'false_discovery_rate': 0.11111111111111116,\n",
            "    'false_negative_rate': 0.07692307692307687,\n",
            "    'false_negatives': 12,\n",
            "    'false_omission_rate': 0.018181818181818188,\n",
            "    'false_positive_rate': 0.027027027027026973,\n",
            "    'false_positives': 18,\n",
            "    'hit_rate': 0.9230769230769231,\n",
            "    'informedness': 0.8960498960498962,\n",
            "    'markedness': 0.8707070707070708,\n",
            "    'matthews_correlation_coefficient': 0.8832875976696266,\n",
            "    'miss_rate': 0.07692307692307687,\n",
            "    'negative_predictive_value': 0.9818181818181818,\n",
            "    'positive_predictive_value': 0.8888888888888888,\n",
            "    'precision': 0.8888888888888888,\n",
            "    'recall': 0.9230769230769231,\n",
            "    'sensitivity': 0.9230769230769231,\n",
            "    'specificity': 0.972972972972973,\n",
            "    'true_negative_rate': 0.972972972972973,\n",
            "    'true_negatives': 648,\n",
            "    'true_positive_rate': 0.9230769230769231,\n",
            "    'true_positives': 144},\n",
            "  Pos-coffee: {   'accuracy': 0.9841849148418491,\n",
            "    'f1_score': 0,\n",
            "    'fall_out': 0.0,\n",
            "    'false_discovery_rate': 1.0,\n",
            "    'false_negative_rate': 1.0,\n",
            "    'false_negatives': 13,\n",
            "    'false_omission_rate': 0.015815085158150888,\n",
            "    'false_positive_rate': 0.0,\n",
            "    'false_positives': 0,\n",
            "    'hit_rate': 0.0,\n",
            "    'informedness': 0.0,\n",
            "    'markedness': -0.015815085158150888,\n",
            "    'matthews_correlation_coefficient': 0,\n",
            "    'miss_rate': 1.0,\n",
            "    'negative_predictive_value': 0.9841849148418491,\n",
            "    'positive_predictive_value': 0,\n",
            "    'precision': 0,\n",
            "    'recall': 0.0,\n",
            "    'sensitivity': 0.0,\n",
            "    'specificity': 1.0,\n",
            "    'true_negative_rate': 1.0,\n",
            "    'true_negatives': 809,\n",
            "    'true_positive_rate': 0.0,\n",
            "    'true_positives': 0},\n",
            "  Pos-gold: {   'accuracy': 0.9951338199513382,\n",
            "    'f1_score': 0,\n",
            "    'fall_out': 0.0,\n",
            "    'false_discovery_rate': 1.0,\n",
            "    'false_negative_rate': 1.0,\n",
            "    'false_negatives': 4,\n",
            "    'false_omission_rate': 0.0048661800486617945,\n",
            "    'false_positive_rate': 0.0,\n",
            "    'false_positives': 0,\n",
            "    'hit_rate': 0.0,\n",
            "    'informedness': 0.0,\n",
            "    'markedness': -0.0048661800486617945,\n",
            "    'matthews_correlation_coefficient': 0,\n",
            "    'miss_rate': 1.0,\n",
            "    'negative_predictive_value': 0.9951338199513382,\n",
            "    'positive_predictive_value': 0,\n",
            "    'precision': 0,\n",
            "    'recall': 0.0,\n",
            "    'sensitivity': 0.0,\n",
            "    'specificity': 1.0,\n",
            "    'true_negative_rate': 1.0,\n",
            "    'true_negatives': 818,\n",
            "    'true_positive_rate': 0.0,\n",
            "    'true_positives': 0},\n",
            "  Pos-housing: {   'accuracy': 1.0,\n",
            "    'f1_score': 0,\n",
            "    'fall_out': 0.0,\n",
            "    'false_discovery_rate': 1.0,\n",
            "    'false_negative_rate': 1.0,\n",
            "    'false_negatives': 0,\n",
            "    'false_omission_rate': 0.0,\n",
            "    'false_positive_rate': 0.0,\n",
            "    'false_positives': 0,\n",
            "    'hit_rate': 0,\n",
            "    'informedness': 0.0,\n",
            "    'markedness': 0.0,\n",
            "    'matthews_correlation_coefficient': 0,\n",
            "    'miss_rate': 1.0,\n",
            "    'negative_predictive_value': 1.0,\n",
            "    'positive_predictive_value': 0,\n",
            "    'precision': 0,\n",
            "    'recall': 0,\n",
            "    'sensitivity': 0,\n",
            "    'specificity': 1.0,\n",
            "    'true_negative_rate': 1.0,\n",
            "    'true_negatives': 822,\n",
            "    'true_positive_rate': 0,\n",
            "    'true_positives': 0},\n",
            "  Pos-heat: {   'accuracy': 0.9987834549878345,\n",
            "    'f1_score': 0,\n",
            "    'fall_out': 0.0,\n",
            "    'false_discovery_rate': 1.0,\n",
            "    'false_negative_rate': 1.0,\n",
            "    'false_negatives': 1,\n",
            "    'false_omission_rate': 0.0012165450121655041,\n",
            "    'false_positive_rate': 0.0,\n",
            "    'false_positives': 0,\n",
            "    'hit_rate': 0.0,\n",
            "    'informedness': 0.0,\n",
            "    'markedness': -0.0012165450121655041,\n",
            "    'matthews_correlation_coefficient': 0,\n",
            "    'miss_rate': 1.0,\n",
            "    'negative_predictive_value': 0.9987834549878345,\n",
            "    'positive_predictive_value': 0,\n",
            "    'precision': 0,\n",
            "    'recall': 0.0,\n",
            "    'sensitivity': 0.0,\n",
            "    'specificity': 1.0,\n",
            "    'true_negative_rate': 1.0,\n",
            "    'true_negatives': 821,\n",
            "    'true_positive_rate': 0.0,\n",
            "    'true_positives': 0}}\n",
            "Saved to: results/experiment_run\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "64YqpLy35w4n"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}