nlp/checklist/checklist_examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de2217ea",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:46.464624Z",
     "start_time": "2022-02-03T15:56:43.304213Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install checklist textblob "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f608c8b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !jupyter nbextension install --py --sys-prefix checklist.viewer\n",
    "# !jupyter nbextension enable --py --sys-prefix checklist.viewer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cba25a59",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc25ad69",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:48.248963Z",
     "start_time": "2022-02-03T15:56:46.466609Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"from textblob import TextBlob\\nimport checklist\\nfrom checklist.editor import Editor\\nfrom checklist.perturb import Perturb\\nfrom checklist.test_types import MFT, INV, DIR\\nfrom checklist.pred_wrapper import PredictorWrapper\";\n",
       "                var nbb_formatted_code = \"from textblob import TextBlob\\nimport checklist\\nfrom checklist.editor import Editor\\nfrom checklist.perturb import Perturb\\nfrom checklist.test_types import MFT, INV, DIR\\nfrom checklist.pred_wrapper import PredictorWrapper\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from textblob import TextBlob\n",
    "import checklist\n",
    "from checklist.editor import Editor\n",
    "from checklist.perturb import Perturb\n",
    "from checklist.test_types import MFT, INV, DIR\n",
    "from checklist.pred_wrapper import PredictorWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9e2398a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:48.468305Z",
     "start_time": "2022-02-03T15:56:48.250428Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"editor = Editor()\";\n",
       "                var nbb_formatted_code = \"editor = Editor()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "editor = Editor()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c069704",
   "metadata": {},
   "source": [
    "# Minimum Functionality Test (MFT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3554c7e9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:48.502887Z",
     "start_time": "2022-02-03T15:56:48.469399Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3499999999999999"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"TextBlob(\\\"Going out to eat is not a bad option\\\").sentiment[0]\";\n",
       "                var nbb_formatted_code = \"TextBlob(\\\"Going out to eat is not a bad option\\\").sentiment[0]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TextBlob(\"Going out to eat is not a bad option\").sentiment[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bde897ef",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:48.512270Z",
     "start_time": "2022-02-03T15:56:48.503667Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"pos = [\\n    \\\"good\\\",\\n    \\\"realistic\\\",\\n    \\\"healthy\\\",\\n    \\\"attractive\\\",\\n    \\\"appealing\\\",\\n    \\\"acceptable\\\",\\n    \\\"best\\\",\\n    \\\"feasible\\\",\\n    \\\"easy\\\",\\n    \\\"ideal\\\",\\n    \\\"affordable\\\",\\n    \\\"economical\\\",\\n    \\\"recommended\\\",\\n    \\\"exciting\\\",\\n    \\\"inexpensive\\\",\\n    \\\"obvious\\\",\\n    \\\"great\\\",\\n    \\\"appropriate\\\",\\n    \\\"effective\\\",\\n    \\\"excellent\\\",\\n]\\n\\nneg = [\\n    \\\"bad\\\",\\n    \\\"unhealthy\\\",\\n    \\\"expensive\\\",\\n    \\\"boring\\\",\\n    \\\"terrible\\\",\\n    \\\"worst\\\",\\n    \\\"unfeasible\\\",\\n    \\\"unappropriate\\\",\\n    \\\"awful\\\",\\n    \\\"time-consuming\\\",\\n]\";\n",
       "                var nbb_formatted_code = \"pos = [\\n    \\\"good\\\",\\n    \\\"realistic\\\",\\n    \\\"healthy\\\",\\n    \\\"attractive\\\",\\n    \\\"appealing\\\",\\n    \\\"acceptable\\\",\\n    \\\"best\\\",\\n    \\\"feasible\\\",\\n    \\\"easy\\\",\\n    \\\"ideal\\\",\\n    \\\"affordable\\\",\\n    \\\"economical\\\",\\n    \\\"recommended\\\",\\n    \\\"exciting\\\",\\n    \\\"inexpensive\\\",\\n    \\\"obvious\\\",\\n    \\\"great\\\",\\n    \\\"appropriate\\\",\\n    \\\"effective\\\",\\n    \\\"excellent\\\",\\n]\\n\\nneg = [\\n    \\\"bad\\\",\\n    \\\"unhealthy\\\",\\n    \\\"expensive\\\",\\n    \\\"boring\\\",\\n    \\\"terrible\\\",\\n    \\\"worst\\\",\\n    \\\"unfeasible\\\",\\n    \\\"unappropriate\\\",\\n    \\\"awful\\\",\\n    \\\"time-consuming\\\",\\n]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pos = [\n",
    "    \"good\",\n",
    "    \"realistic\",\n",
    "    \"healthy\",\n",
    "    \"attractive\",\n",
    "    \"appealing\",\n",
    "    \"acceptable\",\n",
    "    \"best\",\n",
    "    \"feasible\",\n",
    "    \"easy\",\n",
    "    \"ideal\",\n",
    "    \"affordable\",\n",
    "    \"economical\",\n",
    "    \"recommended\",\n",
    "    \"exciting\",\n",
    "    \"inexpensive\",\n",
    "    \"obvious\",\n",
    "    \"great\",\n",
    "    \"appropriate\",\n",
    "    \"effective\",\n",
    "    \"excellent\",\n",
    "]\n",
    "\n",
    "neg = [\n",
    "    \"bad\",\n",
    "    \"unhealthy\",\n",
    "    \"expensive\",\n",
    "    \"boring\",\n",
    "    \"terrible\",\n",
    "    \"worst\",\n",
    "    \"unfeasible\",\n",
    "    \"unappropriate\",\n",
    "    \"awful\",\n",
    "    \"time-consuming\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "749c7d3c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:48.519439Z",
     "start_time": "2022-02-03T15:56:48.513040Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Staying at home is not a great option.',\n",
       " 'Staying at home is not a great option.']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"editor.template(\\n    \\\"Staying at home is not {a:pos} option.\\\", pos=[\\\"good\\\", \\\"great\\\"], nsamples=2\\n).data\";\n",
       "                var nbb_formatted_code = \"editor.template(\\n    \\\"Staying at home is not {a:pos} option.\\\", pos=[\\\"good\\\", \\\"great\\\"], nsamples=2\\n).data\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "editor.template(\n",
    "    \"Staying at home is not {a:pos} option.\", pos=[\"good\", \"great\"], nsamples=2\n",
    ").data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "98edbf78",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:51.851282Z",
     "start_time": "2022-02-03T15:56:48.520704Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/khuyen/Data-science/venv/lib/python3.8/site-packages/torch/cuda/__init__.py:80: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at  ../c10/cuda/CUDAFunctions.cpp:112.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n",
      "/home/khuyen/Data-science/venv/lib/python3.8/site-packages/checklist/text_generation.py:171: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  to_pred = torch.tensor(to_pred, device=self.device).to(torch.int64)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['sleep is not a good option.', 'Rest is not a great option.']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"editor.template(\\\"{mask} is not {a:pos} option.\\\", pos=[\\\"good\\\", \\\"great\\\"], nsamples=2).data\";\n",
       "                var nbb_formatted_code = \"editor.template(\\\"{mask} is not {a:pos} option.\\\", pos=[\\\"good\\\", \\\"great\\\"], nsamples=2).data\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "editor.template(\"{mask} is not {a:pos} option.\", pos=[\"good\", \"great\"], nsamples=2).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d8dd1970",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:54:56.106739Z",
     "start_time": "2022-02-03T16:54:55.467954Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 45;\n",
       "                var nbb_unformatted_code = \"samples = editor.template(\\n    \\\"{mask} is not {a:pos} option.\\\", pos=pos, labels=0, save=True, nsamples=100\\n)\\nsamples += editor.template(\\n    \\\"{mask} is not {a:neg} option.\\\", neg=neg, labels=1, save=True, nsamples=100\\n)\";\n",
       "                var nbb_formatted_code = \"samples = editor.template(\\n    \\\"{mask} is not {a:pos} option.\\\", pos=pos, labels=0, save=True, nsamples=100\\n)\\nsamples += editor.template(\\n    \\\"{mask} is not {a:neg} option.\\\", neg=neg, labels=1, save=True, nsamples=100\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "samples = editor.template(\n",
    "    \"{mask} is not {a:pos} option.\", pos=pos, labels=0, save=True, nsamples=100\n",
    ")\n",
    "samples += editor.template(\n",
    "    \"{mask} is not {a:neg} option.\", neg=neg, labels=1, save=True, nsamples=100\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9c505c5d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T18:54:54.326129Z",
     "start_time": "2022-02-03T18:54:54.310145Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 48;\n",
       "                var nbb_unformatted_code = \"test = MFT(\\n    samples.data, labels=samples.labels, name=\\\"Test negation\\\", capability=\\\"Negation\\\"\\n)\";\n",
       "                var nbb_formatted_code = \"test = MFT(\\n    samples.data, labels=samples.labels, name=\\\"Test negation\\\", capability=\\\"Negation\\\"\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = MFT(\n",
    "    samples.data, labels=samples.labels, name=\"Test negation\", capability=\"Negation\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c23e90e9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T18:57:50.539957Z",
     "start_time": "2022-02-03T18:57:50.523774Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 51;\n",
       "                var nbb_unformatted_code = \"TextBlob(\\\"good\\\").sentiment[0]\";\n",
       "                var nbb_formatted_code = \"TextBlob(\\\"good\\\").sentiment[0]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TextBlob(\"good\").sentiment[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "6282a89b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T19:00:22.344616Z",
     "start_time": "2022-02-03T19:00:22.327901Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 56;\n",
       "                var nbb_unformatted_code = \"(-0.3 + 1) / 2\";\n",
       "                var nbb_formatted_code = \"(-0.3 + 1) / 2\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(-0.3 + 1) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "79ac6eba",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:52.342289Z",
     "start_time": "2022-02-03T15:56:52.334921Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"import numpy as np\\n\\n\\ndef predict_proba(inputs):\\n    p1 = np.array([(TextBlob(x).sentiment[0] + 1) / 2.0 for x in inputs]).reshape(-1, 1)\\n    p0 = 1 - p1\\n    return np.hstack((p0, p1))\\n\\n\\nwrapped_pp = PredictorWrapper.wrap_softmax(predict_proba)\";\n",
       "                var nbb_formatted_code = \"import numpy as np\\n\\n\\ndef predict_proba(inputs):\\n    p1 = np.array([(TextBlob(x).sentiment[0] + 1) / 2.0 for x in inputs]).reshape(-1, 1)\\n    p0 = 1 - p1\\n    return np.hstack((p0, p1))\\n\\n\\nwrapped_pp = PredictorWrapper.wrap_softmax(predict_proba)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def predict_proba(inputs):\n",
    "    p1 = np.array([(TextBlob(x).sentiment[0] + 1) / 2.0 for x in inputs]).reshape(-1, 1)\n",
    "    p0 = 1 - p1\n",
    "    return np.hstack((p0, p1))\n",
    "\n",
    "\n",
    "wrapped_pp = PredictorWrapper.wrap_softmax(predict_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c2ab621f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:52.347977Z",
     "start_time": "2022-02-03T15:56:52.343025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.15, 0.85]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"predict_proba([\\\"good\\\"])\";\n",
       "                var nbb_formatted_code = \"predict_proba([\\\"good\\\"])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predict_proba([\"good\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d3ca1a2e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T19:07:16.039928Z",
     "start_time": "2022-02-03T19:07:16.023240Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0]), array([[0.5, 0.5]]))"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 59;\n",
       "                var nbb_unformatted_code = \"wrapped_pp([\\\"Training is not an unfeasible option.\\\"])\";\n",
       "                var nbb_formatted_code = \"wrapped_pp([\\\"Training is not an unfeasible option.\\\"])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wrapped_pp([\"Training is not an unfeasible option.\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7f25be7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T18:54:59.750687Z",
     "start_time": "2022-02-03T18:54:59.639871Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 200 examples\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 49;\n",
       "                var nbb_unformatted_code = \"test.run(wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test.run(wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.run(wrapped_pp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "76b55ee0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T18:55:00.637884Z",
     "start_time": "2022-02-03T18:55:00.622822Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test cases:      200\n",
      "Fails (rate):    115 (57.5%)\n",
      "\n",
      "Example fails:\n",
      "0.0 Shipping is not an awful option.\n",
      "----\n",
      "0.5 Pinterest is not a time-consuming option.\n",
      "----\n",
      "0.5 Training is not an unfeasible option.\n",
      "----\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 50;\n",
       "                var nbb_unformatted_code = \"test.summary()\";\n",
       "                var nbb_formatted_code = \"test.summary()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1ce46e1f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:52.651483Z",
     "start_time": "2022-02-03T15:56:52.388566Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc1640dd01014768bd1048a1d61aeb83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "TestSummarizer(stats={'npassed': 85, 'nfailed': 115, 'nfiltered': 0}, summarizer={'name': 'Test negation', 'de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"test.visual_summary()\";\n",
       "                var nbb_formatted_code = \"test.visual_summary()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.visual_summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca291992",
   "metadata": {},
   "source": [
    "# Invariance Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5315deaa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.015314Z",
     "start_time": "2022-02-03T15:56:52.652418Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"import spacy\\n\\nnlp = spacy.load(\\\"en_core_web_sm\\\")\";\n",
       "                var nbb_formatted_code = \"import spacy\\n\\nnlp = spacy.load(\\\"en_core_web_sm\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f157a2d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.020288Z",
     "start_time": "2022-02-03T15:56:53.016029Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"data = [\\n    \\\"The cake is great.\\\",\\n    \\\"awful\\\",\\n    \\\"Michael had fun traveling to Mexico\\\",\\n    \\\"Anna hates party.\\\",\\n    \\\"This laptop is not very good\\\",\\n]\";\n",
       "                var nbb_formatted_code = \"data = [\\n    \\\"The cake is great.\\\",\\n    \\\"awful\\\",\\n    \\\"Michael had fun traveling to Mexico\\\",\\n    \\\"Anna hates party.\\\",\\n    \\\"This laptop is not very good\\\",\\n]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = [\n",
    "    \"The cake is great.\",\n",
    "    \"awful\",\n",
    "    \"Michael had fun traveling to Mexico\",\n",
    "    \"Anna hates party.\",\n",
    "    \"This laptop is not very good\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "683c17a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.030795Z",
     "start_time": "2022-02-03T15:56:53.020950Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"pdata = list(nlp.pipe(data))\";\n",
       "                var nbb_formatted_code = \"pdata = list(nlp.pipe(data))\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pdata = list(nlp.pipe(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "98a78cc7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.035660Z",
     "start_time": "2022-02-03T15:56:53.031614Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[The cake is great.,\n",
       " awful,\n",
       " Michael had fun traveling to Mexico,\n",
       " Anna hates party.,\n",
       " This laptop is not very good]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 20;\n",
       "                var nbb_unformatted_code = \"pdata\";\n",
       "                var nbb_formatted_code = \"pdata\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "13cb6ace",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.042770Z",
     "start_time": "2022-02-03T15:56:53.036585Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"def test_invariant(data: list, method: callable, wrapped_predict: callable):\\n    t = Perturb.perturb(data, method)\\n    print(\\\"First sample before and after pertubation:\\\")\\n    print(\\\"\\\\n\\\".join(t.data[0]))\\n    print(\\\"\\\\nSummary:\\\")\\n    test = INV(**t)\\n    test.run(wrapped_predict)\\n    test.summary()\";\n",
       "                var nbb_formatted_code = \"def test_invariant(data: list, method: callable, wrapped_predict: callable):\\n    t = Perturb.perturb(data, method)\\n    print(\\\"First sample before and after pertubation:\\\")\\n    print(\\\"\\\\n\\\".join(t.data[0]))\\n    print(\\\"\\\\nSummary:\\\")\\n    test = INV(**t)\\n    test.run(wrapped_predict)\\n    test.summary()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def test_invariant(data: list, method: callable, wrapped_predict: callable):\n",
    "    t = Perturb.perturb(data, method)\n",
    "    print(\"First sample before and after pertubation:\")\n",
    "    print(\"\\n\".join(t.data[0]))\n",
    "    print(\"\\nSummary:\")\n",
    "    test = INV(**t)\n",
    "    test.run(wrapped_predict)\n",
    "    test.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b0f8f29",
   "metadata": {},
   "source": [
    "## Add Punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6da0e109",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.049396Z",
     "start_time": "2022-02-03T15:56:53.043497Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "The cake is great.\n",
      "The cake is great\n",
      "\n",
      "Summary:\n",
      "Predicting 10 examples\n",
      "Test cases:      5\n",
      "Fails (rate):    0 (0.0%)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 22;\n",
       "                var nbb_unformatted_code = \"test_invariant(pdata, Perturb.punctuation, wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test_invariant(pdata, Perturb.punctuation, wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_invariant(pdata, Perturb.punctuation, wrapped_pp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ff4b6e3",
   "metadata": {},
   "source": [
    "## Add Typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5f0cd910",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.056045Z",
     "start_time": "2022-02-03T15:56:53.050149Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "The cake is great.\n",
      "The cake is rgeat.\n",
      "\n",
      "Summary:\n",
      "Predicting 10 examples\n",
      "Test cases:      5\n",
      "Fails (rate):    2 (40.0%)\n",
      "\n",
      "Example fails:\n",
      "0.4 This laptop is not very good\n",
      "1.0 This laptop i snot very good\n",
      "\n",
      "----\n",
      "0.9 The cake is great.\n",
      "0.5 The cake is rgeat.\n",
      "\n",
      "----\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 23;\n",
       "                var nbb_unformatted_code = \"test_invariant(data, Perturb.add_typos, wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test_invariant(data, Perturb.add_typos, wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_invariant(data, Perturb.add_typos, wrapped_pp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84fc775d",
   "metadata": {},
   "source": [
    "## Change Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f7fe97b0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.064758Z",
     "start_time": "2022-02-03T15:56:53.056987Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "Michael had fun traveling to Mexico\n",
      "George had fun traveling to Mexico\n",
      "Kenneth had fun traveling to Mexico\n",
      "Kevin had fun traveling to Mexico\n",
      "Dustin had fun traveling to Mexico\n",
      "Zachary had fun traveling to Mexico\n",
      "Luis had fun traveling to Mexico\n",
      "Patrick had fun traveling to Mexico\n",
      "Antonio had fun traveling to Mexico\n",
      "Ryan had fun traveling to Mexico\n",
      "James had fun traveling to Mexico\n",
      "\n",
      "Summary:\n",
      "Predicting 22 examples\n",
      "Test cases:      2\n",
      "Fails (rate):    0 (0.0%)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 24;\n",
       "                var nbb_unformatted_code = \"test_invariant(pdata, Perturb.change_names, wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test_invariant(pdata, Perturb.change_names, wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_invariant(pdata, Perturb.change_names, wrapped_pp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a1d97e1",
   "metadata": {},
   "source": [
    "## Change Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c6a5510d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.072598Z",
     "start_time": "2022-02-03T15:56:53.065695Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "Michael had fun traveling to Mexico\n",
      "Michael had fun traveling to Philippines\n",
      "Michael had fun traveling to Brazil\n",
      "Michael had fun traveling to Myanmar\n",
      "Michael had fun traveling to Sudan\n",
      "Michael had fun traveling to Nigeria\n",
      "Michael had fun traveling to Algeria\n",
      "Michael had fun traveling to Iran\n",
      "Michael had fun traveling to Turkey\n",
      "Michael had fun traveling to France\n",
      "Michael had fun traveling to Uganda\n",
      "\n",
      "Summary:\n",
      "Predicting 11 examples\n",
      "Test cases:      1\n",
      "Fails (rate):    0 (0.0%)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 25;\n",
       "                var nbb_unformatted_code = \"test_invariant(pdata, Perturb.change_location, wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test_invariant(pdata, Perturb.change_location, wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_invariant(pdata, Perturb.change_location, wrapped_pp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f50842ed",
   "metadata": {},
   "source": [
    "## Change to related nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f22f14f4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.122986Z",
     "start_time": "2022-02-03T15:56:53.073459Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package brown to /home/khuyen/nltk_data...\n",
      "[nltk_data]   Package brown is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/khuyen/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 26;\n",
       "                var nbb_unformatted_code = \"import nltk\\n\\nnltk.download(\\\"brown\\\")\\nnltk.download(\\\"punkt\\\")\";\n",
       "                var nbb_formatted_code = \"import nltk\\n\\nnltk.download(\\\"brown\\\")\\nnltk.download(\\\"punkt\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "nltk.download(\"brown\")\n",
    "nltk.download(\"punkt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "98a46b9b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.129726Z",
     "start_time": "2022-02-03T15:56:53.123762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 27;\n",
       "                var nbb_unformatted_code = \"def find_first_noun(text: str):\\n    spacy_text = nlp(text)\\n    nouns = [word.text for word in spacy_text if word.tag_ == \\\"NN\\\"]\\n    if len(nouns) >= 1:\\n        return nouns[0]\\n    return nouns\";\n",
       "                var nbb_formatted_code = \"def find_first_noun(text: str):\\n    spacy_text = nlp(text)\\n    nouns = [word.text for word in spacy_text if word.tag_ == \\\"NN\\\"]\\n    if len(nouns) >= 1:\\n        return nouns[0]\\n    return nouns\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def find_first_noun(text: str):\n",
    "    spacy_text = nlp(text)\n",
    "    nouns = [word.text for word in spacy_text if word.tag_ == \"NN\"]\n",
    "    if len(nouns) >= 1:\n",
    "        return nouns[0]\n",
    "    return nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "977f4f28",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:53.137319Z",
     "start_time": "2022-02-03T15:56:53.131323Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 28;\n",
       "                var nbb_unformatted_code = \"def change_to_related_nouns(sent: str, num_words: int = 5):\\n    noun = find_first_noun(sent)\\n    if noun:\\n        related_nouns = editor.related_words(sent, noun)[:num_words]\\n        return [sent.replace(noun, new_word) for new_word in related_nouns]\\n    return sent\";\n",
       "                var nbb_formatted_code = \"def change_to_related_nouns(sent: str, num_words: int = 5):\\n    noun = find_first_noun(sent)\\n    if noun:\\n        related_nouns = editor.related_words(sent, noun)[:num_words]\\n        return [sent.replace(noun, new_word) for new_word in related_nouns]\\n    return sent\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def change_to_related_nouns(sent: str, num_words: int = 5):\n",
    "    noun = find_first_noun(sent)\n",
    "    if noun:\n",
    "        related_nouns = editor.related_words(sent, noun)[:num_words]\n",
    "        return [sent.replace(noun, new_word) for new_word in related_nouns]\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2f50d360",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:56:57.746197Z",
     "start_time": "2022-02-03T15:56:57.710678Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cake'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 30;\n",
       "                var nbb_unformatted_code = \"text = data[0]\\nfind_first_noun(text)\";\n",
       "                var nbb_formatted_code = \"text = data[0]\\nfind_first_noun(text)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text = data[0]\n",
    "find_first_noun(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8f0090fe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T15:57:00.331824Z",
     "start_time": "2022-02-03T15:56:58.716172Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "The cake is great.\n",
      "The game is great.\n",
      "The movie is great.\n",
      "The book is great.\n",
      "The food is great.\n",
      "The story is great.\n",
      "\n",
      "Summary:\n",
      "Predicting 20 examples\n",
      "Test cases:      4\n",
      "Fails (rate):    0 (0.0%)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 31;\n",
       "                var nbb_unformatted_code = \"test_invariant(data, change_to_related_nouns, wrapped_pp)\";\n",
       "                var nbb_formatted_code = \"test_invariant(data, change_to_related_nouns, wrapped_pp)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_invariant(data, change_to_related_nouns, wrapped_pp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c37012d4",
   "metadata": {},
   "source": [
    "# Directional Expectation Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c2b1553a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:10:13.250627Z",
     "start_time": "2022-02-03T16:10:13.237081Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 32;\n",
       "                var nbb_unformatted_code = \"from checklist.expect import Expect\";\n",
       "                var nbb_formatted_code = \"from checklist.expect import Expect\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from checklist.expect import Expect"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cefc48c1",
   "metadata": {},
   "source": [
    "## Expect Changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "84b26566",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:10:13.988400Z",
     "start_time": "2022-02-03T16:10:13.983180Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 33;\n",
       "                var nbb_unformatted_code = \"def changed_pred(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\\n    return pred != orig_pred\\n\\n\\nexpect_fn = Expect.pairwise(changed_pred)\";\n",
       "                var nbb_formatted_code = \"def changed_pred(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\\n    return pred != orig_pred\\n\\n\\nexpect_fn = Expect.pairwise(changed_pred)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def changed_pred(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    return pred != orig_pred\n",
    "\n",
    "\n",
    "expect_fn = Expect.pairwise(changed_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ed982564",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:10:21.117614Z",
     "start_time": "2022-02-03T16:10:21.098155Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['The cake is great.', 'The cake is not great.'],\n",
       " ['Michael had fun traveling to Mexico',\n",
       "  \"Michael didn't have fun traveling to Mexico\"]]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 35;\n",
       "                var nbb_unformatted_code = \"t = Perturb.perturb(pdata, Perturb.add_negation)\\nt.data[0:2]\";\n",
       "                var nbb_formatted_code = \"t = Perturb.perturb(pdata, Perturb.add_negation)\\nt.data[0:2]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = Perturb.perturb(pdata, Perturb.add_negation)\n",
    "t.data[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "be9d8166",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:15:18.438067Z",
     "start_time": "2022-02-03T16:15:18.411961Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 6 examples\n",
      "Test cases:      3\n",
      "Fails (rate):    2 (66.7%)\n",
      "\n",
      "Example fails:\n",
      "0.5 Anna hates party.\n",
      "0.1 Anna doesn't hate party.\n",
      "\n",
      "----\n",
      "0.7 Michael had fun traveling to Mexico\n",
      "0.7 Michael didn't have fun traveling to Mexico\n",
      "\n",
      "----\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 36;\n",
       "                var nbb_unformatted_code = \"test = DIR(**t, expect=expect_fn)\\ntest.run(wrapped_pp)\\ntest.summary()\";\n",
       "                var nbb_formatted_code = \"test = DIR(**t, expect=expect_fn)\\ntest.run(wrapped_pp)\\ntest.summary()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = DIR(**t, expect=expect_fn)\n",
    "test.run(wrapped_pp)\n",
    "test.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc5f4178",
   "metadata": {},
   "source": [
    "## Expect Monotone Decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9c454e34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:24:12.886443Z",
     "start_time": "2022-02-03T16:24:12.843950Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 37;\n",
       "                var nbb_unformatted_code = \"data2 = [\\n    \\\"The cheesecake is great.\\\",\\n    \\\"awesome\\\",\\n    \\\"Michael had fun traveling to Mexico\\\",\\n    \\\"Anna loves party.\\\",\\n    \\\"This laptop is very good\\\",\\n]\\n\\npdata2 = list(nlp.pipe(data2))\";\n",
       "                var nbb_formatted_code = \"data2 = [\\n    \\\"The cheesecake is great.\\\",\\n    \\\"awesome\\\",\\n    \\\"Michael had fun traveling to Mexico\\\",\\n    \\\"Anna loves party.\\\",\\n    \\\"This laptop is very good\\\",\\n]\\n\\npdata2 = list(nlp.pipe(data2))\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data2 = [\n",
    "    \"The cheesecake is great.\",\n",
    "    \"awesome\",\n",
    "    \"Michael had fun traveling to Mexico\",\n",
    "    \"Anna loves party.\",\n",
    "    \"This laptop is very good\",\n",
    "]\n",
    "\n",
    "pdata2 = list(nlp.pipe(data2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b1e599ab",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:25:11.533751Z",
     "start_time": "2022-02-03T16:25:11.517294Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 38;\n",
       "                var nbb_unformatted_code = \"monotonic_decreasing = Expect.monotonic(label=1, increasing=False, tolerance=0.1)\";\n",
       "                var nbb_formatted_code = \"monotonic_decreasing = Expect.monotonic(label=1, increasing=False, tolerance=0.1)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "monotonic_decreasing = Expect.monotonic(label=1, increasing=False, tolerance=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ead6cd48",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-03T16:28:21.623445Z",
     "start_time": "2022-02-03T16:28:21.587734Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First sample before and after pertubation:\n",
      "The cheesecake is great.\n",
      "The cheesecake is not great.\n",
      "\n",
      "Summary:\n",
      "Predicting 8 examples\n",
      "Test cases:      4\n",
      "After filtering: 2 (50.0%)\n",
      "Fails (rate):    1 (50.0%)\n",
      "\n",
      "Example fails:\n",
      "0.5 Anna loves party.\n",
      "0.8 Anna doesn't love party.\n",
      "\n",
      "----\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 42;\n",
       "                var nbb_unformatted_code = \"t = Perturb.perturb(pdata2, Perturb.add_negation)\\n\\nprint(\\\"First sample before and after pertubation:\\\")\\nprint(\\\"\\\\n\\\".join(t.data[0]))\\nprint(\\\"\\\\nSummary:\\\")\\n\\ntest = DIR(**t, expect=monotonic_decreasing)\\ntest.run(wrapped_pp)\\ntest.summary()\";\n",
       "                var nbb_formatted_code = \"t = Perturb.perturb(pdata2, Perturb.add_negation)\\n\\nprint(\\\"First sample before and after pertubation:\\\")\\nprint(\\\"\\\\n\\\".join(t.data[0]))\\nprint(\\\"\\\\nSummary:\\\")\\n\\ntest = DIR(**t, expect=monotonic_decreasing)\\ntest.run(wrapped_pp)\\ntest.summary()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = Perturb.perturb(pdata2, Perturb.add_negation)\n",
    "\n",
    "print(\"First sample before and after pertubation:\")\n",
    "print(\"\\n\".join(t.data[0]))\n",
    "print(\"\\nSummary:\")\n",
    "\n",
    "test = DIR(**t, expect=monotonic_decreasing)\n",
    "test.run(wrapped_pp)\n",
    "test.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}