3.BiLSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:20.241109Z",
     "start_time": "2020-12-31T05:17:18.712511Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 17686071886566804613\n",
      "]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news</th>\n",
       "      <th>is_relevant</th>\n",
       "      <th>Armed Assault</th>\n",
       "      <th>Bombing/Explosion</th>\n",
       "      <th>Kidnapping</th>\n",
       "      <th>Other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Two Lashkar e Jhangvi LeJ militants Asim alias...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Commander Southern Command Lieutenant Gene...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Meanwhile the underground organization Manipur...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Islamic State IS in the latest issue of its on...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A senior Muttahida Qaumi Movement MQM worker i...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                news  is_relevant  \\\n",
       "0  Two Lashkar e Jhangvi LeJ militants Asim alias...         True   \n",
       "1  The Commander Southern Command Lieutenant Gene...         True   \n",
       "2  Meanwhile the underground organization Manipur...         True   \n",
       "3  Islamic State IS in the latest issue of its on...         True   \n",
       "4  A senior Muttahida Qaumi Movement MQM worker i...         True   \n",
       "\n",
       "   Armed Assault  Bombing/Explosion  Kidnapping  Other  \n",
       "0           True              False       False  False  \n",
       "1          False              False       False   True  \n",
       "2          False               True       False  False  \n",
       "3           True              False       False  False  \n",
       "4          False              False       False   True  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import tensorflow as tf\n",
    "import re\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing import text, sequence\n",
    "\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras import metrics\n",
    "import keras.backend as K\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "\n",
    "data = pd.read_csv(\"Data_MachineLearning/df_reduced.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:20.647592Z",
     "start_time": "2020-12-31T05:17:20.242290Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAD4CAYAAAD7CAEUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVdUlEQVR4nO3dfZBd9X3f8ffH4hlTBEahiiQicFS78sQWiozx2E0dU/OYGNI6rpg0VimNMg3MmMYzjXAzwSSlgzuJienYxMSoEa4Nxo+omJbImEkmnTEgMAYEJqxBGMmAZIMhtlNsyLd/3N/Ctbxa3SP27r2L3q+ZO/s733POPd/VvavPnoc9N1WFJEmDesWoG5AkzS0GhySpE4NDktSJwSFJ6sTgkCR1st+oGxiGo446qpYuXTrqNiRpTrnjjju+U1UL9rTcyzI4li5dyubNm0fdhiTNKUkeGWQ5D1VJkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjp5Wf7l+Fy1dN2XRrbtrZeeMbJtS5pb3OOQJHVicEiSOjE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiSOjE4JEmdGBySpE6GFhxJDkpyW5KvJ9mS5OJWPzbJrUkmknw6yQGtfmCbnmjzl/Y914Wt/kCSU4bVsyRpz4a5x/Es8PaqegOwAjg1yYnAB4HLqurngaeAc9vy5wJPtfplbTmSLAdWA68DTgU+mmTeEPuWJE1jaMFRPd9vk/u3RwFvBz7b6huAs9r4zDZNm39SkrT6tVX1bFU9DEwAJwyrb0nS9IZ6jiPJvCR3ATuATcA3ge9V1XNtkW3AojZeBDwK0OY/Dbyqvz7FOv3bWptkc5LNO3fuHMJ3I0mCIQdHVT1fVSuAxfT2El47xG1dWVWrqmrVggULhrUZSdrnzcpVVVX1PeAW4M3A/CSTnzy4GNjextuBJQBt/uHAd/vrU6wjSZplw7yqakGS+W18MPAO4H56AfKuttga4Po23timafO/UlXV6qvbVVfHAsuA24bVtyRpesP8zPGFwIZ2BdQrgOuq6oYk9wHXJvkvwNeAq9ryVwGfSDIBPEnvSiqqakuS64D7gOeA86rq+SH2LUmaxtCCo6ruBo6fov4QU1wVVVX/D/j13TzXJcAlM92jJKk7/3JcktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRO9ht1A+No6bovjboFSRpbQ9vjSLIkyS1J7kuyJcl7W/0DSbYnuas9Tu9b58IkE0keSHJKX/3UVptIsm5YPUuS9myYexzPAe+rqjuTHAbckWRTm3dZVf1x/8JJlgOrgdcBPwt8Ock/abM/ArwD2AbcnmRjVd03xN4lSbsxtOCoqseAx9r475LcDyyaZpUzgWur6lng4SQTwAlt3kRVPQSQ5Nq2rMEhSSMwKyfHkywFjgdubaXzk9ydZH2SI1ptEfBo32rbWm139V23sTbJ5iSbd+7cOdPfgiSpGXpwJHkl8Dnggqp6BrgCeDWwgt4eyZ/MxHaq6sqqWlVVqxYsWDATTylJmsJQr6pKsj+90PhkVX0eoKqe6Jv/58ANbXI7sKRv9cWtxjR1SdIsG+ZVVQGuAu6vqg/11Rf2LfZrwL1tvBFYneTAJMcCy4DbgNuBZUmOTXIAvRPoG4fVtyRpesPc43gL8JvAPUnuarX3A2cnWQEUsBX4bYCq2pLkOnonvZ8Dzquq5wGSnA/cBMwD1lfVliH2LUmaxjCvqvobIFPMunGadS4BLpmifuN060mSZo+3HJEkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInAwVHkl8YdiOSpLlh0D2Ojya5LcnvJDl8qB1JksbaQMFRVf8M+A1gCXBHkk8lecd06yRZkuSWJPcl2ZLkva1+ZJJNSR5sX49o9SS5PMlEkruTrOx7rjVt+QeTrNnr71aS9JINfI6jqh4Efh/4PeCfA5cn+UaSf7mbVZ4D3ldVy4ETgfOSLAfWATdX1TLg5jYNcBqwrD3WAldAL2iAi4A3AScAF02GjSRp9g16juP1SS4D7gfeDvxqVf3TNr5sqnWq6rGqurON/66tuwg4E9jQFtsAnNXGZwJXV89XgflJFgKnAJuq6smqegrYBJza+TuVJM2I/QZc7r8DHwfeX1V/P1msqm8n+f09rZxkKXA8cCtwdFU91mY9DhzdxouAR/tW29Zqu6vvuo219PZUOOaYYwb6piRJ3Q16qOoM4FOToZHkFUkOAaiqT0y3YpJXAp8DLqiqZ/rnVVUB1bnrKVTVlVW1qqpWLViwYCaeUpI0hUGD48vAwX3Th7TatJLsTy80PllVn2/lJ9ohKNrXHa2+nd7J90mLW213dUnSCAwaHAdV1fcnJ9r4kOlWSBLgKuD+qvpQ36yNwOSVUWuA6/vq72lXV50IPN0Oad0EnJzkiHZS/ORWkySNwKDnOH6QZOXkye4kvwj8/R7WeQvwm8A9Se5qtfcDlwLXJTkXeAR4d5t3I3A6MAH8EDgHoKqeTPJHwO1tuT+sqicH7FuSNMMGDY4LgM8k+TYQ4B8D/3q6Farqb9qyUzlpiuULOG83z7UeWD9gr5KkIRooOKrq9iSvBV7TSg9U1Y+H15YkaVwNuscB8EZgaVtnZRKq6uqhdCVJGlsDBUeSTwCvBu4Cnm/lAgwOSdrHDLrHsQpY3s5DSJL2YYNejnsvvRPikqR93KB7HEcB9yW5DXh2slhV7xxKV5KksTVocHxgmE1IkuaOQS/H/askPwcsq6ovt/tUzRtua5KkcTTobdV/C/gs8LFWWgR8cUg9SZLG2KAnx8+jdwuRZ+CFD3X6mWE1JUkaX4MGx7NV9aPJiST7MUO3Q5ckzS2DBsdfJXk/cHD7rPHPAP9reG1JksbVoMGxDtgJ3AP8Nr072e7xk/8kSS8/g15V9Q/An7eHJGkfNui9qh5minMaVXXcjHckSRprXe5VNekg4NeBI2e+HUnSuBvoHEdVfbfvsb2q/hQ4Y7itSZLG0aCHqlb2Tb6C3h5Il8/ykCS9TAz6n/+f9I2fA7by4meFS5L2IYNeVfXLw25EkjQ3DHqo6nenm19VH5qZdiRJ467LVVVvBDa26V8FbgMeHEZTkqTxNehfji8GVlbV+6rqfcAvAsdU1cVVdfFUKyRZn2RHknv7ah9Isj3JXe1xet+8C5NMJHkgySl99VNbbSLJur37NiVJM2XQ4Dga+FHf9I9abTp/AZw6Rf2yqlrRHjcCJFkOrAZe19b5aJJ5SeYBHwFOA5YDZ7dlJUkjMuihqquB25J8oU2fBWyYboWq+uskSwd8/jOBa6vqWeDhJBPACW3eRFU9BJDk2rbsfQM+ryRphg36B4CXAOcAT7XHOVX1X/dym+cnubsdyjqi1RYBj/Yts63VdleXJI3IoIeqAA4BnqmqDwPbkhy7F9u7Ang1sAJ4jJ/8+5CXJMnaJJuTbN65c+dMPa0kaReDfnTsRcDvARe20v7A/+y6sap6oqqe77vb7uThqO3Akr5FF7fa7upTPfeVVbWqqlYtWLCga2uSpAENusfxa8A7gR8AVNW3gcO6bizJwl2ec/KKq43A6iQHtj2ZZfQu970dWJbk2CQH0DuBvhFJ0sgMenL8R1VVSQogyaF7WiHJNcDbgKOSbAMuAt6WZAW9W7RvpfehUFTVliTX0Tvp/RxwXlU9357nfOAmYB6wvqq2DPzdSZJm3KDBcV2SjwHzk/wW8O/Yw4c6VdXZU5Svmmb5S4BLpqjfSO8TByVJY2CPwZEkwKeB1wLPAK8B/qCqNg25N0nSGNpjcLRDVDdW1S8AhoUk7eMGPTl+Z5I3DrUTSdKcMOg5jjcB/ybJVnpXVoXezsjrh9WYJGk8TRscSY6pqm8Bp0y3nCRp37GnPY4v0rsr7iNJPldV/2oWepIkjbE9neNI3/i4YTYiSZob9hQctZuxJGkftadDVW9I8gy9PY+D2xhePDn+j4banSRp7EwbHFU1b7YakSTNDV1uqy5JksEhSerG4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSepk0A9y0svc0nVfGsl2t156xki2K2nvucchSerE4JAkdTK04EiyPsmOJPf21Y5MsinJg+3rEa2eJJcnmUhyd5KVfeusacs/mGTNsPqVJA1mmHscfwGcukttHXBzVS0Dbm7TAKcBy9pjLXAF9IIGuAh4E3ACcNFk2EiSRmNowVFVfw08uUv5TGBDG28AzuqrX109XwXmJ1kInAJsqqonq+opYBM/HUaSpFk02+c4jq6qx9r4ceDoNl4EPNq33LZW2139pyRZm2Rzks07d+6c2a4lSS8Y2cnxqipm8HPMq+rKqlpVVasWLFgwU08rSdrFbAfHE+0QFO3rjlbfDizpW25xq+2uLkkakdkOjo3A5JVRa4Dr++rvaVdXnQg83Q5p3QScnOSIdlL85FaTJI3I0P5yPMk1wNuAo5Jso3d11KXAdUnOBR4B3t0WvxE4HZgAfgicA1BVTyb5I+D2ttwfVtWuJ9wlSbNoaMFRVWfvZtZJUyxbwHm7eZ71wPoZbE2S9BL4l+OSpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiSOjE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiSOjE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiSOhlJcCTZmuSeJHcl2dxqRybZlOTB9vWIVk+Sy5NMJLk7ycpR9CxJ6hnlHscvV9WKqlrVptcBN1fVMuDmNg1wGrCsPdYCV8x6p5KkF4zToaozgQ1tvAE4q69+dfV8FZifZOEI+pMkMbrgKOAvk9yRZG2rHV1Vj7Xx48DRbbwIeLRv3W2t9hOSrE2yOcnmnTt3DqtvSdrn7Tei7b61qrYn+RlgU5Jv9M+sqkpSXZ6wqq4ErgRYtWpVp3UlSYMbyR5HVW1vX3cAXwBOAJ6YPATVvu5oi28HlvStvrjVJEkjMOvBkeTQJIdNjoGTgXuBjcCattga4Po23gi8p11ddSLwdN8hLUnSLBvFoaqjgS8kmdz+p6rq/yS5HbguybnAI8C72/I3AqcDE8APgXNmv2VJ0qRZD46qegh4wxT17wInTVEv4LxZaE2SNIBxuhxXkjQHGBySpE4MDklSJ6P6Ow4JgKXrvjSS7W699IyRbFd6OXCPQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSerE4JAkdWJwSJI6MTgkSZ0YHJKkTgwOSVInBockqRM/yEn7pFF9gBT4IVKa+9zjkCR1YnBIkjqZM4eqkpwKfBiYB3y8qi4dcUvSXvFz1jXXzYk9jiTzgI8ApwHLgbOTLB9tV5K0b5orexwnABNV9RBAkmuBM4H7RtqVNIeM8oKAUXEvazjmSnAsAh7tm94GvKl/gSRrgbVt8vtJHtiL7RwFfGevOhy+ce4Nxrs/e9t749zfHnvLB2epk582zv9usPv+fm6QledKcOxRVV0JXPlSniPJ5qpaNUMtzahx7g3Guz9723vj3J+97b2X2t+cOMcBbAeW9E0vbjVJ0iybK8FxO7AsybFJDgBWAxtH3JMk7ZPmxKGqqnouyfnATfQux11fVVuGsKmXdKhryMa5Nxjv/uxt741zf/a2917aYf2qmqlGJEn7gLlyqEqSNCYMDklSJwZHk+TUJA8kmUiybgTbX59kR5J7+2pHJtmU5MH29YhWT5LLW693J1k55N6WJLklyX1JtiR577j0l+SgJLcl+Xrr7eJWPzbJra2HT7eLKkhyYJueaPOXDqu3vh7nJflakhvGsLetSe5JcleSza028te1bW9+ks8m+UaS+5O8eYx6e037N5t8PJPkgjHq7z+2n4d7k1zTfk5m7n1XVfv8g94J928CxwEHAF8Hls9yD78ErATu7av9N2BdG68DPtjGpwP/GwhwInDrkHtbCKxs48OAv6V365eR99e28co23h+4tW3zOmB1q/8Z8B/a+HeAP2vj1cCnZ+G1/V3gU8ANbXqcetsKHLVLbeSva9veBuDft/EBwPxx6W2XPucBj9P747mR90fvD6YfBg7ue7/925l8383KP+y4P4A3Azf1TV8IXDiCPpbyk8HxALCwjRcCD7Txx4Czp1pulvq8HnjHuPUHHALcSe+uAt8B9tv19aV3Zd6b23i/tlyG2NNi4Gbg7cAN7T+OseitbWcrPx0cI39dgcPbf34Zt96m6PVk4P+OS3+8eKeNI9v76AbglJl833moqmeqW5osGlEv/Y6uqsfa+HHg6DYeWb9tN/Z4er/Zj0V/7VDQXcAOYBO9vcfvVdVzU2z/hd7a/KeBVw2rN+BPgf8E/EObftUY9QZQwF8muSO92/bAeLyuxwI7gf/RDvN9PMmhY9LbrlYD17TxyPurqu3AHwPfAh6j9z66gxl83xkcc0T1fh0Y6bXTSV4JfA64oKqe6Z83yv6q6vmqWkHvt/sTgNeOoo9dJfkVYEdV3THqXqbx1qpaSe/O0+cl+aX+mSN8Xfejd+j2iqo6HvgBvUM/49DbC9p5gncCn9l13qj6a+dVzqQXvj8LHAqcOpPbMDh6xvWWJk8kWQjQvu5o9VnvN8n+9ELjk1X1+XHrD6CqvgfcQm83fH6SyT9w7d/+C721+YcD3x1SS28B3plkK3AtvcNVHx6T3oAXfjulqnYAX6AXvOPwum4DtlXVrW36s/SCZBx663cacGdVPdGmx6G/fwE8XFU7q+rHwOfpvRdn7H1ncPSM6y1NNgJr2ngNvXMLk/X3tCs1TgSe7ts9nnFJAlwF3F9VHxqn/pIsSDK/jQ+md+7lfnoB8q7d9DbZ87uAr7TfDGdcVV1YVYuraim999RXquo3xqE3gCSHJjlsckzvWP29jMHrWlWPA48meU0rnUTvYxRG3tsuzubFw1STfYy6v28BJyY5pP3sTv7bzdz7bjZOHs2FB72rHv6W3vHx/zyC7V9D73jkj+n9tnUuveOMNwMPAl8GjmzLht4HW30TuAdYNeTe3kpvl/tu4K72OH0c+gNeD3yt9XYv8AetfhxwGzBB7zDCga1+UJueaPOPm6XX9228eFXVWPTW+vh6e2yZfN+Pw+vatrcC2Nxe2y8CR4xLb22bh9L7zfzwvtpY9AdcDHyj/Ux8AjhwJt933nJEktSJh6okSZ0YHJKkTgwOSVInBockqRODQ5LUicEhSerE4JAkdfL/AcgR50z4IdcgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.news.str.split().str.len().plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:20.669315Z",
     "start_time": "2020-12-31T05:17:20.650891Z"
    }
   },
   "outputs": [],
   "source": [
    "def f1_multilabel(labels, predictions):\n",
    "    \n",
    "    f1 = 0.\n",
    "    count = 0\n",
    "    for lab, pred in zip(labels, predictions):\n",
    "        lab_den = [i for i, l in enumerate(lab) if l == 1]\n",
    "        pred_den = [i for i, p in enumerate(pred) if p == 1]\n",
    "        den = len(lab_den) + len(pred_den)\n",
    "        if den>0:\n",
    "            count+=1\n",
    "            num = 2*sum([1 if l==p==1 else 0 for l, p in zip(lab,pred)])\n",
    "            f1+=(num/den)\n",
    "\n",
    "    if count == 0:\n",
    "        return 0.\n",
    "    else:\n",
    "        return f1/count\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def accuracy_multilabel(labels, predictions):\n",
    "\n",
    "    acc = 0.\n",
    "    count = 0\n",
    "    for lab, pred in zip(labels, predictions):\n",
    "\n",
    "        lab_den = [i for i, l in enumerate(lab) if l == 1]\n",
    "        pred_den = [i for i, p in enumerate(pred) if p == 1]\n",
    "        den = len(set(lab_den + pred_den))\n",
    "\n",
    "        if den>0:\n",
    "            count+=1\n",
    "            num = sum([1 if l==p==1 else 0 for l, p in zip(lab,pred)])\n",
    "            acc+=(num/den)\n",
    "            \n",
    "    if count == 0:\n",
    "        return 0.\n",
    "    else:\n",
    "        return acc/count\n",
    "\n",
    "\n",
    "\n",
    "def precision_multilabel(labels, predictions):\n",
    "    prec = 0.\n",
    "    count = 0\n",
    "    for lab, pred in zip(labels, predictions):\n",
    "        pred_den = [i for i, p in enumerate(pred) if p == 1]\n",
    "        den = len(pred_den)\n",
    "        \n",
    "        if den>0:\n",
    "            count+=1 \n",
    "            num = sum([1 if l==p==1 else 0 for l, p in zip(lab,pred)])\n",
    "            prec+=(num/den)\n",
    "            \n",
    "    if count == 0:\n",
    "        return 0.\n",
    "    else:\n",
    "        return  prec/count\n",
    "\n",
    "\n",
    "def recall_multilabel(labels, predictions):\n",
    "    rec = 0.\n",
    "    count = 0\n",
    "    for lab, pred in zip(labels, predictions):\n",
    "        lab_den = [i for i, l in enumerate(lab) if l == 1]\n",
    "        den = len(lab_den)\n",
    "        \n",
    "        if den>0:\n",
    "            count+=1\n",
    "            num = sum([1 if l==p==1 else 0 for l, p in zip(lab,pred)])\n",
    "            rec+=(num/den)\n",
    "    if count == 0:\n",
    "        return 0.\n",
    "    else:\n",
    "        return  rec/count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:21.459162Z",
     "start_time": "2020-12-31T05:17:20.671210Z"
    }
   },
   "outputs": [],
   "source": [
    "def cleanHtml(sentence):  # remove html-tags\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, ' ', str(sentence))\n",
    "    return cleantext\n",
    "\n",
    "def cleanPunc(sentence): # clean the word of any punctuation or special characters\n",
    "    cleaned = re.sub(r'[?|!|\\'|\"|#]',r'',sentence)\n",
    "    cleaned = re.sub(r'[.|,|)|(|\\|/]',r' ',cleaned)\n",
    "    cleaned = cleaned.strip()\n",
    "    cleaned = cleaned.replace(\"\\n\",\" \")\n",
    "    return cleaned\n",
    "\n",
    "def keepAlpha(sentence):  # Remove non-alphabetic characters\n",
    "    alpha_sent = \"\"\n",
    "    for word in sentence.split():\n",
    "        alpha_word = re.sub('[^a-z A-Z]+', ' ', word)\n",
    "        alpha_sent += alpha_word\n",
    "        alpha_sent += \" \"\n",
    "    alpha_sent = alpha_sent.strip()\n",
    "    return alpha_sent\n",
    "\n",
    "data['news'] = data['news'].str.lower()\n",
    "data['news'] = data['news'].apply(cleanHtml)\n",
    "data['news'] = data['news'].apply(cleanPunc)\n",
    "data['news'] = data['news'].apply(keepAlpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:21.465565Z",
     "start_time": "2020-12-31T05:17:21.460280Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5954, 6)\n",
      "(1489, 6)\n"
     ]
    }
   ],
   "source": [
    "df_train = data.sample(frac=0.8,random_state=200) #random state is a seed value\n",
    "df_test = data.drop(df_train.index)\n",
    "\n",
    "print(df_train.shape)\n",
    "print(df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:21.485966Z",
     "start_time": "2020-12-31T05:17:21.466502Z"
    }
   },
   "outputs": [],
   "source": [
    "x_train = df_train['news'].values\n",
    "y_train = df_train.iloc[:,1:].values\n",
    "\n",
    "x_test = df_test['news'].values\n",
    "y_test = df_test.iloc[:,1:].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:22.079052Z",
     "start_time": "2020-12-31T05:17:21.488115Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=20000)\n",
    "tokenizer.fit_on_texts(list(x_train))\n",
    "\n",
    "train_seq = tokenizer.texts_to_sequences(x_train)\n",
    "train_pad = sequence.pad_sequences(train_seq, maxlen=300)\n",
    "\n",
    "test_seq = tokenizer.texts_to_sequences(x_test)\n",
    "test_pad = sequence.pad_sequences(test_seq, maxlen=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:22.090317Z",
     "start_time": "2020-12-31T05:17:22.080879Z"
    }
   },
   "outputs": [],
   "source": [
    "def f1_score(y_true, y_pred):\n",
    "    y_pred = K.round(y_pred)\n",
    "    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n",
    "    # tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n",
    "    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n",
    "    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n",
    "\n",
    "    p = tp / (tp + fp + K.epsilon())\n",
    "    r = tp / (tp + fn + K.epsilon())\n",
    "\n",
    "    f1 = 2*p*r / (p+r+K.epsilon())\n",
    "    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n",
    "    return K.mean(f1)\n",
    "\n",
    "\n",
    "def recall_m(y_true, y_pred):\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "def precision_m(y_true, y_pred):\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "def f1_m(y_true, y_pred):\n",
    "    precision = precision_m(y_true, y_pred)\n",
    "    recall = recall_m(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n",
    "\n",
    "\n",
    "\n",
    "### Defining the custom metric function F1\n",
    "def custom_f1(y_true, y_pred):    \n",
    "    def recall_m(y_true, y_pred):\n",
    "        TP = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        Positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        \n",
    "        recall = TP / (Positives+K.epsilon())    \n",
    "        return recall \n",
    "    \n",
    "    \n",
    "    def precision_m(y_true, y_pred):\n",
    "        TP = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        Pred_Positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    \n",
    "        precision = TP / (Pred_Positives+K.epsilon())\n",
    "        return precision \n",
    "    \n",
    "    precision, recall = precision_m(y_true, y_pred), recall_m(y_true, y_pred)\n",
    "    \n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n",
    "\n",
    "\n",
    "from tensorflow.keras.callbacks import Callback\n",
    "\n",
    "### Defining the Callback Metrics\n",
    "class Metrics(Callback):\n",
    "    def __init__(self, validation):   \n",
    "        super(Metrics, self).__init__()\n",
    "        self.validation = validation    \n",
    "            \n",
    "        print('validation shape', len(self.validation[0]))\n",
    "        \n",
    "    def on_train_begin(self, logs={}):        \n",
    "        self.val_f1s = []\n",
    "        self.val_recalls = []\n",
    "        self.val_precisions = []\n",
    "     \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        val_targ = self.validation[1]   \n",
    "        val_predict = (np.asarray(self.model.predict(self.validation[0]))).round()        \n",
    "    \n",
    "        val_f1 = round(f1_score(val_targ, val_predict), 6)\n",
    "        val_recall = round(recall_score(val_targ, val_predict), 6)     \n",
    "        val_precision = round(precision_score(val_targ, val_predict), 6)\n",
    "        \n",
    "        self.val_f1s.append(val_f1)\n",
    "        self.val_recalls.append(val_recall)\n",
    "        self.val_precisions.append(val_precision)\n",
    " \n",
    "        print(f' — val_f1: {val_f1} — val_precision: {val_precision}, — val_recall: {val_recall}')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:22.272935Z",
     "start_time": "2020-12-31T05:17:22.091359Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import recall_score, precision_score, f1_score, accuracy_score,hamming_loss\n",
    "#                          recall_score(df_target[col], df_pred[col], average='macro'),\n",
    "#                         precision_score(df_target[col], df_pred[col], average='macro'),\n",
    "#                         f1_score(df_target[col], df_pred[col], average='macro')\n",
    "\n",
    "def ac(y_true, y_pred): \n",
    "    y_true, y_pred = tf.make_tensor_proto(y_true),  tf.make_tensor_proto(y_pred)\n",
    "    y_true, y_pred = tf.make_ndarray(y_true),  tf.make_ndarray(y_pred)\n",
    "#     y_true, y_pred = y_true.numpy(), y_pred.numpy()\n",
    "    return accuracy_score(y_true, y_pred)\n",
    "def rec (y_true, y_pred): \n",
    "    return recall_score(y_true, y_pred, average='macro')\n",
    "def re (y_true, y_pred): \n",
    "    return precision_score(y_true, y_pred, average='macro'),\n",
    "def f1 (y_true, y_pred): \n",
    "    return f1_score(y_true, y_pred, average='macro')\n",
    "\n",
    "import tensorflow_addons as tfa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:17:22.747442Z",
     "start_time": "2020-12-31T05:17:22.274069Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 300)]             0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 300, 256)          5120000   \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 200)               285600    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 100)               20100     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 5)                 505       \n",
      "=================================================================\n",
      "Total params: 5,426,205\n",
      "Trainable params: 5,426,205\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "inputs = Input(shape=(300, ))\n",
    "x = Embedding(20000, 256)(inputs)\n",
    "x = Bidirectional(LSTM(100))(x)\n",
    "x = Dropout(0.1)(x)\n",
    "x = Dense(100, activation=\"relu\")(x)\n",
    "x = Dropout(0.1)(x)\n",
    "outputs = Dense(5, activation=\"sigmoid\")(x)\n",
    "model = Model(inputs=inputs, outputs=outputs)\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "#               metrics=['accuracy',\n",
    "#                        'binary_accuracy',\n",
    "# #                       \"top_k_categorical_accuracy\", \n",
    "#                        metrics.AUC(),\n",
    "#                        f1_score]\n",
    "              metrics = ['acc',\n",
    "#                          precision_m, recall_m, f1_m,\n",
    "#                          metrics.Precision(),\n",
    "#                          metrics.Recall(),\n",
    "                         tfa.metrics.F1Score(num_classes=5, average='macro')\n",
    "#                          ac, \n",
    "#                          rec, pre,f1\n",
    " \n",
    "                        \n",
    "                        ])\n",
    "\n",
    "\n",
    "# model.compile(loss='binary_crossentropy',\n",
    "#               optimizer='adam',\n",
    "#               metrics = [])\n",
    "\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:21:29.047187Z",
     "start_time": "2020-12-31T05:17:22.749554Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "6/6 [==============================] - 11s 2s/step - loss: 0.6739 - acc: 0.4534 - f1_score: 0.1135 - val_loss: 0.5446 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 2/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.4672 - acc: 0.9558 - f1_score: 0.0791 - val_loss: 0.2882 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 3/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.3121 - acc: 0.9478 - f1_score: 0.0837 - val_loss: 0.2693 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 4/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.2854 - acc: 0.9258 - f1_score: 0.0832 - val_loss: 0.2677 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 5/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.2916 - acc: 0.9166 - f1_score: 0.0897 - val_loss: 0.2645 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 6/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.2804 - acc: 0.9649 - f1_score: 0.0791 - val_loss: 0.2634 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 7/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.2850 - acc: 0.9628 - f1_score: 0.0803 - val_loss: 0.2603 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 8/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.2805 - acc: 0.9721 - f1_score: 0.0783 - val_loss: 0.2577 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 9/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.2766 - acc: 0.9632 - f1_score: 0.0818 - val_loss: 0.2554 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 10/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.2711 - acc: 0.9665 - f1_score: 0.0812 - val_loss: 0.2491 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 11/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.2506 - acc: 0.9631 - f1_score: 0.0750 - val_loss: 0.2335 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 12/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.2294 - acc: 0.9616 - f1_score: 0.0759 - val_loss: 0.2047 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 13/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1889 - acc: 0.8778 - f1_score: 0.0789 - val_loss: 0.1948 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 14/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1729 - acc: 0.7595 - f1_score: 0.0874 - val_loss: 0.2015 - val_acc: 0.9983 - val_f1_score: 0.0657\n",
      "Epoch 15/200\n",
      "6/6 [==============================] - 8s 1s/step - loss: 0.1549 - acc: 0.7034 - f1_score: 0.0921 - val_loss: 0.1976 - val_acc: 1.0000 - val_f1_score: 0.0656\n",
      "Epoch 16/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1496 - acc: 0.7791 - f1_score: 0.0857 - val_loss: 0.2102 - val_acc: 0.9866 - val_f1_score: 0.0658\n",
      "Epoch 17/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1406 - acc: 0.6470 - f1_score: 0.0988 - val_loss: 0.2040 - val_acc: 0.9966 - val_f1_score: 0.0658\n",
      "Epoch 18/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1319 - acc: 0.6797 - f1_score: 0.0947 - val_loss: 0.2167 - val_acc: 0.9983 - val_f1_score: 0.0657\n",
      "Epoch 19/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1185 - acc: 0.6263 - f1_score: 0.0991 - val_loss: 0.2092 - val_acc: 0.9983 - val_f1_score: 0.0657\n",
      "Epoch 20/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1182 - acc: 0.6823 - f1_score: 0.0938 - val_loss: 0.2235 - val_acc: 0.9983 - val_f1_score: 0.0657\n",
      "Epoch 21/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1120 - acc: 0.6114 - f1_score: 0.1018 - val_loss: 0.2239 - val_acc: 0.9883 - val_f1_score: 0.0663\n",
      "Epoch 22/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1010 - acc: 0.5635 - f1_score: 0.1060 - val_loss: 0.2423 - val_acc: 0.8557 - val_f1_score: 0.0740\n",
      "Epoch 23/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.1010 - acc: 0.5027 - f1_score: 0.1164 - val_loss: 0.2391 - val_acc: 0.7383 - val_f1_score: 0.0826\n",
      "Epoch 24/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0954 - acc: 0.4701 - f1_score: 0.1212 - val_loss: 0.2560 - val_acc: 0.4983 - val_f1_score: 0.1085\n",
      "Epoch 25/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0923 - acc: 0.4451 - f1_score: 0.1278 - val_loss: 0.2570 - val_acc: 0.4564 - val_f1_score: 0.1134\n",
      "Epoch 26/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0848 - acc: 0.4237 - f1_score: 0.1316 - val_loss: 0.2695 - val_acc: 0.3523 - val_f1_score: 0.1323\n",
      "Epoch 27/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0761 - acc: 0.3881 - f1_score: 0.1403 - val_loss: 0.2639 - val_acc: 0.3473 - val_f1_score: 0.1429\n",
      "Epoch 28/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0708 - acc: 0.3623 - f1_score: 0.1425 - val_loss: 0.2699 - val_acc: 0.3272 - val_f1_score: 0.1489\n",
      "Epoch 29/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0661 - acc: 0.3771 - f1_score: 0.1412 - val_loss: 0.2787 - val_acc: 0.3003 - val_f1_score: 0.1497\n",
      "Epoch 30/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0608 - acc: 0.3712 - f1_score: 0.1435 - val_loss: 0.2694 - val_acc: 0.3607 - val_f1_score: 0.1425\n",
      "Epoch 31/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0579 - acc: 0.3806 - f1_score: 0.1400 - val_loss: 0.2855 - val_acc: 0.3154 - val_f1_score: 0.1523\n",
      "Epoch 32/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0569 - acc: 0.3806 - f1_score: 0.1468 - val_loss: 0.2953 - val_acc: 0.2919 - val_f1_score: 0.1565\n",
      "Epoch 33/200\n",
      "6/6 [==============================] - 7s 1s/step - loss: 0.0503 - acc: 0.3522 - f1_score: 0.1575 - val_loss: 0.2900 - val_acc: 0.3406 - val_f1_score: 0.1360\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fd6d40dc070>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early_stop = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=20)\n",
    "# my_metric = Metrics()\n",
    "model.fit(train_pad, y_train, \n",
    "          batch_size=1024,\n",
    "          \n",
    "          epochs=200, \n",
    "          validation_split=0.1, \n",
    "#           callbacks=[]\n",
    "          callbacks=early_stop\n",
    "#           callbacks=[early_stop, my_metric]\n",
    "         )\n",
    "          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:21:30.435570Z",
     "start_time": "2020-12-31T05:21:29.050029Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 1s 536ms/step\n"
     ]
    }
   ],
   "source": [
    "outputs = model.predict([test_pad], batch_size=1024, verbose=1)\n",
    "# model.evaluate(test_pad, outputs, batch_size=32, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:21:30.443465Z",
     "start_time": "2020-12-31T05:21:30.438564Z"
    }
   },
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open('bilstm_output.pt','wb') as file:\n",
    "#     pickle.dump(outputs, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:22:02.547439Z",
     "start_time": "2020-12-31T05:22:02.541129Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.8409304e-05, 3.5519930e-05, 2.3704039e-05, 9.4633087e-06,\n",
       "        4.8667316e-05],\n",
       "       [6.3919229e-05, 7.7372228e-05, 4.1510772e-05, 1.9276651e-05,\n",
       "        1.0046235e-04],\n",
       "       [9.2872918e-01, 7.4782056e-01, 7.3134303e-03, 6.5694720e-02,\n",
       "        1.5378863e-01],\n",
       "       ...,\n",
       "       [1.7347837e-05, 2.3062770e-05, 1.6391099e-05, 6.4238316e-06,\n",
       "        3.1488456e-05],\n",
       "       [9.9776745e-01, 6.4662367e-02, 9.8081553e-01, 1.1590719e-02,\n",
       "        3.1181842e-02],\n",
       "       [6.4316504e-05, 7.5728021e-05, 4.2713316e-05, 1.9066358e-05,\n",
       "        9.9659155e-05]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:25:28.900572Z",
     "start_time": "2020-12-31T05:25:28.896344Z"
    }
   },
   "outputs": [],
   "source": [
    "pred = outputs >= 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:30:15.265491Z",
     "start_time": "2020-12-31T05:30:15.258767Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True,  True, False, False, False],\n",
       "       [ True,  True, False, False, False],\n",
       "       [ True,  True, False, False, False],\n",
       "       ...,\n",
       "       [ True,  True, False, False, False],\n",
       "       [ True,  True, False, False, False],\n",
       "       [ True, False,  True, False, False]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred[pred[:,0]==True]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:29:59.582166Z",
     "start_time": "2020-12-31T05:29:59.548230Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "[ True False False False False]\n",
      "101\n"
     ]
    }
   ],
   "source": [
    "# pred[pred[:,0]== False]=0\n",
    "i=0\n",
    "for row in pred:\n",
    "    \n",
    "    if row[0]== True and ~row[1:].any():\n",
    "        print(row)\n",
    "        i +=1\n",
    "print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:22:46.569052Z",
     "start_time": "2020-12-31T05:22:44.915587Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold 0.1\n",
      "threshold 0.2\n",
      "threshold 0.3\n",
      "threshold 0.4\n",
      "threshold 0.5\n",
      "threshold 0.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yibo/miniconda3/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold 0.7\n",
      "threshold 0.8\n",
      "threshold 0.9\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>hamming</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>EMR</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>all</td>\n",
       "      <td>0.738079</td>\n",
       "      <td>0.690828</td>\n",
       "      <td>0.456947</td>\n",
       "      <td>0.550059</td>\n",
       "      <td>0.573441</td>\n",
       "      <td>0.396224</td>\n",
       "      <td>0.452517</td>\n",
       "      <td>0.102619</td>\n",
       "      <td>0.358935</td>\n",
       "      <td>0.738079</td>\n",
       "      <td>0.688088</td>\n",
       "      <td>0.466406</td>\n",
       "      <td>0.41688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.752183</td>\n",
       "      <td>0.621849</td>\n",
       "      <td>0.347418</td>\n",
       "      <td>0.445783</td>\n",
       "      <td>0.524795</td>\n",
       "      <td>0.335358</td>\n",
       "      <td>0.391145</td>\n",
       "      <td>0.102619</td>\n",
       "      <td>0.29396</td>\n",
       "      <td>0.752183</td>\n",
       "      <td>0.620167</td>\n",
       "      <td>0.3841</td>\n",
       "      <td>0.346254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.825038</td>\n",
       "      <td>0.786389</td>\n",
       "      <td>0.802430</td>\n",
       "      <td>0.142377</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.1</td>\n",
       "      <td>Armed Assault</td>\n",
       "      <td>0.856279</td>\n",
       "      <td>0.856279</td>\n",
       "      <td>0.856279</td>\n",
       "      <td>0.856279</td>\n",
       "      <td>0.804889</td>\n",
       "      <td>0.662221</td>\n",
       "      <td>0.697358</td>\n",
       "      <td>0.143721</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.1</td>\n",
       "      <td>Bombing/Explosion</td>\n",
       "      <td>0.946273</td>\n",
       "      <td>0.946273</td>\n",
       "      <td>0.946273</td>\n",
       "      <td>0.946273</td>\n",
       "      <td>0.791611</td>\n",
       "      <td>0.817076</td>\n",
       "      <td>0.803679</td>\n",
       "      <td>0.053727</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.9</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.790831</td>\n",
       "      <td>0.832687</td>\n",
       "      <td>0.808791</td>\n",
       "      <td>0.120887</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.9</td>\n",
       "      <td>Armed Assault</td>\n",
       "      <td>0.913365</td>\n",
       "      <td>0.913365</td>\n",
       "      <td>0.913365</td>\n",
       "      <td>0.913365</td>\n",
       "      <td>0.518202</td>\n",
       "      <td>0.814295</td>\n",
       "      <td>0.513252</td>\n",
       "      <td>0.086635</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.9</td>\n",
       "      <td>Bombing/Explosion</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.942915</td>\n",
       "      <td>0.690194</td>\n",
       "      <td>0.850507</td>\n",
       "      <td>0.741981</td>\n",
       "      <td>0.057085</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>0.9</td>\n",
       "      <td>Kidnapping</td>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.491941</td>\n",
       "      <td>0.495938</td>\n",
       "      <td>0.016118</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>0.9</td>\n",
       "      <td>Other</td>\n",
       "      <td>0.942243</td>\n",
       "      <td>0.942243</td>\n",
       "      <td>0.942243</td>\n",
       "      <td>0.942243</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.471122</td>\n",
       "      <td>0.485131</td>\n",
       "      <td>0.057757</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold               type  accuracy  recall(micro)  precision(micro)  \\\n",
       "0         0.1                all  0.738079       0.690828          0.456947   \n",
       "1         0.1              multi  0.752183       0.621849          0.347418   \n",
       "2         0.1        is_relevant  0.857623       0.857623          0.857623   \n",
       "3         0.1      Armed Assault  0.856279       0.856279          0.856279   \n",
       "4         0.1  Bombing/Explosion  0.946273       0.946273          0.946273   \n",
       "..        ...                ...       ...            ...               ...   \n",
       "58        0.9        is_relevant  0.879113       0.879113          0.879113   \n",
       "59        0.9      Armed Assault  0.913365       0.913365          0.913365   \n",
       "60        0.9  Bombing/Explosion  0.942915       0.942915          0.942915   \n",
       "61        0.9         Kidnapping  0.983882       0.983882          0.983882   \n",
       "62        0.9              Other  0.942243       0.942243          0.942243   \n",
       "\n",
       "    f1(micro)  recall(macro)  precision(macro)  f1(macro)   hamming  \\\n",
       "0    0.550059       0.573441          0.396224   0.452517  0.102619   \n",
       "1    0.445783       0.524795          0.335358   0.391145  0.102619   \n",
       "2    0.857623       0.825038          0.786389   0.802430  0.142377   \n",
       "3    0.856279       0.804889          0.662221   0.697358  0.143721   \n",
       "4    0.946273       0.791611          0.817076   0.803679  0.053727   \n",
       "..        ...            ...               ...        ...       ...   \n",
       "58   0.879113       0.790831          0.832687   0.808791  0.120887   \n",
       "59   0.913365       0.518202          0.814295   0.513252  0.086635   \n",
       "60   0.942915       0.690194          0.850507   0.741981  0.057085   \n",
       "61   0.983882       0.500000          0.491941   0.495938  0.016118   \n",
       "62   0.942243       0.500000          0.471122   0.485131  0.057757   \n",
       "\n",
       "   accuracy(custom)       EMR recall(custom) precision(custom) f1(custom)  \n",
       "0          0.358935  0.738079       0.688088          0.466406    0.41688  \n",
       "1           0.29396  0.752183       0.620167            0.3841   0.346254  \n",
       "2              None      None           None              None       None  \n",
       "3              None      None           None              None       None  \n",
       "4              None      None           None              None       None  \n",
       "..              ...       ...            ...               ...        ...  \n",
       "58             None      None           None              None       None  \n",
       "59             None      None           None              None       None  \n",
       "60             None      None           None              None       None  \n",
       "61             None      None           None              None       None  \n",
       "62             None      None           None              None       None  \n",
       "\n",
       "[63 rows x 15 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns = ['threshold','type','accuracy', \n",
    "                             'recall(micro)', 'precision(micro)', 'f1(micro)', \n",
    "                             'recall(macro)', 'precision(macro)', 'f1(macro)', \n",
    "                             'hamming',\n",
    "\n",
    "                            'accuracy(custom)', 'EMR',\n",
    "                             'recall(custom)', 'precision(custom)', 'f1(custom)',\n",
    "                            ])\n",
    "targets = y_test\n",
    "\n",
    "Threshold = np.around(np.linspace(0.1, 0.9, 9),1)\n",
    "\n",
    "for threshold in Threshold:\n",
    "    print('threshold', threshold)\n",
    "    pred = outputs >= threshold\n",
    "    \n",
    "    \n",
    "    # ToDo  加了就等于屏蔽relevant\n",
    "    pred[:,]\n",
    "\n",
    "#     accuracy = accuracy_score(targets, outputs)\n",
    "#     f1_score_micro = f1_score(targets, outputs, average='micro')\n",
    "#     f1_score_macro = f1_score(targets, outputs, average='macro')\n",
    "#     print(f\"Accuracy Score = {accuracy}\")\n",
    "#     print(f\"F1 Score (Micro) = {f1_score_micro}\")\n",
    "#     print(f\"F1 Score (Macro) = {f1_score_macro}\")\n",
    "\n",
    "\n",
    "    df.loc[len(df)] = [threshold, \n",
    "                       'all', \n",
    "                       accuracy_score(targets, pred),\n",
    "                       recall_score(targets, pred, average='micro'),\n",
    "                       precision_score(targets, pred, average='micro'),\n",
    "                       f1_score(targets, pred, average='micro'),\n",
    "                       recall_score(targets, pred, average='macro'),\n",
    "                       precision_score(targets, pred, average='macro'),\n",
    "                       f1_score(targets, pred, average='macro'),\n",
    "                       hamming_loss(targets, pred),\n",
    "                       \n",
    "                        accuracy_multilabel(targets, pred),                         \n",
    "                        accuracy_score(np.array(targets), np.array(pred)),\n",
    "                        recall_multilabel(targets, pred),\n",
    "                        precision_multilabel(targets, pred),\n",
    "                        f1_multilabel(targets, pred)  \n",
    "                       ]\n",
    "    \n",
    "    \n",
    "    df.loc[len(df)] = [threshold, \n",
    "                       'multi', \n",
    "                       accuracy_score(targets[:,1:], pred[:,1:]),\n",
    "                       recall_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       precision_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       f1_score(targets[:,1:], pred[:,1:], average='micro'),\n",
    "                       recall_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       precision_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       f1_score(targets[:,1:], pred[:,1:], average='macro'),\n",
    "                       hamming_loss(targets, pred),\n",
    "                       \n",
    "                        accuracy_multilabel(targets[:,1:], pred[:,1:]),                         \n",
    "                        accuracy_score(np.array(targets[:,1:]), np.array(pred[:,1:])),\n",
    "                        recall_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                        precision_multilabel(targets[:,1:], pred[:,1:]),\n",
    "                        f1_multilabel(targets[:,1:], pred[:,1:])  \n",
    "                       ]\n",
    "    \n",
    "    \n",
    "    \n",
    "    df_target = pd.DataFrame(targets)\n",
    "    df_target.columns = ['is_relevant',\t'Armed Assault',\t'Bombing/Explosion',\t'Kidnapping',\t'Other']\n",
    "\n",
    "    df_pred = pd.DataFrame(pred)\n",
    "    df_pred.columns = df_target.columns\n",
    "    \n",
    "    for col in df_pred.columns:\n",
    "\n",
    "        df.loc[len(df)] = [threshold, \n",
    "                           col, \n",
    "                           accuracy_score(df_target[col], df_pred[col]),\n",
    "                           recall_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           precision_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           f1_score(df_target[col], df_pred[col], average='micro'),\n",
    "                           recall_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           precision_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           f1_score(df_target[col], df_pred[col], average='macro'),\n",
    "                           hamming_loss(df_target[col], df_pred[col]),\n",
    "                           \n",
    "                            None, None, None, None, None\n",
    "                           \n",
    "                           ]\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:09:35.146544Z",
     "start_time": "2020-12-31T05:09:35.120930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>hamming</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>EMR</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.1</td>\n",
       "      <td>all</td>\n",
       "      <td>0.689053</td>\n",
       "      <td>0.726331</td>\n",
       "      <td>0.365872</td>\n",
       "      <td>0.486620</td>\n",
       "      <td>0.593398</td>\n",
       "      <td>0.330761</td>\n",
       "      <td>0.403296</td>\n",
       "      <td>0.139154</td>\n",
       "      <td>0.303165</td>\n",
       "      <td>0.689053</td>\n",
       "      <td>0.726228</td>\n",
       "      <td>0.368779</td>\n",
       "      <td>0.370996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.2</td>\n",
       "      <td>all</td>\n",
       "      <td>0.711216</td>\n",
       "      <td>0.665680</td>\n",
       "      <td>0.430622</td>\n",
       "      <td>0.522952</td>\n",
       "      <td>0.508875</td>\n",
       "      <td>0.352145</td>\n",
       "      <td>0.397539</td>\n",
       "      <td>0.110275</td>\n",
       "      <td>0.333926</td>\n",
       "      <td>0.711216</td>\n",
       "      <td>0.667189</td>\n",
       "      <td>0.432724</td>\n",
       "      <td>0.39477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.3</td>\n",
       "      <td>all</td>\n",
       "      <td>0.732707</td>\n",
       "      <td>0.634615</td>\n",
       "      <td>0.479866</td>\n",
       "      <td>0.546497</td>\n",
       "      <td>0.471955</td>\n",
       "      <td>0.374058</td>\n",
       "      <td>0.401989</td>\n",
       "      <td>0.095635</td>\n",
       "      <td>0.35273</td>\n",
       "      <td>0.732707</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.482234</td>\n",
       "      <td>0.410704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.4</td>\n",
       "      <td>all</td>\n",
       "      <td>0.762257</td>\n",
       "      <td>0.582840</td>\n",
       "      <td>0.554149</td>\n",
       "      <td>0.568133</td>\n",
       "      <td>0.398510</td>\n",
       "      <td>0.423946</td>\n",
       "      <td>0.383901</td>\n",
       "      <td>0.080457</td>\n",
       "      <td>0.371162</td>\n",
       "      <td>0.762257</td>\n",
       "      <td>0.585162</td>\n",
       "      <td>0.538194</td>\n",
       "      <td>0.421272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.5</td>\n",
       "      <td>all</td>\n",
       "      <td>0.767629</td>\n",
       "      <td>0.542899</td>\n",
       "      <td>0.577953</td>\n",
       "      <td>0.559878</td>\n",
       "      <td>0.352085</td>\n",
       "      <td>0.349732</td>\n",
       "      <td>0.337503</td>\n",
       "      <td>0.077502</td>\n",
       "      <td>0.365141</td>\n",
       "      <td>0.767629</td>\n",
       "      <td>0.546499</td>\n",
       "      <td>0.562162</td>\n",
       "      <td>0.41503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.6</td>\n",
       "      <td>all</td>\n",
       "      <td>0.760242</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0.311459</td>\n",
       "      <td>0.350364</td>\n",
       "      <td>0.316030</td>\n",
       "      <td>0.076830</td>\n",
       "      <td>0.3454</td>\n",
       "      <td>0.760242</td>\n",
       "      <td>0.50418</td>\n",
       "      <td>0.586592</td>\n",
       "      <td>0.400603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.7</td>\n",
       "      <td>all</td>\n",
       "      <td>0.760242</td>\n",
       "      <td>0.463018</td>\n",
       "      <td>0.636179</td>\n",
       "      <td>0.535959</td>\n",
       "      <td>0.280014</td>\n",
       "      <td>0.381171</td>\n",
       "      <td>0.308980</td>\n",
       "      <td>0.072801</td>\n",
       "      <td>0.337587</td>\n",
       "      <td>0.760242</td>\n",
       "      <td>0.46813</td>\n",
       "      <td>0.635693</td>\n",
       "      <td>0.395901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.8</td>\n",
       "      <td>all</td>\n",
       "      <td>0.740094</td>\n",
       "      <td>0.392012</td>\n",
       "      <td>0.674300</td>\n",
       "      <td>0.495790</td>\n",
       "      <td>0.208167</td>\n",
       "      <td>0.374025</td>\n",
       "      <td>0.242554</td>\n",
       "      <td>0.072398</td>\n",
       "      <td>0.297084</td>\n",
       "      <td>0.740094</td>\n",
       "      <td>0.397074</td>\n",
       "      <td>0.674772</td>\n",
       "      <td>0.371316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>0.9</td>\n",
       "      <td>all</td>\n",
       "      <td>0.742109</td>\n",
       "      <td>0.347633</td>\n",
       "      <td>0.712121</td>\n",
       "      <td>0.467197</td>\n",
       "      <td>0.176252</td>\n",
       "      <td>0.298316</td>\n",
       "      <td>0.205985</td>\n",
       "      <td>0.071995</td>\n",
       "      <td>0.275184</td>\n",
       "      <td>0.742109</td>\n",
       "      <td>0.35162</td>\n",
       "      <td>0.70202</td>\n",
       "      <td>0.351188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold type  accuracy  recall(micro)  precision(micro)  f1(micro)  \\\n",
       "0         0.1  all  0.689053       0.726331          0.365872   0.486620   \n",
       "7         0.2  all  0.711216       0.665680          0.430622   0.522952   \n",
       "14        0.3  all  0.732707       0.634615          0.479866   0.546497   \n",
       "21        0.4  all  0.762257       0.582840          0.554149   0.568133   \n",
       "28        0.5  all  0.767629       0.542899          0.577953   0.559878   \n",
       "35        0.6  all  0.760242       0.500000          0.590909   0.541667   \n",
       "42        0.7  all  0.760242       0.463018          0.636179   0.535959   \n",
       "49        0.8  all  0.740094       0.392012          0.674300   0.495790   \n",
       "56        0.9  all  0.742109       0.347633          0.712121   0.467197   \n",
       "\n",
       "    recall(macro)  precision(macro)  f1(macro)   hamming accuracy(custom)  \\\n",
       "0        0.593398          0.330761   0.403296  0.139154         0.303165   \n",
       "7        0.508875          0.352145   0.397539  0.110275         0.333926   \n",
       "14       0.471955          0.374058   0.401989  0.095635          0.35273   \n",
       "21       0.398510          0.423946   0.383901  0.080457         0.371162   \n",
       "28       0.352085          0.349732   0.337503  0.077502         0.365141   \n",
       "35       0.311459          0.350364   0.316030  0.076830           0.3454   \n",
       "42       0.280014          0.381171   0.308980  0.072801         0.337587   \n",
       "49       0.208167          0.374025   0.242554  0.072398         0.297084   \n",
       "56       0.176252          0.298316   0.205985  0.071995         0.275184   \n",
       "\n",
       "         EMR recall(custom) precision(custom) f1(custom)  \n",
       "0   0.689053       0.726228          0.368779   0.370996  \n",
       "7   0.711216       0.667189          0.432724    0.39477  \n",
       "14  0.732707       0.636364          0.482234   0.410704  \n",
       "21  0.762257       0.585162          0.538194   0.421272  \n",
       "28  0.767629       0.546499          0.562162    0.41503  \n",
       "35  0.760242        0.50418          0.586592   0.400603  \n",
       "42  0.760242        0.46813          0.635693   0.395901  \n",
       "49  0.740094       0.397074          0.674772   0.371316  \n",
       "56  0.742109        0.35162           0.70202   0.351188  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.type == 'all']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:31:06.155162Z",
     "start_time": "2020-12-31T05:31:06.129616Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>hamming</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>EMR</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.1</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.752183</td>\n",
       "      <td>0.621849</td>\n",
       "      <td>0.347418</td>\n",
       "      <td>0.445783</td>\n",
       "      <td>0.524795</td>\n",
       "      <td>0.335358</td>\n",
       "      <td>0.391145</td>\n",
       "      <td>0.102619</td>\n",
       "      <td>0.29396</td>\n",
       "      <td>0.752183</td>\n",
       "      <td>0.620167</td>\n",
       "      <td>0.3841</td>\n",
       "      <td>0.346254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.2</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.791135</td>\n",
       "      <td>0.532213</td>\n",
       "      <td>0.439815</td>\n",
       "      <td>0.481622</td>\n",
       "      <td>0.418814</td>\n",
       "      <td>0.381301</td>\n",
       "      <td>0.385974</td>\n",
       "      <td>0.082337</td>\n",
       "      <td>0.336158</td>\n",
       "      <td>0.791135</td>\n",
       "      <td>0.530825</td>\n",
       "      <td>0.469745</td>\n",
       "      <td>0.367554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.3</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.813298</td>\n",
       "      <td>0.476190</td>\n",
       "      <td>0.512048</td>\n",
       "      <td>0.493469</td>\n",
       "      <td>0.357390</td>\n",
       "      <td>0.393775</td>\n",
       "      <td>0.358250</td>\n",
       "      <td>0.074144</td>\n",
       "      <td>0.35589</td>\n",
       "      <td>0.813298</td>\n",
       "      <td>0.480669</td>\n",
       "      <td>0.527105</td>\n",
       "      <td>0.374269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.4</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.822028</td>\n",
       "      <td>0.445378</td>\n",
       "      <td>0.555944</td>\n",
       "      <td>0.494557</td>\n",
       "      <td>0.330931</td>\n",
       "      <td>0.434373</td>\n",
       "      <td>0.348468</td>\n",
       "      <td>0.070383</td>\n",
       "      <td>0.356529</td>\n",
       "      <td>0.822028</td>\n",
       "      <td>0.452456</td>\n",
       "      <td>0.561869</td>\n",
       "      <td>0.370275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.5</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.826729</td>\n",
       "      <td>0.394958</td>\n",
       "      <td>0.607759</td>\n",
       "      <td>0.478778</td>\n",
       "      <td>0.288909</td>\n",
       "      <td>0.475694</td>\n",
       "      <td>0.318123</td>\n",
       "      <td>0.067965</td>\n",
       "      <td>0.340889</td>\n",
       "      <td>0.826729</td>\n",
       "      <td>0.402299</td>\n",
       "      <td>0.604803</td>\n",
       "      <td>0.350667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.6</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.826058</td>\n",
       "      <td>0.364146</td>\n",
       "      <td>0.640394</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>0.265210</td>\n",
       "      <td>0.518629</td>\n",
       "      <td>0.303385</td>\n",
       "      <td>0.066085</td>\n",
       "      <td>0.322917</td>\n",
       "      <td>0.826058</td>\n",
       "      <td>0.372518</td>\n",
       "      <td>0.638614</td>\n",
       "      <td>0.331975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.7</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.826058</td>\n",
       "      <td>0.322129</td>\n",
       "      <td>0.688623</td>\n",
       "      <td>0.438931</td>\n",
       "      <td>0.233935</td>\n",
       "      <td>0.473790</td>\n",
       "      <td>0.283400</td>\n",
       "      <td>0.064338</td>\n",
       "      <td>0.296714</td>\n",
       "      <td>0.826058</td>\n",
       "      <td>0.330199</td>\n",
       "      <td>0.688623</td>\n",
       "      <td>0.305634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.8</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.820013</td>\n",
       "      <td>0.260504</td>\n",
       "      <td>0.704545</td>\n",
       "      <td>0.380368</td>\n",
       "      <td>0.190135</td>\n",
       "      <td>0.354536</td>\n",
       "      <td>0.247057</td>\n",
       "      <td>0.065547</td>\n",
       "      <td>0.247358</td>\n",
       "      <td>0.820013</td>\n",
       "      <td>0.26907</td>\n",
       "      <td>0.704545</td>\n",
       "      <td>0.254083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>0.9</td>\n",
       "      <td>multi</td>\n",
       "      <td>0.807925</td>\n",
       "      <td>0.140056</td>\n",
       "      <td>0.746269</td>\n",
       "      <td>0.235849</td>\n",
       "      <td>0.107296</td>\n",
       "      <td>0.366071</td>\n",
       "      <td>0.146557</td>\n",
       "      <td>0.067696</td>\n",
       "      <td>0.144076</td>\n",
       "      <td>0.807925</td>\n",
       "      <td>0.149948</td>\n",
       "      <td>0.746269</td>\n",
       "      <td>0.146084</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold   type  accuracy  recall(micro)  precision(micro)  f1(micro)  \\\n",
       "1         0.1  multi  0.752183       0.621849          0.347418   0.445783   \n",
       "8         0.2  multi  0.791135       0.532213          0.439815   0.481622   \n",
       "15        0.3  multi  0.813298       0.476190          0.512048   0.493469   \n",
       "22        0.4  multi  0.822028       0.445378          0.555944   0.494557   \n",
       "29        0.5  multi  0.826729       0.394958          0.607759   0.478778   \n",
       "36        0.6  multi  0.826058       0.364146          0.640394   0.464286   \n",
       "43        0.7  multi  0.826058       0.322129          0.688623   0.438931   \n",
       "50        0.8  multi  0.820013       0.260504          0.704545   0.380368   \n",
       "57        0.9  multi  0.807925       0.140056          0.746269   0.235849   \n",
       "\n",
       "    recall(macro)  precision(macro)  f1(macro)   hamming accuracy(custom)  \\\n",
       "1        0.524795          0.335358   0.391145  0.102619          0.29396   \n",
       "8        0.418814          0.381301   0.385974  0.082337         0.336158   \n",
       "15       0.357390          0.393775   0.358250  0.074144          0.35589   \n",
       "22       0.330931          0.434373   0.348468  0.070383         0.356529   \n",
       "29       0.288909          0.475694   0.318123  0.067965         0.340889   \n",
       "36       0.265210          0.518629   0.303385  0.066085         0.322917   \n",
       "43       0.233935          0.473790   0.283400  0.064338         0.296714   \n",
       "50       0.190135          0.354536   0.247057  0.065547         0.247358   \n",
       "57       0.107296          0.366071   0.146557  0.067696         0.144076   \n",
       "\n",
       "         EMR recall(custom) precision(custom) f1(custom)  \n",
       "1   0.752183       0.620167            0.3841   0.346254  \n",
       "8   0.791135       0.530825          0.469745   0.367554  \n",
       "15  0.813298       0.480669          0.527105   0.374269  \n",
       "22  0.822028       0.452456          0.561869   0.370275  \n",
       "29  0.826729       0.402299          0.604803   0.350667  \n",
       "36  0.826058       0.372518          0.638614   0.331975  \n",
       "43  0.826058       0.330199          0.688623   0.305634  \n",
       "50  0.820013        0.26907          0.704545   0.254083  \n",
       "57  0.807925       0.149948          0.746269   0.146084  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.type == 'multi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-31T05:23:13.959930Z",
     "start_time": "2020-12-31T05:23:13.945726Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>type</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall(micro)</th>\n",
       "      <th>precision(micro)</th>\n",
       "      <th>f1(micro)</th>\n",
       "      <th>recall(macro)</th>\n",
       "      <th>precision(macro)</th>\n",
       "      <th>f1(macro)</th>\n",
       "      <th>hamming</th>\n",
       "      <th>accuracy(custom)</th>\n",
       "      <th>EMR</th>\n",
       "      <th>recall(custom)</th>\n",
       "      <th>precision(custom)</th>\n",
       "      <th>f1(custom)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.1</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.857623</td>\n",
       "      <td>0.825038</td>\n",
       "      <td>0.786389</td>\n",
       "      <td>0.802430</td>\n",
       "      <td>0.142377</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.2</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.862995</td>\n",
       "      <td>0.862995</td>\n",
       "      <td>0.862995</td>\n",
       "      <td>0.862995</td>\n",
       "      <td>0.820477</td>\n",
       "      <td>0.793736</td>\n",
       "      <td>0.805614</td>\n",
       "      <td>0.137005</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.3</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.863667</td>\n",
       "      <td>0.863667</td>\n",
       "      <td>0.863667</td>\n",
       "      <td>0.863667</td>\n",
       "      <td>0.814064</td>\n",
       "      <td>0.795209</td>\n",
       "      <td>0.803906</td>\n",
       "      <td>0.136333</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.4</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.813493</td>\n",
       "      <td>0.799506</td>\n",
       "      <td>0.806107</td>\n",
       "      <td>0.133647</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.5</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.866353</td>\n",
       "      <td>0.807793</td>\n",
       "      <td>0.800357</td>\n",
       "      <td>0.803965</td>\n",
       "      <td>0.133647</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.6</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.871054</td>\n",
       "      <td>0.871054</td>\n",
       "      <td>0.871054</td>\n",
       "      <td>0.871054</td>\n",
       "      <td>0.807364</td>\n",
       "      <td>0.808775</td>\n",
       "      <td>0.808066</td>\n",
       "      <td>0.128946</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.7</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.804655</td>\n",
       "      <td>0.818773</td>\n",
       "      <td>0.811354</td>\n",
       "      <td>0.124244</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>0.8</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.875756</td>\n",
       "      <td>0.798955</td>\n",
       "      <td>0.820897</td>\n",
       "      <td>0.809066</td>\n",
       "      <td>0.124244</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.9</td>\n",
       "      <td>is_relevant</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.879113</td>\n",
       "      <td>0.790831</td>\n",
       "      <td>0.832687</td>\n",
       "      <td>0.808791</td>\n",
       "      <td>0.120887</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold         type  accuracy  recall(micro)  precision(micro)  \\\n",
       "2         0.1  is_relevant  0.857623       0.857623          0.857623   \n",
       "9         0.2  is_relevant  0.862995       0.862995          0.862995   \n",
       "16        0.3  is_relevant  0.863667       0.863667          0.863667   \n",
       "23        0.4  is_relevant  0.866353       0.866353          0.866353   \n",
       "30        0.5  is_relevant  0.866353       0.866353          0.866353   \n",
       "37        0.6  is_relevant  0.871054       0.871054          0.871054   \n",
       "44        0.7  is_relevant  0.875756       0.875756          0.875756   \n",
       "51        0.8  is_relevant  0.875756       0.875756          0.875756   \n",
       "58        0.9  is_relevant  0.879113       0.879113          0.879113   \n",
       "\n",
       "    f1(micro)  recall(macro)  precision(macro)  f1(macro)   hamming  \\\n",
       "2    0.857623       0.825038          0.786389   0.802430  0.142377   \n",
       "9    0.862995       0.820477          0.793736   0.805614  0.137005   \n",
       "16   0.863667       0.814064          0.795209   0.803906  0.136333   \n",
       "23   0.866353       0.813493          0.799506   0.806107  0.133647   \n",
       "30   0.866353       0.807793          0.800357   0.803965  0.133647   \n",
       "37   0.871054       0.807364          0.808775   0.808066  0.128946   \n",
       "44   0.875756       0.804655          0.818773   0.811354  0.124244   \n",
       "51   0.875756       0.798955          0.820897   0.809066  0.124244   \n",
       "58   0.879113       0.790831          0.832687   0.808791  0.120887   \n",
       "\n",
       "   accuracy(custom)   EMR recall(custom) precision(custom) f1(custom)  \n",
       "2              None  None           None              None       None  \n",
       "9              None  None           None              None       None  \n",
       "16             None  None           None              None       None  \n",
       "23             None  None           None              None       None  \n",
       "30             None  None           None              None       None  \n",
       "37             None  None           None              None       None  \n",
       "44             None  None           None              None       None  \n",
       "51             None  None           None              None       None  \n",
       "58             None  None           None              None       None  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.type == 'is_relevant']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df.type == 'all']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.677353Z",
     "start_time": "2020-12-30T03:27:28.181Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from torch import Tensor\n",
    "from sklearn.metrics import roc_curve, auc, hamming_loss, accuracy_score\n",
    "import pdb\n",
    "\n",
    "CLASSIFICATION_THRESHOLD: float = 0.5  # Best keep it in [0.0, 1.0] range\n",
    "\n",
    "# def accuracy(out, labels):\n",
    "#     outputs = np.argmax(out, axis=1)\n",
    "#     return np.sum(outputs == labels)\n",
    "\n",
    "\n",
    "\n",
    "def fbeta(y_pred, y_true,\n",
    "#     y_pred: Tensor,\n",
    "#     y_true: Tensor,\n",
    "    beta: float = 1,\n",
    "    eps: float = 1e-9,\n",
    "):\n",
    "    \"Computes the f_beta between `preds` and `targets`\"\n",
    "    beta2 = beta ** 2\n",
    "#     if sigmoid:\n",
    "#         y_pred = y_pred.sigmoid()\n",
    "#     y_pred = (y_pred > thresh)\n",
    "#     .float()\n",
    "#     y_true = y_true\n",
    "#     .float()\n",
    "#     TP = (y_pred * y_true).sum(dim=1)\n",
    "#     prec = TP / (y_pred.sum(dim=1) + eps)\n",
    "#     rec = TP / (y_true.sum(dim=1) + eps)\n",
    "    \n",
    "    TP = (y_pred * y_true).sum(axis=1)\n",
    "    prec = TP / (y_pred.sum(axis=1) + eps)\n",
    "    rec = TP / (y_true.sum(axis=1) + eps)\n",
    "    \n",
    "    res = (prec * rec) / (prec * beta2 + rec + eps) * (1 + beta2)\n",
    "    return res.mean().item()\n",
    "\n",
    "\n",
    "def roc_auc(y_pred: Tensor, y_true: Tensor):\n",
    "    # ROC-AUC calcualation\n",
    "    # Compute ROC curve and ROC area for each class\n",
    "    fpr = dict()\n",
    "    tpr = dict()\n",
    "    roc_auc = dict()\n",
    "\n",
    "    y_true = y_true.detach().cpu().numpy()\n",
    "    y_pred = y_pred.detach().cpu().numpy()\n",
    "\n",
    "    # Compute micro-average ROC curve and ROC area\n",
    "    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_true.ravel(), y_pred.ravel())\n",
    "    roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n",
    "\n",
    "    return roc_auc[\"micro\"]\n",
    "\n",
    "\n",
    "def Hamming_loss(y_pred, y_true,\n",
    "#     y_pred: Tensor,\n",
    "#     y_true: Tensor,\n",
    "#     sigmoid: bool = True,\n",
    "#     thresh: float = CLASSIFICATION_THRESHOLD,\n",
    "    sample_weight=None,\n",
    "):\n",
    "#     if sigmoid:\n",
    "#         y_pred = y_pred.sigmoid()\n",
    "#     y_pred = (y_pred > thresh)\n",
    "#     .float()\n",
    "    return hamming_loss(y_true, y_pred, sample_weight=sample_weight)\n",
    "\n",
    "\n",
    "def Exact_Match_Ratio(y_pred, y_true,\n",
    "#     y_pred: Tensor,\n",
    "#     y_true: Tensor,\n",
    "#     sigmoid: bool = True,\n",
    "#     thresh: float = CLASSIFICATION_THRESHOLD,\n",
    "    normalize: bool = True,\n",
    "    sample_weight=None,\n",
    "):\n",
    "#     if sigmoid:\n",
    "#         y_pred = y_pred.sigmoid()\n",
    "#     y_pred = (y_pred > thresh)\n",
    "#     .float()\n",
    "    return accuracy_score(\n",
    "        y_true, y_pred, normalize=normalize, sample_weight=sample_weight\n",
    "    )\n",
    "\n",
    "\n",
    "def F1(\n",
    "#     y_pred: Tensor, y_true: Tensor, \n",
    "       y_pred, y_true,\n",
    "       threshold: float = CLASSIFICATION_THRESHOLD):\n",
    "    return fbeta(y_pred, y_true, thresh=threshold, beta=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.677808Z",
     "start_time": "2020-12-30T03:27:28.183Z"
    }
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.678217Z",
     "start_time": "2020-12-30T03:27:28.186Z"
    }
   },
   "outputs": [],
   "source": [
    "# EXACT MATCH RATIO\n",
    "print(\"EMR: \", accuracy_score(np.array(targets), np.array(pred)))\n",
    "#ACCURACY\n",
    "print(\"Accuracy: \", accuracy_multilabel(targets, pred))\n",
    "#PRECISION\n",
    "print(\"Precision: \", precision_multilabel(targets, pred))\n",
    "#RECALL\n",
    "print(\"Recall: \", recall_multilabel(targets, pred))\n",
    "#F1-SCORE\n",
    "print(\"F1-score: \", f1_multilabel(targets, pred))\n",
    "[recall_score(targets, pred, average='weighted'),\n",
    " precision_score(targets, pred, average='weighted'),\n",
    " f1_score(targets, pred, average='micro'),\n",
    "f1_score(targets, pred, average='macro'),\n",
    "f1_score(targets, pred, average = 'weighted')]\n",
    "\n",
    "# print(\"accuracy_score:\", accuracy_score(targets, pred))\n",
    "# print(\"Hamming_loss:\", hamming_loss(targets, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.678640Z",
     "start_time": "2020-12-30T03:27:28.188Z"
    }
   },
   "outputs": [],
   "source": [
    "recall_score(targets, pred, average='micro'),\n",
    "# precision_score(targets, pred, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.679065Z",
     "start_time": "2020-12-30T03:27:28.191Z"
    }
   },
   "outputs": [],
   "source": [
    "targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.679520Z",
     "start_time": "2020-12-30T03:27:28.193Z"
    }
   },
   "outputs": [],
   "source": [
    "targets1.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.679952Z",
     "start_time": "2020-12-30T03:27:28.197Z"
    }
   },
   "outputs": [],
   "source": [
    "recall_multilabel(targets1, pred1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.680382Z",
     "start_time": "2020-12-30T03:27:28.199Z"
    }
   },
   "outputs": [],
   "source": [
    "recall_multilabel(targets.astype(int), pred.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.680827Z",
     "start_time": "2020-12-30T03:27:28.201Z"
    }
   },
   "outputs": [],
   "source": [
    "recall_score(targets1, pred1, average='weighted'),\n",
    "# precision_score(targets1, pred1, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.681329Z",
     "start_time": "2020-12-30T03:27:28.206Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = pred; y_true = targets;eps = 1e-9; beta=1\n",
    "\n",
    "TP = (y_pred * y_true).sum(axis=1)\n",
    "prec = TP / (y_pred.sum(axis=1) + eps)\n",
    "rec = TP / (y_true.sum(axis=1) + eps)\n",
    "res = (prec * rec) / (prec * beta + rec + eps) * (1 + beta)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.681746Z",
     "start_time": "2020-12-30T03:27:28.212Z"
    }
   },
   "outputs": [],
   "source": [
    "TP.mean(), prec.mean(), rec.mean(), res.mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-30T03:27:31.682190Z",
     "start_time": "2020-12-30T03:27:28.215Z"
    }
   },
   "outputs": [],
   "source": [
    "pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}