labeling_data/training_to_label_with_BERT_and_Cleanlab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training to label with BERT and Cleanlab\n",
    "In this notebook, we'll use the labels that we have generated and checked with [Cleanlab](https://github.com/cgnorthcutt/cleanlab) to build a classifier using BERT embeddings to label the remaining data. We'll check our results with Cleanlab and adjust as necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0412 21:01:54.837924 4709707200 file_utils.py:38] PyTorch version 1.4.0 available.\n",
      "I0412 21:01:54.839154 4709707200 file_utils.py:54] TensorFlow version 2.1.0 available.\n",
      "I0412 21:01:55.094659 4709707200 textcleaner.py:37] 'pattern' package not found; tag filters are not available for English\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from random import sample\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np \n",
    "import cleanlab\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from tensorflow.keras import models, layers\n",
    "import pandas as pd\n",
    "import torch\n",
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import inspect\n",
    "from pathlib import Path \n",
    "currentdir = Path.cwd()\n",
    "parentdir = os.path.dirname(currentdir)\n",
    "sys.path.insert(0,parentdir) \n",
    "\n",
    "from mlyoucanuse.embeddings import get_embeddings_index\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>82955</td>\n",
       "      <td>606008</td>\n",
       "      <td>politician</td>\n",
       "      <td>person involved in politics, person who holds ...</td>\n",
       "      <td>1</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>189290</td>\n",
       "      <td>33481</td>\n",
       "      <td>military officer</td>\n",
       "      <td>member of an armed force or uniformed service ...</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>131512</td>\n",
       "      <td>8887</td>\n",
       "      <td>farmer</td>\n",
       "      <td>person that works in agriculture</td>\n",
       "      <td>1</td>\n",
       "      <td>da</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1734662</td>\n",
       "      <td>2012</td>\n",
       "      <td>cartographer</td>\n",
       "      <td>person preparing geographical maps</td>\n",
       "      <td>1</td>\n",
       "      <td>ia</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>294126</td>\n",
       "      <td>323</td>\n",
       "      <td>land surveyor</td>\n",
       "      <td>profession</td>\n",
       "      <td>0</td>\n",
       "      <td>tr</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id  occupation_count        occupation  \\\n",
       "0    82955            606008        politician   \n",
       "1   189290             33481  military officer   \n",
       "2   131512              8887            farmer   \n",
       "3  1734662              2012      cartographer   \n",
       "4   294126               323     land surveyor   \n",
       "\n",
       "                                         description  in_google_news  \\\n",
       "0  person involved in politics, person who holds ...               1   \n",
       "1  member of an armed force or uniformed service ...               0   \n",
       "2                   person that works in agriculture               1   \n",
       "3                 person preparing geographical maps               1   \n",
       "4                                         profession               0   \n",
       "\n",
       "  language_detected    source  label labeled_by  label_error_reason  \n",
       "0                en  wikidata      1      human                 NaN  \n",
       "1                en  wikidata     -1        NaN                 NaN  \n",
       "2                da  wikidata      1      human                 NaN  \n",
       "3                ia  wikidata      1      human                 NaN  \n",
       "4                tr  wikidata     -1        NaN                 NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df = pd.read_csv('occupations.wikidata.all.gnews.labeled.final.csv', sep='\\t')\n",
    "occs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive examples: 2,432; Negative examples 1,651\n"
     ]
    }
   ],
   "source": [
    "pos_eg = occs_df.query('label==1')['occupation'].tolist()\n",
    "neg_eg = occs_df.query('label==0')['occupation'].tolist()\n",
    "print(f\"Positive examples: {len(pos_eg):,}; Negative examples {len(neg_eg):,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0412 21:02:01.401956 4709707200 tokenization_utils.py:418] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt from cache at /Users/todd/.cache/torch/transformers/cee054f6aafe5e2cf816d2228704e326446785f940f5451a5b26033516a4ac3d.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-large-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0412 21:02:02.178060 4709707200 configuration_utils.py:254] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-config.json from cache at /Users/todd/.cache/torch/transformers/90deb4d9dd705272dc4b3db1364d759d551d72a9f70a91f60e3a1f5e278b985d.960df871e8c3e87cb781ccdbdd12a3c1e0ad83b173a9754bd183354449046e38\n",
      "I0412 21:02:02.179345 4709707200 configuration_utils.py:290] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": 0,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 1024,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 4096,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0412 21:02:02.651582 4709707200 modeling_utils.py:458] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-pytorch_model.bin from cache at /Users/todd/.cache/torch/transformers/56c451878be53ca1e310764d1e8312301f3d921378919467947ddd53fef6ba2b.b5f1c2070e0a0c189ca3b08270b0cb5bd0635b7319e74e93bd0dc26689953c27\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertModel.from_pretrained('bert-large-cased')\n",
    "model.eval()\n",
    "model.to(device)\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Here's an example of how we'll use BERT to provide embeddings for the text of an occupation label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 12, 1024])\n",
      "torch.Size([1024])\n"
     ]
    }
   ],
   "source": [
    "input_ids = torch.tensor(tokenizer.encode(\"The quick brown fox jumped over the lazy dog.\")).unsqueeze(0).to(device)  # Batch size 1\n",
    "outputs = model(input_ids)\n",
    "last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "last_hidden_states.detach()\n",
    "print(last_hidden_states.shape)\n",
    "text_embedding = last_hidden_states[0][0]\n",
    "print(text_embedding.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate X; BERT embeddings for each Occupation we labeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2432/2432 [05:53<00:00,  6.88it/s]\n",
      "100%|██████████| 1651/1651 [03:55<00:00,  7.01it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4083, 1024)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X =[]\n",
    "with torch.no_grad():\n",
    "    for name in tqdm(pos_eg, total=len(pos_eg)):\n",
    "        input_ids = torch.tensor(tokenizer.encode(name)).unsqueeze(0).to(device)  # Batch size 1\n",
    "        outputs = model(input_ids)\n",
    "        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "        X.append(last_hidden_states[0][0].detach().cpu().numpy())\n",
    "    for name in tqdm(neg_eg, total=len(neg_eg)):\n",
    "        input_ids = torch.tensor(tokenizer.encode(name)).unsqueeze(0).to(device)  # Batch size 1\n",
    "        outputs = model(input_ids)\n",
    "        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "        X.append(last_hidden_states[0][0].detach().cpu().numpy())\n",
    "X = np.array(X)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6891/6891 [17:26<00:00,  6.59it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6891, 1024)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unlabeled = occs_df.query(\"label==-1\")['occupation'].tolist()\n",
    "unlabeled_X = []\n",
    "with torch.no_grad():\n",
    "    for name in tqdm(unlabeled, total=len(unlabeled)):\n",
    "        input_ids = torch.tensor(tokenizer.encode(name)).unsqueeze(0).to(device)  # Batch size 1\n",
    "        outputs = model(input_ids)\n",
    "        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "        unlabeled_X.append(last_hidden_states[0][0].detach().cpu().numpy())\n",
    "unlabeled_X = np.array(unlabeled_X)\n",
    "unlabeled_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split our data into train, validation and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4083, 1024), (4083,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.concatenate([np.ones(len(pos_eg), dtype=np.int64), np.zeros(len(neg_eg), dtype=np.int64)])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=.8, random_state=12)\n",
    "X_test, X_validation, y_test, y_validation = train_test_split(X_test, y_test, train_size=.5, random_state=12)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a simple model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential([\n",
    "    layers.Dense(512, input_shape=(1024,), activation='relu'),\n",
    "    layers.Dropout(.2),\n",
    "    layers.Dense(1, activation='sigmoid')\n",
    "])\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(X_train,\n",
    "                    y_train,\n",
    "                    epochs=20,\n",
    "                    batch_size=128,\n",
    "                    validation_data=(X_validation, y_validation),\n",
    "                    verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate and adjust as necessary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3266/3266 [==============================] - 0s 35us/sample - loss: 0.0725 - accuracy: 0.9715\n",
      "Train results: loss 0.073 acc: 0.972\n",
      "409/409 [==============================] - 0s 39us/sample - loss: 0.3905 - accuracy: 0.8704\n",
      "Validation results: loss 0.391 acc: 0.870\n",
      "408/408 [==============================] - 0s 47us/sample - loss: 0.3022 - accuracy: 0.8971\n",
      "Test results: loss 0.302 acc: 0.897\n"
     ]
    }
   ],
   "source": [
    "res = model.evaluate(X_train, y_train)\n",
    "print(f\"Train results: loss {res[0]:.3f} acc: {res[1]:.3f}\")\n",
    "res =  model.evaluate(X_validation, y_validation)\n",
    "print(f\"Validation results: loss {res[0]:.3f} acc: {res[1]:.3f}\")\n",
    "res = model.evaluate(X_test, y_test)\n",
    "print(f\"Test results: loss {res[0]:.3f} acc: {res[1]:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU1dn38e/NLoKILCqLgBE0CsgygEo0GA2CGlBcAiFRHuICcdeoJEYlJuRNosljTNSIW0zEoCGRB6MGNxDXyICIoBARQYcAIsomO9zvH6cGeobp2bqru2f697muvrq7+lTV3TU9ddc5deqUuTsiIpK/6mQ7ABERyS4lAhGRPKdEICKS55QIRETynBKBiEieUyIQEclzSgSSVmb2rJldmO6y2WRmy8zs1BiW62Z2RPT6j2Z2c2XKVmM9I83suerGWc5yB5hZUbqXK5lXL9sBSPaZ2aaEt42BbcCu6P2l7j6pssty98FxlK3t3H1MOpZjZh2Bj4D67r4zWvYkoNJ/Q8k/SgSCuzcpfm1my4CL3P2F0uXMrF7xzkVEag81DUlSxVV/M7vRzFYBD5tZczP7p5mtMbMvotftEuaZaWYXRa9HmdmrZnZHVPYjMxtczbKdzGyWmW00sxfM7G4zezRJ3JWJ8Wdm9lq0vOfMrGXC598zs+VmttbMbipn+/Qzs1VmVjdh2tlmNj963dfM3jCzdWa20sz+YGYNkizrT2b284T310fz/NfMRpcqe4aZvW1mG8zsEzMbn/DxrOh5nZltMrPji7dtwvwnmNlsM1sfPZ9Q2W1THjP7ajT/OjNbaGZDEj473czei5a5wsx+GE1vGf191pnZ52b2iplpv5Rh2uBSkUOAg4AOwCWE38zD0fvDgC3AH8qZvx+wGGgJ/Bp40MysGmUfA94CWgDjge+Vs87KxPgd4H+A1kADoHjHdDRwb7T8NtH62lEGd/838CXwjVLLfSx6vQu4Jvo+xwOnAD8oJ26iGAZF8XwT6AyUPj/xJXABcCBwBjDWzM6KPjspej7Q3Zu4+xulln0Q8DRwV/Tdfgs8bWYtSn2HfbZNBTHXB54CnovmuwKYZGZHRkUeJDQzNgW6Ai9F068DioBWwMHAjwGNe5NhSgRSkd3Are6+zd23uPtad/+7u292943ABODr5cy/3N3vd/ddwCPAoYR/+EqXNbPDgD7ALe6+3d1fBaYlW2ElY3zY3f/j7luAJ4Ae0fRzgX+6+yx33wbcHG2DZP4KjAAws6bA6dE03H2Ou7/p7jvdfRlwXxlxlOX8KL4F7v4lIfElfr+Z7v6uu+929/nR+iqzXAiJ4wN3/0sU11+BRcC3Esok2zblOQ5oAvwy+hu9BPyTaNsAO4CjzewAd//C3ecmTD8U6ODuO9z9FdcAaBmnRCAVWePuW4vfmFljM7svajrZQGiKODCxeaSUVcUv3H1z9LJJFcu2AT5PmAbwSbKAKxnjqoTXmxNiapO47GhHvDbZughH/8PMrCEwDJjr7sujOLpEzR6rojh+QagdVKREDMDyUt+vn5nNiJq+1gNjKrnc4mUvLzVtOdA24X2ybVNhzO6emDQTl3sOIUkuN7OXzez4aPrtwBLgOTNbambjKvc1JJ2UCKQipY/OrgOOBPq5+wHsbYpI1tyTDiuBg8ysccK09uWUTyXGlYnLjtbZIllhd3+PsMMbTMlmIQhNTIuAzlEcP65ODITmrUSPEWpE7d29GfDHhOVWdDT9X0KTWaLDgBWViKui5bYv1b6/Z7nuPtvdhxKajaYSahq4+0Z3v87dDweGANea2SkpxiJVpEQgVdWU0Oa+LmpvvjXuFUZH2IXAeDNrEB1NfqucWVKJcQpwppl9LTqxexsV/588BlxFSDh/KxXHBmCTmR0FjK1kDE8Ao8zs6CgRlY6/KaGGtNXM+hISULE1hKasw5Ms+xmgi5l9x8zqmdm3gaMJzTip+Deh9nCDmdU3swGEv9Hk6G820syaufsOwjbZDWBmZ5rZEdG5oPWE8yrlNcVJDJQIpKruBPYDPgPeBP6VofWOJJxwXQv8HHiccL1DWaodo7svBC4j7NxXAl8QTmaWp7iN/iV3/yxh+g8JO+mNwP1RzJWJ4dnoO7xEaDZ5qVSRHwC3mdlG4Baio+to3s2EcyKvRT1xjiu17LXAmYRa01rgBuDMUnFXmbtvJ+z4BxO2+z3ABe6+KCryPWBZ1EQ2hvD3hHAy/AVgE/AGcI+7z0glFqk603kZqYnM7HFgkbvHXiMRqe1UI5Aawcz6mNlXzKxO1L1yKKGtWURSpCuLpaY4BPgH4cRtETDW3d/ObkgitYOahkRE8pyahkRE8lyNaxpq2bKld+zYMdthiIjUKHPmzPnM3VuV9VmNSwQdO3aksLAw22GIiNQoZlb6ivI91DQkIpLnlAhERPKcEoGISJ6rcecIRCTzduzYQVFREVu3bq24sGRVo0aNaNeuHfXr16/0PEoEIlKhoqIimjZtSseOHUl+XyHJNndn7dq1FBUV0alTp0rPF2vTkJkNMrPFZrakrHHGzex/zWxe9PiPma2LI45Jk6BjR6hTJzxP0m28Rapk69attGjRQkkgx5kZLVq0qHLNLbYaQXQTkLsJt9srAmab2bRo/HYA3P2ahPJXAD3THcekSXDJJbA5uqXJ8uXhPcDIkcnnE5GSlARqhur8neKsEfQFlrj70miI2smEgcKSGUF0i790uummvUmg2ObNYbqIiMSbCNpS8nZ7RZS8Hd4eZtYB6MS+464Xf36JmRWaWeGaNWuqFMTHH1dtuojknrVr19KjRw969OjBIYccQtu2bfe83759e7nzFhYWcuWVV1a4jhNOOCEtsc6cOZMzzzwzLcvKlFzpPjocmBLdtHwf7j7R3QvcvaBVqzKvkE7qsNI3+atguoikLt3n5Vq0aMG8efOYN28eY8aM4ZprrtnzvkGDBuzcuTPpvAUFBdx1110VruP1119PLcgaLM5EsIKS911tR/L7og4nhmYhgAkToHHjktMaNw7TRST9is/LLV8O7nvPy6W7k8aoUaMYM2YM/fr144YbbuCtt97i+OOPp2fPnpxwwgksXrwYKHmEPn78eEaPHs2AAQM4/PDDSySIJk2a7Ck/YMAAzj33XI466ihGjhxJ8SjNzzzzDEcddRS9e/fmyiuvrPDI//PPP+ess86ie/fuHHfcccyfPx+Al19+eU+NpmfPnmzcuJGVK1dy0kkn0aNHD7p27corr7yS3g1Wjji7j84GOptZJ0ICGE7Je6sCEN3LtTnhNnVpV3xC+KabQnPQYYeFJKATxSLxKO+8XLr/74qKinj99depW7cuGzZs4JVXXqFevXq88MIL/PjHP+bvf//7PvMsWrSIGTNmsHHjRo488kjGjh27T5/7t99+m4ULF9KmTRv69+/Pa6+9RkFBAZdeeimzZs2iU6dOjBgxosL4br31Vnr27MnUqVN56aWXuOCCC5g3bx533HEHd999N/3792fTpk00atSIiRMnctppp3HTTTexa9cuNpfeiDGKLRG4+04zuxyYDtQFHnL3hWZ2G1Do7tOiosOByR7jjRFGjtSOXyRTMnle7rzzzqNu3boArF+/ngsvvJAPPvgAM2PHjh1lznPGGWfQsGFDGjZsSOvWrVm9ejXt2rUrUaZv3757pvXo0YNly5bRpEkTDj/88D3980eMGMHEiRPLje/VV1/dk4y+8Y1vsHbtWjZs2ED//v259tprGTlyJMOGDaNdu3b06dOH0aNHs2PHDs466yx69OiR0rapiljPEbj7M+7exd2/4u4Tomm3JCQB3H28u+9zjYGI1EyZPC+3//7773l98803c/LJJ7NgwQKeeuqppH3pGzZsuOd13bp1yzy/UJkyqRg3bhwPPPAAW7ZsoX///ixatIiTTjqJWbNm0bZtW0aNGsWf//zntK6zPLlyslhEaolsnZdbv349bduGjol/+tOf0r78I488kqVLl7Js2TIAHn/88QrnOfHEE5kUnRyZOXMmLVu25IADDuDDDz+kW7du3HjjjfTp04dFixaxfPlyDj74YC6++GIuuugi5s6dm/bvkIwSgYik1ciRMHEidOgAZuF54sT4m2dvuOEGfvSjH9GzZ8+0H8ED7Lffftxzzz0MGjSI3r1707RpU5o1a1buPOPHj2fOnDl0796dcePG8cgjjwBw55130rVrV7p37079+vUZPHgwM2fO5Nhjj6Vnz548/vjjXHXVVWn/DsnUuHsWFxQUuG5MI5JZ77//Pl/96lezHUbWbdq0iSZNmuDuXHbZZXTu3Jlrrrmm4hkzrKy/l5nNcfeCssqrRiAiUkn3338/PXr04JhjjmH9+vVceuml2Q4pLTT6qIhIJV1zzTU5WQNIlWoEIiJ5TolARCTPKRGIiOQ5JQIRkTynRCAiOe/kk09m+vTpJabdeeedjB07Nuk8AwYMoLir+emnn866dfveAHH8+PHccccd5a576tSpvPfenvtpccstt/DCCy9UJfwy5dJw1UoEIpLzRowYweTJk0tMmzx5cqUGfoMwauiBBx5YrXWXTgS33XYbp556arWWlauUCEQk55177rk8/fTTe25Cs2zZMv773/9y4oknMnbsWAoKCjjmmGO49dZby5y/Y8eOfPbZZwBMmDCBLl268LWvfW3PUNUQrhHo06cPxx57LOeccw6bN2/m9ddfZ9q0aVx//fX06NGDDz/8kFGjRjFlyhQAXnzxRXr27Em3bt0YPXo027Zt27O+W2+9lV69etGtWzcWLVpU7vfL9nDVuo5ARKrk6qth3rz0LrNHD7jzzuSfH3TQQfTt25dnn32WoUOHMnnyZM4//3zMjAkTJnDQQQexa9cuTjnlFObPn0/37t3LXM6cOXOYPHky8+bNY+fOnfTq1YvevXsDMGzYMC6++GIAfvKTn/Dggw9yxRVXMGTIEM4880zOPffcEsvaunUro0aN4sUXX6RLly5ccMEF3HvvvVx99dUAtGzZkrlz53LPPfdwxx138MADDyT9ftkerlo1AhGpERKbhxKbhZ544gl69epFz549WbhwYYlmnNJeeeUVzj77bBo3bswBBxzAkCFD9ny2YMECTjzxRLp168akSZNYuHBhufEsXryYTp060aVLFwAuvPBCZs2atefzYcOGAdC7d+89A9Ul8+qrr/K9730PKHu46rvuuot169ZRr149+vTpw8MPP8z48eN59913adq0abnLrgzVCESkSso7co/T0KFDueaaa5g7dy6bN2+md+/efPTRR9xxxx3Mnj2b5s2bM2rUqKTDT1dk1KhRTJ06lWOPPZY//elPzJw5M6V4i4eyTmUY63HjxnHGGWfwzDPP0L9/f6ZPn75nuOqnn36aUaNGce2113LBBRekFKtqBCJSIzRp0oSTTz6Z0aNH76kNbNiwgf33359mzZqxevVqnn322XKXcdJJJzF16lS2bNnCxo0beeqpp/Z8tnHjRg499FB27NixZ+hogKZNm7Jx48Z9lnXkkUeybNkylixZAsBf/vIXvv71r1fru2V7uGrVCESkxhgxYgRnn332niai4mGbjzrqKNq3b0///v3Lnb9Xr158+9vf5thjj6V169b06dNnz2c/+9nP6NevH61ataJfv357dv7Dhw/n4osv5q677tpzkhigUaNGPPzww5x33nns3LmTPn36MGbMmGp9r+J7KXfv3p3GjRuXGK56xowZ1KlTh2OOOYbBgwczefJkbr/9durXr0+TJk3ScgMbDUMtIhXSMNQ1i4ahFhGRKlEiEBHJc0oEIlIpNa0ZOV9V5++kRCAiFWrUqBFr165VMshx7s7atWtp1KhRleZTryERqVC7du0oKipizZo12Q5FKtCoUSPatWtXpXliTQRmNgj4HVAXeMDdf1lGmfOB8YAD77j7d+KMSUSqrn79+nTq1CnbYUhMYksEZlYXuBv4JlAEzDazae7+XkKZzsCPgP7u/oWZtY4rHhERKVuc5wj6Akvcfam7bwcmA0NLlbkYuNvdvwBw909jjEdERMoQZyJoC3yS8L4ompaoC9DFzF4zszejpqR9mNklZlZoZoVqoxQRSa9s9xqqB3QGBgAjgPvNbJ+7R7j7RHcvcPeCVq1aZThEEZHaLc5EsAJon/C+XTQtUREwzd13uPtHwH8IiUFERDIkzkQwG+hsZp3MrAEwHJhWqsxUQm0AM2tJaCpaGmNMIiJSSmyJwN13ApcD04H3gSfcfaGZ3WZmxXeDmA6sNbP3gBnA9e6+Nq6YRERkXxp9VEQkD2j0URERSUqJQEQkzykRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ5TIhARyXNKBCIieU6JQEQkzykRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ5TIhARyXNKBCIieU6JQEQkzykRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ6LNRGY2SAzW2xmS8xsXBmfjzKzNWY2L3pcFGc8IiKyr3pxLdjM6gJ3A98EioDZZjbN3d8rVfRxd788rjhERKR8cdYI+gJL3H2pu28HJgNDY1yfiIhUQ5yJoC3wScL7omhaaeeY2Xwzm2Jm7ctakJldYmaFZla4Zs2aOGIVEclb2T5Z/BTQ0d27A88Dj5RVyN0nunuBuxe0atUqowGKiNR2cSaCFUDiEX67aNoe7r7W3bdFbx8AescYj4iIlCHORDAb6GxmncysATAcmJZYwMwOTXg7BHg/xnhERKQMsfUacvedZnY5MB2oCzzk7gvN7Dag0N2nAVea2RBgJ/A5MCqueEREpGzm7tmOoUoKCgq8sLAw22GIiNQoZjbH3QvK+izbJ4tFRCTLlAhERPKcEoGISJ5TIpBKWbECxo2D7duzHYmIpJsSgVTKXXfBr34FM2ZkOxIRSTclAqmQO0yZEl4/91x2YxGR9FMikArNmwdLl0L9+koEIrWREoFUaMoUqFsXrr4aFiyAlSuzHZGIpJMSgZTLHf72NxgwAL7znTDt+eezGpKIpJkSgZRrwQL44AM491zo3h1at1bzkEhto0Qg5ZoyBczg7LOhTh345jdDjWD37mxHJiLpokQg5ZoyBU46CQ4+OLwfOBA+/RTmz89uXCKSPkoEktR774XHuefunfbNb4ZnNQ+J1B5KBJLU3/8enocN2zvt0EOhWzclApHaRIlAkpoyBfr3hzZtSk4fOBBeeQU2b85OXCKSXkoEUqb//CecB0hsFio2cGAYc2jWrMzHJSLpp0QgZSqrWajYiSdCw4ZqHhKpLZQIpExTpkC/fnDYYft+tt9+oSeREoFI7aBEIPtYuhTmzi27WajYwIGwcGEYnlqkJli8WNe/JKNEIPsobhY655zkZQYODM8abkJqgr/+FY46Cn7602xHkpuUCGQfU6ZA797QqVPyMt26hYvM1DwkuW75chg7FurVg1/+EhYtynZEuadSicDM9jezOtHrLmY2xMzqxxuaZMPy5fDWW+U3C0EYdmLgQA03Iblt1y644ILwG33lFWjcGMaMCYMpyl6VrRHMAhqZWVvgOeB7wJ/iCkqy5x//CM/lNQsVGzgQPvss3K9AJBfdcUfo5vz738Nxx4W77L38Mvz5z9mOLLdUNhGYu28GhgH3uPt5wDEVzmQ2yMwWm9kSMxtXTrlzzMzNrKCS8UhMpkyBY4+Fzp0rLnvqqeFZzUOSi+bOhZtvhvPOC7UCgIsughNOgOuuCwcxElQ6EZjZ8cBI4OloWt0KZqgL3A0MBo4GRpjZ0WWUawpcBfy7skFLPFasgNdfr7hZqNghh4ShqZUIJNds3hzun9G6Nfzxj6EpE8IIuvfdB+vXww03ZDfGXFLZRHA18CPgSXdfaGaHAxXdxrwvsMTdl7r7dmAyMLSMcj8DfgVsrWQsEpMnnwzPlU0EEJqHXn0VvvwynphEquP660N30UcegYMOKvlZ166hRvDww7o6vlilEoG7v+zuQ9z9V9FJ48/c/coKZmsLfJLwviiatoeZ9QLau/vTlMPMLjGzQjMrXLNmTWVClmqYMgWOOSZ0s6usgQNhx47Q7iq116uvhp5i//pXtiOp2NNPwz33hJ39KaeUXeaWW6BjR7j0Uti2LaPh5aTK9hp6zMwOMLP9gQXAe2Z2fSorjhLKb4HrKirr7hPdvcDdC1q1apXKaiWJ1avD0VFVagMAX/saNGqk5qHarLAQTj893K3unHNCr7Jc9emnMHp0aLKcMCF5ucaNQ7JYtAhuvz1z8eWqyjYNHe3uG4CzgGeBToSeQ+VZAbRPeN8umlasKdAVmGlmy4DjgGk6YZwdTz4ZutRVNREUDzehC8tqpwUL4LTToEWLkBAOOQTOOCMMSphr3OH73w/t/489FsbDKs/gweFE8s9/DkuWZCbG6tqyBX79a/jkk4rLVkdlE0H96LqBs4Bp7r4DqKgn7mygs5l1MrMGwHBgWvGH7r7e3Vu6e0d37wi8CQxx98Iqf4tK2LgxVBmlbFOmwJFHhqahqho4MNzApqgo/XFJ9nzwQbgRUaNG8OKL4SLD6dPDidfTToOVK7MdYUn33Qf//GfYYVb2d3znnSFh/OAHuXltwc6dcP/9cMQRcOONe7t3p1tlE8F9wDJgf2CWmXUANpQ3g7vvBC4HpgPvA09EJ5pvM7Mh1Q+5en75SzjzTPjFL3LzD55Na9bAzJmhNlDcu6IqNNxE7fPxx6F78M6d8MILcPjhYfoRR8Azz4TfzODB4eg7FyxaBNdeG36Ll19e+fnatAn7hOefD8NQ5Ar3sNPv2hUuuSQM/vjyy3DVVbGt0Kv1AOpVd95UHr179/bq2LrV/TvfcQf30aPdt2+v1mJqpfvvD9vl7berN//u3e6HHOI+fHh645LsWLnS/Ygj3Js1c587t+wy06e716vnPmBA+N/Kpm3b3Hv1cm/Rwv2//636/Dt3uvfp4966tfvnn6c/vqqaMcO9X7/wP/nVr7o/+WT4H0sVUOjJ9ufJPihRCJoRTuwWRo/fAM0qM2+6H9VNBO5hY95yS/jWp5zi/sUX1V5UrXLaae5f+UpqP7YLLgj/iLt2pS8uybzPPnPv2tV9//3dX3ut/LKPPhr+l847L+xMs+VHPwpx/OMf1V/G22+7163rfuml6YurOjEMGhS+S7t27g8+6L5jR/qWn45E8Hfgp8Dh0eNW4B+VmTfdj1QSQbFHHnGvXz9k26VLKy7/6KPuHTq4m4XnRx9NOYScsXZtOLK78cbUllO8UygsTE9cknnr17sXFLg3bOj+4ouVm+eOO8Lf/fLL03PUWlUvvxz+L7///dSXde214btUlADT7cMP97ZWNG/ufvvt7ps3p3896UgE8yozLROPdCQC91D9OvDAUB18883k5R591L1x47Clih+NG9eeZPDww+E7zZ6d2nJWrQrL+cUv0hKWZNiXX7qfeGI4KHjqqarNe9112fnbr1vnfthhoRlr48bUl7dxo3v79qFGlImm41WrQgKtX999v/1CzSbOVop0JII3gK8lvO8PvFGZedP9SFcicHd//333Tp3cGzVynzKl7DIdOpRMAsWPDh3SFkZWnXFG+C7pOJrr0SO0GUvNsnVraB6sU8d98uSqz79rl/vIkeH/4qGH0h9fMiNHhuac8g7kqmrq1PA9fvWr9C2ztPXrQxP1/vvvbY5asSK+9RVLRyI4FniH0HNoGfA20L0y86b7kc5E4O7+6afuxx8ftsSvf73vDtGs7ERgltYwsmLdunA0ct116VneDTeE5aXj6EwyY8cO97PPDr/pBx+s/nK2bXMfODDs2Kpao6iOxx4LMf/0p+lf9llnhSP0yjQbV8XWre533unesqXvObeyeHF611GelBPBnsJwAHBA9Prqqsybrke6E4F7aI87//ywNS65pGS1sDbXCP7yl/Bd3ngjPct74YWwvH/+Mz3Lk3jt2uX+3e+Gv9nvfpf68jZscO/dO+xEX3899eUls3x56NF0wgnpPZla7OOP3Zs0cT/99PTUlDdtcr/rrr37km98w/2tt1JfblWlLRGUmBE+ru68qTziSATu4Z+iuPfBwIHhaNm9dp8jGDo09E5IV0+fLVvCTuDKK9OzPInP7t3uY8aE3/OECelb7urVoc3+oINC02u67dzp/vWvhx31hx+mf/nFfvvbsG3+9rfqL2PVKvef/CRsCwiJa/r07JxUd48vEXxS3XlTecSVCIo9+GA4Yda1azjycK+dvYY2bAi9Q666Kr3LHTTI/aij0rtMSa/du/ee4B03Lv3L//DD0AnjsMPci4rSu+xf/jLE/fDD6V1uaTt2uPfs6X7ooXsPCitr8eLQstCwYdhnnHVW5nsilUU1gip64YVQ9TzkkNR70+Sqv/41/PVfeSW9yy0+kipOopJ7xo/32Lt8zpkTjtq7dk1fT5g5c8I5qHPPzcxR9VtvhR355ZdXrvxrr4WdvllIApdcktlzABWpdiIANhKGkij92AjsLG/euB6ZSATu7gsXunfsGJo6nnwyI6vMqHPOCUc76b4AbMGC8Kt64IH0LlfS4ze/CX+fUaPiv/jv+efDjvukk0KzYVXt3Om+ZIn700+HA4wjjnBv0yZc+5Ipl18eduzJ2vR37QoXshV3ODnooNActGpV5mKsrFhqBNl6ZCoRuIc/Zt++4Yfw299mr20v3TZtCgnussvSv+zdu8M/6/nnp3/Zkpr77vOMXwlcXPMcNiz5Oj/7LBxNP/RQaKo6+2z3o492b9DAS5yba906XP+TSevWhd9zjx4lT0xv2RK2Z5cuIbaOHcMJ4U2bMhtfVZSXCOqlOFRRrXbwwTBjBnzve2FAq0WLws2vDzww25Gl5tlnw7C2VR1yujLMwsBf06bBrl1Qt9wbmkpcPv8c5swJQ0cXPz7+OAwh/eijmfu7DB8Oq1bBNdfA2LHhvgaLF5d8rF27t3z9+vCVr4SRcM84IzwXP1q2rN6giKlo1gx+97swXPXvfx/ufXzPPfCHP4R7HxQUwOOPw7BhUK8G700tJIqao6CgwAsLYxmpOqndu2HcuHADi2bN4Oqrw6OmJoThw+Gll8IwwnHsEP7613C/2Lfegj590r/8mmrlynBDlAMOSO8ObcOGcKP24h3+7JpRvVQAABJoSURBVNmwdOnez484IuywjjsujGS5337pW3dl3XhjGB662CGH7N3Bd+my93WnTrm3Q3UPIxfPmBH+bps3h4R2/fXw9a9nPjlVl5nNcfcy7/eiRFAF8+bBbbeFm7jU1ISwZQu0agXf/W64qXcc1qwJNw3/+c/hppviWUdN8eWXMHky3HtvOEIHaNAg/A2KH61bl/++WbO9O5svvwy/w8Sd/uLFe9fXoUPY6ffpE5579YLmzTP/vUtzD3fAa9w47PibNct2RFWzbBkMGgTHHw8//GH17tuRbUoEafbOOyEh/OMf4eiuOCHkwj9cRaZOhbPPDuOvn3pqfOvp1QuaNs3fexm//37Y+f/5z2HM/mOOCc0KdeqERLlmTWhaKH69Zg1s2lT2surXDwmhceNwpL97d5jepk3JnX7v3qGcSFmUCGJSExPCd78bbkC+cmXYwcRl3Dj4zW9CW3XTpvGtJ5ds3x5qi/feGxJg/frhPMzYseHezhU1IWzZUjIxlE4WGzbA0Ufv3em3aZOZ7yW1Q3mJIOu9gKr6yGSvocp6553QHRPcDzjA/eabS97gIlcuSNu61b1p0/QM2VuRF18M22PatPjXlW0ffRSuSm/dOnznTp3ChU+rV2c7MpG9UPfRzCgrIdx3X+4MUfHUU2H9zz4b/7q2bg1dVK+4Iv51ZcPOnWFMpTPOCAm+Th33b33L/ZlndHMeyU3lJQI1DcVg/nz42c/CDeHNyr5HcocO4QRUJo0aBf/3f7B6dThhGbfTTw9t2osWxb+uTFm9Gh58ECZOhOXLQ++Xiy6Ciy8O95UVyVXlNQ3lWEet2qF7d/jb3+Ddd8PrsixfntmYtm8PSWDo0MwkAQjXE1xzTfiuHTpkZp1V5R66A27YEB4bN+59XfqxZEm4PmLHDjj55NCd+Kyz4j3XIpIJSgQx6tYt7ADL2umbhZN+XbuGcl27hkebNqn3S/7iC/joo/BYujQ8L1wI69bFcxFZMgMHhufnnw9Hzdmya1eonT36aLh4qfQOv7gXTnmKe+5cdhlceikcdVT8cYtkipqGYjZpUriIZ/PmvdMaNIBvfCPsoN59N1x5Wax5871JITFBJPZE2rYtNCuV3tkXP69bVzKG5s3h8MPh2GNDj5ZM1QjcoX17OOEEeOKJzKwz0c6doQ//hAmheapTp3DV6gEH7H00bVryfbLpDRtmPn6RdFLTUBaNHBmeb7opXOJ/2GFhx1Q8HeCzz8IR+4IFITEsWACPPRb6nxdr2xbatYOiIvjvf0ued2jYEDp2DDv7448Pz5067X1k64K34uEmpk7N7HATO3aEo/9f/CI053TrFoYBOOccDXkhUpZYawRmNgj4HVAXeMDdf1nq8zHAZcAuYBNwibu/V94ya1qNoLrcw05/wYK9CWLFipAMinf0xc+HHhouVMpFkyfDiBHw739D377xrmvbNnjkEfh//y/UmHr1gptvhiFDcnf7iGRKVmoEZlYXuBv4JlAEzDazaaV29I+5+x+j8kOA3wKD4oqpJjELzSrt28PgwdmOpvpOPTV8l+eeiy8RbN0KDzwQBgQsKoJ+/cKgYKefXnPGgRHJpjiPk/oCS9x9qbtvByYDQxMLuPuGhLf7AzXrhIVUqGXLcGT+3HPpX/bmzfC//xtqRldcEZrHpk+HN94II1cqCYhUTpyJoC3wScL7omhaCWZ2mZl9CPwauLKsBZnZJWZWaGaFa9asiSVYic/AgWHnvGFDxWUrY9OmMJJlp05hePCjjgojQ86aFdalBCBSNVlvOXX3u939K8CNwE+SlJno7gXuXtBKo2rVOAMHhh48M2emtpz168OJ9g4dwrDGPXvCq6+GIbUHDFACEKmuOHsNrQDaJ7xvF01LZjJwb4zxSJYcfzzsv39oHhoyJHk59zC42ief7H18/PHe1wsWhFrFmWeGk8Bxn3wWyRdxJoLZQGcz60RIAMOB7yQWMLPO7v5B9PYM4AOk1mnYMByxT58edubJdvRFReHEb+l527cP3W6//W0YMyaccxCR9IktEbj7TjO7HJhO6D76kLsvNLPbCIMfTQMuN7NTgR3AF8CFccUj2TVwIDz9dOjTX6xOnXB9RPv2YVjls8/e21OqeOefjdsTiuSbWC8oc/dngGdKTbsl4fVVca5fcsfo0eEWhC1a7N3RH3po7t2WUCQfZf1ksVRs0qTQNbJOnfA8aVK2I6q6Jk3gBz8IzTsnnBASgZKASG7Qv2KOKz1W0fLl4T2UHKZCRKS6VCPIcTfdVHLAOgjv8/2m8CKSPkoEOe7jj6s2XUSkqpQIclyyu17pblgiki5KBDluwgRo3LjktMaNw3QRkXRQIshxI0eG++N26BD603foEN7rRLGIpIsSQQ0wcmQYX3/37vBc1SRQG7qfikh81H20llP3UxGpiGoEtZy6n4pIRZQIajl1PxWRiigR1HLqfioiFVEiqOXU/VREKqJEUMup+6mIVES9hvLAyJHa8YtIcqoRSIV0HYJI7aYagZRL1yGI1H6qEUi5dB2CSO2nRCDl0nUIIrWfEoGUS9chiNR+SgRSLl2HIFL7KRFIuXQdgkjtp15DUiFdhyBSu8VaIzCzQWa22MyWmNm4Mj6/1szeM7P5ZvaimXWIMx4REdlXbInAzOoCdwODgaOBEWZ2dKlibwMF7t4dmAL8Oq54pObSBW0i8YqzRtAXWOLuS919OzAZGJpYwN1nuHtxL/U3gXYxxiNZksqOvPiCtuXLwX3vBW1KBiLpE2ciaAt8kvC+KJqWzPeBZ2OMR7Ig1R25LmgTiV9O9Boys+8CBcDtST6/xMwKzaxwzZo1mQ1OUpLqjlwXtInEL85EsAJon/C+XTStBDM7FbgJGOLu28pakLtPdPcCdy9o1apVLMFKPFLdkeuCNpH4xZkIZgOdzayTmTUAhgPTEguYWU/gPkIS+DTGWCRLUt2R64I2kfjFlgjcfSdwOTAdeB94wt0XmtltZjYkKnY70AT4m5nNM7NpSRYnNVSqO3Jd0CYSP3P3bMdQJQUFBV5YWJjtMKQKJk0K5wQ+/jjUBCZMyOyOPNvrF8kFZjbH3QvK+kxXFkvssnllsu6nIFKxnOg1JBKXdHQ/1QVtUtupRiC1Wqq9llSjkHygGoHUaqn2WtIFbZIPlAikVku115IuaJN8oEQgtVqq3U91QZvkAyUCqfVGjoRly2D37vBclbZ9XdAm+UCJQKQcuqBN8oF6DYlUQHdok9pONQKRmOk6BMl1SgQiMUrXjXWUTCROSgQiMUrXlc26S5vESYlAJEbpuA5BF7VJ3JQIRGKUjusQ0pFM1LQk5VEiEIlROq5DSDWZqGlJKqJEIBKjdFyHkGoy0QisUhElApGYpXJlc/H8qSSTdI3AmkqNQokkt+kOZSK1XMeOYeddWocOITHFPX/pobwh1Gh0hXZmlXeHMtUIRGq5bI/Amgu9nlQjKZ8SgUgtl+0RWLM9lLdOlldMiUAkD2RzBNZ0dKFN5Yg+F2okuU6JQETKlWqNItVEkuoRfbZrJDWBEoGIVCiVGkWqiSTVI/ps10hqAiUCEYldKokk1SP6bNdIaoJYE4GZDTKzxWa2xMzGlfH5SWY218x2mtm5ccYiIjVTqkf02a6RQO7XKGJLBGZWF7gbGAwcDYwws6NLFfsYGAU8FlccIlKzpWOYjmzWSGpCjSLOGkFfYIm7L3X37cBkYGhiAXdf5u7zgd0xxiEiNVi2bxeaao2kJvRaijMRtAU+SXhfFE2rMjO7xMwKzaxwzZo1aQlORGqOVIfpSEW2L8jLhBpxstjdJ7p7gbsXtGrVKtvhiEgeyfYFeZkQZyJYAbRPeN8umiYiUqNk84I8iP9kc5yJYDbQ2cw6mVkDYDgwLcb1iYjknFRrFJk42Rzr6KNmdjpwJ1AXeMjdJ5jZbUChu08zsz7Ak0BzYCuwyt2PKW+ZGn1URPJJqqO/Fitv9FENQy0iksPq1Ak1gdLMQlNVZWkYahGRGioTJ5uVCEREclg6TjZXRIlARCSHZeKCunrpW5SIiMRh5Mh4L6JTjUBEJM8pEYiI5DklAhGRPKdEICKS55QIRETyXI27stjM1gBlXHCdE1oCn2U7iHIovtTkenyQ+zEqvtSkEl8Hdy9z+OYalwhymZkVJruEOxcovtTkenyQ+zEqvtTEFZ+ahkRE8pwSgYhInlMiSK+J2Q6gAoovNbkeH+R+jIovNbHEp3MEIiJ5TjUCEZE8p0QgIpLnlAiqyMzam9kMM3vPzBaa2VVllBlgZuvNbF70uCXDMS4zs3ejde9zOzcL7jKzJWY238x6ZTC2IxO2yzwz22BmV5cqk/HtZ2YPmdmnZrYgYdpBZva8mX0QPTdPMu+FUZkPzOzCDMV2u5ktiv5+T5rZgUnmLfe3EHOM481sRcLf8fQk8w4ys8XR73FcBuN7PCG2ZWY2L8m8sW7DZPuUjP7+3F2PKjyAQ4Fe0eumwH+Ao0uVGQD8M4sxLgNalvP56cCzgAHHAf/OUpx1gVWEC12yuv2Ak4BewIKEab8GxkWvxwG/KmO+g4Cl0XPz6HXzDMQ2EKgXvf5VWbFV5rcQc4zjgR9W4jfwIXA40AB4p/T/U1zxlfr8N8At2diGyfYpmfz9qUZQRe6+0t3nRq83Au8DbbMbVZUNBf7swZvAgWZ2aBbiOAX40N2zfqW4u88CPi81eSjwSPT6EeCsMmY9DXje3T939y+A54FBccfm7s+5+87o7ZtAu3Sus6qSbL/K6Asscfel7r4dmEzY7mlVXnxmZsD5wF/Tvd7KKGefkrHfnxJBCsysI9AT+HcZHx9vZu+Y2bNmdkxGAwMHnjOzOWZ2SRmftwU+SXhfRHaS2XCS//Nlc/sVO9jdV0avVwEHl1EmF7blaEINrywV/RbidnnUfPVQkqaNXNh+JwKr3f2DJJ9nbBuW2qdk7PenRFBNZtYE+DtwtbtvKPXxXEJzx7HA74GpGQ7va+7eCxgMXGZmJ2V4/RUyswbAEOBvZXyc7e23Dw/18Jzra21mNwE7gUlJimTzt3Av8BWgB7CS0PySi0ZQfm0gI9uwvH1K3L8/JYJqMLP6hD/YJHf/R+nP3X2Du2+KXj8D1DezlpmKz91XRM+fAk8Sqt+JVgDtE963i6Zl0mBgrruvLv1BtrdfgtXFTWbR86dllMnatjSzUcCZwMhoR7GPSvwWYuPuq919l7vvBu5Psu6s/hbNrB4wDHg8WZlMbMMk+5SM/f6UCKooak98EHjf3X+bpMwhUTnMrC9hO6/NUHz7m1nT4teEk4oLShWbBlwQ9R46DlifUAXNlKRHYdncfqVMA4p7YVwI/F8ZZaYDA82sedT0MTCaFiszGwTcAAxx981JylTmtxBnjInnnc5Osu7ZQGcz6xTVEocTtnumnAoscveisj7MxDYsZ5+Sud9fXGfCa+sD+BqhijYfmBc9TgfGAGOiMpcDCwk9IN4ETshgfIdH630niuGmaHpifAbcTeit8S5QkOFtuD9hx94sYVpWtx8hKa0EdhDaWb8PtABeBD4AXgAOisoWAA8kzDsaWBI9/idDsS0htA0X/wb/GJVtAzxT3m8hg9vvL9Hvaz5hp3Zo6Rij96cTesp8GFeMZcUXTf9T8e8uoWxGt2E5+5SM/f40xISISJ5T05CISJ5TIhARyXNKBCIieU6JQEQkzykRiIjkOSUCkYiZ7bKSI6OmbSRMM+uYOPKlSC6pl+0ARHLIFnfvke0gRDJNNQKRCkTj0f86GpP+LTM7Ipre0cxeigZVe9HMDoumH2zhHgHvRI8TokXVNbP7ozHnnzOz/aLyV0Zj0c83s8lZ+pqSx5QIRPbar1TT0LcTPlvv7t2APwB3RtN+Dzzi7t0Jg77dFU2/C3jZw6B5vQhXpAJ0Bu5292OAdcA50fRxQM9oOWPi+nIiyejKYpGImW1y9yZlTF8GfMPdl0aDg61y9xZm9hlh2IQd0fSV7t7SzNYA7dx9W8IyOhLGje8cvb8RqO/uPzezfwGbCKOsTvVowD2RTFGNQKRyPMnrqtiW8HoXe8/RnUEY+6kXMDsaEVMkY5QIRCrn2wnPb0SvXyeMlgkwEnglev0iMBbAzOqaWbNkCzWzOkB7d58B3Ag0A/aplYjESUceInvtZyVvYP4vdy/uQtrczOYTjupHRNOuAB42s+uBNcD/RNOvAiaa2fcJR/5jCSNflqUu8GiULAy4y93Xpe0biVSCzhGIVCA6R1Dg7p9lOxaROKhpSEQkz6lGICKS51QjEBHJc0oEIiJ5TolARCTPKRGIiOQ5JQIRkTz3/wEH3YQ6taWniQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "history_dict = history.history\n",
    "loss_values = history_dict['loss']\n",
    "val_loss_values = history_dict['val_loss']\n",
    "epochs = range(1, len(loss_values) + 1)\n",
    "plt.plot(epochs, loss_values, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss_values, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_train_x = np.vstack((X_train, X_test)) \n",
    "final_train_y = np.hstack((y_train, y_test))\n",
    "\n",
    "model = models.Sequential([\n",
    "    layers.Dense(512, input_shape=(1024,), activation='relu'),\n",
    "    layers.Dropout(0.2),\n",
    "    layers.Dense(1, activation='sigmoid')\n",
    "])\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(final_train_x,\n",
    "                    final_train_y,\n",
    "                    epochs=7,\n",
    "                    batch_size=128,\n",
    "                    validation_data=(X_validation, y_validation),\n",
    "                   verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6891it [02:21, 48.60it/s]\n"
     ]
    }
   ],
   "source": [
    "predictions = {}\n",
    "for idx, word in tqdm(enumerate(unlabeled)):\n",
    "    res = model.predict(unlabeled_X[idx].reshape(1, -1))\n",
    "    predictions[word] = res[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update our dataset with the BERT predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6891/6891 [00:21<00:00, 314.38it/s]\n"
     ]
    }
   ],
   "source": [
    "for occ in tqdm(predictions, total=len(predictions)):\n",
    "    if occ in occs_df['occupation'].tolist():\n",
    "        the_row = occs_df[occs_df['occupation'] == occ]\n",
    "        the_index = the_row.index[0]\n",
    "        occs_df.loc[the_index, 'label'] = int(round(predictions[occ]))\n",
    "        occs_df.loc[the_index, 'labeled_by'] = 'bert'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Cleanlab to find potentially problematic labels\n",
    "    * Get out-of-sample predicted probabilities using cross-validation\n",
    "    * Compute confident joint\n",
    "    * Find label errors    \n",
    "    \n",
    "## Step 1: Get out-of-sample predicted probabilities using cross-validation    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_y = [int(round(val)) for  key,val in predictions.items() ]\n",
    "\n",
    "all_X = np.vstack((X, unlabeled_X)) \n",
    "all_y = np.hstack((y, labeled_y))\n",
    "\n",
    "# using a simple, non-optimized logistic regression classifier; the cross-validation will expose label weaknesses\n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    all_X, all_y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Compute confident joint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_confident_joint(psx, y, verbose=False):\n",
    "    # Verify inputs\n",
    "    psx = np.asarray(psx)\n",
    "\n",
    "    # Find the number of unique classes if K is not given\n",
    "    K = len(np.unique(y))\n",
    "\n",
    "    # Estimate the probability thresholds for confident counting\n",
    "    # You can specify these thresholds yourself if you want\n",
    "    # as you may want to optimize them using a validation set.\n",
    "    # By default (and provably so) they are set to the average class prob.\n",
    "    thresholds = [np.mean(psx[:,k][y == k]) for k in range(K)] # P(s^=k|s=k)\n",
    "    thresholds = np.asarray(thresholds)\n",
    "\n",
    "    # Compute confident joint\n",
    "    confident_joint = np.zeros((K, K), dtype = int)\n",
    "    for i, row in enumerate(psx):\n",
    "        y_label = y[i]\n",
    "        # Find out how many classes each example is confidently labeled as\n",
    "        confident_bins = row >= thresholds - 1e-6\n",
    "        num_confident_bins = sum(confident_bins)\n",
    "        # If more than one conf class, inc the count of the max prob class\n",
    "        if num_confident_bins == 1:\n",
    "            confident_joint[y_label][np.argmax(confident_bins)] += 1\n",
    "        elif num_confident_bins > 1:\n",
    "            confident_joint[y_label][np.argmax(row)] += 1\n",
    "\n",
    "    # Normalize confident joint (use cleanlab, trust me on this)\n",
    "    confident_joint = cleanlab.latent_estimation.calibrate_confident_joint(\n",
    "        confident_joint, y)\n",
    "    if verbose:\n",
    "        cleanlab.util.print_joint_matrix(confident_joint)\n",
    "    return confident_joint\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, all_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Find label errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_label_errors(confident_joint, y, verbose=False):\n",
    "    # We arbitrarily choose at least 5 examples left in every class.\n",
    "    # Regardless of whether some of them might be label errors.\n",
    "    MIN_NUM_PER_CLASS = 5\n",
    "    # Leave at least MIN_NUM_PER_CLASS examples per class.\n",
    "    # NOTE prune_count_matrix is transposed (relative to confident_joint)\n",
    "    prune_count_matrix = cleanlab.pruning.keep_at_least_n_per_class(\n",
    "        prune_count_matrix=confident_joint.T,\n",
    "        n=MIN_NUM_PER_CLASS,\n",
    "    )\n",
    "    K = len(np.unique(y)) # number of unique classes\n",
    "    y_counts = np.bincount(y)\n",
    "    noise_masks_per_class = []\n",
    "    # For each row in the transposed confident joint\n",
    "    for k in range(K):\n",
    "        noise_mask = np.zeros(len(psx), dtype=bool)\n",
    "        psx_k = psx[:, k]\n",
    "        if y_counts[k] > MIN_NUM_PER_CLASS:  # Don't prune if not MIN_NUM_PER_CLASS\n",
    "            for j in range(K):  # noisy label index (k is the true label index)\n",
    "                if k != j:  # Only prune for noise rates, not diagonal entries\n",
    "                    num2prune = prune_count_matrix[k][j]\n",
    "                    if num2prune > 0:\n",
    "                        # num2prune'th largest p(classk) - p(class j)\n",
    "                        # for x with noisy label j\n",
    "                        margin = psx_k - psx[:, j]\n",
    "                        y_filter = y == j\n",
    "                        threshold = -np.partition(\n",
    "                            -margin[y_filter], num2prune - 1\n",
    "                        )[num2prune - 1]\n",
    "                        noise_mask = noise_mask | (y_filter & (margin >= threshold))\n",
    "            noise_masks_per_class.append(noise_mask)\n",
    "        else:\n",
    "            noise_masks_per_class.append(np.zeros(len(s), dtype=bool))\n",
    "\n",
    "    # Boolean label error mask\n",
    "    label_errors_bool = np.stack(noise_masks_per_class).any(axis=0)\n",
    "\n",
    "     # Remove label errors if given label == model prediction\n",
    "    for i, pred_label in enumerate(psx.argmax(axis=1)):\n",
    "        # np.all let's this work for multi_label and single label\n",
    "        if label_errors_bool[i] and np.all(pred_label == y[i]):\n",
    "            label_errors_bool[i] = False\n",
    "\n",
    "    # Convert boolean mask to an ordered list of indices for label errors\n",
    "    label_errors_idx = np.arange(len(y))[label_errors_bool]\n",
    "    # self confidence is the holdout probability that an example\n",
    "    # belongs to its given class label\n",
    "    self_confidence = np.array(\n",
    "        [np.mean(psx[i][y[i]]) for i in label_errors_idx]\n",
    "    )\n",
    "    margin = self_confidence - psx[label_errors_bool].max(axis=1)\n",
    "    label_errors_idx = label_errors_idx[np.argsort(margin)]\n",
    "    label_errors_idx.sort()\n",
    "    if verbose:\n",
    "        print('Indices of label errors found by confident learning:')\n",
    "        print('Note label errors are sorted by likelihood of being an error')\n",
    "        print('but here we just sort them by index for comparison with above.')\n",
    "        print(np.array(label_errors_idx))\n",
    "    return label_errors_idx\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, all_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print out questionable labels and their existing scores for manual review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"playwright\", 1\n",
      "\"Goliard\", 1\n",
      "\"feminist\", 1\n",
      "\"sculptor\", 1\n",
      "\"academic\", 1\n",
      "\"astronaut\", 1\n",
      "\"printer\", 1\n",
      "\"industrialist\", 1\n",
      "\"bobsledder\", 1\n",
      "\"paleontologist\", 1\n",
      "\"anarchist\", 1\n",
      "\"pirate\", 1\n",
      "\"rikishi\", 1\n",
      "\"crank\", 1\n",
      "\"minister\", 1\n",
      "\"duke\", 1\n",
      "\"mullah\", 1\n",
      "\"count\", 1\n",
      "\"knight\", 1\n",
      "\"runner\", 1\n",
      "\"sniper\", 1\n",
      "\"bartender\", 1\n",
      "\"socialist\", 1\n",
      "\"private sector employee\", 1\n",
      "\"venture capitalist\", 1\n",
      "\"mime\", 1\n",
      "\"Patriot\", 1\n",
      "\"corporal\", 1\n",
      "\"gladiator\", 1\n",
      "\"healer\", 1\n",
      "\"prior\", 1\n",
      "\"unemployed\", 1\n",
      "\"baritone\", 1\n",
      "\"counsel\", 1\n",
      "\"Mohel\", 1\n",
      "\"Gendarme\", 1\n",
      "\"Pythia\", 1\n",
      "\"Painter\", 1\n",
      "\"Studienrat\", 1\n",
      "\"censor\", 1\n",
      "\"General\", 1\n",
      "\"Freedom Fighters\", 1\n",
      "\"comic\", 1\n",
      "\"boss\", 1\n",
      "\"hobo\", 1\n",
      "\"Bedel\", 1\n",
      "\"virgin\", 1\n",
      "\"Shah\", 1\n",
      "\"Hakim\", 1\n",
      "\"judicial commissioner\", 1\n",
      "\"partner\", 1\n",
      "\"champion\", 1\n",
      "\"Gurkha\", 1\n",
      "\"Rangatira\", 1\n",
      "\"Poeta\", 1\n",
      "\"Revolutionary\", 1\n",
      "\"Mawla\", 1\n",
      "\"Monsieur\", 1\n",
      "\"Korporal\", 1\n",
      "\"Fundi\", 1\n",
      "\"Patrick\", 1\n",
      "\"Cacique\", 1\n",
      "\"Archer\", 1\n",
      "\"Sepoy\", 1\n",
      "\"Thuggee\", 1\n",
      "\"Richter\", 1\n",
      "\"airport security\", 1\n",
      "\"spouse\", 1\n",
      "\"parent\", 1\n",
      "\"MasterChef\", 1\n",
      "\"Sexton\", 1\n",
      "\"Hostess\", 1\n",
      "\"Lascar\", 1\n",
      "\"Met Éireann\", 1\n",
      "\"patron\", 1\n",
      "\"Censor\", 1\n",
      "\"Performer\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Kabalista\", 1\n",
      "\"Maalimine\", 1\n",
      "\"Buddhist\", 1\n",
      "\"inebriate\", 1\n",
      "\"operador\", 1\n",
      "\"Occitanista\", 1\n",
      "\"Student\", 1\n",
      "\"sports official\", 1\n",
      "\"salonnière\", 1\n",
      "\"literary historian\", 1\n",
      "\"preface author\", 1\n",
      "\"vase painter\", 1\n",
      "\"Four Evangelists\", 1\n",
      "\"bhikkhuni\", 1\n",
      "\"fitness professional\", 1\n",
      "\"stylite\", 1\n",
      "\"fictional pirate\", 1\n",
      "\"local politics\", 1\n",
      "\"Federal Social Court\", 1\n",
      "\"liberal profession\", 1\n",
      "\"role-playing game\", 1\n",
      "\"tea master\", 1\n",
      "\"anarcha-feminist\", 1\n",
      "\"canut\", 1\n",
      "\"Stolnik\", 1\n",
      "\"voice acting\", 1\n",
      "\"Rutgers University\", 1\n",
      "\"book printer\", 1\n",
      "\"shield-maiden\", 1\n",
      "\"driving instructor\", 1\n",
      "\"social work\", 1\n",
      "\"Agricultural labour\", 1\n",
      "\"water sport\", 1\n",
      "\"Cursus honorum\", 1\n",
      "\"Barbouze\", 1\n",
      "\"violin making and maintenance\", 1\n",
      "\"Civil Guard (Spain)\", 1\n",
      "\"brand management\", 1\n",
      "\"natural philosophy\", 1\n",
      "\"field hockey\", 1\n",
      "\"Bakushin\", 1\n",
      "\"senior management\", 1\n",
      "\"hydraulic engineering\", 1\n",
      "\"yaksha\", 1\n",
      "\"water burial\", 1\n",
      "\"Bomberos Voluntarios (Guatemala)\", 1\n",
      "\"Tampere University of Technology\", 1\n",
      "\"forensic science\", 1\n",
      "\"management consulting\", 1\n",
      "\"Doctor in Engineering\", 1\n",
      "\"Brigade RISTA - Electronic Warfare Intelligence\", 1\n",
      "\"computational engineering\", 1\n",
      "\"record label\", 1\n",
      "\"Münzwardein\", 1\n",
      "\"security consultant\", 1\n",
      "\"secondary school\", 1\n",
      "\"Tannaim\", 1\n",
      "\"Pennsylvania State University\", 1\n",
      "\"film screenwriter\", 1\n",
      "\"seal cutting\", 1\n",
      "\"art studies\", 1\n",
      "\"amateur wrestling\", 1\n",
      "\"golf instructor\", 1\n",
      "\"Mwami of Rwanda\", 1\n",
      "\"collaborationism\", 1\n",
      "\"Buddhist Publication Society\", 1\n",
      "\"human geography\", 1\n",
      "\"arriero\", 1\n",
      "\"pagan clergy\", 1\n",
      "\"knyaz\", 1\n",
      "\"information technology consulting\", 1\n",
      "\"Kaumātua\", 1\n",
      "\"oidor\", 1\n",
      "\"Pakistan Air Force\", 1\n",
      "\"Erivan Governorate\", 1\n",
      "\"Wokou\", 1\n",
      "\"onmyōji\", 1\n",
      "\"arts administration\", 1\n",
      "\"Humanista\", 1\n",
      "\"alcohol intoxication\", 1\n",
      "\"King's Fund\", 1\n",
      "\"fine art\", 1\n",
      "\"Andragogy\", 1\n",
      "\"Department of Physics, National Taiwan University\", 1\n",
      "\"Religious Society of Friends\", 1\n",
      "\"speedcubing\", 1\n",
      "\"china painting\", 1\n",
      "\"Italian singer-songwriter\", 1\n",
      "\"Maciste\", 1\n",
      "\"esoteric tarot\", 1\n",
      "\"physical education\", 1\n",
      "\"Indian Information Service\", 1\n",
      "\"Refractory clergy\", 1\n",
      "\"site manager\", 1\n",
      "\"well drilling\", 1\n",
      "\"palynology\", 1\n",
      "\"Asset management\", 1\n",
      "\"copyright troll\", 1\n",
      "\"The Charlotte Observer\", 1\n",
      "\"street vendor\", 1\n",
      "\"lumières\", 1\n",
      "\"paranormal phenomenon\", 1\n",
      "\"IT project manager\", 1\n",
      "\"Revolutionary leadership and family medicine education.\", 1\n",
      "\"Legal aid\", 1\n",
      "\"alumni association\", 1\n",
      "\"magnicide\", 1\n",
      "\"Reper\", 1\n",
      "\"Insurgente\", 1\n",
      "\"S.T.A.R.S.\", 1\n",
      "\"Nigun\", 1\n",
      "\"Kuge\", 1\n",
      "\"clinical psychology\", 1\n",
      "\"First Lady of Afghanistan\", 1\n",
      "\"philosophy of science\", 1\n",
      "\"Williams-Sonoma\", 1\n",
      "\"Civil Police\", 1\n",
      "\"child care\", 1\n",
      "\"Shadow Theatre\", 1\n",
      "\"Ripon Community Hospital\", 1\n",
      "\"Pensioner Guards\", 1\n",
      "\"American frontier\", 1\n",
      "\"Italian military internees\", 1\n",
      "\"Paramilitar\", 1\n",
      "\"mostassaf\", 1\n",
      "\"Natali Castillo\", 1\n",
      "\"community health worker\", 1\n",
      "\"bleachery\", 1\n",
      "\"sustainable development\", 1\n",
      "\"Facade engineering\", 1\n",
      "\"prison chaplain\", 1\n",
      "\"Dutch municipal executive\", 1\n",
      "\"beer pouring\", 1\n",
      "\"Gigathlon\", 1\n",
      "\"psychic detective\", 1\n",
      "\"Major Penitentiary\", 1\n",
      "\"Nobiles\", 1\n",
      "\"business intelligence\", 1\n",
      "\"tonoi\", 1\n",
      "\"Utah Highway Patrol\", 1\n",
      "\"Misogamy\", 1\n",
      "\"forced disappearance\", 1\n",
      "\"cornett\", 1\n",
      "\"Mashinostroitel\", 1\n",
      "\"video on demand\", 1\n",
      "\"Foodstirs\", 1\n",
      "\"Andragogue\", 1\n",
      "\"Heyoka\", 1\n",
      "\"lighting technology engineer\", 1\n",
      "\"Field of guidance and achievement\", 1\n",
      "\"male prostitution\", 1\n",
      "\"Cioban\", 1\n",
      "\"timple\", 1\n",
      "\"Foresight (futures studies)\", 1\n",
      "\"port chaplain\", 1\n",
      "\"motivational speaking\", 1\n",
      "\"petroleum-gas industry\", 1\n",
      "\"Italian studies\", 1\n",
      "\"Perm Governorate\", 1\n",
      "\"wingsuit flying\", 1\n",
      "\"estrada\", 1\n",
      "\"Protestant theology\", 1\n",
      "\"National Players\", 1\n",
      "\"railway engineering\", 1\n",
      "\"mumble rap\", 1\n",
      "\"Ledger art\", 1\n",
      "\"Glacier National Park\", 1\n",
      "\"Ishkhan\", 1\n",
      "\"ship breaking\", 1\n",
      "\"United States Army\", 1\n",
      "\"Tai Situpa\", 1\n",
      "\"volunteer fire department\", 1\n",
      "\"Rugby league match officials\", 1\n",
      "\"nuclear power\", 1\n",
      "\"Catholic priests in public office\", 1\n",
      "\"What? Where? When?\", 1\n",
      "\"edaphology\", 1\n",
      "\"organizational leadership\", 1\n",
      "\"Bariton\", 1\n",
      "\"Henri Blanvalet\", 1\n",
      "\"Medical Specimens\", 1\n",
      "\"political analysis\", 1\n",
      "\"Recurring status\", 1\n",
      "\"A Few Good Men\", 1\n",
      "\"Doshu\", 1\n",
      "\"short story\", 1\n",
      "\"patroon\", 1\n",
      "\"infante\", 1\n",
      "\"consumer protection\", 1\n",
      "\"letter delivery\", 1\n",
      "\"Daikan\", 1\n",
      "\"Communist Party of India\", 1\n",
      "\"Medal of Honor for St. Petersburg\", 1\n",
      "\"Yobidashi\", 1\n",
      "\"Regista\", 1\n",
      "\"food industry\", 1\n",
      "\"deltiology\", 1\n",
      "\"Iraq Football Association\", 1\n",
      "\"prison escape\", 1\n",
      "\"Korean Social Democratic Party\", 1\n",
      "\"folk dance\", 1\n",
      "\"Musketeers of the Guard\", 1\n",
      "\"Uryadnik\", 1\n",
      "\"graphics engineer\", 1\n",
      "\"Newquay Tretherras\", 1\n",
      "\"Željko Ražnatović\", 1\n",
      "\"war elephant\", 1\n",
      "\"horsecar\", 1\n",
      "\"Chinese calligraphy\", 1\n",
      "\"Human Resources Management\", 1\n",
      "\"traditional song\", 1\n",
      "\"Bojnice Castle\", 1\n",
      "\"public transport authority\", 1\n",
      "\"sigillography\", 1\n",
      "\"ultra-triathlon\", 1\n",
      "\"Republican Guard\", 1\n",
      "\"sysop\", 1\n",
      "\"people smuggling\", 1\n",
      "\"Byzantine Empire under the Palaiologos dynasty\", 1\n",
      "\"archival science\", 1\n",
      "\"authority control\", 1\n",
      "\"борец за право голоса женщин\", 1\n",
      "\"Wine retail\", 1\n",
      "\"קרימינולוג\", 1\n",
      "\"Daftar Gubernur Nusa Tenggara Barat\", 1\n",
      "\"folklivsskildrare\", 1\n",
      "\"strojní zámečník\", 1\n",
      "\"főkapitány\", 1\n",
      "\"hymnolog\", 1\n",
      "\"antineoplásico\", 1\n",
      "\"sadelmakare\", 1\n",
      "\"kyrkobyggare\", 1\n",
      "\"folkbildare\", 1\n",
      "\"skogsägare\", 1\n",
      "\"ר\"מ\", 1\n",
      "\"золотопромышленник\", 1\n",
      "\"unknown parameters category\", 1\n",
      "\"青衣\", 1\n",
      "\"Jyraou\", 1\n",
      "\"Self-Presentation\", 1\n",
      "\"IT specialist\", 1\n",
      "\"esporter\", 1\n",
      "\"cantante de tonada\", 1\n",
      "\"指導棋士\", 1\n",
      "\"Hacendado\", 1\n",
      "\"пейзажист\", 1\n",
      "\"Эколог\", 1\n",
      "\"Военная коллегия пугачёвцев\", 1\n",
      "\"зайсанг\", 1\n",
      "\"Daftar Menteri Perdagangan Indonesia\", 1\n",
      "\"TV-kock\", 1\n",
      "\"Physical Education and Sport.\", 1\n",
      "\"Formann\", 1\n",
      "\"弓道家\", 1\n",
      "\"sovereign\", 0\n",
      "\"combined track and field event athlete\", 0\n",
      "\"public figure\", 0\n",
      "\"Polymath\", 0\n",
      "\"royalty\", 0\n",
      "\"Margrave\", 0\n",
      "\"conscientious objection\", 0\n",
      "\"Kohen\", 0\n",
      "\"bushi\", 0\n",
      "\"celebrity\", 0\n",
      "\"satrap\", 0\n",
      "\"insurance\", 0\n",
      "\"mother\", 0\n",
      "\"space marine\", 0\n",
      "\"Strategos\", 0\n",
      "\"Musicians\", 0\n",
      "\"father\", 0\n",
      "\"daimyo\", 0\n",
      "\"geometer\", 0\n",
      "\"pacemaker\", 0\n",
      "\"Raja\", 0\n",
      "\"Polier\", 0\n",
      "\"acharya\", 0\n",
      "\"janissary\", 0\n",
      "\"herald\", 0\n",
      "\"Berthold\", 0\n",
      "\"syndicalist\", 0\n",
      "\"cadre\", 0\n",
      "\"Catholicos\", 0\n",
      "\"homicide\", 0\n",
      "\"carabinier\", 0\n",
      "\"Franciscan\", 0\n",
      "\"peasant\", 0\n",
      "\"bass guitar\", 0\n",
      "\"circus\", 0\n",
      "\"Jägermeister\", 0\n",
      "\"guild\", 0\n",
      "\"sports figure\", 0\n",
      "\"Freiherr\", 0\n",
      "\"railway\", 0\n",
      "\"principal of Uppsala University\", 0\n",
      "\"heir presumptive\", 0\n",
      "\"fountain pen\", 0\n",
      "\"victim\", 0\n",
      "\"Organista\", 0\n",
      "\"Ingeniero\", 0\n",
      "\"fan\", 0\n",
      "\"Rinpoche\", 0\n",
      "\"curandero\", 0\n",
      "\"Vogt\", 0\n",
      "\"Spiker\", 0\n",
      "\"hedge fund\", 0\n",
      "\"cinephile\", 0\n",
      "\"sled dog\", 0\n",
      "\"commissary\", 0\n",
      "\"gentry\", 0\n",
      "\"dayan\", 0\n",
      "\"supercentenarian\", 0\n",
      "\"streamer\", 0\n",
      "\"show business\", 0\n",
      "\"proofreading\", 0\n",
      "\"voluntary service\", 0\n",
      "\"marines\", 0\n",
      "\"Human Resource\", 0\n",
      "\"commercial\", 0\n",
      "\"consulting\", 0\n",
      "\"professional\", 0\n",
      "\"human\", 0\n",
      "\"Ohio State University\", 0\n",
      "\"dramaturgy\", 0\n",
      "\"chronicle\", 0\n",
      "\"duathlon\", 0\n",
      "\"Rosicrucian\", 0\n",
      "\"demimonde\", 0\n",
      "\"artistic gymnastics\", 0\n",
      "\"headmaster in France\", 0\n",
      "\"percussion instrument\", 0\n",
      "\"Young Guard\", 0\n",
      "\"land surveyor in Poland\", 0\n",
      "\"triumvirate\", 0\n",
      "\"Maharaja\", 0\n",
      "\"clergy\", 0\n",
      "\"demagogy\", 0\n",
      "\"scholar of the bible as literature\", 0\n",
      "\"mechanics\", 0\n",
      "\"proletariat\", 0\n",
      "\"remix\", 0\n",
      "\"revenger\", 0\n",
      "\"bookkeeping\", 0\n",
      "\"Nobile\", 0\n",
      "\"essay\", 0\n",
      "\"interior ministry\", 0\n",
      "\"plenipotentiary\", 0\n",
      "\"Member of Parliament in the Parliament of England\", 0\n",
      "\"Heimat\", 0\n",
      "\"jewellery design\", 0\n",
      "\"radio broadcasting\", 0\n",
      "\"Tattooed Lady\", 0\n",
      "\"wage\", 0\n",
      "\"French high civil service\", 0\n",
      "\"mythical creature\", 0\n",
      "\"Generalissimo\", 0\n",
      "\"letterpress printing\", 0\n",
      "\"Cresta Run\", 0\n",
      "\"Dijkgraaf\", 0\n",
      "\"oblate\", 0\n",
      "\"finance\", 0\n",
      "\"epigraph\", 0\n",
      "\"charcoal burner\", 0\n",
      "\"eulogy\", 0\n",
      "\"cavalier\", 0\n",
      "\"Naik\", 0\n",
      "\"Winzer\", 0\n",
      "\"tsarina\", 0\n",
      "\"Birkebeiner\", 0\n",
      "\"galley slave\", 0\n",
      "\"erudition\", 0\n",
      "\"Grafik\", 0\n",
      "\"Lawyers\", 0\n",
      "\"Discalced Carmelites\", 0\n",
      "\"dauphin of France\", 0\n",
      "\"transport\", 0\n",
      "\"crossword\", 0\n",
      "\"Grunewald\", 0\n",
      "\"member of the Argentine Chamber of Deputies\", 0\n",
      "\"bookstore\", 0\n",
      "\"Bachiller\", 0\n",
      "\"Diplom\", 0\n",
      "\"health administration\", 0\n",
      "\"Nazi\", 0\n",
      "\"fertility deity\", 0\n",
      "\"pseudonym\", 0\n",
      "\"regicide\", 0\n",
      "\"mythological king\", 0\n",
      "\"Naturaliste\", 0\n",
      "\"vignette\", 0\n",
      "\"marxist\", 0\n",
      "\"screenplay\", 0\n",
      "\"electronic warfare\", 0\n",
      "\"felony\", 0\n",
      "\"supervisory board\", 0\n",
      "\"racehorse owners and breeders\", 0\n",
      "\"Privy Council of Thailand\", 0\n",
      "\"Norwegian School of Sport Sciences\", 0\n",
      "\"massage\", 0\n",
      "\"voice\", 0\n",
      "\"mayor of Albacete\", 0\n",
      "\"antisemite\", 0\n",
      "\"Named Professor\", 0\n",
      "\"Iceman\", 0\n",
      "\"District Chief Executive\", 0\n",
      "\"Marketing management\", 0\n",
      "\"restaurant\", 0\n",
      "\"grief counseling\", 0\n",
      "\"labour movement\", 0\n",
      "\"grocery store\", 0\n",
      "\"exegesis\", 0\n",
      "\"King Momo\", 0\n",
      "\"seinen\", 0\n",
      "\"streaker\", 0\n",
      "\"comic strip\", 0\n",
      "\"manufactory\", 0\n",
      "\"Faujdar\", 0\n",
      "\"Assist\", 0\n",
      "\"cinematograph\", 0\n",
      "\"management consulting industry\", 0\n",
      "\"Aconitum\", 0\n",
      "\"Category:Composers\", 0\n",
      "\"Club Deportivo Guadalajara\", 0\n",
      "\"wanker\", 0\n",
      "\"roller skating\", 0\n",
      "\"Claretians\", 0\n",
      "\"music organisation\", 0\n",
      "\"Visor\", 0\n",
      "\"active duty\", 0\n",
      "\"Candidate of Biology Sciences\", 0\n",
      "\"Professors\", 0\n",
      "\"dialogue\", 0\n",
      "\"portrait\", 0\n",
      "\"Gagik Tsarukyan\", 0\n",
      "\"Pensador\", 0\n",
      "\"harpsichord\", 0\n",
      "\"Once Caldas\", 0\n",
      "\"Zehntner\", 0\n",
      "\"member of the Senate of France\", 0\n",
      "\"Screen Actors Guild\", 0\n",
      "\"Idol\", 0\n",
      "\"pension\", 0\n",
      "\"Akim\", 0\n",
      "\"maritime transport\", 0\n",
      "\"bronze sculpture\", 0\n",
      "\"Nadadores\", 0\n",
      "\"odalisque\", 0\n",
      "\"maltster\", 0\n",
      "\"person of short stature\", 0\n",
      "\"interaction design\", 0\n",
      "\"signatory\", 0\n",
      "\"Rector of Charles University\", 0\n",
      "\"Insertion\", 0\n",
      "\"General of the Infantry\", 0\n",
      "\"fortepiano\", 0\n",
      "\"Democratic Party\", 0\n",
      "\"Varela Project\", 0\n",
      "\"MV Explorer\", 0\n",
      "\"Captain of the guard\", 0\n",
      "\"reiki master\", 0\n",
      "\"kelvin\", 0\n",
      "\"Counselor-in-Training\", 0\n",
      "\"pauperism\", 0\n",
      "\"Director-General of the National Heritage Board\", 0\n",
      "\"Holocaust denial\", 0\n",
      "\"Judge Advocate General of the Armed Forces\", 0\n",
      "\"accused\", 0\n",
      "\"software\", 0\n",
      "\"Mirza\", 0\n",
      "\"rescue\", 0\n",
      "\"Barclays\", 0\n",
      "\"proxy\", 0\n",
      "\"female\", 0\n",
      "\"informal attire\", 0\n",
      "\"vegetarian\", 0\n",
      "\"Leadman\", 0\n",
      "\"agricultural cooperative\", 0\n",
      "\"Caesar\", 0\n",
      "\"proselyte\", 0\n",
      "\"personality\", 0\n",
      "\"infomercial\", 0\n",
      "\"sinecure\", 0\n",
      "\"lur\", 0\n",
      "\"passenger\", 0\n",
      "\"Sonderkommando\", 0\n",
      "\"masonic lodge\", 0\n",
      "\"doctor's office\", 0\n",
      "\"General Officer Commanding\", 0\n",
      "\"spectrometer\", 0\n",
      "\"Member of the Congress of Deputies of Spain\", 0\n",
      "\"missiology\", 0\n",
      "\"Columbia University Libraries\", 0\n",
      "\"Faqih\", 0\n",
      "\"Deputy Commander, ROK/US Combined Forces Command\", 0\n",
      "\"Delegado\", 0\n",
      "\"schoolbook publisher\", 0\n",
      "\"Methodist\", 0\n",
      "\"herrnhutare\", 0\n",
      "\"rotary tiller\", 0\n",
      "\"Werkmeister\", 0\n",
      "\"fyrmästare\", 0\n",
      "\"canonge lectoral\", 0\n",
      "\"Cronista\", 0\n",
      "\"Vorstand\", 0\n",
      "\"Patrono\", 0\n",
      "\"Pilote\", 0\n",
      "\"pietist\", 0\n",
      "\"JK\", 0\n",
      "\"patostreamer\", 0\n",
      "\"Ambassadeur\", 0\n",
      "\"kryddkrämare\", 0\n",
      "\"Bokser\", 0\n",
      "\"Procura\", 0\n",
      "\"Creative\", 0\n",
      "\"ski instructor\", 0\n",
      "\"prophets of Islam\", 0\n",
      "\"AV Idol\", 0\n",
      "\"Celtic studies\", 0\n",
      "\"art exhibition curator\", 0\n",
      "\"fictional cowboy\", 0\n",
      "\"golf course designer\", 0\n",
      "\"Mosso d'Esquadra\", 0\n",
      "\"public administration\", 0\n",
      "\"food manufacturing\", 0\n",
      "\"Schout-bij-nacht\", 0\n",
      "\"Nguyen dynasty\", 0\n",
      "\"former comfort women\", 0\n",
      "\"Bishops of Lyon\", 0\n",
      "\"health minister\", 0\n",
      "\"public accounting\", 0\n",
      "\"door-to-door\", 0\n",
      "\"Spanish Navy\", 0\n",
      "\"Industrial Workers of the World\", 0\n",
      "\"Traffic engineer\", 0\n",
      "\"network management\", 0\n",
      "\"White Fathers\", 0\n",
      "\"medical assistant\", 0\n",
      "\"policy analysis\", 0\n",
      "\"broker maritime\", 0\n",
      "\"English boxing\", 0\n",
      "\"weather manipulation\", 0\n",
      "\"Indian Civil Service\", 0\n",
      "\"environmental education\", 0\n",
      "\"electronic data processing\", 0\n",
      "\"construction engineering\", 0\n",
      "\"Sequoia National Park\", 0\n",
      "\"youth sports minister\", 0\n",
      "\"high tech\", 0\n",
      "\"Hellanodikai\", 0\n",
      "\"Artistic cycling\", 0\n",
      "\"Mechurchletukhutsesi\", 0\n",
      "\"Film Writers Association\", 0\n",
      "\"antique shop\", 0\n",
      "\"industrial ecology\", 0\n",
      "\"Apostoł\", 0\n",
      "\"Tournament director\", 0\n",
      "\"backing vocal\", 0\n",
      "\"Trubadurzy\", 0\n",
      "\"YouNow\", 0\n",
      "\"sports tourism\", 0\n",
      "\"Landwirtschaftsrat\", 0\n",
      "\"hemslöjdsinstruktör\", 0\n",
      "\"Logistyka dystrybucji\", 0\n",
      "\"Государственный советник\", 0\n",
      "\"Спацијализам\", 0\n",
      "\"Военный\", 0\n",
      "\"Rakennusarkkitehti\", 0\n",
      "\"maadleja\", 0\n",
      "\"népnevelő\", 0\n"
     ]
    }
   ],
   "source": [
    "idx_name = {idx : name for idx, name in enumerate(pos_eg + neg_eg + unlabeled)}\n",
    "# we'll sort the bad labels for easier review\n",
    "questionable_good = []\n",
    "questionable_bad = []\n",
    "for bad_idx in label_errors_idx:\n",
    "    true_name = idx_name[bad_idx]\n",
    "    the_idx = occs_df.index[ occs_df['occupation'] == true_name].tolist()[0]\n",
    "    if int(occs_df.iloc[the_idx]['label']) ==0:\n",
    "        questionable_bad.append((occs_df.iloc[the_idx]['occupation'], occs_df.iloc[the_idx]['label']))\n",
    "    if int(occs_df.iloc[the_idx]['label']) ==1:\n",
    "        questionable_good.append((occs_df.iloc[the_idx]['occupation'], occs_df.iloc[the_idx]['label']))\n",
    "for name, label in questionable_good + questionable_bad:\n",
    "    print('\"{}\", {}'.format(name, label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update the dataframe with manually adjusted labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 31/84 [00:00<00:00, 309.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 84/84 [00:00<00:00, 316.71it/s]\n",
      "100%|██████████| 94/94 [00:00<00:00, 330.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "bad_bert_occs = [\"shamakhi dancers\", \"circus arts\", \"yaksha\", \"Protestant theology\", \"architectural photography\", \"lumières\", \"consumer protection\", \"Jerome Mayo Greenberg\", \"role-playing game\", \"media literacy\", \"Pensioner Guards\", \"social anthropology\", \"Aviation historians\", \"short story\", \"communication medium\", \"bleachery\", \"video on demand\", \"armed struggle\", \"traditional song\", \"Rugby league match officials\", \"zootechnics\", \"special effects\", \"plastic surgery\", \"male prostitution\", \"deported French resistance\", \"china painting\", \"Perm Governorate\", \"Aalto University\", \"A Few Good Men\", \"beer pouring\", \"transport planning\", \"Chinese calligraphy\", \"local government\", \"natural philosophy\", \"petroleum-gas industry\", \"Transactions of the Royal Society of Tropical Medicine and Hygiene\", \"local history\", \"fackskolelärare\", \"digital photography\", \"social engagement\", \"Principle of Aberdeen University\", \"local authority\", \"computational science\", \"animal control service\", \"skolförestånderska\", \"Academy of Finland\", \"sigillography\", \"voice-over\", \"musical instrument\", \"Salinas de Ibargoiti\", \"Saint Petersburg State University\", \"Tobolsk Governorate\", \"General Francos's opposition\", \"Erivan Governorate\", \"staff and line\", \"Người dẫn chương trình truyền hình (chương trình TV)\", \"製本職人\", \"29 January 28\", \"Brigade RISTA - Electronic Warfare Intelligence\", \"Bomberos Voluntarios (Guatemala)\", \"London South Bank University\", \"supreme court\", \"Gubkin University\", \"High Priests of Amun\", \"community service\", \"Postgraduate Work\", \"Old lesbians: Gendered histories and persistent challenges.\", \"Canvas print\", \"theatrical makeup\", \"list of type designers\", \"Nazi plunder\", \"Domestikos\", \"Catholicos of All Armenians\", \"conservation movement\", \"arts administration\", \"Medical Specimens\", \"political analysis\", \"Shadow Theatre\", \"capo dei capi\", \"Letrados\", \"La Révolution prolétarienne\", \"Ña Catita\", \"Sofer\", \"skogsägare\", \"akademifogde\", \"organizational leadership\", \"Carabinieri\", \"furniture construction\", \"capitano del popolo\", \"Željko Ražnatović\", \"土木技師\", \"space exploration\", \"major police MVD\", \"dollmaking\"]\n",
    "bert_good_occs = [\"Ambassadeur\", \"Chief Executive\", \"Named Professor\", \"folk hero\", \"cinephile\", \"peasant\", \"District Chief Executive\", \"scholar of the bible as literature\", \"Captain of the guard\", \"marxist\", \"Werkmeister\", \"Franciscan\", \"public figure\", \"land surveyor in Poland\", \"Member of the Congress of Deputies of Spain\", \"Jägermeister\", \"galley slave\", \"Member of Parliament in the Parliament of England\", \"Young Guard\", \"madam\", \"pietist\", \"Nazi\", \"patriarch\", \"Judge Advocate General of the Armed Forces\", \"competition judge\", \"diving instructor\", \"King of Jerusalem\", \"Tournament director\", \"Deputy Commander, ROK/US Combined Forces Command\", \"proselyte\", \"principal of Uppsala University\", \"headmaster in France\", \"reiki master\", \"Spiker\", \"combined track and field event athlete\", \"mayor of Albacete\", \"medical assistant\", \"research assistant\", \"Universal Esperanto Association committee member\", \"groom\", \"personality\", \"celebrity\", \"centenarian\", \"Naturaliste\", \"superhero\", \"sinecure\", \"General Officer Commanding\", \"revenger\", \"Candidate of Biology Sciences\", \"human\", \"Freiherr\", \"tsarina\", \"member of the Senate of France\", \"erudite\", \"delegate\", \"passenger\", \"youth sports minister\", \"vegetarian\", \"wanker\", \"streamer\", \"mythological king\", \"internet celebrity\", \"Polymath\", \"charcoal burner\", \"Director-General of the National Heritage Board\", \"General of the Infantry\", \"streaker\", \"pacemaker\", \"person of short stature\", \"Iceman\", \"vase painter\", \"mother\", \"patostreamer\", \"schoolbook publisher\", \"registrar\", \"supercentenarian\", \"fado singer\", \"Tattooed Lady\", \"maltster\", \"cure doctor\", \"Jedi\", \"elder\", \"building contractor\", \"court Jew\"]\n",
    "\n",
    "print(f\"Dataframe length before update {len(occs_df):,}\")\n",
    "for occ in tqdm(bert_good_occs, total=len(bert_good_occs)):\n",
    "    if occ in occs_df['occupation'].tolist():\n",
    "        the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "        the_index = the_row.index[0]\n",
    "        occs_df.loc[the_index, 'label'] = 1\n",
    "        occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'\n",
    "for occ in tqdm(bad_bert_occs, total=len(bad_bert_occs)):\n",
    "    if occ in occs_df['occupation'].tolist():\n",
    "        the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "        the_index = the_row.index[0]\n",
    "        occs_df.loc[the_index, 'label'] = 0\n",
    "        occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'    \n",
    "print(f\"Dataframe length after update {len(occs_df):,}\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleanlab, 2nd pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_label_names(idx_label_map, label_errors_idx):\n",
    "    label_error_names = [idx_label_map.get(tmp) for tmp in label_errors_idx]\n",
    "\n",
    "    # We'll sort the labels alphabetically and by label value for easier manual review\n",
    "    label_error_names.sort()\n",
    "    pos_labels = []\n",
    "    neg_labels = []\n",
    "    for name in label_error_names:\n",
    "        key_name = name.replace('_', ' ')\n",
    "        val = int(occs_df[occs_df['occupation'] == key_name].label )\n",
    "        if val ==1:\n",
    "            pos_labels.append((key_name, val))\n",
    "        else:\n",
    "            neg_labels.append((key_name, val))\n",
    "    for key_name, val in pos_labels + neg_labels:\n",
    "        print('\"{}\", {}'.format(key_name, val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0412 22:33:45.533503 4709707200 configuration_utils.py:254] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-config.json from cache at /Users/todd/.cache/torch/transformers/90deb4d9dd705272dc4b3db1364d759d551d72a9f70a91f60e3a1f5e278b985d.960df871e8c3e87cb781ccdbdd12a3c1e0ad83b173a9754bd183354449046e38\n",
      "I0412 22:33:45.535501 4709707200 configuration_utils.py:290] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": 0,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 1024,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 4096,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0412 22:33:45.958415 4709707200 modeling_utils.py:458] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-pytorch_model.bin from cache at /Users/todd/.cache/torch/transformers/56c451878be53ca1e310764d1e8312301f3d921378919467947ddd53fef6ba2b.b5f1c2070e0a0c189ca3b08270b0cb5bd0635b7319e74e93bd0dc26689953c27\n",
      "100%|██████████| 2085/2085 [04:35<00:00,  7.58it/s]\n",
      "100%|██████████| 1435/1435 [03:20<00:00,  7.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"Abogado\", 1\n",
      "\"Advokat\", 1\n",
      "\"Archer\", 1\n",
      "\"Assistant Secretary\", 1\n",
      "\"Avocat\", 1\n",
      "\"Bedel\", 1\n",
      "\"Bokor\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Burgess\", 1\n",
      "\"Cacique\", 1\n",
      "\"Censor\", 1\n",
      "\"Dempster\", 1\n",
      "\"Dewan\", 1\n",
      "\"Drayman\", 1\n",
      "\"Evangelist\", 1\n",
      "\"Expeditor\", 1\n",
      "\"Fechter\", 1\n",
      "\"Freedom Fighters\", 1\n",
      "\"Fundi\", 1\n",
      "\"Gendarme\", 1\n",
      "\"General\", 1\n",
      "\"Grammarian\", 1\n",
      "\"Gurkha\", 1\n",
      "\"Hakim\", 1\n",
      "\"Heir apparent\", 1\n",
      "\"Hofmeister\", 1\n",
      "\"Imagineer\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Juris Doctor\", 1\n",
      "\"Jägermeister\", 1\n",
      "\"Khatib\", 1\n",
      "\"Lascar\", 1\n",
      "\"MasterChef\", 1\n",
      "\"Mawla\", 1\n",
      "\"Met Éireann\", 1\n",
      "\"Mohel\", 1\n",
      "\"Mormon missionary\", 1\n",
      "\"Operator\", 1\n",
      "\"Patrick\", 1\n",
      "\"Poeta\", 1\n",
      "\"Pythia\", 1\n",
      "\"Rangatira\", 1\n",
      "\"Reichsmarschall\", 1\n",
      "\"Revolutionary\", 1\n",
      "\"Richter\", 1\n",
      "\"Rynda\", 1\n",
      "\"Schiffer\", 1\n",
      "\"Sepoy\", 1\n",
      "\"Shah\", 1\n",
      "\"Shihan\", 1\n",
      "\"Soldat\", 1\n",
      "\"Student\", 1\n",
      "\"Thuggee\", 1\n",
      "\"Tulku\", 1\n",
      "\"Zen master\", 1\n",
      "\"abbey\", 1\n",
      "\"academic\", 1\n",
      "\"artisan\", 1\n",
      "\"associate\", 1\n",
      "\"autopilot\", 1\n",
      "\"ballet dancer\", 1\n",
      "\"banker\", 1\n",
      "\"baritone\", 1\n",
      "\"baron\", 1\n",
      "\"bartender\", 1\n",
      "\"billionaire\", 1\n",
      "\"communist\", 1\n",
      "\"cook\", 1\n",
      "\"corporal\", 1\n",
      "\"counsel\", 1\n",
      "\"count\", 1\n",
      "\"counterintelligence\", 1\n",
      "\"cowboy\", 1\n",
      "\"crank\", 1\n",
      "\"dairy farmer\", 1\n",
      "\"dealer\", 1\n",
      "\"dresser\", 1\n",
      "\"duce\", 1\n",
      "\"expert\", 1\n",
      "\"feminist\", 1\n",
      "\"fundraiser\", 1\n",
      "\"ghost\", 1\n",
      "\"gigolo\", 1\n",
      "\"gladiator\", 1\n",
      "\"goddess\", 1\n",
      "\"gymnast\", 1\n",
      "\"hobo\", 1\n",
      "\"human\", 1\n",
      "\"inebriate\", 1\n",
      "\"intensivist\", 1\n",
      "\"jackaroo\", 1\n",
      "\"knight\", 1\n",
      "\"kumu hula\", 1\n",
      "\"lady\", 1\n",
      "\"lama\", 1\n",
      "\"lawyer\", 1\n",
      "\"mannequin\", 1\n",
      "\"mason\", 1\n",
      "\"mullah\", 1\n",
      "\"narrator\", 1\n",
      "\"nuncio\", 1\n",
      "\"paraglider\", 1\n",
      "\"parent\", 1\n",
      "\"partner\", 1\n",
      "\"pasha\", 1\n",
      "\"pastry shop\", 1\n",
      "\"patron saint\", 1\n",
      "\"paymaster\", 1\n",
      "\"photographer\", 1\n",
      "\"psychiatrist\", 1\n",
      "\"recorder\", 1\n",
      "\"referee\", 1\n",
      "\"rikishi\", 1\n",
      "\"runner\", 1\n",
      "\"santero\", 1\n",
      "\"sculptor\", 1\n",
      "\"shipyard\", 1\n",
      "\"signwriter\", 1\n",
      "\"spammer\", 1\n",
      "\"suffragan bishop\", 1\n",
      "\"sultan\", 1\n",
      "\"tourist\", 1\n",
      "\"trainer\", 1\n",
      "\"unemployed\", 1\n",
      "\"venture capitalist\", 1\n",
      "\"virgin\", 1\n",
      "\"Assist\", 0\n",
      "\"Boomer\", 0\n",
      "\"Buddha\", 0\n",
      "\"Celedon\", 0\n",
      "\"Creative\", 0\n",
      "\"Cronista\", 0\n",
      "\"English\", 0\n",
      "\"Faqih\", 0\n",
      "\"Faujdar\", 0\n",
      "\"Francisco Franco\", 0\n",
      "\"Georg\", 0\n",
      "\"Idol\", 0\n",
      "\"Laity\", 0\n",
      "\"Leadman\", 0\n",
      "\"Maharaja\", 0\n",
      "\"Majordomo\", 0\n",
      "\"Margrave\", 0\n",
      "\"Minutemen\", 0\n",
      "\"Mirza\", 0\n",
      "\"Mujahid\", 0\n",
      "\"Musicians\", 0\n",
      "\"Naik\", 0\n",
      "\"Organista\", 0\n",
      "\"Polier\", 0\n",
      "\"Private\", 0\n",
      "\"Professors\", 0\n",
      "\"Raja\", 0\n",
      "\"Rinpoche\", 0\n",
      "\"Rolf\", 0\n",
      "\"Screen Actors Guild\", 0\n",
      "\"St. Louis\", 0\n",
      "\"Strategos\", 0\n",
      "\"Venerable\", 0\n",
      "\"Winzer\", 0\n",
      "\"Zamindar\", 0\n",
      "\"accounting\", 0\n",
      "\"accused\", 0\n",
      "\"acharya\", 0\n",
      "\"antisemite\", 0\n",
      "\"athletics\", 0\n",
      "\"bikers\", 0\n",
      "\"blog\", 0\n",
      "\"bronze sculpture\", 0\n",
      "\"cadaver\", 0\n",
      "\"cadre\", 0\n",
      "\"café\", 0\n",
      "\"cartoon\", 0\n",
      "\"cavalier\", 0\n",
      "\"cinematograph\", 0\n",
      "\"comic strip\", 0\n",
      "\"commercial\", 0\n",
      "\"consulting\", 0\n",
      "\"cornet\", 0\n",
      "\"curandero\", 0\n",
      "\"daimyo\", 0\n",
      "\"demimonde\", 0\n",
      "\"dialogue\", 0\n",
      "\"donor\", 0\n",
      "\"drama\", 0\n",
      "\"drum kit\", 0\n",
      "\"fado\", 0\n",
      "\"fan\", 0\n",
      "\"father\", 0\n",
      "\"felony\", 0\n",
      "\"female\", 0\n",
      "\"foley\", 0\n",
      "\"fountain pen\", 0\n",
      "\"geometer\", 0\n",
      "\"hedge fund\", 0\n",
      "\"herald\", 0\n",
      "\"homicide\", 0\n",
      "\"hostage\", 0\n",
      "\"insurance\", 0\n",
      "\"janissary\", 0\n",
      "\"kelvin\", 0\n",
      "\"lever\", 0\n",
      "\"literacy\", 0\n",
      "\"magi\", 0\n",
      "\"massage\", 0\n",
      "\"monster\", 0\n",
      "\"mythical creature\", 0\n",
      "\"nigger\", 0\n",
      "\"oblate\", 0\n",
      "\"patronage\", 0\n",
      "\"pharmacy\", 0\n",
      "\"portrait\", 0\n",
      "\"professional\", 0\n",
      "\"pseudonym\", 0\n",
      "\"qadi\", 0\n",
      "\"remix\", 0\n",
      "\"rescue\", 0\n",
      "\"rotary tiller\", 0\n",
      "\"royalty\", 0\n",
      "\"satrap\", 0\n",
      "\"skeleton\", 0\n",
      "\"technology\", 0\n",
      "\"versatile\", 0\n",
      "\"victim\", 0\n",
      "\"viola\", 0\n",
      "\"volleyball\", 0\n",
      "\"wiper\", 0\n",
      "\"witchcraft\", 0\n"
     ]
    }
   ],
   "source": [
    "good_occs = occs_df.query(\"in_google_news ==1 and label ==1\")['occupation'].tolist()\n",
    "bad_occs = occs_df.query(\"in_google_news ==1 and label ==0\")['occupation'].tolist()\n",
    "y = np.concatenate([np.ones(len(good_occs), dtype=np.int32), np.zeros(len(bad_occs), dtype=np.int32)])\n",
    "\n",
    "X =[]\n",
    "model = BertModel.from_pretrained('bert-large-cased')\n",
    "model.eval()\n",
    "model.to(device)\n",
    "with torch.no_grad():\n",
    "    for name in tqdm(good_occs, total=len(good_occs)):\n",
    "        input_ids = torch.tensor(tokenizer.encode(name)).unsqueeze(0).to(device)  # Batch size 1\n",
    "        outputs = model(input_ids)\n",
    "        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "        X.append(last_hidden_states[0][0].detach().cpu().numpy())\n",
    "    for name in tqdm(bad_occs, total=len(bad_occs)):\n",
    "        input_ids = torch.tensor(tokenizer.encode(name)).unsqueeze(0).to(device)  # Batch size 1\n",
    "        outputs = model(input_ids)\n",
    "        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
    "        X.append(last_hidden_states[0][0].detach().cpu().numpy())\n",
    "X = np.array(X)\n",
    "X.shape\n",
    "\n",
    "all_words = good_occs + bad_occs\n",
    "\n",
    "# We'll save the mapping of index to words for decoding\n",
    "idx_label_map = {}\n",
    "for idx, name in enumerate(all_words):\n",
    "    idx_label_map[idx]= name\n",
    "    \n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    X, y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, y)\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, y)\n",
    "\n",
    "print_label_names(idx_label_map, label_errors_idx)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"Abogado\", 1\n",
      "\"Advokat\", 1\n",
      "\"Archer\", 1\n",
      "\"Assistant Secretary\", 1\n",
      "\"Avocat\", 1\n",
      "\"Bedel\", 1\n",
      "\"Bokor\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Burgess\", 1\n",
      "\"Cacique\", 1\n",
      "\"Censor\", 1\n",
      "\"Dempster\", 1\n",
      "\"Dewan\", 1\n",
      "\"Drayman\", 1\n",
      "\"Evangelist\", 1\n",
      "\"Expeditor\", 1\n",
      "\"Fechter\", 1\n",
      "\"Freedom Fighters\", 1\n",
      "\"Fundi\", 1\n",
      "\"Gendarme\", 1\n",
      "\"General\", 1\n",
      "\"Grammarian\", 1\n",
      "\"Gurkha\", 1\n",
      "\"Hakim\", 1\n",
      "\"Heir apparent\", 1\n",
      "\"Hofmeister\", 1\n",
      "\"Imagineer\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Juris Doctor\", 1\n",
      "\"Jägermeister\", 1\n",
      "\"Khatib\", 1\n",
      "\"Lascar\", 1\n",
      "\"MasterChef\", 1\n",
      "\"Mawla\", 1\n",
      "\"Met Éireann\", 1\n",
      "\"Mohel\", 1\n",
      "\"Mormon missionary\", 1\n",
      "\"Operator\", 1\n",
      "\"Patrick\", 1\n",
      "\"Poeta\", 1\n",
      "\"Pythia\", 1\n",
      "\"Rangatira\", 1\n",
      "\"Reichsmarschall\", 1\n",
      "\"Revolutionary\", 1\n",
      "\"Richter\", 1\n",
      "\"Rynda\", 1\n",
      "\"Schiffer\", 1\n",
      "\"Sepoy\", 1\n",
      "\"Shah\", 1\n",
      "\"Shihan\", 1\n",
      "\"Soldat\", 1\n",
      "\"Student\", 1\n",
      "\"Thuggee\", 1\n",
      "\"Tulku\", 1\n",
      "\"Zen master\", 1\n",
      "\"abbey\", 1\n",
      "\"academic\", 1\n",
      "\"artisan\", 1\n",
      "\"associate\", 1\n",
      "\"autopilot\", 1\n",
      "\"ballet dancer\", 1\n",
      "\"banker\", 1\n",
      "\"baritone\", 1\n",
      "\"baron\", 1\n",
      "\"bartender\", 1\n",
      "\"billionaire\", 1\n",
      "\"communist\", 1\n",
      "\"cook\", 1\n",
      "\"corporal\", 1\n",
      "\"counsel\", 1\n",
      "\"count\", 1\n",
      "\"counterintelligence\", 1\n",
      "\"cowboy\", 1\n",
      "\"crank\", 1\n",
      "\"dairy farmer\", 1\n",
      "\"dealer\", 1\n",
      "\"dresser\", 1\n",
      "\"duce\", 1\n",
      "\"expert\", 1\n",
      "\"feminist\", 1\n",
      "\"fundraiser\", 1\n",
      "\"ghost\", 1\n",
      "\"gigolo\", 1\n",
      "\"gladiator\", 1\n",
      "\"goddess\", 1\n",
      "\"gymnast\", 1\n",
      "\"hobo\", 1\n",
      "\"human\", 1\n",
      "\"inebriate\", 1\n",
      "\"intensivist\", 1\n",
      "\"jackaroo\", 1\n",
      "\"knight\", 1\n",
      "\"kumu hula\", 1\n",
      "\"lady\", 1\n",
      "\"lama\", 1\n",
      "\"lawyer\", 1\n",
      "\"mannequin\", 1\n",
      "\"mason\", 1\n",
      "\"mullah\", 1\n",
      "\"narrator\", 1\n",
      "\"nuncio\", 1\n",
      "\"paraglider\", 1\n",
      "\"parent\", 1\n",
      "\"partner\", 1\n",
      "\"pasha\", 1\n",
      "\"pastry shop\", 1\n",
      "\"patron saint\", 1\n",
      "\"paymaster\", 1\n",
      "\"photographer\", 1\n",
      "\"psychiatrist\", 1\n",
      "\"recorder\", 1\n",
      "\"referee\", 1\n",
      "\"rikishi\", 1\n",
      "\"runner\", 1\n",
      "\"santero\", 1\n",
      "\"sculptor\", 1\n",
      "\"shipyard\", 1\n",
      "\"signwriter\", 1\n",
      "\"spammer\", 1\n",
      "\"suffragan bishop\", 1\n",
      "\"sultan\", 1\n",
      "\"tourist\", 1\n",
      "\"trainer\", 1\n",
      "\"unemployed\", 1\n",
      "\"venture capitalist\", 1\n",
      "\"virgin\", 1\n",
      "\"Assist\", 0\n",
      "\"Boomer\", 0\n",
      "\"Buddha\", 0\n",
      "\"Celedon\", 0\n",
      "\"Creative\", 0\n",
      "\"Cronista\", 0\n",
      "\"English\", 0\n",
      "\"Faqih\", 0\n",
      "\"Faujdar\", 0\n",
      "\"Francisco Franco\", 0\n",
      "\"Georg\", 0\n",
      "\"Idol\", 0\n",
      "\"Laity\", 0\n",
      "\"Leadman\", 0\n",
      "\"Maharaja\", 0\n",
      "\"Majordomo\", 0\n",
      "\"Margrave\", 0\n",
      "\"Minutemen\", 0\n",
      "\"Mirza\", 0\n",
      "\"Mujahid\", 0\n",
      "\"Musicians\", 0\n",
      "\"Naik\", 0\n",
      "\"Organista\", 0\n",
      "\"Polier\", 0\n",
      "\"Private\", 0\n",
      "\"Professors\", 0\n",
      "\"Raja\", 0\n",
      "\"Rinpoche\", 0\n",
      "\"Rolf\", 0\n",
      "\"Screen Actors Guild\", 0\n",
      "\"St. Louis\", 0\n",
      "\"Strategos\", 0\n",
      "\"Venerable\", 0\n",
      "\"Winzer\", 0\n",
      "\"Zamindar\", 0\n",
      "\"accounting\", 0\n",
      "\"accused\", 0\n",
      "\"acharya\", 0\n",
      "\"antisemite\", 0\n",
      "\"athletics\", 0\n",
      "\"bikers\", 0\n",
      "\"blog\", 0\n",
      "\"bronze sculpture\", 0\n",
      "\"cadaver\", 0\n",
      "\"cadre\", 0\n",
      "\"café\", 0\n",
      "\"cartoon\", 0\n",
      "\"cavalier\", 0\n",
      "\"cinematograph\", 0\n",
      "\"comic strip\", 0\n",
      "\"commercial\", 0\n",
      "\"consulting\", 0\n",
      "\"cornet\", 0\n",
      "\"curandero\", 0\n",
      "\"daimyo\", 0\n",
      "\"demimonde\", 0\n",
      "\"dialogue\", 0\n",
      "\"donor\", 0\n",
      "\"drama\", 0\n",
      "\"drum kit\", 0\n",
      "\"fado\", 0\n",
      "\"fan\", 0\n",
      "\"father\", 0\n",
      "\"felony\", 0\n",
      "\"female\", 0\n",
      "\"foley\", 0\n",
      "\"fountain pen\", 0\n",
      "\"geometer\", 0\n",
      "\"hedge fund\", 0\n",
      "\"herald\", 0\n",
      "\"homicide\", 0\n",
      "\"hostage\", 0\n",
      "\"insurance\", 0\n",
      "\"janissary\", 0\n",
      "\"kelvin\", 0\n",
      "\"lever\", 0\n",
      "\"literacy\", 0\n",
      "\"magi\", 0\n",
      "\"massage\", 0\n",
      "\"monster\", 0\n",
      "\"mythical creature\", 0\n",
      "\"nigger\", 0\n",
      "\"oblate\", 0\n",
      "\"patronage\", 0\n",
      "\"pharmacy\", 0\n",
      "\"portrait\", 0\n",
      "\"professional\", 0\n",
      "\"pseudonym\", 0\n",
      "\"qadi\", 0\n",
      "\"remix\", 0\n",
      "\"rescue\", 0\n",
      "\"rotary tiller\", 0\n",
      "\"royalty\", 0\n",
      "\"satrap\", 0\n",
      "\"skeleton\", 0\n",
      "\"technology\", 0\n",
      "\"versatile\", 0\n",
      "\"victim\", 0\n",
      "\"viola\", 0\n",
      "\"volleyball\", 0\n",
      "\"wiper\", 0\n",
      "\"witchcraft\", 0\n"
     ]
    }
   ],
   "source": [
    "print_label_names(idx_label_map, label_errors_idx)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:00<00:00, 259.82it/s]\n",
      "100%|██████████| 5/5 [00:00<00:00, 232.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n",
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "bad_occs_manual=[ \"Freedom Fighters\", \"Met Éireann\", \"autopilot\", \"counterintelligence\", \"pastry shop\", ] \n",
    "good_occs_manual = [ \"Faujdar\", \"MV Explorer\", \"Naik\", \"Raja\", \"Rinpoche\", \"Zamindar\", \"daimyo\", \"father\", ]\n",
    "print(f\"Dataframe length before update {len(occs_df):,}\")\n",
    "for occ in tqdm(good_occs_manual, total=len(good_occs_manual)):\n",
    "    if occ in occs_df['occupation'].tolist():\n",
    "        the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "        the_index = the_row.index[0]\n",
    "        occs_df.loc[the_index, 'label'] = 1\n",
    "        occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'\n",
    "for occ in tqdm(bad_occs_manual, total=len(bad_occs_manual)):\n",
    "    if occ in occs_df['occupation'].tolist():\n",
    "        the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "        the_index = the_row.index[0]\n",
    "        occs_df.loc[the_index, 'label'] = 0\n",
    "        occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'    \n",
    "print(f\"Dataframe length after update {len(occs_df):,}\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "occs_df.to_csv('occupations.wikidata.all.labeled.csv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>82955</td>\n",
       "      <td>606008</td>\n",
       "      <td>politician</td>\n",
       "      <td>person involved in politics, person who holds ...</td>\n",
       "      <td>1</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>189290</td>\n",
       "      <td>33481</td>\n",
       "      <td>military officer</td>\n",
       "      <td>member of an armed force or uniformed service ...</td>\n",
       "      <td>0</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>bert</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>131512</td>\n",
       "      <td>8887</td>\n",
       "      <td>farmer</td>\n",
       "      <td>person that works in agriculture</td>\n",
       "      <td>1</td>\n",
       "      <td>da</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1734662</td>\n",
       "      <td>2012</td>\n",
       "      <td>cartographer</td>\n",
       "      <td>person preparing geographical maps</td>\n",
       "      <td>1</td>\n",
       "      <td>ia</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>294126</td>\n",
       "      <td>323</td>\n",
       "      <td>land surveyor</td>\n",
       "      <td>profession</td>\n",
       "      <td>0</td>\n",
       "      <td>tr</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>bert</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id  occupation_count        occupation  \\\n",
       "0    82955            606008        politician   \n",
       "1   189290             33481  military officer   \n",
       "2   131512              8887            farmer   \n",
       "3  1734662              2012      cartographer   \n",
       "4   294126               323     land surveyor   \n",
       "\n",
       "                                         description  in_google_news  \\\n",
       "0  person involved in politics, person who holds ...               1   \n",
       "1  member of an armed force or uniformed service ...               0   \n",
       "2                   person that works in agriculture               1   \n",
       "3                 person preparing geographical maps               1   \n",
       "4                                         profession               0   \n",
       "\n",
       "  language_detected    source  label labeled_by  label_error_reason  \n",
       "0                en  wikidata      1      human                 NaN  \n",
       "1                en  wikidata      1       bert                 NaN  \n",
       "2                da  wikidata      1      human                 NaN  \n",
       "3                ia  wikidata      1      human                 NaN  \n",
       "4                tr  wikidata      1       bert                 NaN  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The dataset is now labeled and verified. Next we will use it to generate negative examples in preparing the Employers dataset"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}