labeling_data/correcting_GoogleNews_labels_with_Cleanlab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correcting GoogleNews labels with Cleanlab\n",
    "In this notebook, we will use [Cleanlab](https://github.com/cgnorthcutt/cleanlab) to assess how well we did with our labeling effort.\n",
    "We'll update the data set and correct some inconsistencies that Cleanlab indicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from random import sample\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np \n",
    "import cleanlab\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import inspect\n",
    "from pathlib import Path \n",
    "currentdir = Path.cwd()\n",
    "parentdir = os.path.dirname(currentdir)\n",
    "sys.path.insert(0,parentdir) \n",
    "\n",
    "from mlyoucanuse.embeddings import get_embeddings_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3000000/3000000 [00:01<00:00, 1784789.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 15s, sys: 8.05 s, total: 2min 23s\n",
      "Wall time: 2min 28s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "gnews_embed = get_embeddings_index('GoogleNews', parent_dir=parentdir, embedding_dimensions=300)\n",
    "gnews_vocab = {tmp for tmp in tqdm(gnews_embed.keys())} \n",
    "sample(gnews_vocab, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of occupations: 10,974\n"
     ]
    }
   ],
   "source": [
    "occs_df = pd.read_csv('occupations.wikidata.all.gnews.labeled.csv', sep='\\t')\n",
    "print(f\"Number of occupations: {len(occs_df):,}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10969</th>\n",
       "      <td>10526680</td>\n",
       "      <td>1</td>\n",
       "      <td>hovkamrerare</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>sl</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10970</th>\n",
       "      <td>10526703</td>\n",
       "      <td>1</td>\n",
       "      <td>Hovrättspresident</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>sv</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10971</th>\n",
       "      <td>66486266</td>\n",
       "      <td>1</td>\n",
       "      <td>activista taurí</td>\n",
       "      <td>subclase de activista</td>\n",
       "      <td>0</td>\n",
       "      <td>oc</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10972</th>\n",
       "      <td>360443</td>\n",
       "      <td>1</td>\n",
       "      <td>Escort</td>\n",
       "      <td>Wikimedia disambiguation page</td>\n",
       "      <td>1</td>\n",
       "      <td>oc</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10973</th>\n",
       "      <td>87252988</td>\n",
       "      <td>1</td>\n",
       "      <td>弓道家</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id  occupation_count         occupation  \\\n",
       "10969  10526680                 1       hovkamrerare   \n",
       "10970  10526703                 1  Hovrättspresident   \n",
       "10971  66486266                 1    activista taurí   \n",
       "10972    360443                 1             Escort   \n",
       "10973  87252988                 1                弓道家   \n",
       "\n",
       "                         description  in_google_news language_detected  \\\n",
       "10969                            NaN               0                sl   \n",
       "10970                            NaN               0                sv   \n",
       "10971          subclase de activista               0                oc   \n",
       "10972  Wikimedia disambiguation page               1                oc   \n",
       "10973                            NaN               0           Unknown   \n",
       "\n",
       "         source  label labeled_by  label_error_reason  \n",
       "10969  wikidata     -1        NaN                 NaN  \n",
       "10970  wikidata     -1        NaN                 NaN  \n",
       "10971  wikidata     -1        NaN                 NaN  \n",
       "10972  wikidata      1      human                 NaN  \n",
       "10973  wikidata     -1        NaN                 NaN  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of single occupations in GNews and Wikidata: 2,937\n",
      "Number of compound occupations in GNews and Wikidata: 3,520\n",
      "Number of positive examples: 1,950; negative examples: 1,570\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(['pensioner', 'mascot', 'firefighter', 'gallerist', 'student'],\n",
       " ['Nanga', 'newspaper', 'ballet', 'delegate', 'videography'])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_occupation_names = occs_df['occupation'].tolist()\n",
    "all_occ_names_gnews_format = [ tmp.replace(' ', '_') for tmp in all_occupation_names ]\n",
    "\n",
    "print(f\"Number of single occupations in GNews and Wikidata: {len(gnews_vocab & set(all_occupation_names)):,}\")\n",
    "print(f\"Number of compound occupations in GNews and Wikidata: {len(gnews_vocab & set(all_occ_names_gnews_format)):,}\")\n",
    "\n",
    "good_occs = occs_df.query(\"in_google_news ==1 and label ==1\")['occupation'].tolist()\n",
    "bad_occs = occs_df.query(\"in_google_news ==1 and label ==0 \")['occupation'].tolist()\n",
    "print(f\"Number of positive examples: {len(good_occs):,}; negative examples: {len(bad_occs):,}\") \n",
    "\n",
    "sample(good_occs, 5), sample(bad_occs, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3520, 300), (3520,))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.concatenate([np.ones(len(good_occs), dtype=np.int32), np.zeros(len(bad_occs), dtype=np.int32)])\n",
    "\n",
    "all_words = good_occs + bad_occs\n",
    "all_words = [tmp.replace(' ', '_') for tmp in all_words]\n",
    "\n",
    "# We'll save the mapping of index to words for decoding\n",
    "idx_label_map = {}\n",
    "for idx, name in enumerate(all_words):\n",
    "    idx_label_map[idx]= name\n",
    "    \n",
    "X = [gnews_embed[tmp] for tmp in all_words]\n",
    "X = np.array(X)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Cleanlab to find potentially problematic labels\n",
    "    * Get out-of-sample predicted probabilities using cross-validation\n",
    "    * Compute confident joint\n",
    "    * Find label errors    \n",
    "    \n",
    "## Step 1: Get out-of-sample predicted probabilities using cross-validation    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using a simple, non-optimized logistic regression classifier; the cross-validation will expose label weaknesses\n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    X, y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Compute confident joint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Joint Label Noise Distribution Matrix P(s,y) of shape (2, 2)\n",
      " p(s,y)\ty=0\ty=1\n",
      "\t---\t---\n",
      "s=0 |\t1469\t101\n",
      "s=1 |\t117\t1833\n",
      "\tTrace(matrix) = 3302\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def compute_confident_joint(psx, y):\n",
    "    # Verify inputs\n",
    "    psx = np.asarray(psx)\n",
    "\n",
    "    # Find the number of unique classes if K is not given\n",
    "    K = len(np.unique(y))\n",
    "\n",
    "    # Estimate the probability thresholds for confident counting\n",
    "    # You can specify these thresholds yourself if you want\n",
    "    # as you may want to optimize them using a validation set.\n",
    "    # By default (and provably so) they are set to the average class prob.\n",
    "    thresholds = [np.mean(psx[:,k][y == k]) for k in range(K)] # P(s^=k|s=k)\n",
    "    thresholds = np.asarray(thresholds)\n",
    "\n",
    "    # Compute confident joint\n",
    "    confident_joint = np.zeros((K, K), dtype = int)\n",
    "    for i, row in enumerate(psx):\n",
    "        y_label = y[i]\n",
    "        # Find out how many classes each example is confidently labeled as\n",
    "        confident_bins = row >= thresholds - 1e-6\n",
    "        num_confident_bins = sum(confident_bins)\n",
    "        # If more than one conf class, inc the count of the max prob class\n",
    "        if num_confident_bins == 1:\n",
    "            confident_joint[y_label][np.argmax(confident_bins)] += 1\n",
    "        elif num_confident_bins > 1:\n",
    "            confident_joint[y_label][np.argmax(row)] += 1\n",
    "\n",
    "    # Normalize confident joint (use cleanlab, trust me on this)\n",
    "    confident_joint = cleanlab.latent_estimation.calibrate_confident_joint(\n",
    "        confident_joint, y)\n",
    "    cleanlab.util.print_joint_matrix(confident_joint)\n",
    "    return confident_joint\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Find label errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indices of label errors found by confident learning:\n",
      "Note label errors are sorted by likelihood of being an error\n",
      "but here we just sort them by index for comparison with above.\n",
      "[  84  138  212  251  314  363  375  433  449  468  528  541  595  729\n",
      "  732  774  781  835  837  838  852  855  856  891  923  932  939  973\n",
      "  988  999 1009 1041 1091 1102 1155 1188 1193 1196 1206 1218 1224 1228\n",
      " 1237 1241 1245 1261 1270 1298 1313 1317 1329 1361 1366 1378 1393 1398\n",
      " 1399 1416 1458 1477 1485 1495 1513 1529 1556 1559 1571 1578 1582 1585\n",
      " 1586 1591 1595 1597 1603 1621 1658 1660 1665 1675 1701 1703 1709 1715\n",
      " 1718 1722 1723 1726 1727 1741 1750 1754 1756 1757 1763 1776 1780 1786\n",
      " 1797 1803 1804 1806 1813 1814 1823 1826 1828 1834 1837 1854 1858 1859\n",
      " 1892 1901 1906 1912 1933 1952 1954 1959 1961 1963 1972 1973 1974 1983\n",
      " 1986 1988 1996 2000 2001 2002 2003 2004 2007 2022 2029 2039 2043 2066\n",
      " 2084 2085 2106 2124 2136 2143 2145 2151 2159 2168 2170 2202 2207 2210\n",
      " 2219 2299 2300 2301 2319 2344 2354 2381 2393 2402 2414 2423 2438 2455\n",
      " 2482 2500 2506 2507 2512 2542 2549 2575 2600 2620 2629 2647 2676 2681\n",
      " 2686 2704 2735 2750 2756 2784 2834 2843 2864 2898 2904 2918 2926 2953\n",
      " 3033 3057 3083 3164 3174 3187 3231 3241 3260 3280 3281 3324 3326 3329\n",
      " 3349 3357 3389 3450 3452 3469 3484 3486]\n"
     ]
    }
   ],
   "source": [
    "def find_label_errors(confident_joint, y):\n",
    "    # We arbitrarily choose at least 5 examples left in every class.\n",
    "    # Regardless of whether some of them might be label errors.\n",
    "    MIN_NUM_PER_CLASS = 5\n",
    "    # Leave at least MIN_NUM_PER_CLASS examples per class.\n",
    "    # NOTE prune_count_matrix is transposed (relative to confident_joint)\n",
    "    prune_count_matrix = cleanlab.pruning.keep_at_least_n_per_class(\n",
    "        prune_count_matrix=confident_joint.T,\n",
    "        n=MIN_NUM_PER_CLASS,\n",
    "    )\n",
    "    K = len(np.unique(y)) # number of unique classes\n",
    "    y_counts = np.bincount(y)\n",
    "    noise_masks_per_class = []\n",
    "    # For each row in the transposed confident joint\n",
    "    for k in range(K):\n",
    "        noise_mask = np.zeros(len(psx), dtype=bool)\n",
    "        psx_k = psx[:, k]\n",
    "        if y_counts[k] > MIN_NUM_PER_CLASS:  # Don't prune if not MIN_NUM_PER_CLASS\n",
    "            for j in range(K):  # noisy label index (k is the true label index)\n",
    "                if k != j:  # Only prune for noise rates, not diagonal entries\n",
    "                    num2prune = prune_count_matrix[k][j]\n",
    "                    if num2prune > 0:\n",
    "                        # num2prune'th largest p(classk) - p(class j)\n",
    "                        # for x with noisy label j\n",
    "                        margin = psx_k - psx[:, j]\n",
    "                        y_filter = y == j\n",
    "                        threshold = -np.partition(\n",
    "                            -margin[y_filter], num2prune - 1\n",
    "                        )[num2prune - 1]\n",
    "                        noise_mask = noise_mask | (y_filter & (margin >= threshold))\n",
    "            noise_masks_per_class.append(noise_mask)\n",
    "        else:\n",
    "            noise_masks_per_class.append(np.zeros(len(s), dtype=bool))\n",
    "\n",
    "    # Boolean label error mask\n",
    "    label_errors_bool = np.stack(noise_masks_per_class).any(axis=0)\n",
    "\n",
    "     # Remove label errors if given label == model prediction\n",
    "    for i, pred_label in enumerate(psx.argmax(axis=1)):\n",
    "        # np.all let's this work for multi_label and single label\n",
    "        if label_errors_bool[i] and np.all(pred_label == y[i]):\n",
    "            label_errors_bool[i] = False\n",
    "\n",
    "    # Convert boolean mask to an ordered list of indices for label errors\n",
    "    label_errors_idx = np.arange(len(y))[label_errors_bool]\n",
    "    # self confidence is the holdout probability that an example\n",
    "    # belongs to its given class label\n",
    "    self_confidence = np.array(\n",
    "        [np.mean(psx[i][y[i]]) for i in label_errors_idx]\n",
    "    )\n",
    "    margin = self_confidence - psx[label_errors_bool].max(axis=1)\n",
    "    label_errors_idx = label_errors_idx[np.argsort(margin)]\n",
    "\n",
    "    print('Indices of label errors found by confident learning:')\n",
    "    print('Note label errors are sorted by likelihood of being an error')\n",
    "    print('but here we just sort them by index for comparison with above.')\n",
    "    label_errors_idx.sort()\n",
    "    print(np.array(label_errors_idx))\n",
    "    return label_errors_idx\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's print out the questionable occupations and their labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"Aristocrat\", 1\n",
      "\"Bankier\", 1\n",
      "\"Bard\", 1\n",
      "\"Baru\", 1\n",
      "\"Boxer\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Cacique\", 1\n",
      "\"Cadet\", 1\n",
      "\"Carabinieri\", 1\n",
      "\"Carpenter\", 1\n",
      "\"Certified Public Accountant\", 1\n",
      "\"Confucian scholar\", 1\n",
      "\"Curé\", 1\n",
      "\"Derebey\", 1\n",
      "\"Dewan\", 1\n",
      "\"Dichter\", 1\n",
      "\"Drayman\", 1\n",
      "\"Equestrian\", 1\n",
      "\"Fakir\", 1\n",
      "\"Fundi\", 1\n",
      "\"Hakim\", 1\n",
      "\"Hostess\", 1\n",
      "\"Investment Advisor\", 1\n",
      "\"Jesuit\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Juris Doctor\", 1\n",
      "\"Mahant\", 1\n",
      "\"MasterChef\", 1\n",
      "\"Media Composer\", 1\n",
      "\"Mohel\", 1\n",
      "\"Noble\", 1\n",
      "\"Pir\", 1\n",
      "\"Public Protector\", 1\n",
      "\"Regent\", 1\n",
      "\"Regidor\", 1\n",
      "\"Roman legionary\", 1\n",
      "\"Russian oligarch\", 1\n",
      "\"Sexton\", 1\n",
      "\"Shah\", 1\n",
      "\"Shogun\", 1\n",
      "\"Smuggler\", 1\n",
      "\"Statesman\", 1\n",
      "\"Student\", 1\n",
      "\"Stunt\", 1\n",
      "\"Swami\", 1\n",
      "\"Visor\", 1\n",
      "\"Womanizer\", 1\n",
      "\"abbey\", 1\n",
      "\"academic\", 1\n",
      "\"alewife\", 1\n",
      "\"augur\", 1\n",
      "\"autopilot\", 1\n",
      "\"ayatollah\", 1\n",
      "\"bicycle motocross\", 1\n",
      "\"brazier\", 1\n",
      "\"brewer\", 1\n",
      "\"burglary\", 1\n",
      "\"censor\", 1\n",
      "\"chess\", 1\n",
      "\"child\", 1\n",
      "\"choir\", 1\n",
      "\"comic book\", 1\n",
      "\"communist\", 1\n",
      "\"count\", 1\n",
      "\"drugstore operator\", 1\n",
      "\"duce\", 1\n",
      "\"equestrian\", 1\n",
      "\"existentialist\", 1\n",
      "\"fashion\", 1\n",
      "\"feminist\", 1\n",
      "\"finance\", 1\n",
      "\"foley artist\", 1\n",
      "\"fundraiser\", 1\n",
      "\"gaucho\", 1\n",
      "\"geisha\", 1\n",
      "\"gendarmerie\", 1\n",
      "\"gnostic\", 1\n",
      "\"harp\", 1\n",
      "\"humanitarian\", 1\n",
      "\"immigrant\", 1\n",
      "\"lineworker\", 1\n",
      "\"management\", 1\n",
      "\"medium\", 1\n",
      "\"militant\", 1\n",
      "\"mime\", 1\n",
      "\"naturopathic practitioner\", 1\n",
      "\"noble\", 1\n",
      "\"orthopedics\", 1\n",
      "\"paleoecologist\", 1\n",
      "\"paparazzi\", 1\n",
      "\"pastoralist\", 1\n",
      "\"patrician\", 1\n",
      "\"picador\", 1\n",
      "\"polyglot\", 1\n",
      "\"pope\", 1\n",
      "\"prince consort\", 1\n",
      "\"prior\", 1\n",
      "\"prompter\", 1\n",
      "\"rebbetzin\", 1\n",
      "\"reindeer herder\", 1\n",
      "\"rentier\", 1\n",
      "\"semiotician\", 1\n",
      "\"shareholder\", 1\n",
      "\"spouse\", 1\n",
      "\"stabler\", 1\n",
      "\"streamer\", 1\n",
      "\"sumo\", 1\n",
      "\"syndicalist\", 1\n",
      "\"tamer\", 1\n",
      "\"terrorist\", 1\n",
      "\"tinker\", 1\n",
      "\"tourist\", 1\n",
      "\"troll\", 1\n",
      "\"usher\", 1\n",
      "\"vizier\", 1\n",
      "\"warfare\", 1\n",
      "\"zionist\", 1\n",
      "\"Archdruid\", 0\n",
      "\"Attaché\", 0\n",
      "\"Brother\", 0\n",
      "\"CEO\", 0\n",
      "\"Chief Executive\", 0\n",
      "\"Holocaust survivor\", 0\n",
      "\"Jedi Master\", 0\n",
      "\"Lascar\", 0\n",
      "\"Maker\", 0\n",
      "\"Mawla\", 0\n",
      "\"Meister\", 0\n",
      "\"Minutemen\", 0\n",
      "\"Nuncio\", 0\n",
      "\"Owner\", 0\n",
      "\"ProPublica\", 0\n",
      "\"Procurator fiscal\", 0\n",
      "\"Seabee\", 0\n",
      "\"Shihan\", 0\n",
      "\"Speaker\", 0\n",
      "\"Stasi\", 0\n",
      "\"Subedar\", 0\n",
      "\"Trekker\", 0\n",
      "\"Ubisoft Montreal\", 0\n",
      "\"United Parcel Service\", 0\n",
      "\"Venerable\", 0\n",
      "\"Warraq\", 0\n",
      "\"amanuensis\", 0\n",
      "\"billionaire\", 0\n",
      "\"bookworm\", 0\n",
      "\"boss\", 0\n",
      "\"carillonneur\", 0\n",
      "\"centenarian\", 0\n",
      "\"cinephile\", 0\n",
      "\"commoner\", 0\n",
      "\"dame\", 0\n",
      "\"deserter\", 0\n",
      "\"detective\", 0\n",
      "\"dilettante\", 0\n",
      "\"disc jockey\", 0\n",
      "\"drover\", 0\n",
      "\"druid\", 0\n",
      "\"employee\", 0\n",
      "\"employer\", 0\n",
      "\"equerry\", 0\n",
      "\"escapology\", 0\n",
      "\"federalist\", 0\n",
      "\"fifer\", 0\n",
      "\"folk hero\", 0\n",
      "\"fugitive\", 0\n",
      "\"galley slave\", 0\n",
      "\"geotechnical engineering\", 0\n",
      "\"goddess\", 0\n",
      "\"godparent\", 0\n",
      "\"groom\", 0\n",
      "\"heir\", 0\n",
      "\"heir presumptive\", 0\n",
      "\"hero\", 0\n",
      "\"honorary consul\", 0\n",
      "\"intermediary\", 0\n",
      "\"jockey\", 0\n",
      "\"judoka\", 0\n",
      "\"karateka\", 0\n",
      "\"keynote speaker\", 0\n",
      "\"kumu hula\", 0\n",
      "\"lama\", 0\n",
      "\"layperson\", 0\n",
      "\"lur\", 0\n",
      "\"maestro\", 0\n",
      "\"master\", 0\n",
      "\"mate\", 0\n",
      "\"matriarch\", 0\n",
      "\"millionaire\", 0\n",
      "\"mule\", 0\n",
      "\"mystic\", 0\n",
      "\"naval attaché\", 0\n",
      "\"navy\", 0\n",
      "\"nuncio\", 0\n",
      "\"nurse\", 0\n",
      "\"nymph\", 0\n",
      "\"pasha\", 0\n",
      "\"pastry shop\", 0\n",
      "\"pigeon fancier\", 0\n",
      "\"pilgrim\", 0\n",
      "\"pirate\", 0\n",
      "\"prelate\", 0\n",
      "\"prima donna\", 0\n",
      "\"publican\", 0\n",
      "\"representative\", 0\n",
      "\"seminarian\", 0\n",
      "\"sexual predator\", 0\n",
      "\"sidekick\", 0\n",
      "\"smoker\", 0\n",
      "\"socialite\", 0\n",
      "\"suffragette\", 0\n",
      "\"terminology\", 0\n",
      "\"trustee\", 0\n",
      "\"tsar\", 0\n",
      "\"versatile\", 0\n",
      "\"villain\", 0\n",
      "\"visitor\", 0\n",
      "\"wanker\", 0\n"
     ]
    }
   ],
   "source": [
    "def print_label_names(idx_label_map, label_errors_idx):\n",
    "    label_error_names = [idx_label_map.get(tmp) for tmp in label_errors_idx]\n",
    "\n",
    "    # We'll sort the labels alphabetically and by label value for easier manual review\n",
    "    label_error_names.sort()\n",
    "    pos_labels = []\n",
    "    neg_labels = []\n",
    "    for name in label_error_names:\n",
    "        key_name = name.replace('_', ' ')\n",
    "        val = int(occs_df[occs_df['occupation'] == key_name].label )\n",
    "        if val ==1:\n",
    "            pos_labels.append((key_name, val))\n",
    "        else:\n",
    "            neg_labels.append((key_name, val))\n",
    "    for key_name, val in pos_labels + neg_labels:\n",
    "        print('\"{}\", {}'.format(key_name, val))\n",
    "\n",
    "print_label_names(idx_label_map, label_errors_idx)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2277</th>\n",
       "      <td>24729786</td>\n",
       "      <td>43</td>\n",
       "      <td>publican</td>\n",
       "      <td>owner or manager of a pub or public house</td>\n",
       "      <td>1</td>\n",
       "      <td>ga</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       item_id  occupation_count occupation  \\\n",
       "2277  24729786                43   publican   \n",
       "\n",
       "                                    description  in_google_news  \\\n",
       "2277  owner or manager of a pub or public house               1   \n",
       "\n",
       "     language_detected    source  label labeled_by  label_error_reason  \n",
       "2277                ga  wikidata      0      human                 NaN  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df[occs_df['occupation'] =='publican']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8791</th>\n",
       "      <td>40561</td>\n",
       "      <td>1</td>\n",
       "      <td>sumo</td>\n",
       "      <td>full-contact wrestling sport</td>\n",
       "      <td>1</td>\n",
       "      <td>la</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  occupation_count occupation                   description  \\\n",
       "8791    40561                 1       sumo  full-contact wrestling sport   \n",
       "\n",
       "      in_google_news language_detected    source  label labeled_by  \\\n",
       "8791               1                la  wikidata      1      human   \n",
       "\n",
       "      label_error_reason  \n",
       "8791                 NaN  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df[occs_df['occupation'] =='sumo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2271</th>\n",
       "      <td>499134</td>\n",
       "      <td>33</td>\n",
       "      <td>amanuensis</td>\n",
       "      <td>person employed to write or type what another ...</td>\n",
       "      <td>1</td>\n",
       "      <td>es</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  occupation_count  occupation  \\\n",
       "2271   499134                33  amanuensis   \n",
       "\n",
       "                                            description  in_google_news  \\\n",
       "2271  person employed to write or type what another ...               1   \n",
       "\n",
       "     language_detected    source  label labeled_by  label_error_reason  \n",
       "2271                es  wikidata      0      human                 NaN  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df[occs_df['occupation'] =='amanuensis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8,\n",
       " ['intelligence officer',\n",
       "  'security guard',\n",
       "  'amanuensis',\n",
       "  'private sector employee',\n",
       "  'chauffeur / chauffeuse',\n",
       "  'waste collector',\n",
       "  'factory employee',\n",
       "  'supercargo'])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "possibly_mislabeled =[]\n",
    "for idx, row in occs_df.iterrows():\n",
    "    if row['label'] in (0, -1):\n",
    "        if 'person employed' in str(row['description']):\n",
    "            possibly_mislabeled.append(row['occupation'])\n",
    "len(possibly_mislabeled), possibly_mislabeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7773</th>\n",
       "      <td>915830</td>\n",
       "      <td>1</td>\n",
       "      <td>supercargo</td>\n",
       "      <td>person employed on board a vessel by the owner...</td>\n",
       "      <td>0</td>\n",
       "      <td>it</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  occupation_count  occupation  \\\n",
       "7773   915830                 1  supercargo   \n",
       "\n",
       "                                            description  in_google_news  \\\n",
       "7773  person employed on board a vessel by the owner...               0   \n",
       "\n",
       "     language_detected    source  label labeled_by  label_error_reason  \n",
       "7773                it  wikidata     -1        NaN                 NaN  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df[occs_df['occupation'] =='supercargo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1197</th>\n",
       "      <td>842782</td>\n",
       "      <td>344</td>\n",
       "      <td>detective</td>\n",
       "      <td>investigator, either a member of a police agen...</td>\n",
       "      <td>1</td>\n",
       "      <td>ia</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  occupation_count occupation  \\\n",
       "1197   842782               344  detective   \n",
       "\n",
       "                                            description  in_google_news  \\\n",
       "1197  investigator, either a member of a police agen...               1   \n",
       "\n",
       "     language_detected    source  label labeled_by  label_error_reason  \n",
       "1197                ia  wikidata      0      human                 NaN  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df[occs_df['occupation'] =='detective']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's update the dataframe with our cleaned labels (after manual review, not shown here)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "manual_good_occs =['Archdruid','Attaché','CEO','Gurkha','Hauptmann','Heir apparent','Holocaust survivor','Jedi Master','Lascar','Maker','Mawla','Meister','Official Assignee','Oligarch','Owner','Pictor','Procurator fiscal','Rynda','Seabee','Shihan','Speaker','Stasi','Subedar','Trekker','amanuensis','autodidact','baritone','billionaire','bookworm','boss','carillonneur','commoner','dame','debutante','deserter','detective','disc jockey','drover','druid','employee','employer','empresario','equerry','federalist','fifer','fugitive','godparent','heir','honorary consul','intermediary','jockey','judoka','karateka','keynote speaker','kumu hula','lama','layperson','liar','maestro','mannequin','master','mate','matriarch','millionaire','mystic','naval attaché','nuncio','nurse','nymph','pigeon fancier','pilgrim','pirate','prelate','prima donna','publican','recluse','representative','seminarian','sexual predator','sidekick','smoker','socialite','suffragette','trustee','tsar','villain', 'publican', 'detective', 'intelligence officer',  'security guard',  'amanuensis',  'private sector employee',  'chauffeur / chauffeuse',  'waste collector',  'factory employee', 'supercargo']\n",
    "manual_bad_occs = ['Baru','Indie','Stunt','Visor','bicycle motocross','burglary','carmen','chess','choir','comic book','fashion','finance','gendarmerie','grips','harp','management','orthopedics','schoolbook publisher','stagehands','streamer','sumo','syndicalist','warfare','woodworkers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/96 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 96/96 [00:00<00:00, 158.34it/s]\n",
      "100%|██████████| 24/24 [00:00<00:00, 315.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def update_labels(occs_df, manual_good_occs, manual_bad_occs):\n",
    "    print(f\"Dataframe length before update {len(occs_df):,}\")\n",
    "    for occ in tqdm(manual_good_occs, total=len(manual_good_occs)):\n",
    "        if occ in occs_df['occupation'].tolist():\n",
    "            the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "            the_index = the_row.index[0]\n",
    "            occs_df.loc[the_index, 'label'] = 1\n",
    "            occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'\n",
    "    for occ in tqdm(manual_bad_occs, total=len(manual_bad_occs)):\n",
    "        if occ in occs_df['occupation'].tolist():\n",
    "            the_row = occs_df[occs_df['occupation'] == occ ]\n",
    "            the_index = the_row.index[0]\n",
    "            occs_df.loc[the_index, 'label'] = 0\n",
    "            occs_df.loc[the_index, 'labeled_by'] = 'cleanlab'    \n",
    "    print(f\"Dataframe length after update {len(occs_df):,}\")     \n",
    "    return occs_df\n",
    "\n",
    "occs_df = update_labels(occs_df, manual_good_occs, manual_bad_occs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's cycle the Cleanlab processing againg and review the suggested corrections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Joint Label Noise Distribution Matrix P(s,y) of shape (2, 2)\n",
      " p(s,y)\ty=0\ty=1\n",
      "\t---\t---\n",
      "s=0 |\t1476\t28\n",
      "s=1 |\t64\t1952\n",
      "\tTrace(matrix) = 3428\n",
      "\n",
      "Indices of label errors found by confident learning:\n",
      "Note label errors are sorted by likelihood of being an error\n",
      "but here we just sort them by index for comparison with above.\n",
      "[ 140  216  255  318  370  382  458  538  552  562  746  793  855  859\n",
      "  872  912  944  952  995 1022 1067 1130 1187 1226 1229 1260 1274 1350\n",
      " 1408 1420 1442 1523 1537 1566 1567 1627 1632 1643 1648 1722 1727 1738\n",
      " 1762 1764 1779 1787 1813 1815 1816 1822 1845 1864 1884 1885 1887 1898\n",
      " 1907 1919 1923 1930 1952 1964 1969 1977 2024 2052 2101 2105 2170 2203\n",
      " 2208 2289 2331 2374 2406 2421 2526 2531 2641 2659 2696 2714 2758 2759\n",
      " 2850 2880 3040 3062 3080 3265 3452 3491]\n",
      "\"Aristocrat\", 1\n",
      "\"Bankier\", 1\n",
      "\"Bard\", 1\n",
      "\"Bedel\", 1\n",
      "\"Bokor\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Cacique\", 1\n",
      "\"Carabinieri\", 1\n",
      "\"Carpenter\", 1\n",
      "\"Censor\", 1\n",
      "\"Confucian scholar\", 1\n",
      "\"Curé\", 1\n",
      "\"Derebey\", 1\n",
      "\"Dewan\", 1\n",
      "\"Dragoman\", 1\n",
      "\"Drayman\", 1\n",
      "\"Druggist\", 1\n",
      "\"Equestrian\", 1\n",
      "\"Fakir\", 1\n",
      "\"Hostess\", 1\n",
      "\"Jesuit\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Mahant\", 1\n",
      "\"Media Composer\", 1\n",
      "\"Noble\", 1\n",
      "\"Pir\", 1\n",
      "\"Regidor\", 1\n",
      "\"Shogun\", 1\n",
      "\"Slayer\", 1\n",
      "\"Statesman\", 1\n",
      "\"Swami\", 1\n",
      "\"Womanizer\", 1\n",
      "\"academic\", 1\n",
      "\"adapter\", 1\n",
      "\"alewife\", 1\n",
      "\"augur\", 1\n",
      "\"ayatollah\", 1\n",
      "\"bey\", 1\n",
      "\"conscript\", 1\n",
      "\"count\", 1\n",
      "\"drugstore operator\", 1\n",
      "\"duce\", 1\n",
      "\"emir\", 1\n",
      "\"equestrian\", 1\n",
      "\"fundraiser\", 1\n",
      "\"general\", 1\n",
      "\"gnostic\", 1\n",
      "\"humanitarian\", 1\n",
      "\"lineworker\", 1\n",
      "\"medium\", 1\n",
      "\"mime\", 1\n",
      "\"noble\", 1\n",
      "\"paparazzi\", 1\n",
      "\"pastoralist\", 1\n",
      "\"prince consort\", 1\n",
      "\"prior\", 1\n",
      "\"shareholder\", 1\n",
      "\"stabler\", 1\n",
      "\"tamer\", 1\n",
      "\"terrorist\", 1\n",
      "\"tinker\", 1\n",
      "\"tourist\", 1\n",
      "\"usher\", 1\n",
      "\"zionist\", 1\n",
      "\"Brother\", 0\n",
      "\"Chief Executive\", 0\n",
      "\"Compo\", 0\n",
      "\"Custos\", 0\n",
      "\"Hustling\", 0\n",
      "\"Jedi\", 0\n",
      "\"Pooter\", 0\n",
      "\"ProPublica\", 0\n",
      "\"Santa Claus\", 0\n",
      "\"Venerable\", 0\n",
      "\"associate\", 0\n",
      "\"centenarian\", 0\n",
      "\"chair\", 0\n",
      "\"cinephile\", 0\n",
      "\"dilettante\", 0\n",
      "\"folk hero\", 0\n",
      "\"groom\", 0\n",
      "\"heir presumptive\", 0\n",
      "\"hero\", 0\n",
      "\"madam\", 0\n",
      "\"pastry shop\", 0\n",
      "\"registrar\", 0\n",
      "\"stallion\", 0\n",
      "\"survivor\", 0\n",
      "\"suspect\", 0\n",
      "\"versatile\", 0\n",
      "\"visitor\", 0\n",
      "\"wanker\", 0\n"
     ]
    }
   ],
   "source": [
    "good_occs = occs_df.query(\"in_google_news ==1 and label ==1\")['occupation'].tolist()\n",
    "bad_occs = occs_df.query(\"in_google_news ==1 and label ==0 \")['occupation'].tolist()\n",
    "y = np.concatenate([np.ones(len(good_occs), dtype=np.int32), np.zeros(len(bad_occs), dtype=np.int32)])\n",
    "\n",
    "all_words = good_occs + bad_occs\n",
    "all_words = [tmp.replace(' ', '_') for tmp in all_words]\n",
    "\n",
    "# We'll save the mapping of index to words for decoding\n",
    "idx_label_map = {}\n",
    "for idx, name in enumerate(all_words):\n",
    "    idx_label_map[idx]= name\n",
    "    \n",
    "X = [gnews_embed[tmp] for tmp in all_words]\n",
    "X = np.array(X)\n",
    "\n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    X, y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))\n",
    "\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, y)\n",
    "\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, y)\n",
    "\n",
    "print_label_names(idx_label_map, label_errors_idx)   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manually adjusting the labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from latest multipass\n",
    "bad_occs_second_pass =[ \"Aristocrats\", \"Carabinieri\", \"Certified Public Accountants\", \"Confessors\", \"Distillers\", \"Geographers\", \"Investment Advisors\", \"Raphaël\", \"Sextons\", \"audio\", \"comic books\", \"conquistadors\", \"counts\", \"fashions\", \"freemasons\", \"gentlemans\", \"geometer\", \"hair\", \"house\", \"humanitarian\", \"interior\", \"jewel\", \"journal\", \"leather\", \"lighting\", \"lyric\", \"magic\", \"naturopathic practitioners\", \"neurology\", \"parachute\", \"paralympic athletes\", \"physical\", \"playbill\", \"plenipotentiary\", \"podiatry\", \"production\", \"radiation\", \"ring\", \"school\", \"screen\", \"shoe\", \"sing\", \"social\", \"solo\", \"sophists\", \"sound\", \"spectrometer\", \"speech\", \"station\", \"steel\", \"structure\", \"super\", \"tamer\", \"tatoo\", \"taxi\", \"technical\", \"theatrical\", \"ticketing\", \"track\", \"traditional healers\", \"watercolors\", \"website\", \"wedding\"]\n",
    "good_occs_second_pass = [ \"Brother\", \"Chief Executive\", \"Dempster\", \"Freedom Fighters\", \"Jedi\", \"Met Éireann\", \"Patrick\", \"Santa Claus\", \"Schiffer\", \"ascetic\", \"associate\", \"centenarian\", \"crank\", \"delegate\", \"dilettante\", \"erudite\", \"fado singer\", \"folk hero\", \"groom\", \"hero\", \"inebriate\", \"madam\", \"mule\", \"pastry shop\", \"patriarch\", \"pipe organ\", \"registrar\", \"stallion\", \"superhero\", \"suspect\", \"virgin\", \"woodworkers\", ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:00<00:00, 269.20it/s]\n",
      "100%|██████████| 63/63 [00:00<00:00, 1357.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n",
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "occs_df = update_labels(occs_df, good_occs_second_pass, bad_occs_second_pass)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2nd pass Cleanlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Joint Label Noise Distribution Matrix P(s,y) of shape (2, 2)\n",
      " p(s,y)\ty=0\ty=1\n",
      "\t---\t---\n",
      "s=0 |\t1466\t14\n",
      "s=1 |\t56\t1984\n",
      "\tTrace(matrix) = 3450\n",
      "\n",
      "Indices of label errors found by confident learning:\n",
      "Note label errors are sorted by likelihood of being an error\n",
      "but here we just sort them by index for comparison with above.\n",
      "[ 140  217  281  372  384  462  543  558  617  752  861  865  878  918\n",
      "  957 1018 1029 1074 1137 1194 1275 1283 1306 1418 1430 1447 1452 1453\n",
      " 1548 1553 1575 1577 1578 1643 1655 1659 1739 1744 1755 1781 1793 1797\n",
      " 1807 1829 1834 1837 1843 1866 1886 1909 1917 1942 1946 1986 1991 1999\n",
      " 2058 2096 2123 2220 2346 2389 2421 2540 2564 2766 2999 3069 3294 3454]\n",
      "\"Aristocrat\", 1\n",
      "\"Bankier\", 1\n",
      "\"Bard\", 1\n",
      "\"Bedel\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Cacique\", 1\n",
      "\"Confucian scholar\", 1\n",
      "\"Curé\", 1\n",
      "\"Derebey\", 1\n",
      "\"Dewan\", 1\n",
      "\"Dragoman\", 1\n",
      "\"Drayman\", 1\n",
      "\"Equestrian\", 1\n",
      "\"Fakir\", 1\n",
      "\"Frigate Captain\", 1\n",
      "\"Fundi\", 1\n",
      "\"Gendarme\", 1\n",
      "\"Jesuit\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Mahant\", 1\n",
      "\"Media Composer\", 1\n",
      "\"Pir\", 1\n",
      "\"Procurator\", 1\n",
      "\"Public Protector\", 1\n",
      "\"Regidor\", 1\n",
      "\"Shogun\", 1\n",
      "\"Statesman\", 1\n",
      "\"Swami\", 1\n",
      "\"Womanizer\", 1\n",
      "\"abbey\", 1\n",
      "\"academic\", 1\n",
      "\"alewife\", 1\n",
      "\"augur\", 1\n",
      "\"bey\", 1\n",
      "\"count\", 1\n",
      "\"criminal\", 1\n",
      "\"drugstore operator\", 1\n",
      "\"duce\", 1\n",
      "\"equestrian\", 1\n",
      "\"fundraiser\", 1\n",
      "\"gnostic\", 1\n",
      "\"medium\", 1\n",
      "\"mime\", 1\n",
      "\"paparazzi\", 1\n",
      "\"picador\", 1\n",
      "\"pipe organ\", 1\n",
      "\"prince consort\", 1\n",
      "\"prior\", 1\n",
      "\"prompter\", 1\n",
      "\"rentier\", 1\n",
      "\"semiotician\", 1\n",
      "\"shareholder\", 1\n",
      "\"stabler\", 1\n",
      "\"usher\", 1\n",
      "\"wainwright\", 1\n",
      "\"zionist\", 1\n",
      "\"Bana\", 0\n",
      "\"Custos\", 0\n",
      "\"Debut\", 0\n",
      "\"Hofmeister\", 0\n",
      "\"Hustling\", 0\n",
      "\"Venerable\", 0\n",
      "\"chair\", 0\n",
      "\"cinephile\", 0\n",
      "\"demon\", 0\n",
      "\"pasha\", 0\n",
      "\"survivor\", 0\n",
      "\"versatile\", 0\n",
      "\"visitor\", 0\n",
      "\"wanker\", 0\n"
     ]
    }
   ],
   "source": [
    "good_occs = occs_df.query(\"in_google_news ==1 and label ==1\")['occupation'].tolist()\n",
    "bad_occs = occs_df.query(\"in_google_news ==1 and label ==0 \")['occupation'].tolist()\n",
    "y = np.concatenate([np.ones(len(good_occs), dtype=np.int32), np.zeros(len(bad_occs), dtype=np.int32)])\n",
    "\n",
    "all_words = good_occs + bad_occs\n",
    "all_words = [tmp.replace(' ', '_') for tmp in all_words]\n",
    "\n",
    "# We'll save the mapping of index to words for decoding\n",
    "idx_label_map = {}\n",
    "for idx, name in enumerate(all_words):\n",
    "    idx_label_map[idx]= name\n",
    "    \n",
    "X = [gnews_embed[tmp] for tmp in all_words]\n",
    "X = np.array(X)\n",
    "\n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    X, y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))\n",
    "\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, y)\n",
    "\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, y)\n",
    "\n",
    "print_label_names(idx_label_map, label_errors_idx) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Third Pass Cleanlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 240.78it/s]\n",
      "100%|██████████| 9/9 [00:00<00:00, 1507.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n",
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Joint Label Noise Distribution Matrix P(s,y) of shape (2, 2)\n",
      " p(s,y)\ty=0\ty=1\n",
      "\t---\t---\n",
      "s=0 |\t1457\t18\n",
      "s=1 |\t49\t1996\n",
      "\tTrace(matrix) = 3453\n",
      "\n",
      "Indices of label errors found by confident learning:\n",
      "Note label errors are sorted by likelihood of being an error\n",
      "but here we just sort them by index for comparison with above.\n",
      "[ 140  217  337  373  385  545  560  755  869  882  922  961 1004 1032\n",
      " 1129 1140 1197 1287 1423 1435 1457 1476 1553 1580 1582 1602 1660 1664\n",
      " 1744 1749 1760 1779 1784 1786 1802 1812 1839 1841 1842 1848 1871 1891\n",
      " 1911 1945 1947 1951 1991 1996 2004 2222 2240 2348 2390 2421 2433 2540\n",
      " 2564 2670 2706 2766 2926 3047 3069 3079 3174 3454 3482]\n",
      "\"Advokat\", 1\n",
      "\"Aristocrat\", 1\n",
      "\"Bard\", 1\n",
      "\"Bedel\", 1\n",
      "\"Buddhist\", 1\n",
      "\"Cacique\", 1\n",
      "\"Confucian scholar\", 1\n",
      "\"Constructor\", 1\n",
      "\"Curé\", 1\n",
      "\"Derebey\", 1\n",
      "\"Dewan\", 1\n",
      "\"Dragoman\", 1\n",
      "\"Drayman\", 1\n",
      "\"Druggist\", 1\n",
      "\"Equestrian\", 1\n",
      "\"Fakir\", 1\n",
      "\"Frigate Captain\", 1\n",
      "\"Jesuit\", 1\n",
      "\"Jihadi\", 1\n",
      "\"Mahant\", 1\n",
      "\"Media Composer\", 1\n",
      "\"Noble\", 1\n",
      "\"Pir\", 1\n",
      "\"Public Protector\", 1\n",
      "\"Regidor\", 1\n",
      "\"Shah\", 1\n",
      "\"Shogun\", 1\n",
      "\"Soldat\", 1\n",
      "\"Womanizer\", 1\n",
      "\"academic\", 1\n",
      "\"alewife\", 1\n",
      "\"bey\", 1\n",
      "\"count\", 1\n",
      "\"drugstore operator\", 1\n",
      "\"duce\", 1\n",
      "\"equestrian\", 1\n",
      "\"gaucho\", 1\n",
      "\"gnostic\", 1\n",
      "\"medium\", 1\n",
      "\"mime\", 1\n",
      "\"paparazzi\", 1\n",
      "\"pastoralist\", 1\n",
      "\"prior\", 1\n",
      "\"revolutionary\", 1\n",
      "\"shareholder\", 1\n",
      "\"stabler\", 1\n",
      "\"tinker\", 1\n",
      "\"usher\", 1\n",
      "\"zionist\", 1\n",
      "\"Cossack\", 0\n",
      "\"Custos\", 0\n",
      "\"Debut\", 0\n",
      "\"Hustling\", 0\n",
      "\"Literato\", 0\n",
      "\"Politico\", 0\n",
      "\"Pooter\", 0\n",
      "\"ProPublica\", 0\n",
      "\"Venerable\", 0\n",
      "\"Wali\", 0\n",
      "\"chair\", 0\n",
      "\"cinephile\", 0\n",
      "\"cracker\", 0\n",
      "\"goddess\", 0\n",
      "\"jury\", 0\n",
      "\"versatile\", 0\n",
      "\"visitor\", 0\n",
      "\"wanker\", 0\n"
     ]
    }
   ],
   "source": [
    "bad_occs_third_pass = [ \"pipe organ\", \"furniture\", \"gondoliers\", \"hat\", \"laypersons\", \"paint\", \"panel\", \"set\", \"soap\" ]\n",
    "good_occs_third_pass =[ \"Hofmeister\", \"Pythia\", \"dragoon\", \"pasha\", \"rikishi\", \"survivor\"]\n",
    "\n",
    "occs_df = update_labels(occs_df, good_occs_third_pass, bad_occs_third_pass)\n",
    "good_occs = occs_df.query(\"in_google_news ==1 and label ==1\")['occupation'].tolist()\n",
    "bad_occs = occs_df.query(\"in_google_news ==1 and label ==0 \")['occupation'].tolist()\n",
    "y = np.concatenate([np.ones(len(good_occs), dtype=np.int32), np.zeros(len(bad_occs), dtype=np.int32)])\n",
    "\n",
    "all_words = good_occs + bad_occs\n",
    "all_words = [tmp.replace(' ', '_') for tmp in all_words]\n",
    "\n",
    "# We'll save the mapping of index to words for decoding\n",
    "idx_label_map = {}\n",
    "for idx, name in enumerate(all_words):\n",
    "    idx_label_map[idx]= name\n",
    "    \n",
    "X = [gnews_embed[tmp] for tmp in all_words]\n",
    "X = np.array(X)\n",
    "\n",
    "psx = cleanlab.latent_estimation.estimate_cv_predicted_probabilities(\n",
    "    X, y, clf=LogisticRegression(max_iter=1000, multi_class='auto', solver='lbfgs'))\n",
    "\n",
    "\n",
    "confident_joint = compute_confident_joint(psx, y)\n",
    "\n",
    "\n",
    "label_errors_idx = find_label_errors(confident_joint, y)\n",
    "\n",
    "print_label_names(idx_label_map, label_errors_idx) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fourth Update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:00<00:00, 244.52it/s]\n",
      "100%|██████████| 5/5 [00:00<00:00, 2823.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n",
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "bad_occs_fourth_pass = [  \"furniture\",   \"hat\",   \"paint\",   \"panel\",   \"set\"]\n",
    "good_occs_fourth_pass =[ \"dragoon\", \"goddess\", \"santero\"]\n",
    "\n",
    "occs_df = update_labels(occs_df, good_occs_fourth_pass, bad_occs_fourth_pass)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Another way of spot checking; random sampling "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>occupation_count</th>\n",
       "      <th>occupation</th>\n",
       "      <th>description</th>\n",
       "      <th>in_google_news</th>\n",
       "      <th>language_detected</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>labeled_by</th>\n",
       "      <th>label_error_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>13418253</td>\n",
       "      <td>5490</td>\n",
       "      <td>philologist</td>\n",
       "      <td>person who practices philology</td>\n",
       "      <td>1</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3914</th>\n",
       "      <td>220344</td>\n",
       "      <td>2</td>\n",
       "      <td>Pythia</td>\n",
       "      <td>priestess of the Temple of Apollo at Delphi</td>\n",
       "      <td>1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>cleanlab</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9282</th>\n",
       "      <td>212927</td>\n",
       "      <td>1</td>\n",
       "      <td>Una</td>\n",
       "      <td>river in Bosnia and Herzegovina and Croatia</td>\n",
       "      <td>1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5329</th>\n",
       "      <td>1043452</td>\n",
       "      <td>2</td>\n",
       "      <td>maintenance</td>\n",
       "      <td>operational and functional checks, servicing, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>fr</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9576</th>\n",
       "      <td>37559037</td>\n",
       "      <td>1</td>\n",
       "      <td>Sergent</td>\n",
       "      <td>family name</td>\n",
       "      <td>1</td>\n",
       "      <td>fr</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2215</th>\n",
       "      <td>1124183</td>\n",
       "      <td>25</td>\n",
       "      <td>lumberjack</td>\n",
       "      <td>craftsmen who perform the initial harvesting o...</td>\n",
       "      <td>1</td>\n",
       "      <td>lb</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4258</th>\n",
       "      <td>18362</td>\n",
       "      <td>4</td>\n",
       "      <td>theoretical physics</td>\n",
       "      <td>branch of physics</td>\n",
       "      <td>1</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5165</th>\n",
       "      <td>567086</td>\n",
       "      <td>1</td>\n",
       "      <td>Annihilator</td>\n",
       "      <td>Canadian metal band</td>\n",
       "      <td>1</td>\n",
       "      <td>la</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>1</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9598</th>\n",
       "      <td>6958747</td>\n",
       "      <td>1</td>\n",
       "      <td>work</td>\n",
       "      <td>particular form of activity, sold by many peop...</td>\n",
       "      <td>1</td>\n",
       "      <td>en</td>\n",
       "      <td>wikidata</td>\n",
       "      <td>0</td>\n",
       "      <td>human</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       item_id  occupation_count           occupation  \\\n",
       "442   13418253              5490          philologist   \n",
       "3914    220344                 2               Pythia   \n",
       "9282    212927                 1                  Una   \n",
       "5329   1043452                 2          maintenance   \n",
       "9576  37559037                 1              Sergent   \n",
       "2215   1124183                25           lumberjack   \n",
       "4258     18362                 4  theoretical physics   \n",
       "5165    567086                 1          Annihilator   \n",
       "9598   6958747                 1                 work   \n",
       "\n",
       "                                            description  in_google_news  \\\n",
       "442                      person who practices philology               1   \n",
       "3914        priestess of the Temple of Apollo at Delphi               1   \n",
       "9282        river in Bosnia and Herzegovina and Croatia               1   \n",
       "5329  operational and functional checks, servicing, ...               1   \n",
       "9576                                        family name               1   \n",
       "2215  craftsmen who perform the initial harvesting o...               1   \n",
       "4258                                  branch of physics               1   \n",
       "5165                                Canadian metal band               1   \n",
       "9598  particular form of activity, sold by many peop...               1   \n",
       "\n",
       "     language_detected    source  label labeled_by  label_error_reason  \n",
       "442                 en  wikidata      1      human                 NaN  \n",
       "3914           Unknown  wikidata      1   cleanlab                 NaN  \n",
       "9282           Unknown  wikidata      0      human                 NaN  \n",
       "5329                fr  wikidata      1      human                 NaN  \n",
       "9576                fr  wikidata      1      human                 NaN  \n",
       "2215                lb  wikidata      1      human                 NaN  \n",
       "4258                en  wikidata      0      human                 NaN  \n",
       "5165                la  wikidata      1      human                 NaN  \n",
       "9598                en  wikidata      0      human                 NaN  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occs_df.query('in_google_news==1 and (label ==1 or label==0)').sample(9)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_good_occs =['tamer', 'Rangatira' ]\n",
    "random_bad_occs = ['ethnographers','clarinetists', 'importers', 'models']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00, 232.97it/s]\n",
      "100%|██████████| 4/4 [00:00<00:00, 2404.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe length before update 10,974\n",
      "Dataframe length after update 10,974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "occs_df = update_labels(occs_df, random_good_occs, random_bad_occs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Once we're satisfied, we'll export and move on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "occs_df.to_csv('occupations.wikidata.all.gnews.labeled.final.csv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now let's build a classifier and label the rest of the Wikidata occupations, see: `training_to_label_with_BERT_and_Cleanlab.ipynb`"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}