notebooks/finetune_bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from torchnlp.datasets import imdb_dataset # run pip install pytorch-nlp if you dont have this\n",
    "from tamnun.bert import BertClassifier, BertVectorizer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import classification_report, accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data = imdb_dataset(train=True, test=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract the texts and the labels from the imdb datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 25000, 25000, 25000)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_texts, train_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), train_data)))\n",
    "test_texts, test_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), test_data)))\n",
    "\n",
    "len(train_texts), len(train_labels), len(test_texts), len(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the target variable to a binary vector neg=0, pos=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25000,), (25000,), 0.5, 0.5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y = np.array(train_labels) == 'pos'\n",
    "test_y = np.array(test_labels) == 'pos'\n",
    "train_y.shape, test_y.shape, np.mean(train_y), np.mean(test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-Tune BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5:\n",
      "6249/6250 batch loss: 0.5317028760910034    avg loss: 0.2504522643852234\n",
      "Epoch 2/5:\n",
      "6249/6250 batch loss: 0.4307538568973541 6  avg loss: 0.12871999006032944\n",
      "Epoch 3/5:\n",
      "6249/6250 batch loss: 0.006784558296203613  avg loss: 0.07053191775560379\n",
      "Epoch 4/5:\n",
      "6249/6250 batch loss: 0.003382742404937744   avg loss: 0.04482248139500618\n",
      "Epoch 5/5:\n",
      "6249/6250 batch loss: 0.0027724504470825195  avg loss: 0.03316646183252334\n"
     ]
    }
   ],
   "source": [
    "clf = make_pipeline(BertVectorizer(do_truncate=True), BertClassifier(num_of_classes=2, lr=1e-5)).fit(train_texts, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = clf.predict(test_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.93604\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.93      0.94     12663\n",
      "           1       0.93      0.94      0.94     12337\n",
      "\n",
      "   micro avg       0.94      0.94      0.94     25000\n",
      "   macro avg       0.94      0.94      0.94     25000\n",
      "weighted avg       0.94      0.94      0.94     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy:', accuracy_score(predicted, test_y))\n",
    "print(classification_report(predicted, test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}