Copy_of_DeBERTa.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cgray1117/NLPTransformersProject/blob/main/Copy_of_DeBERTa.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "COO2drs2a4FM"
      },
      "outputs": [],
      "source": [
        "# Install libraries\n",
        "!pip install imblearn\n",
        "!pip install transformers\n",
        "!pip install tokenizers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7US-NV-SQhS0"
      },
      "outputs": [],
      "source": [
        "# Import libraries\n",
        "import numpy as np\n",
        "import regex as re\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import statistics\n",
        "import math\n",
        "import os\n",
        "\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import RepeatedKFold\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.python.keras\n",
        "import tensorflow.python.keras.backend as K\n",
        "\n",
        "import tokenizers\n",
        "from transformers import DebertaTokenizer, TFDebertaModel\n",
        "from collections import Counter\n",
        "from imblearn.over_sampling import RandomOverSampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0mIvs6LEC8bT"
      },
      "outputs": [],
      "source": [
        "MODEL_NAME = 'microsoft/deberta-base'\n",
        "MAX_LEN = 256\n",
        "ARTIFACTS_PATH = '../artifacts/'\n",
        "EPOCHS = 5\n",
        "if not os.path.exists(ARTIFACTS_PATH):\n",
        "    os.makedirs(ARTIFACTS_PATH)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a7XUPg3X8EiK"
      },
      "source": [
        "## Data NLP"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lp09sfeUSPO1"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('project_data.csv')\n",
        "\n",
        "# Identify null and remove\n",
        "df[\"Label\"].isnull().sum()\n",
        "df['Label'].replace('', np.nan, inplace=True)\n",
        "df.dropna(subset=['Label'], inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ktS_IqeYfyha"
      },
      "outputs": [],
      "source": [
        "# Drop extra columns\n",
        "df = df.drop('Username', axis=1)\n",
        "df = df.drop('UserLocation', axis=1)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PTeC11bm997-"
      },
      "outputs": [],
      "source": [
        "tknzr = TweetTokenizer()\n",
        "\n",
        "# Tokenize each tweet\n",
        "df['tokenized'] = df['Text'].map(lambda t: tknzr.tokenize(t))\n",
        "\n",
        "# lowecase, strip and ensure we only include words\n",
        "df['tokenized'] = df['tokenized'].map(\n",
        "    lambda t: [word.lower().strip() for word in t if word.isalpha()])\n",
        "\n",
        "# Importing stopwords\n",
        "nltk.download(\"stopwords\")\n",
        "\n",
        "stopwords_en = stopwords.words('english')\n",
        "\n",
        "# Instantiate the WordNetLemmatizer\n",
        "wordnet_lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# lemmarize and remove stopwords\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "df['tokenized'] = df['tokenized'].map(\n",
        "    lambda t: [wordnet_lemmatizer.lemmatize(word) for word in t \n",
        "               if word not in stopwords_en])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "78dPJDJI-5O8"
      },
      "outputs": [],
      "source": [
        "df['tokenized'][:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8yKTwu2p-4nq"
      },
      "outputs": [],
      "source": [
        "token_tweets = []\n",
        "for lst in df['tokenized']:\n",
        "  txt = \" \".join(lst)\n",
        "  token_tweets.append(txt)\n",
        "\n",
        "X_data = np.array(token_tweets)\n",
        "y_data = df[[\"Label\"]].to_numpy().reshape(-1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTpyqcJb8MZF"
      },
      "source": [
        "## EDA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_T7jP3FyTzuZ"
      },
      "outputs": [],
      "source": [
        "categories = df[['Label']].values.reshape(-1)\n",
        "counter_categories = Counter(categories)\n",
        "category_names = counter_categories.keys()\n",
        "category_values = counter_categories.values()\n",
        "y_pos = np.arange(len(category_names))\n",
        "plt.figure(1, figsize=(10, 5))\n",
        "plt.bar(y_pos, category_values, align='center', alpha=0.5)\n",
        "plt.xticks(y_pos, category_names)\n",
        "plt.ylabel('Number of texts')\n",
        "plt.xlabel('Labels')\n",
        "plt.title('Distribution of texts per category')\n",
        "plt.gca().yaxis.grid(True)\n",
        "plt.show()\n",
        "print(counter_categories)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v_mVIghx_Fli"
      },
      "outputs": [],
      "source": [
        "n_texts = len(X_data)\n",
        "print('Texts in dataset: %d' % n_texts)\n",
        "n_categories = len(df['Label'].unique())\n",
        "print('Number of categories: %d' % n_categories)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5aTB4JKG_baT"
      },
      "source": [
        "## Decode, and Create Deberta"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FMu_4n3n_eBo"
      },
      "outputs": [],
      "source": [
        "def deberta_encode(texts, tokenizer): # Create encoding function\n",
        "    ct = len(texts) # Assign \"ct\" to number of rows in data\n",
        "    input_ids = np.ones((ct, MAX_LEN), dtype='int32') # Assign \"input_ids\" to ct x MAX_LEN array of ones\n",
        "    attention_mask = np.zeros((ct, MAX_LEN), dtype='int32') # Assign \"attention_mask\" to ct x MAX_LEN array of zeros\n",
        "    token_type_ids = np.zeros((ct, MAX_LEN), dtype='int32') # # Assign \"token_type_ids\" to ct x MAX_LEN arry of zeros\n",
        "\n",
        "    for k, text in enumerate(texts): # Iterating through rows of data with \"text\" = row and \"k\" = iteration number \n",
        "        tok_text = tokenizer.tokenize(text) # Assign \"tok_text\" to tokenized row using DeBERTa tokenizer\n",
        "        enc_text = tokenizer.convert_tokens_to_ids(tok_text[:(MAX_LEN-2)]) # Truncate and convert tokens to numerical ids\n",
        "        input_length = len(enc_text) + 2 # Assign \"input_length\" to 2 + rows of encoded text \n",
        "        input_length = input_length if input_length < MAX_LEN else MAX_LEN # Ensure input length is <= MAX_LEN\n",
        "        input_ids[k,:input_length] = np.asarray([0] + enc_text + [2], dtype='int32') # Place encoded text in input_id array\n",
        "        attention_mask[k,:input_length] = 1 # Set attention_mask of encoded text to 1\n",
        "\n",
        "    return {'input_word_ids': input_ids,'input_mask': attention_mask,\n",
        "            'input_type_ids': token_type_ids}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KLU9PYZTACvu"
      },
      "outputs": [],
      "source": [
        "# Commented out lines here were for oversmapling\n",
        "X_train, X_rem, y_train, y_rem = train_test_split(X_data, y_data, test_size=0.3, random_state=444)\n",
        "\n",
        "X_valid, X_test, y_valid, y_test = train_test_split(X_rem, y_rem, test_size=0.5)\n",
        "\n",
        "#oversample = RandomOverSampler(sampling_strategy='minority')\n",
        "#X_train = pd.DataFrame(X_train)\n",
        "#y_train = pd.DataFrame(y_train)\n",
        "#X_train, y_train = oversample.fit_resample(X_train, y_train)\n",
        "#X_train = X_train.to_numpy().reshape(-1)\n",
        "#y_train = y_train.to_numpy().reshape(-1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nJBZPNM9AIXY"
      },
      "outputs": [],
      "source": [
        "tokenizer = DebertaTokenizer.from_pretrained(MODEL_NAME)\n",
        "\n",
        "X_train = deberta_encode(X_train, tokenizer)\n",
        "X_test = deberta_encode(X_test, tokenizer)\n",
        "X_valid = deberta_encode(X_valid, tokenizer)\n",
        "\n",
        "y_valid = np.asarray(y_valid, dtype='int32').reshape(-1)\n",
        "y_train = np.asarray(y_train, dtype='int32').reshape(-1)\n",
        "y_test = np.asarray(y_test, dtype='int32').reshape(-1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "21r70gx48W0H"
      },
      "source": [
        "## Build Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VC4kZadcALI_"
      },
      "outputs": [],
      "source": [
        "def build_model(n_categories):\n",
        "      input_word_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_word_ids') # Creating inputs for keras model\n",
        "      input_mask = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_mask')\n",
        "      input_type_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_type_ids')\n",
        "\n",
        "      # Import Deberta model from HuggingFace\n",
        "      deberta_model = TFDebertaModel.from_pretrained(MODEL_NAME) # Initialize model\n",
        "      x = deberta_model(input_word_ids, attention_mask=input_mask, token_type_ids=input_type_ids) # Assign \"x\" to model with inputs\n",
        "\n",
        "      x = x[0] # Slice out the embeddings output\n",
        "\n",
        "      x = tf.keras.layers.Dropout(0.1)(x)\n",
        "      x = tf.keras.layers.Flatten()(x)\n",
        "      x = tf.keras.layers.Dense(256, activation='relu')(x)\n",
        "      x = tf.keras.layers.Dense(n_categories, activation='softmax')(x)\n",
        "\n",
        "      model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=x)\n",
        "      model.compile(\n",
        "          optimizer=tf.keras.optimizers.Adam(lr=1e-5),\n",
        "          loss='sparse_categorical_crossentropy',\n",
        "          metrics=['accuracy'])\n",
        "\n",
        "      return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ux7taYQyAQoO"
      },
      "outputs": [],
      "source": [
        "model = build_model(n_categories)\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VSyQLM6iAYHN"
      },
      "source": [
        "## Train and Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "IwTd_aLrAY5T",
        "outputId": "a4f1b04a-0532-4e45-be9d-a98c79839f2a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            " 57/206 [=======>......................] - ETA: 2:35:57 - loss: 0.7133 - accuracy: 0.6996"
          ]
        }
      ],
      "source": [
        "# the code in this block was adpated from https://stackoverflow.com/questions/48118111/get-loss-values-for-each-training-instance-keras\n",
        "class LossHistory(keras.callbacks.Callback):\n",
        "    def on_train_begin(self, logs={}):\n",
        "        self.losses = []\n",
        "    def on_batch_end(self, batch, logs={}):\n",
        "        self.losses.append(logs.get('loss'))\n",
        "\n",
        "history = LossHistory()\n",
        "model.fit(X_train, y_train, batch_size=16, epochs=EPOCHS, verbose=1, validation_data=(X_valid, y_valid), callbacks=[history])\n",
        "print(history.losses)\n",
        "\n",
        "plt.plot(history.losses, linestyle = 'dotted')\n",
        "#plt.ylabel('Number of texts')\n",
        "plt.title('Distribution of Loss')\n",
        "plt.show()\n",
        "# history = model.fit(X_train,y_train,epochs=EPOCHS,batch_size=16,verbose=1,validation_data=(X_test, y_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z9jbDLNs-CFj"
      },
      "outputs": [],
      "source": [
        "# Get training and test loss histories\n",
        "train_loss = model.history['loss']\n",
        "val_loss = model.history['val_loss']\n",
        "\n",
        "# Create count of the number of epochs\n",
        "epoch_count = range(1, len(train_loss) + 1)\n",
        "\n",
        "plt.plot(epoch_count, train_loss, linestyle = 'dotted')\n",
        "plt.plot(epoch_count, val_loss, linestyle = 'dotted')\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.legend(['Training Loss', 'Validation Loss'])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P4reUcgk5Tdx"
      },
      "outputs": [],
      "source": [
        "# Get training and test accuracy histories\n",
        "train_acc = history.history['acc']\n",
        "val_acc = history.history['val_acc']\n",
        "\n",
        "# Create count of the number of epochs\n",
        "epoch_acc = range(1, len(train_acc) + 1)\n",
        "\n",
        "plt.plot(epoch_count, train_loss, linestyle = 'dotted')\n",
        "plt.plot(epoch_count, val_loss, linestyle = 'dotted')\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.legend(['Training Loss', 'Validation Loss'])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ESFA-o5p8c3e"
      },
      "source": [
        "## Visualizations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AGnzixxtAi-v"
      },
      "outputs": [],
      "source": [
        "def plot_confusion_matrix(X_test, y_test, model):\n",
        "    y_pred = model.predict(X_test)\n",
        "    y_pred = [np.argmax(i) for i in model.predict(X_test)]\n",
        "\n",
        "    con_mat = tf.math.confusion_matrix(labels=y_test, predictions=y_pred).numpy()\n",
        "\n",
        "    con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)\n",
        "    label_names = list(range(len(con_mat_norm)))\n",
        "\n",
        "    con_mat_df = pd.DataFrame(con_mat_norm,\n",
        "                              index=label_names, \n",
        "                              columns=label_names)\n",
        "\n",
        "    figure = plt.figure(figsize=(10, 10))\n",
        "    sns.heatmap(con_mat_df, cmap=plt.cm.Blues, annot=True)\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lYdnT3T9AlXD"
      },
      "outputs": [],
      "source": [
        "plot_confusion_matrix(X_test, y_test, model)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "foXRwrlp8fW7"
      },
      "source": [
        "## Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p8qRWfJ-Q0Kt"
      },
      "outputs": [],
      "source": [
        "y_pred = model.predict(X_test)\n",
        "y_pred = [np.argmax(i) for i in model.predict(X_test)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vT6suI4vQ0PR"
      },
      "outputs": [],
      "source": [
        "print(\"precision: \" + str(precision_score(y_test, y_pred)))\n",
        "print(\"recall: \" + str(recall_score(y_test, y_pred)))\n",
        "print(\"f1: \" + str(f1_score(y_test, y_pred)))\n",
        "print(\"accuracy: \" + str(model.evaluate(X_test, y_test)[1]*100))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}