03 - Data Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3bca3ee6-5fb5-447b-80b5-9ba4a4f5d84b",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c86be30b",
   "metadata": {},
   "source": [
    "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/juanesgutiz19/ia-project-bankruptcy-prediction/blob/master/03%20-%20Data%20Preprocessing.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8ddc3b58-9794-4540-9ec4-225df7cbfdd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e81b3721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyarrow in c:\\users\\user\\anaconda3\\lib\\site-packages (9.0.0)\n",
      "Requirement already satisfied: numpy>=1.16.6 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pyarrow) (1.21.5)\n"
     ]
    }
   ],
   "source": [
    "!pip install pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "37dd7ea6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: fastparquet in c:\\users\\user\\anaconda3\\lib\\site-packages (0.8.2)\n",
      "Requirement already satisfied: packaging in c:\\users\\user\\anaconda3\\lib\\site-packages (from fastparquet) (21.3)\n",
      "Requirement already satisfied: pandas>=1.1.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from fastparquet) (1.4.2)\n",
      "Requirement already satisfied: cramjam>=2.3.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from fastparquet) (2.5.0)\n",
      "Requirement already satisfied: fsspec in c:\\users\\user\\anaconda3\\lib\\site-packages (from fastparquet) (2022.2.0)\n",
      "Requirement already satisfied: numpy>=1.18 in c:\\users\\user\\anaconda3\\lib\\site-packages (from fastparquet) (1.21.5)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas>=1.1.0->fastparquet) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas>=1.1.0->fastparquet) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from python-dateutil>=2.8.1->pandas>=1.1.0->fastparquet) (1.16.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\users\\user\\anaconda3\\lib\\site-packages (from packaging->fastparquet) (3.0.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install fastparquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f2e59e42-4376-485f-aeb3-f1e856f5074c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bankrupt?</th>\n",
       "      <th>ROA(C) before interest and depreciation before interest</th>\n",
       "      <th>ROA(A) before interest and % after tax</th>\n",
       "      <th>ROA(B) before interest and depreciation after tax</th>\n",
       "      <th>Operating Gross Margin</th>\n",
       "      <th>Realized Sales Gross Margin</th>\n",
       "      <th>Operating Profit Rate</th>\n",
       "      <th>Pre-tax net Interest Rate</th>\n",
       "      <th>After-tax net Interest Rate</th>\n",
       "      <th>Non-industry income and expenditure/revenue</th>\n",
       "      <th>...</th>\n",
       "      <th>Net Income to Total Assets</th>\n",
       "      <th>Total assets to GNP price</th>\n",
       "      <th>No-credit Interval</th>\n",
       "      <th>Gross Profit to Sales</th>\n",
       "      <th>Net Income to Stockholder's Equity</th>\n",
       "      <th>Liability to Equity</th>\n",
       "      <th>Degree of Financial Leverage (DFL)</th>\n",
       "      <th>Interest Coverage Ratio (Interest expense to EBIT)</th>\n",
       "      <th>Net Income Flag</th>\n",
       "      <th>Equity to Liability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.370594</td>\n",
       "      <td>0.424389</td>\n",
       "      <td>Low</td>\n",
       "      <td>0.601457</td>\n",
       "      <td>High</td>\n",
       "      <td>0.998969</td>\n",
       "      <td>0.796887</td>\n",
       "      <td>0.808809</td>\n",
       "      <td>0.302646</td>\n",
       "      <td>...</td>\n",
       "      <td>0.716845</td>\n",
       "      <td>0.009219</td>\n",
       "      <td>0.622879</td>\n",
       "      <td>0.601453</td>\n",
       "      <td>0.827890</td>\n",
       "      <td>0.290202</td>\n",
       "      <td>0.026601</td>\n",
       "      <td>0.564050</td>\n",
       "      <td>1</td>\n",
       "      <td>0.016469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.464291</td>\n",
       "      <td>0.538214</td>\n",
       "      <td>Medium</td>\n",
       "      <td>0.610235</td>\n",
       "      <td>High</td>\n",
       "      <td>0.998946</td>\n",
       "      <td>0.797380</td>\n",
       "      <td>0.809301</td>\n",
       "      <td>0.303556</td>\n",
       "      <td>...</td>\n",
       "      <td>0.795297</td>\n",
       "      <td>0.008323</td>\n",
       "      <td>0.623652</td>\n",
       "      <td>0.610237</td>\n",
       "      <td>0.839969</td>\n",
       "      <td>0.283846</td>\n",
       "      <td>0.264577</td>\n",
       "      <td>0.570175</td>\n",
       "      <td>1</td>\n",
       "      <td>0.020794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.426071</td>\n",
       "      <td>0.499019</td>\n",
       "      <td>Medium</td>\n",
       "      <td>0.601450</td>\n",
       "      <td>High</td>\n",
       "      <td>0.998857</td>\n",
       "      <td>0.796403</td>\n",
       "      <td>0.808388</td>\n",
       "      <td>0.302035</td>\n",
       "      <td>...</td>\n",
       "      <td>0.774670</td>\n",
       "      <td>0.040003</td>\n",
       "      <td>0.623841</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>0.836774</td>\n",
       "      <td>0.290189</td>\n",
       "      <td>0.026555</td>\n",
       "      <td>0.563706</td>\n",
       "      <td>1</td>\n",
       "      <td>0.016474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.399844</td>\n",
       "      <td>0.451265</td>\n",
       "      <td>Medium</td>\n",
       "      <td>0.583541</td>\n",
       "      <td>High</td>\n",
       "      <td>0.998700</td>\n",
       "      <td>0.796967</td>\n",
       "      <td>0.808966</td>\n",
       "      <td>0.303350</td>\n",
       "      <td>...</td>\n",
       "      <td>0.739555</td>\n",
       "      <td>0.003252</td>\n",
       "      <td>0.622929</td>\n",
       "      <td>0.583538</td>\n",
       "      <td>0.834697</td>\n",
       "      <td>0.281721</td>\n",
       "      <td>0.026697</td>\n",
       "      <td>0.564663</td>\n",
       "      <td>1</td>\n",
       "      <td>0.023982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.465022</td>\n",
       "      <td>0.538432</td>\n",
       "      <td>Medium</td>\n",
       "      <td>0.598783</td>\n",
       "      <td>High</td>\n",
       "      <td>0.998973</td>\n",
       "      <td>0.797366</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.303475</td>\n",
       "      <td>...</td>\n",
       "      <td>0.795016</td>\n",
       "      <td>0.003878</td>\n",
       "      <td>0.623521</td>\n",
       "      <td>0.598782</td>\n",
       "      <td>0.839973</td>\n",
       "      <td>0.278514</td>\n",
       "      <td>0.024752</td>\n",
       "      <td>0.575617</td>\n",
       "      <td>1</td>\n",
       "      <td>0.035490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 96 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Bankrupt?   ROA(C) before interest and depreciation before interest  \\\n",
       "0          1                                           0.370594          \n",
       "1          1                                           0.464291          \n",
       "2          1                                           0.426071          \n",
       "3          1                                           0.399844          \n",
       "4          1                                           0.465022          \n",
       "\n",
       "    ROA(A) before interest and % after tax  \\\n",
       "0                                 0.424389   \n",
       "1                                 0.538214   \n",
       "2                                 0.499019   \n",
       "3                                 0.451265   \n",
       "4                                 0.538432   \n",
       "\n",
       "   ROA(B) before interest and depreciation after tax   Operating Gross Margin  \\\n",
       "0                                                Low                 0.601457   \n",
       "1                                             Medium                 0.610235   \n",
       "2                                             Medium                 0.601450   \n",
       "3                                             Medium                 0.583541   \n",
       "4                                             Medium                 0.598783   \n",
       "\n",
       "   Realized Sales Gross Margin   Operating Profit Rate  \\\n",
       "0                         High                0.998969   \n",
       "1                         High                0.998946   \n",
       "2                         High                0.998857   \n",
       "3                         High                0.998700   \n",
       "4                         High                0.998973   \n",
       "\n",
       "    Pre-tax net Interest Rate   After-tax net Interest Rate  \\\n",
       "0                    0.796887                      0.808809   \n",
       "1                    0.797380                      0.809301   \n",
       "2                    0.796403                      0.808388   \n",
       "3                    0.796967                      0.808966   \n",
       "4                    0.797366                      0.809304   \n",
       "\n",
       "    Non-industry income and expenditure/revenue  ...  \\\n",
       "0                                      0.302646  ...   \n",
       "1                                      0.303556  ...   \n",
       "2                                      0.302035  ...   \n",
       "3                                      0.303350  ...   \n",
       "4                                      0.303475  ...   \n",
       "\n",
       "    Net Income to Total Assets   Total assets to GNP price  \\\n",
       "0                     0.716845                    0.009219   \n",
       "1                     0.795297                    0.008323   \n",
       "2                     0.774670                    0.040003   \n",
       "3                     0.739555                    0.003252   \n",
       "4                     0.795016                    0.003878   \n",
       "\n",
       "    No-credit Interval   Gross Profit to Sales  \\\n",
       "0             0.622879                0.601453   \n",
       "1             0.623652                0.610237   \n",
       "2             0.623841                0.601449   \n",
       "3             0.622929                0.583538   \n",
       "4             0.623521                0.598782   \n",
       "\n",
       "    Net Income to Stockholder's Equity   Liability to Equity  \\\n",
       "0                             0.827890              0.290202   \n",
       "1                             0.839969              0.283846   \n",
       "2                             0.836774              0.290189   \n",
       "3                             0.834697              0.281721   \n",
       "4                             0.839973              0.278514   \n",
       "\n",
       "    Degree of Financial Leverage (DFL)  \\\n",
       "0                             0.026601   \n",
       "1                             0.264577   \n",
       "2                             0.026555   \n",
       "3                             0.026697   \n",
       "4                             0.024752   \n",
       "\n",
       "    Interest Coverage Ratio (Interest expense to EBIT)   Net Income Flag  \\\n",
       "0                                           0.564050                   1   \n",
       "1                                           0.570175                   1   \n",
       "2                                           0.563706                   1   \n",
       "3                                           0.564663                   1   \n",
       "4                                           0.575617                   1   \n",
       "\n",
       "    Equity to Liability  \n",
       "0              0.016469  \n",
       "1              0.020794  \n",
       "2              0.016474  \n",
       "3              0.023982  \n",
       "4              0.035490  \n",
       "\n",
       "[5 rows x 96 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_parquet('https://mlearner.s3.amazonaws.com/data/raw_data.parquet')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49b7abf1-bd98-4c5b-8d29-11967aa07f2f",
   "metadata": {},
   "source": [
    "## Reparando la data faltante"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "713470b2-442a-4397-8e79-6152bc294de3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " Operating Gross Margin                  501\n",
       " Total Asset Return Growth Rate Ratio    501\n",
       " Accounts Receivable Turnover            501\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k = df.isna().sum()\n",
    "k[k!=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e6536685-4d7f-4eca-ba2a-010666163fe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' Operating Gross Margin',\n",
       " ' Total Asset Return Growth Rate Ratio',\n",
       " ' Accounts Receivable Turnover']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_na = df.columns[df.isna().any()].tolist()\n",
    "list_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ac8b9bf9-0f9d-4b81-bf8a-efcaa4614b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in list_na:\n",
    "    df[i].fillna(df[i].mean(),inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f612b029-ecf5-48c4-8848-9086fda73e06",
   "metadata": {},
   "source": [
    "## One hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5e4ef9f9-3dd4-4e3f-8640-f1cead7936bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_onehot(x):\n",
    "    values = np.unique(x)\n",
    "    r = np.r_[[np.argwhere(i==values)[0][0] for i in x]]\n",
    "    return np.eye(len(values))[r].astype(int)\n",
    "\n",
    "def replace_column_with_onehot(df, col):\n",
    "    assert sum(df[col].isna())==0, \"column must have no NaN values\"\n",
    "    values = np.unique(df[col]\n",
    "                      )\n",
    "    k = to_onehot(df[col].values)\n",
    "    r = pd.DataFrame(k, columns=[\"%s_%s\"%(col, values[i]) for i in range(k.shape[1])], index=df.index).join(df)\n",
    "    del(r[col])\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d2f8b02f-578a-4588-b709-38468db8555c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' ROA(B) before interest and depreciation after tax',\n",
       " ' Realized Sales Gross Margin',\n",
       " ' Revenue Per Share (Yuan ¥)',\n",
       " ' Interest Expense Ratio',\n",
       " ' Debt ratio %',\n",
       " ' Current Assets/Total Assets',\n",
       " ' Working Capital/Equity',\n",
       " ' Cash Flow to Sales',\n",
       " ' Cash Flow to Liability',\n",
       " ' CFO to Assets']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_cat = [i for i in df.columns if not i in df._get_numeric_data()]\n",
    "list_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84da3d74-4642-43c1-a126-fc06bce9fa2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "print(len(list_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "09a314b2-23a2-4514-9a6f-6b655b4e3b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in list_cat:\n",
    "    df_aux = pd.get_dummies(df[i], prefix=i)\n",
    "    df.drop(i,inplace=True,axis=1)\n",
    "    df = df.join(df_aux)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aeb62a3f-0fb2-4dbd-a659-08f42ec5ae02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bankrupt?</th>\n",
       "      <th>ROA(C) before interest and depreciation before interest</th>\n",
       "      <th>ROA(A) before interest and % after tax</th>\n",
       "      <th>Operating Gross Margin</th>\n",
       "      <th>Operating Profit Rate</th>\n",
       "      <th>Pre-tax net Interest Rate</th>\n",
       "      <th>After-tax net Interest Rate</th>\n",
       "      <th>Non-industry income and expenditure/revenue</th>\n",
       "      <th>Continuous interest rate (after tax)</th>\n",
       "      <th>Operating Expense Rate</th>\n",
       "      <th>...</th>\n",
       "      <th>Working Capital/Equity_Low</th>\n",
       "      <th>Working Capital/Equity_Medium</th>\n",
       "      <th>Cash Flow to Sales_High</th>\n",
       "      <th>Cash Flow to Sales_Low</th>\n",
       "      <th>Cash Flow to Liability_High</th>\n",
       "      <th>Cash Flow to Liability_Low</th>\n",
       "      <th>Cash Flow to Liability_Medium</th>\n",
       "      <th>CFO to Assets_High</th>\n",
       "      <th>CFO to Assets_Low</th>\n",
       "      <th>CFO to Assets_Medium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.370594</td>\n",
       "      <td>0.424389</td>\n",
       "      <td>0.601457</td>\n",
       "      <td>0.998969</td>\n",
       "      <td>0.796887</td>\n",
       "      <td>0.808809</td>\n",
       "      <td>0.302646</td>\n",
       "      <td>0.780985</td>\n",
       "      <td>1.256969e-04</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.464291</td>\n",
       "      <td>0.538214</td>\n",
       "      <td>0.610235</td>\n",
       "      <td>0.998946</td>\n",
       "      <td>0.797380</td>\n",
       "      <td>0.809301</td>\n",
       "      <td>0.303556</td>\n",
       "      <td>0.781506</td>\n",
       "      <td>2.897851e-04</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.426071</td>\n",
       "      <td>0.499019</td>\n",
       "      <td>0.601450</td>\n",
       "      <td>0.998857</td>\n",
       "      <td>0.796403</td>\n",
       "      <td>0.808388</td>\n",
       "      <td>0.302035</td>\n",
       "      <td>0.780284</td>\n",
       "      <td>2.361297e-04</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.399844</td>\n",
       "      <td>0.451265</td>\n",
       "      <td>0.583541</td>\n",
       "      <td>0.998700</td>\n",
       "      <td>0.796967</td>\n",
       "      <td>0.808966</td>\n",
       "      <td>0.303350</td>\n",
       "      <td>0.781241</td>\n",
       "      <td>1.078888e-04</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.465022</td>\n",
       "      <td>0.538432</td>\n",
       "      <td>0.598783</td>\n",
       "      <td>0.998973</td>\n",
       "      <td>0.797366</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.303475</td>\n",
       "      <td>0.781550</td>\n",
       "      <td>7.890000e+09</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 115 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Bankrupt?   ROA(C) before interest and depreciation before interest  \\\n",
       "0          1                                           0.370594          \n",
       "1          1                                           0.464291          \n",
       "2          1                                           0.426071          \n",
       "3          1                                           0.399844          \n",
       "4          1                                           0.465022          \n",
       "\n",
       "    ROA(A) before interest and % after tax   Operating Gross Margin  \\\n",
       "0                                 0.424389                 0.601457   \n",
       "1                                 0.538214                 0.610235   \n",
       "2                                 0.499019                 0.601450   \n",
       "3                                 0.451265                 0.583541   \n",
       "4                                 0.538432                 0.598783   \n",
       "\n",
       "    Operating Profit Rate   Pre-tax net Interest Rate  \\\n",
       "0                0.998969                    0.796887   \n",
       "1                0.998946                    0.797380   \n",
       "2                0.998857                    0.796403   \n",
       "3                0.998700                    0.796967   \n",
       "4                0.998973                    0.797366   \n",
       "\n",
       "    After-tax net Interest Rate   Non-industry income and expenditure/revenue  \\\n",
       "0                      0.808809                                      0.302646   \n",
       "1                      0.809301                                      0.303556   \n",
       "2                      0.808388                                      0.302035   \n",
       "3                      0.808966                                      0.303350   \n",
       "4                      0.809304                                      0.303475   \n",
       "\n",
       "    Continuous interest rate (after tax)   Operating Expense Rate  ...  \\\n",
       "0                               0.780985             1.256969e-04  ...   \n",
       "1                               0.781506             2.897851e-04  ...   \n",
       "2                               0.780284             2.361297e-04  ...   \n",
       "3                               0.781241             1.078888e-04  ...   \n",
       "4                               0.781550             7.890000e+09  ...   \n",
       "\n",
       "    Working Capital/Equity_Low   Working Capital/Equity_Medium  \\\n",
       "0                            0                               0   \n",
       "1                            0                               0   \n",
       "2                            0                               0   \n",
       "3                            0                               0   \n",
       "4                            0                               0   \n",
       "\n",
       "    Cash Flow to Sales_High   Cash Flow to Sales_Low  \\\n",
       "0                         1                        0   \n",
       "1                         1                        0   \n",
       "2                         1                        0   \n",
       "3                         1                        0   \n",
       "4                         1                        0   \n",
       "\n",
       "    Cash Flow to Liability_High   Cash Flow to Liability_Low  \\\n",
       "0                             0                            0   \n",
       "1                             0                            0   \n",
       "2                             0                            0   \n",
       "3                             0                            1   \n",
       "4                             0                            0   \n",
       "\n",
       "    Cash Flow to Liability_Medium   CFO to Assets_High   CFO to Assets_Low  \\\n",
       "0                               1                    0                   0   \n",
       "1                               1                    1                   0   \n",
       "2                               1                    0                   0   \n",
       "3                               0                    1                   0   \n",
       "4                               1                    1                   0   \n",
       "\n",
       "    CFO to Assets_Medium  \n",
       "0                      1  \n",
       "1                      0  \n",
       "2                      1  \n",
       "3                      0  \n",
       "4                      0  \n",
       "\n",
       "[5 rows x 115 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "11280ea2-bce3-4348-ac09-d4f095645cd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i for i in df.columns if not i in df._get_numeric_data()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b31cdf8f-9b81-4ffe-9137-218f48fda870",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_parquet('clean_data.parquet',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "994546be-2880-4802-ab08-5e4799d4d6cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}