superconduct_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repositories uploaded!!\n",
      "Second Upload Completed!!\n",
      "Superconductivity Data\n",
      "-------------------------------------------\n",
      "(21263, 82)\n",
      "Training Set:  (6634, 81)\n",
      "Source Set 1:  (6956, 81)\n",
      "Source Set 2:  (7673, 81)\n",
      "Final Source Set:  (14629, 81)\n",
      "Adaboost.R2 Transfer Learning (M + H, L)\n",
      "-------------------------------------------\n",
      "(664, 80) (5970, 80)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidwang/opt/anaconda3/lib/python3.8/site-packages/sklearn/model_selection/_split.py:293: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2 ##STrAdaBoost.R2\n",
    "# from TwoStageTrAdaBoostR2 import TwoStageTrAdaBoostR2 ##two-stage TrAdaBoost.R2\n",
    "\n",
    "import pandas as pd\n",
    "import sys\n",
    "import numpy as np\n",
    "from pandas import DataFrame\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Input, Dense, Activation, Conv2D, Dropout, Flatten\n",
    "from keras import optimizers, utils, initializers, regularizers\n",
    "import keras.backend as K\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.preprocessing import StandardScaler #Importing the StandardScaler\n",
    "\n",
    "from itertools import combinations\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "from math import sqrt\n",
    "\n",
    "#Geo plotting libraries\n",
    "import geopandas as gdp\n",
    "from matplotlib.colors import ListedColormap\n",
    "# import geoplot as glpt\n",
    "\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn import linear_model\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "import matplotlib.lines as mlines\n",
    "\n",
    "import statistics\n",
    "\n",
    "from scipy.stats import rv_continuous\n",
    "from scipy.stats import *\n",
    "\n",
    "from statistics import mean\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.spatial import distance\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "\n",
    "######### Instance Transfer repositories ####################\n",
    "from adapt.instance_based import TwoStageTrAdaBoostR2\n",
    "\n",
    "print(\"Repositories uploaded!!\")\n",
    "\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from adapt.instance_based import KMM\n",
    "\n",
    "print(\"Second Upload Completed!!\")\n",
    "################################### superconduct ###########################################################################################################\n",
    "\n",
    "\n",
    "################################### Superconductivity ###########################################################################################################\n",
    "## Target Data: critical_temp\n",
    "## Correlation col: gmean_ElectronAffinity\n",
    "## Cuts at: 40.0 and 60.0\n",
    "##########################################################################################################################################################\n",
    "\n",
    "superconduct_df = pd.read_csv(\"UCI_regression/superconduct/train.csv\")\n",
    "superconduct_unq_df = pd.read_csv(\"UCI_regression/superconduct/train.csv\")\n",
    "\n",
    "print(\"Superconductivity Data\")\n",
    "print(\"-------------------------------------------\")\n",
    "print(superconduct_df.shape)\n",
    "\n",
    "# print(\"The correlation matrix is: \")\n",
    "# superconduct_df_corr = superconduct_df.corr()['critical_temp'].abs().sort_values()\n",
    "# print(superconduct_df_corr.to_string()) ### Helps to print the entire series\n",
    "\n",
    "# to find where to split the data\n",
    "# print('Min: ', superconduct_df['gmean_ElectronAffinity'].min())\n",
    "# print('Max: ', superconduct_df['gmean_ElectronAffinity'].max())\n",
    "\n",
    "drop_col_superconduct = ['gmean_ElectronAffinity']\n",
    "# superconduct_df['gmean_ElectronAffinity'].sort_values()\n",
    "\n",
    "superconduct_train_df = superconduct_df.loc[(superconduct_df['gmean_ElectronAffinity'] > 40.0) & (superconduct_df['gmean_ElectronAffinity'] <= 60.0)]\n",
    "superconduct_train_df = superconduct_train_df.drop(drop_col_superconduct, axis = 1)\n",
    "superconduct_train_df = superconduct_train_df.reset_index(drop = True)\n",
    "print(\"Training Set: \", superconduct_train_df.shape)\n",
    "\n",
    "superconduct_source1_df = superconduct_df.loc[(superconduct_df['gmean_ElectronAffinity'] > 60.0)]\n",
    "superconduct_source1_df = superconduct_source1_df.drop(drop_col_superconduct, axis = 1)\n",
    "superconduct_source1_df = superconduct_source1_df.reset_index(drop = True)\n",
    "print(\"Source Set 1: \", superconduct_source1_df.shape)\n",
    "\n",
    "superconduct_source2_df = superconduct_df.loc[(superconduct_df['gmean_ElectronAffinity'] <= 40.0)]\n",
    "superconduct_source2_df = superconduct_source2_df.drop(drop_col_superconduct, axis = 1)\n",
    "superconduct_source2_df = superconduct_source2_df.reset_index(drop = True)\n",
    "print(\"Source Set 2: \",superconduct_source2_df.shape)\n",
    "\n",
    "\n",
    "superconduct_source_df = pd.concat([superconduct_source1_df, superconduct_source2_df], ignore_index=True)\n",
    "print(\"Final Source Set: \",superconduct_source_df.shape)\n",
    "\n",
    "#################### Splitting into features and target ####################\n",
    "target_column_superconduct = ['critical_temp']\n",
    "\n",
    "superconduct_train_df_y = superconduct_train_df[target_column_superconduct]\n",
    "superconduct_train_df_X = superconduct_train_df.drop(target_column_superconduct, axis = 1)\n",
    "\n",
    "superconduct_source_df_y = superconduct_source_df[target_column_superconduct]\n",
    "superconduct_source_df_X = superconduct_source_df.drop(target_column_superconduct, axis = 1)\n",
    "\n",
    "\n",
    "########################### Transfer Learning superconduct #####################################################\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "\n",
    "\n",
    "print(\"Adaboost.R2 Transfer Learning (M + H, L)\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_AdaTL_superconduct = []\n",
    "rmselist_AdaTL_superconduct = []\n",
    "\n",
    "r2scorelist_Ada_superconduct = []\n",
    "rmselist_Ada_superconduct = []\n",
    "\n",
    "r2scorelist_KMM_superconduct = []\n",
    "rmselist_KMM_superconduct = []\n",
    "\n",
    "r2scorelist_GBRTL_superconduct = []\n",
    "rmselist_GBRTL_superconduct = []\n",
    "\n",
    "r2scorelist_GBR_superconduct = []\n",
    "rmselist_GBR_superconduct = []\n",
    "\n",
    "r2scorelist_TwoTrAda_superconduct = []\n",
    "rmselist_TwoTrAda_superconduct = []\n",
    "\n",
    "r2scorelist_stradaboost_superconduct = []\n",
    "rmselist_stradaboost_superconduct = []\n",
    "\n",
    "\n",
    "\n",
    "kfold = KFold(n_splits = 10, random_state=42, shuffle=False)\n",
    "\n",
    "for train_ix, test_ix in kfold.split(superconduct_train_df_X):\n",
    "    ############### get data ###############\n",
    "    superconduct_test_df_X, superconduct_tgt_df_X  = superconduct_train_df_X.iloc[train_ix], superconduct_train_df_X.iloc[test_ix] #### Make it opposite, so target size is small.\n",
    "    superconduct_test_df_y, superconduct_tgt_df_y  = superconduct_train_df_y.iloc[train_ix], superconduct_train_df_y.iloc[test_ix] #### Make it opposite, so target size is small.\n",
    "\n",
    "    print(superconduct_tgt_df_X.shape, superconduct_test_df_X.shape)\n",
    "\n",
    "    ############### Merging the datasets ##########################################\n",
    "    superconduct_X_df = pd.concat([superconduct_tgt_df_X, superconduct_source_df_X], ignore_index=True)\n",
    "    superconduct_y_df = pd.concat([superconduct_tgt_df_y, superconduct_source_df_y], ignore_index=True)\n",
    "\n",
    "    superconduct_np_train_X = superconduct_X_df.to_numpy()\n",
    "    superconduct_np_train_y = superconduct_y_df.to_numpy()\n",
    "\n",
    "    superconduct_np_test_X = superconduct_test_df_X.to_numpy()\n",
    "    superconduct_np_test_y = superconduct_test_df_y.to_numpy()\n",
    "\n",
    "    superconduct_np_train_y_list = superconduct_np_train_y.ravel()\n",
    "    superconduct_np_test_y_list = superconduct_np_test_y.ravel()\n",
    "\n",
    "    src_size_superconduct = len(superconduct_source_df_y)\n",
    "    tgt_size_superconduct = len(superconduct_tgt_df_y)\n",
    "\n",
    "    src_idx = np.arange(start=0, stop=(src_size_superconduct - 1), step=1)\n",
    "    tgt_idx = np.arange(start=src_size_superconduct, stop=((src_size_superconduct + tgt_size_superconduct)-1), step=1)\n",
    "\n",
    "\n",
    "    ################### AdaBoost Tl ###################\n",
    "    model_AdaTL_superconduct = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 8), learning_rate=0.01, n_estimators=500)\n",
    "    model_AdaTL_superconduct.fit(superconduct_np_train_X, superconduct_np_train_y_list)\n",
    "\n",
    "    y_pred_AdaTL_superconduct = model_AdaTL_superconduct.predict(superconduct_np_test_X)\n",
    "\n",
    "    mse_AdaTL_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_AdaTL_superconduct))\n",
    "    rmselist_AdaTL_superconduct.append(mse_AdaTL_superconduct)\n",
    "\n",
    "    r2_score_AdaTL_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_AdaTL_superconduct)\n",
    "    r2_score_AdaTL_superconduct = (r2_score_AdaTL_superconduct[0])**2\n",
    "    r2scorelist_AdaTL_superconduct.append(r2_score_AdaTL_superconduct)\n",
    "\n",
    "\n",
    "    ################### AdaBoost ###################\n",
    "    model_Ada_superconduct = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 8), learning_rate=0.01, n_estimators=500)\n",
    "    model_Ada_superconduct.fit(superconduct_tgt_df_X, superconduct_tgt_df_y)\n",
    "\n",
    "    y_pred_ada_superconduct = model_Ada_superconduct.predict(superconduct_np_test_X)\n",
    "\n",
    "    mse_Ada_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_ada_superconduct))\n",
    "    rmselist_Ada_superconduct.append(mse_Ada_superconduct)\n",
    "\n",
    "    r2_score_Ada_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_ada_superconduct)\n",
    "    r2_score_Ada_superconduct = (r2_score_Ada_superconduct[0])**2\n",
    "    r2scorelist_Ada_superconduct.append(r2_score_Ada_superconduct)\n",
    "\n",
    "\n",
    "   ################### KMM ###################\n",
    "    model_KMM_superconduct = KMM(get_estimator = get_estimator)\n",
    "    model_KMM_superconduct.fit(superconduct_np_train_X, superconduct_np_train_y_list, src_idx, tgt_idx)\n",
    "\n",
    "    y_pred_KMM_superconduct = model_KMM_superconduct.predict(superconduct_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_KMM_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_KMM_superconduct))\n",
    "    rmselist_KMM_superconduct.append(mse_KMM_superconduct)\n",
    "\n",
    "    r2_score_KMM_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_KMM_superconduct)\n",
    "    r2_score_KMM_superconduct = (r2_score_KMM_superconduct[0])**2\n",
    "    r2scorelist_KMM_superconduct.append(r2_score_KMM_superconduct)\n",
    "\n",
    "\n",
    "    ################### GBRTL ###################\n",
    "    model_GBRTL_superconduct = GradientBoostingRegressor(learning_rate = 0.01, max_depth = 4, n_estimators = 1000, subsample = 0.5)\n",
    "    model_GBRTL_superconduct.fit(superconduct_np_train_X, superconduct_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_superconduct = model_GBRTL_superconduct.predict(superconduct_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_GBRTL_superconduct))\n",
    "    rmselist_GBRTL_superconduct.append(mse_GBRTL_superconduct)\n",
    "\n",
    "    r2_score_GBRTL_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_GBRTL_superconduct)\n",
    "    r2_score_GBRTL_superconduct = (r2_score_GBRTL_superconduct[0])**2\n",
    "    r2scorelist_GBRTL_superconduct.append(r2_score_GBRTL_superconduct)\n",
    "\n",
    "\n",
    "    ################### GBR ###################\n",
    "    model_GBR_superconduct = GradientBoostingRegressor(learning_rate=0.01, max_depth=4, n_estimators=1000, subsample=0.5)\n",
    "    model_GBR_superconduct.fit(superconduct_tgt_df_X, superconduct_tgt_df_y)\n",
    "\n",
    "    y_pred_GBR_superconduct = model_GBR_superconduct.predict(superconduct_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBR_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_GBR_superconduct))\n",
    "    rmselist_GBR_superconduct.append(mse_GBR_superconduct)\n",
    "\n",
    "    r2_score_GBR_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_GBR_superconduct)\n",
    "    r2_score_GBR_superconduct = (r2_score_GBR_superconduct[0])**2\n",
    "    r2scorelist_GBR_superconduct.append(r2_score_GBR_superconduct)\n",
    "\n",
    "\n",
    "    ################### Two-TrAdaBoost ###################\n",
    "    from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "    model_TwoTrAda_superconduct = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 1000, cv=10) #, kwargs_TwoTrAda)\n",
    "    model_TwoTrAda_superconduct.fit(superconduct_np_train_X, superconduct_np_train_y_list, src_idx, tgt_idx)\n",
    "\n",
    "    y_pred_TwoTrAda_superconduct = model_TwoTrAda_superconduct.predict(superconduct_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_TwoTrAda_superconduct))\n",
    "    rmselist_TwoTrAda_superconduct.append(mse_TwoTrAda_superconduct)\n",
    "\n",
    "    r2_score_TwoTrAda_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_TwoTrAda_superconduct)\n",
    "    r2_score_TwoTrAda_superconduct = (r2_score_TwoTrAda_superconduct[0])**2\n",
    "    r2scorelist_TwoTrAda_superconduct.append(r2_score_TwoTrAda_superconduct)\n",
    "\n",
    "\n",
    "    ################### STrAdaBoost ###################\n",
    "    from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "    sample_size = [len(superconduct_tgt_df_X), len(superconduct_source_df_X)]\n",
    "    n_estimators = 100\n",
    "    steps = 30\n",
    "    fold = 10\n",
    "    random_state = np.random.RandomState(1)\n",
    "\n",
    "\n",
    "    model_stradaboost_superconduct = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_superconduct.fit(superconduct_np_train_X, superconduct_np_train_y_list)\n",
    "    y_pred_stradaboost_superconduct = model_stradaboost_superconduct.predict(superconduct_np_test_X)\n",
    "\n",
    "\n",
    "    mse_stradaboost_superconduct = sqrt(mean_squared_error(superconduct_np_test_y, y_pred_stradaboost_superconduct))\n",
    "    rmselist_stradaboost_superconduct.append(mse_stradaboost_superconduct)\n",
    "\n",
    "    r2_score_stradaboost_superconduct = pearsonr(superconduct_np_test_y_list, y_pred_stradaboost_superconduct)\n",
    "    r2_score_stradaboost_superconduct = (r2_score_stradaboost_superconduct[0])**2\n",
    "    r2scorelist_stradaboost_superconduct.append(r2_score_stradaboost_superconduct)\n",
    "\n",
    "\n",
    "\n",
    "with open('superconduct_rmse.txt', 'w') as superconduct_handle_rmse:\n",
    "    superconduct_handle_rmse.write(\"AdaBoost TL:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_AdaTL_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nAdaBoost:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_Ada_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nKMM:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_KMM_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nGBRT:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_GBRTL_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nGBR:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_GBR_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nTrAdaBoost:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_TwoTrAda_superconduct)\n",
    "\n",
    "    superconduct_handle_rmse.write(\"\\n\\nSTrAdaBoost:\\n \")\n",
    "    superconduct_handle_rmse.writelines(\"%s\\n\" % ele for ele in rmselist_stradaboost_superconduct)\n",
    "\n",
    "\n",
    "with open('superconduct_r2.txt', 'w') as superconduct_handle_r2:\n",
    "    superconduct_handle_r2.write(\"AdaBoost TL:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_AdaTL_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nAdaBoost:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_Ada_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nKMM:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_KMM_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nGBRT:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_GBRTL_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nGBR:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_GBR_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nTrAdaBoost:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_TwoTrAda_superconduct)\n",
    "\n",
    "    superconduct_handle_r2.write(\"\\n\\nSTrAdaBoost:\\n \")\n",
    "    superconduct_handle_r2.writelines(\"%s\\n\" % ele for ele in r2scorelist_stradaboost_superconduct)\n",
    "\n",
    "\n",
    "######################################################################################\n",
    "\n",
    "\n",
    "# print(\"RMSE of Adaboost.R2(TL):\", statistics.mean(rmselist_AdaTL_superconduct))\n",
    "# print(\"R^2 of AdaboostR2(TL):\", statistics.mean(r2scorelist_AdaTL_superconduct))\n",
    "# print(\"\\n\")\n",
    "# print(\"RMSE of Adaboost.R2(TL):\", rmselist_AdaTL_superconduct)\n",
    "# print(\"R^2 of AdaboostR2(TL):\", r2scorelist_AdaTL_superconduct)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}