RA_largedata_results3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repositories uploaded!!\n"
     ]
    }
   ],
   "source": [
    "# from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2 ##STrAdaBoost.R2\n",
    "# from TwoStageTrAdaBoostR2 import TwoStageTrAdaBoostR2 ##two-stage TrAdaBoost.R2\n",
    "\n",
    "import pandas as pd\n",
    "import sys\n",
    "import numpy as np\n",
    "from pandas import DataFrame\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.preprocessing import StandardScaler \n",
    "from sklearn.linear_model import LinearRegression, Ridge\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.lines as mlines\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy.stats.stats import pearsonr\n",
    "from math import sqrt\n",
    "import statistics \n",
    "from scipy.stats import *\n",
    "from scipy.spatial import distance\n",
    "\n",
    "######### Instance Transfer repositories ####################\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from adapt.instance_based import KMM\n",
    "\n",
    "print(\"Repositories uploaded!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ailerons Data\n",
      "Training Set:  (7154, 41)\n",
      "Testing Set:  (6596, 41)\n",
      "-------------------------------------------\n",
      "Target Set:  (358, 41)\n",
      "Source Set:  (6796, 41)\n",
      "Test Set:  (6596, 41)\n"
     ]
    }
   ],
   "source": [
    "################################################################ Ailerons ###############################################\n",
    "target_ailerons = ['goal']\n",
    "colnames_ailerons = ['climbRate', 'Sgz', 'p', 'q', 'curPitch', 'curRoll', 'absRoll', 'diffClb', 'diffRollRate', 'diffDiffClb', 'SeTime1',\n",
    "            'SeTime2', 'SeTime3', 'SeTime4', 'SeTime5', 'SeTime6', 'SeTime7', 'SeTime8', 'SeTime9', 'SeTime10', 'SeTime11', 'SeTime12', \n",
    "            'SeTime13', 'SeTime14', 'diffSeTime1', 'diffSeTime2', 'diffSeTime3', 'diffSeTime4', 'diffSeTime5', 'diffSeTime6', 'diffSeTime7',\n",
    "            'diffSeTime8', 'diffSeTime9', 'diffSeTime10', 'diffSeTime11', 'diffSeTime12', 'diffSeTime13', 'diffSeTime14', 'alpha', 'Se', 'goal']\n",
    "AileronsData_train_df = pd.read_csv('UCI_regression/Ailerons/ailerons.data', header = None, names = colnames_ailerons) \n",
    "print(\"Ailerons Data\")\n",
    "print(\"Training Set: \", AileronsData_train_df.shape)\n",
    "\n",
    "AileronsData_test_df = pd.read_csv('UCI_regression/Ailerons/ailerons.test', header = None, names = colnames_ailerons) \n",
    "print(\"Testing Set: \", AileronsData_test_df.shape)\n",
    "\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "############################################### Standardization ###############################################\n",
    "ailerons_cols = AileronsData_train_df.columns.difference(['goal'])\n",
    "ss = StandardScaler()\n",
    "AileronsData_train_df[ailerons_cols] = ss.fit_transform(AileronsData_train_df[ailerons_cols])\n",
    "AileronsData_test_df[ailerons_cols] = ss.transform(AileronsData_test_df[ailerons_cols]) ## Use the same scale as the training data\n",
    "\n",
    "\n",
    "#################### Splitting with small target set and large source and test set #############\n",
    "AileronsData_source_df, AileronsData_tgt_df = train_test_split(AileronsData_train_df, test_size = 0.05) ## test_size = tgt size\n",
    "# print(AileronsData_df_tgt.shape, AileronsData_df_source.shape, AileronsData_df_test.shape)\n",
    "\n",
    "AileronsData_train_df = AileronsData_train_df.reset_index(drop = True)\n",
    "AileronsData_tgt_df = AileronsData_tgt_df.reset_index(drop = True)\n",
    "AileronsData_source_df = AileronsData_source_df.reset_index(drop = True)\n",
    "print(\"Target Set: \", AileronsData_tgt_df.shape)\n",
    "print(\"Source Set: \", AileronsData_source_df.shape)\n",
    "print(\"Test Set: \", AileronsData_test_df.shape)\n",
    "\n",
    "\n",
    "# ################################# Splitting into predictors and target data #################################\n",
    "# AileronsData_df_test_y = AileronsData_test_df[target_ailerons]\n",
    "# AileronsData_df_test_X = AileronsData_test_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "# AileronsData_df_tgt_y = AileronsData_tgt_df[target_ailerons]\n",
    "# AileronsData_df_tgt_X = AileronsData_tgt_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "# AileronsData_df_source_y = AileronsData_source_df[target_ailerons]\n",
    "# AileronsData_df_source_X = AileronsData_source_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "\n",
    "# ############## Merging the datasets #################\n",
    "# ailerons_X_df = pd.concat([AileronsData_df_source_X, AileronsData_df_tgt_X], ignore_index=True)\n",
    "# ailerons_y_df = pd.concat([AileronsData_df_source_y, AileronsData_df_tgt_y], ignore_index=True)\n",
    "\n",
    "# ailerons_np_train_X = ailerons_X_df.to_numpy()\n",
    "# ailerons_np_train_y = ailerons_y_df.to_numpy()\n",
    "\n",
    "# ailerons_np_test_X = AileronsData_df_test_X.to_numpy()\n",
    "# ailerons_np_test_y = AileronsData_df_test_y.to_numpy()\n",
    "\n",
    "# ailerons_np_train_y_list = ailerons_np_train_y.ravel()\n",
    "# ailerons_np_test_y_list = ailerons_np_test_y.ravel()\n",
    "\n",
    "# src_size_ailerons = len(AileronsData_df_source_y)\n",
    "# tgt_size_ailerons = len(AileronsData_df_tgt_y)\n",
    "\n",
    "# print(\"-------------------------------------------\")\n",
    "# ##############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15, 41)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "######################################## Variance Sampling: Phase 1###################################################\n",
    "kmeans = KMeans(n_clusters = 15, random_state=0).fit(AileronsData_tgt_df)\n",
    "\n",
    "AileronsData_alternate_df = AileronsData_tgt_df.copy()\n",
    "AileronsData_alternate_df_np = AileronsData_tgt_df.to_numpy()\n",
    "\n",
    "idxlist = []\n",
    "AileronsData_new_df_list = []\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in AileronsData_alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "#     print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", mindist)\n",
    "#     print(\"Matrix shape: \", AileronsData_alternate_df_np.shape)\n",
    "    AileronsData_new_df_list.append(rowval)\n",
    "    AileronsData_alternate_df = np.delete(AileronsData_alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "AileronsData_new_df = pd.DataFrame(np.vstack(AileronsData_new_df_list))\n",
    "AileronsData_new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of source dataset before Variance Sampling:  (6796, 41)\n",
      "Shape of source dataset after Variance Sampling:  (6781, 41)\n",
      "----------------------------------------------\n",
      "Shape of target before Variance Sampling:  (358, 41)\n",
      "Shape of target after Variance Sampling:  (373, 41)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "##################################################### Variance Sampling: Phase 2 ################################################\n",
    "\n",
    "AileronsData_alternate_source_df = AileronsData_source_df.copy()\n",
    "AileronsData_alternate_source_df_np = AileronsData_alternate_source_df.to_numpy()\n",
    "\n",
    "idxlist2 = []\n",
    "AileronsData_final_df_list = []\n",
    "\n",
    "for row_nm in AileronsData_new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in AileronsData_alternate_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "#     print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", min_dist)\n",
    "#     print(\"Matrix shape: \", AileronsData_alternate_source_df_np.shape)\n",
    "    AileronsData_final_df_list.append(row_val)\n",
    "    AileronsData_alternate_source_df_np = np.delete(AileronsData_alternate_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "AileronsData_final_df = pd.DataFrame(np.vstack(AileronsData_final_df_list), columns= AileronsData_alternate_source_df.columns)\n",
    "\n",
    "\n",
    "print(\"Shape of source dataset before Variance Sampling: \",AileronsData_source_df.shape)\n",
    "AileronsData_source_df = pd.DataFrame(np.vstack(AileronsData_alternate_source_df_np), columns= AileronsData_source_df.columns)\n",
    "print(\"Shape of source dataset after Variance Sampling: \", AileronsData_source_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "\n",
    "print(\"Shape of target before Variance Sampling: \", AileronsData_tgt_df.shape)\n",
    "AileronsData_tgt_df = pd.concat([AileronsData_tgt_df, AileronsData_final_df], ignore_index=True)\n",
    "print(\"Shape of target after Variance Sampling: \", AileronsData_tgt_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Set:  (373, 41)\n",
      "Source Set:  (1500, 41)\n",
      "Test Set:  (6596, 41)\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "################################## Importance Sampling: Manhattan Distance ######################################################\n",
    "\n",
    "AileronsData_source_df[\"ManDis\"] = \"\"\n",
    "\n",
    "AileronsData_tgt_df_mean = []\n",
    "prow = AileronsData_tgt_df.mean()\n",
    "AileronsData_tgt_df_mean = [prow.climbRate, prow.Sgz, prow.p, prow.q, prow.curPitch, prow.curRoll, prow.absRoll,\n",
    "       prow.diffClb, prow.diffRollRate, prow.diffDiffClb, prow.SeTime1, prow.SeTime2, prow.SeTime3, prow.SeTime4, \n",
    "        prow.SeTime5, prow.SeTime6, prow.SeTime7, prow.SeTime8, prow.SeTime9, prow.SeTime10, prow.SeTime11, prow.SeTime12, \n",
    "        prow.SeTime13, prow.SeTime14, prow.diffSeTime1, prow.diffSeTime2, prow.diffSeTime3, prow.diffSeTime4, prow.diffSeTime5, \n",
    "        prow.diffSeTime6, prow.diffSeTime7, prow.diffSeTime8, prow.diffSeTime9, prow.diffSeTime10, prow.diffSeTime11, prow.diffSeTime12,\n",
    "       prow.diffSeTime13, prow.diffSeTime14, prow.alpha, prow.Se, prow.goal]\n",
    "\n",
    "rowidx = 0\n",
    "\n",
    "for row in AileronsData_source_df.itertuples():\n",
    "    row_list =[row.climbRate, row.Sgz, row.p, row.q, row.curPitch, row.curRoll, row.absRoll,\n",
    "       row.diffClb, row.diffRollRate, row.diffDiffClb, row.SeTime1, row.SeTime2, row.SeTime3, row.SeTime4, \n",
    "        row.SeTime5, row.SeTime6, row.SeTime7, row.SeTime8, row.SeTime9, row.SeTime10, row.SeTime11, row.SeTime12, \n",
    "        row.SeTime13, row.SeTime14, row.diffSeTime1, row.diffSeTime2, row.diffSeTime3, row.diffSeTime4, row.diffSeTime5, \n",
    "        row.diffSeTime6, row.diffSeTime7, row.diffSeTime8, row.diffSeTime9, row.diffSeTime10, row.diffSeTime11, row.diffSeTime12,\n",
    "       row.diffSeTime13, row.diffSeTime14, row.alpha, row.Se, row.goal]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = AileronsData_tgt_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "\n",
    "    AileronsData_source_df.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "AileronsData_source_df = AileronsData_source_df.sort_values(by =['ManDis'])\n",
    "AileronsData_source_df = AileronsData_source_df.head(1500) \n",
    "AileronsData_source_df = AileronsData_source_df.drop(['ManDis'], axis =1)\n",
    "AileronsData_source_df = AileronsData_source_df.reset_index(drop=True)\n",
    "\n",
    "# print(\"Target Set: \",AileronsData_tgt_df.shape)\n",
    "# print(\"Source Set: \",AileronsData_source_df.shape)\n",
    "# print(\"Test Set: \",AileronsData_test_df.shape)\n",
    "\n",
    "\n",
    "################################# Splitting into predictors and target data #################################\n",
    "print(\"Target Set: \", AileronsData_tgt_df.shape)\n",
    "print(\"Source Set: \", AileronsData_source_df.shape)\n",
    "print(\"Test Set: \", AileronsData_test_df.shape)\n",
    "\n",
    "\n",
    "AileronsData_df_test_y = AileronsData_test_df[target_ailerons]\n",
    "AileronsData_df_test_X = AileronsData_test_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "AileronsData_df_tgt_y = AileronsData_tgt_df[target_ailerons]\n",
    "AileronsData_df_tgt_X = AileronsData_tgt_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "AileronsData_df_source_y = AileronsData_source_df[target_ailerons]\n",
    "AileronsData_df_source_X = AileronsData_source_df.drop(target_ailerons, axis = 1)\n",
    "\n",
    "\n",
    "############## Merging the datasets #################\n",
    "ailerons_X_df = pd.concat([AileronsData_df_tgt_X, AileronsData_df_source_X], ignore_index=True)\n",
    "ailerons_y_df = pd.concat([AileronsData_df_tgt_y, AileronsData_df_source_y], ignore_index=True)\n",
    "\n",
    "ailerons_np_train_X = ailerons_X_df.to_numpy()\n",
    "ailerons_np_train_y = ailerons_y_df.to_numpy()\n",
    "\n",
    "ailerons_np_test_X = AileronsData_df_test_X.to_numpy()\n",
    "ailerons_np_test_y = AileronsData_df_test_y.to_numpy()\n",
    "\n",
    "ailerons_np_train_y_list = ailerons_np_train_y.ravel()\n",
    "ailerons_np_test_y_list = ailerons_np_test_y.ravel()\n",
    "\n",
    "src_size_ailerons = len(AileronsData_df_source_y)\n",
    "tgt_size_ailerons = len(AileronsData_df_tgt_y)\n",
    "\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "RMSE of STrAdaboostR2: [0.00018653840674816383, 0.000189810904166031, 0.0001873658474340575, 0.00019085531291090202, 0.00019114973737215766, 0.00019043394736071044, 0.0001888067344590376, 0.00018804314636032746, 0.00018824536253958136, 0.0001927143957780782, 0.00019012721685156454, 0.00018968020765799165, 0.00019056782946398726, 0.00019100182687938224, 0.00018898277228031844, 0.00019125494130398105, 0.00018900343933637447, 0.00018761671245519221, 0.0001917718719682461, 0.00019119037172070315]\n",
      "R^2 of STrAdaboostR2: [0.7994431446922216, 0.79291772428616, 0.7959727769704424, 0.7919700807049479, 0.7885973254825356, 0.7924961778829837, 0.7915936882136941, 0.795523949236591, 0.7945970418686013, 0.7897947537475372, 0.7934872816059522, 0.7915548650511036, 0.792148766070684, 0.788703506201112, 0.7906655665234325, 0.7924082390032214, 0.7917252298714027, 0.7959090954930488, 0.7854179709888617, 0.7903470071690677]\n",
      "\n",
      "\n",
      "RMSE of STrAdaboostR2: 0.0001897580492523394 1.6385462150355857e-06\n",
      "R^2 of STrAdaboostR2: 0.79226370955318 0.0031061025941011814\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#################################### STrAdaBoost.R2 Active Sampling Ailerons ################################################################\n",
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "sample_size = [len(AileronsData_df_tgt_X), len(AileronsData_df_source_X)]\n",
    "n_estimators = 100\n",
    "steps = 30\n",
    "fold = 10\n",
    "random_state = np.random.RandomState(1)\n",
    "\n",
    "r2scorelist_stradaboost_ailerons = []\n",
    "rmselist_stradaboost_ailerons = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_stradaboost_ailerons = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_ailerons.fit(ailerons_np_train_X, ailerons_np_train_y_list)\n",
    "\n",
    "\n",
    "    y_pred_stradaboost_ailerons = model_stradaboost_ailerons.predict(ailerons_np_test_X)\n",
    "\n",
    "    mse_stradaboost_ailerons = sqrt(mean_squared_error(ailerons_np_test_y, y_pred_stradaboost_ailerons))\n",
    "    rmselist_stradaboost_ailerons.append(mse_stradaboost_ailerons)\n",
    "        \n",
    "    r2_score_stradaboost_ailerons = pearsonr(ailerons_np_test_y_list, y_pred_stradaboost_ailerons)\n",
    "    r2_score_stradaboost_ailerons = (r2_score_stradaboost_ailerons[0])**2\n",
    "    r2scorelist_stradaboost_ailerons.append(r2_score_stradaboost_ailerons)\n",
    "\n",
    "print(\"RMSE of STrAdaboostR2:\", rmselist_stradaboost_ailerons)\n",
    "print(\"R^2 of STrAdaboostR2:\", r2scorelist_stradaboost_ailerons)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(\"RMSE of STrAdaboostR2:\", statistics.mean(rmselist_stradaboost_ailerons), statistics.stdev(rmselist_stradaboost_ailerons))\n",
    "print(\"R^2 of STrAdaboostR2:\", statistics.mean(r2scorelist_stradaboost_ailerons), statistics.stdev(r2scorelist_stradaboost_ailerons))\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Two-Stage TrAdaboost.R2\n",
      "-------------------------------------------\n",
      "RMSE of TrAdaboost.R2: 0.00020574786209551332\n",
      "R^2 of TrAdaboost.R2: 0.7413189333258624\n",
      "\n",
      "\n",
      "RMSE of TrAdaboost.R2: [0.00020681854607685043, 0.00020651638834517068, 0.00020383301674030212, 0.00020683286942591622, 0.0002063692404633311, 0.00020680161826133934, 0.00020364534019528337, 0.00020411402277487162, 0.000206621642615898, 0.00020635976828406653, 0.00020632514249374563, 0.00020637385249563253, 0.00020668712168823728, 0.00020655433214305843, 0.00020393791473106047, 0.0002035921161816063, 0.00020649446155580529, 0.00020694081790597585, 0.0002066865427000793, 0.00020345248683203583]\n",
      "R^2 of TrAdaboostR2: [0.7389481264149564, 0.7396235798682649, 0.745455458309211, 0.7389356605217205, 0.740037235075707, 0.7390041251533895, 0.7458853651842333, 0.7447896713826876, 0.7394059008386079, 0.7400556736305642, 0.7401337947231849, 0.7400038569756268, 0.7392396724376006, 0.7395462943100826, 0.7451917559072191, 0.7460176753457975, 0.739764541432313, 0.7387092421464678, 0.7392852245341871, 0.7463458123254262]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### Two-Stage TrAdaBoost.R2 Ailerons #######################################\n",
    "\n",
    "src_idx_ailerons = np.arange(start = 0, stop = (src_size_ailerons - 1), step = 1)\n",
    "tgt_idx_ailerons = np.arange(start = src_size_ailerons, stop = ((src_size_ailerons + tgt_size_ailerons)-1), step = 1)\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "print(\"Two-Stage TrAdaboost.R2\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_TwoTrAda_ailerons = []\n",
    "rmselist_TwoTrAda_ailerons = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_TwoTrAda_ailerons = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 100, cv = 10) \n",
    "    model_TwoTrAda_ailerons.fit(ailerons_np_train_X, ailerons_np_train_y_list, src_idx_ailerons, tgt_idx_ailerons)\n",
    "\n",
    "    y_pred_TwoTrAda_ailerons = model_TwoTrAda_ailerons.predict(ailerons_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_ailerons = sqrt(mean_squared_error(ailerons_np_test_y, y_pred_TwoTrAda_ailerons))\n",
    "    rmselist_TwoTrAda_ailerons.append(mse_TwoTrAda_ailerons)\n",
    "        \n",
    "    r2_score_TwoTrAda_ailerons = pearsonr(ailerons_np_test_y_list, y_pred_TwoTrAda_ailerons)\n",
    "    r2_score_TwoTrAda_ailerons = (r2_score_TwoTrAda_ailerons[0])**2\n",
    "    r2scorelist_TwoTrAda_ailerons.append(r2_score_TwoTrAda_ailerons)\n",
    "\n",
    "\n",
    "print(\"RMSE of TrAdaboost.R2:\", statistics.mean(rmselist_TwoTrAda_ailerons))\n",
    "print(\"R^2 of TrAdaboost.R2:\", statistics.mean(r2scorelist_TwoTrAda_ailerons))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of TrAdaboost.R2:\", rmselist_TwoTrAda_ailerons)\n",
    "print(\"R^2 of TrAdaboostR2:\", r2scorelist_TwoTrAda_ailerons)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost.R2 Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE of Adaboost.R2: 0.0001761917430976294\n",
      "R^2 of Adaboost.R2: 0.8097699164413095\n",
      "\n",
      "\n",
      "RMSE of Adaboost.R2: [0.00017628318682367841, 0.00017584055293828223, 0.00017634410324385614, 0.00017622372690464236, 0.00017604538005660224, 0.00017632961518882105, 0.00017559384898326461, 0.00017618526560104336, 0.00017594284539940966, 0.00017653234805190102, 0.00017617342396156684, 0.00017606639444173904, 0.000176084858111512, 0.00017622582289483174, 0.0001768299014573899, 0.00017651844635589153, 0.000175820852554345, 0.00017596883305018337, 0.00017563652462611293, 0.00017611957287911955, 0.0001766978465605523, 0.00017666096469340104, 0.00017663608447072543, 0.00017663114710269118, 0.0001757538831803806, 0.0001761817413146185, 0.00017634383918371436, 0.0001766047699291948, 0.0001764780839549657, 0.00017499842901444533]\n",
      "R^2 of Adaboost.R2: [0.8097089208457251, 0.8105180497864907, 0.8094016940819106, 0.8095727562546757, 0.810032955429172, 0.8092818777560554, 0.8111499586291837, 0.8097693408067742, 0.8102031236206514, 0.8091117380584097, 0.8098589009691258, 0.8101214854530965, 0.8099923474533364, 0.8097219728869393, 0.8083456877805938, 0.8089375828110345, 0.810613935471705, 0.8102203004408474, 0.8108123669042882, 0.810010380218553, 0.8086873933589545, 0.8087811249824546, 0.8088496414933575, 0.8087675316213655, 0.8108219895635649, 0.8099575706845877, 0.8093495667164528, 0.8088148055417917, 0.8092751837392091, 0.8124073098789772]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### AdaBoostR2 Transfer Learning Ailerons #####################################################\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "print(\"Adaboost.R2 Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_AdaTL_ailerons = []\n",
    "rmselist_AdaTL_ailerons = []\n",
    "\n",
    "for x in range(0, 30):\n",
    "\n",
    "    model_AdaTL_ailerons = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 6), learning_rate = 0.1, n_estimators = 100) \n",
    "    model_AdaTL_ailerons.fit(ailerons_np_train_X, ailerons_np_train_y_list)\n",
    "\n",
    "    y_pred_AdaTL_ailerons = model_AdaTL_ailerons.predict(ailerons_np_test_X)\n",
    "\n",
    "    mse_AdaTL_ailerons = sqrt(mean_squared_error(ailerons_np_test_y, y_pred_AdaTL_ailerons))\n",
    "    rmselist_AdaTL_ailerons.append(mse_AdaTL_ailerons)\n",
    "        \n",
    "    r2_score_AdaTL_ailerons = pearsonr(ailerons_np_test_y_list, y_pred_AdaTL_ailerons)\n",
    "    r2_score_AdaTL_ailerons = (r2_score_AdaTL_ailerons[0])**2\n",
    "    r2scorelist_AdaTL_ailerons.append(r2_score_AdaTL_ailerons)\n",
    "\n",
    "print(\"RMSE of Adaboost.R2:\", statistics.mean(rmselist_AdaTL_ailerons))\n",
    "print(\"R^2 of Adaboost.R2:\", statistics.mean(r2scorelist_AdaTL_ailerons))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of Adaboost.R2:\", rmselist_AdaTL_ailerons)\n",
    "print(\"R^2 of Adaboost.R2:\", r2scorelist_AdaTL_ailerons)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE of GBRTL: 0.0001654395827399936\n",
      "R^2 of GBRTL: 0.8318196264020231\n",
      "\n",
      "\n",
      "RMSE of GBRTL: [0.0001656659832301053, 0.0001655080035961031, 0.00016563006901253418, 0.0001653424573613975, 0.0001654268570127941, 0.000165369585315523, 0.0001653589841636465, 0.00016539590637807439, 0.00016549309690746676, 0.0001653418398216678, 0.00016563954571484176, 0.00016522546755197082, 0.00016530249601359962, 0.00016549258747006396, 0.0001653736781975519, 0.00016554922704477832, 0.00016564238356153062, 0.00016529866896184068, 0.00016537571584277668, 0.00016535910164160495]\n",
      "R^2 of GBRTL: [0.8313632275100481, 0.8317019098900963, 0.831431736835772, 0.8320078717975276, 0.8318424099928636, 0.8319617877754968, 0.8319838443346086, 0.8319111910282008, 0.8317069350591485, 0.8320264396152213, 0.8314067854200656, 0.8322483245092896, 0.8320858269936434, 0.8317125723895374, 0.8319530269492782, 0.8316036589868377, 0.8314168497170955, 0.8321096142661164, 0.8319398652821469, 0.8319786496874683]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################### Gradient Boosting Regression Transfer Learning Ailerons #######################################\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "\n",
    "print(\"Gradient Boosting Regression Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_GBRTL_ailerons = []\n",
    "rmselist_GBRTL_ailerons = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_GBRTL_ailerons = GradientBoostingRegressor(learning_rate = 0.1, max_depth = 6, n_estimators = 100)\n",
    "    model_GBRTL_ailerons.fit(ailerons_np_train_X, ailerons_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_ailerons = model_GBRTL_ailerons.predict(AileronsData_df_test_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_ailerons = sqrt(mean_squared_error(ailerons_np_test_y, y_pred_GBRTL_ailerons))\n",
    "    rmselist_GBRTL_ailerons.append(mse_GBRTL_ailerons)\n",
    "        \n",
    "    r2_score_GBRTL_ailerons = pearsonr(ailerons_np_test_y_list, y_pred_GBRTL_ailerons)\n",
    "    r2_score_GBRTL_ailerons = (r2_score_GBRTL_ailerons[0])**2\n",
    "    r2scorelist_GBRTL_ailerons.append(r2_score_GBRTL_ailerons)\n",
    "\n",
    "    \n",
    "print(\"RMSE of GBRTL:\", statistics.mean(rmselist_GBRTL_ailerons))\n",
    "print(\"R^2 of GBRTL:\", statistics.mean(r2scorelist_GBRTL_ailerons))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of GBRTL:\", rmselist_GBRTL_ailerons)\n",
    "print(\"R^2 of GBRTL:\", r2scorelist_GBRTL_ailerons)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elevators Data\n",
      "(8752, 19)\n",
      "(7847, 19)\n",
      "Target Set:  (438, 19)\n",
      "Source Set:  (8314, 19)\n",
      "Test Set:  (7847, 19)\n"
     ]
    }
   ],
   "source": [
    "####################################### Elevators #########################################################\n",
    "target_elevators = ['Goal']\n",
    "colnames_elevators = ['climbRate', 'Sgz', 'p', 'q', 'curRoll', 'absRoll', 'diffClb', 'diffRollRate', 'diffDiffClb', 'SaTime1', 'SaTime2', \n",
    "                      'SaTime3', 'SaTime4', 'diffSaTime1', 'diffSaTime2', 'diffSaTime3', 'diffSaTime4', 'Sa', 'Goal']\n",
    "Elevators_train_df = pd.read_csv('UCI_regression/Elevators/elevators.data', header = None, names = colnames_elevators)\n",
    "print(\"Elevators Data\")\n",
    "print(Elevators_train_df.shape)\n",
    "\n",
    "Elevators_test_df = pd.read_csv('UCI_regression/Elevators/elevators.test', header = None, names = colnames_elevators)\n",
    "print(Elevators_test_df.shape)\n",
    "\n",
    "############################################### Standardization ###############################################\n",
    "elevators_cols = Elevators_train_df.columns.difference(['Goal'])\n",
    "ss = StandardScaler()\n",
    "Elevators_train_df[elevators_cols] = ss.fit_transform(Elevators_train_df[elevators_cols])\n",
    "Elevators_test_df[elevators_cols] = ss.transform(Elevators_test_df[elevators_cols]) ## Use the same scale as the training data\n",
    "\n",
    "#################### Splitting with small target set and large source and test set #############\n",
    "Elevators_source_df, Elevators_tgt_df = train_test_split(Elevators_train_df, test_size = 0.05) ## test_size = tgt size\n",
    "# print(Elevators_df_tgt.shape, Elevators_df_source.shape, Elevators_df_test.shape)\n",
    "\n",
    "Elevators_tgt_df = Elevators_tgt_df.reset_index(drop = True)\n",
    "Elevators_source_df = Elevators_source_df.reset_index(drop = True)\n",
    "print(\"Target Set: \", Elevators_tgt_df.shape)\n",
    "print(\"Source Set: \", Elevators_source_df.shape)\n",
    "print(\"Test Set: \", Elevators_test_df.shape)\n",
    "\n",
    "################################# Splitting into predictors and target data #################################\n",
    "# Elevators_test_df_y = Elevators_test_df[target_elevators]\n",
    "# Elevators_test_df_X = Elevators_test_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "# Elevators_tgt_df_y = Elevators_tgt_df[target_elevators]\n",
    "# Elevators_tgt_df_X = Elevators_tgt_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "# Elevators_source_df_y = Elevators_source_df[target_elevators]\n",
    "# Elevators_source_df_X = Elevators_source_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "\n",
    "# ############## Merging the datasets #################\n",
    "# elevators_X_df = pd.concat([Elevators_source_df_X, Elevators_tgt_df_X], ignore_index=True)\n",
    "# elevators_y_df = pd.concat([Elevators_source_df_y, Elevators_tgt_df_y], ignore_index=True)\n",
    "\n",
    "# elevators_np_train_X = elevators_X_df.to_numpy()\n",
    "# elevators_np_train_y = elevators_y_df.to_numpy()\n",
    "\n",
    "# elevators_np_test_X = Elevators_test_df_X.to_numpy()\n",
    "# elevators_np_test_y = Elevators_test_df_y.to_numpy()\n",
    "\n",
    "# elevators_np_train_y_list = elevators_np_train_y.ravel()\n",
    "# elevators_np_test_y_list = elevators_np_test_y.ravel()\n",
    "\n",
    "# src_size_elevators = len(Elevators_source_df_y)\n",
    "# tgt_size_elevators = len(Elevators_tgt_df_y)\n",
    "\n",
    "# print(\"---------------------------\")\n",
    "\n",
    "# ###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of source before : (8314, 19)\n",
      "Shape of source after : (8294, 19)\n",
      "----------------------------------------------\n",
      "Shape of target before : (438, 19)\n",
      "Shape of target after : (458, 19)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################################## Variance Sampling: Phase 1 ###################################################\n",
    "kmeans = KMeans(n_clusters = 20, random_state=0).fit(Elevators_tgt_df)\n",
    "\n",
    "Elevators_alternate_df = Elevators_tgt_df.copy()\n",
    "Elevators_alternate_df_np = Elevators_tgt_df.to_numpy()\n",
    "\n",
    "idxlist = []\n",
    "Elevators_new_df_list = []\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in Elevators_alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "#     print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", mindist)\n",
    "#     print(\"Matrix shape: \", Elevators_alternate_df_np.shape)\n",
    "    Elevators_new_df_list.append(rowval)\n",
    "    Elevators_alternate_df = np.delete(Elevators_alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "Elevators_new_df = pd.DataFrame(np.vstack(Elevators_new_df_list))\n",
    "Elevators_new_df.shape\n",
    "\n",
    "##################################################### Variance Sampling: Phase 2 ################################################\n",
    "\n",
    "Elevators_alternate_source_df = Elevators_source_df.copy()\n",
    "Elevators_alternate_source_df_np = Elevators_alternate_source_df.to_numpy()\n",
    "\n",
    "idxlist2 = []\n",
    "Elevators_final_df_list = []\n",
    "\n",
    "for row_nm in Elevators_new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in Elevators_alternate_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "#     print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", min_dist)\n",
    "#     print(\"Matrix shape: \", Elevators_alternate_source_df_np.shape)\n",
    "    Elevators_final_df_list.append(row_val)\n",
    "    Elevators_alternate_source_df_np = np.delete(Elevators_alternate_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "Elevators_final_df = pd.DataFrame(np.vstack(Elevators_final_df_list), columns= Elevators_alternate_source_df.columns)\n",
    "\n",
    "print(\"Shape of source before :\",Elevators_source_df.shape)\n",
    "Elevators_source_df = pd.DataFrame(np.vstack(Elevators_alternate_source_df_np), columns= Elevators_source_df.columns)\n",
    "print(\"Shape of source after :\", Elevators_source_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "print(\"Shape of target before :\", Elevators_tgt_df.shape)\n",
    "Elevators_tgt_df = pd.concat([Elevators_tgt_df, Elevators_final_df], ignore_index=True)\n",
    "print(\"Shape of target after :\", Elevators_tgt_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Set:  (2536, 19)\n",
      "Source Set:  (6216, 19)\n",
      "Test Set:  (7847, 19)\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "################################## Importance Sampling: Manhattan Distance ######################################################\n",
    "\n",
    "Elevators_source_df[\"ManDis\"] = \"\"\n",
    "\n",
    "Elevators_tgt_df_mean = []\n",
    "prow = Elevators_tgt_df.mean()\n",
    "Elevators_tgt_df_mean = [prow.climbRate, prow.Sgz, prow.p, prow.q, prow.curRoll, prow.absRoll,\n",
    "       prow.diffClb, prow.diffRollRate, prow.diffDiffClb, prow.SaTime1, prow.SaTime2, prow.SaTime3, prow.SaTime4, \n",
    "        prow.diffSaTime1, prow.diffSaTime2, prow.diffSaTime3, prow.diffSaTime4, prow.Sa, prow.Goal]\n",
    "\n",
    "rowidx = 0\n",
    "\n",
    "for row in Elevators_source_df.itertuples():\n",
    "    row_list =[row.climbRate, row.Sgz, row.p, row.q, row.curRoll, row.absRoll, row.diffClb, \n",
    "            row.diffRollRate, row.diffDiffClb, row.SaTime1, row.SaTime2, row.SaTime3, row.SaTime4, \n",
    "               row.diffSaTime1, row.diffSaTime2, row.diffSaTime3, row.diffSaTime4, row.Sa, row.Goal]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = Elevators_tgt_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "\n",
    "    Elevators_source_df.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "Elevators_source_df = Elevators_source_df.sort_values(by =['ManDis'])\n",
    "Elevators_tgt_source_df = Elevators_source_df.head(2078) ## 600 instances chosen for Kinematics Dataset\n",
    "Elevators_source_df = Elevators_source_df.iloc[2078:]\n",
    "\n",
    "Elevators_source_df = Elevators_source_df.drop(['ManDis'], axis =1)\n",
    "Elevators_tgt_source_df = Elevators_tgt_source_df.drop(['ManDis'], axis =1)\n",
    "\n",
    "Elevators_tgt_df = pd.concat([Elevators_tgt_df, Elevators_tgt_source_df], ignore_index=True) ### This line is used only for STrAdaBoost.R2 and not for TrAdaBoost.R2\n",
    "\n",
    "Elevators_tgt_df = Elevators_tgt_df.reset_index(drop=True)\n",
    "Elevators_source_df = Elevators_source_df.reset_index(drop=True)\n",
    "\n",
    "\n",
    "#################### Splitting with small target set and large source and test set #############\n",
    "print(\"Target Set: \", Elevators_tgt_df.shape)\n",
    "print(\"Source Set: \", Elevators_source_df.shape)\n",
    "print(\"Test Set: \", Elevators_test_df.shape)\n",
    "\n",
    "\n",
    "Elevators_test_df_y = Elevators_test_df[target_elevators]\n",
    "Elevators_test_df_X = Elevators_test_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "Elevators_tgt_df_y = Elevators_tgt_df[target_elevators]\n",
    "Elevators_tgt_df_X = Elevators_tgt_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "Elevators_source_df_y = Elevators_source_df[target_elevators]\n",
    "Elevators_source_df_X = Elevators_source_df.drop(target_elevators, axis = 1)\n",
    "\n",
    "\n",
    "############## Merging the datasets #################\n",
    "elevators_X_df = pd.concat([Elevators_tgt_df_X, Elevators_source_df_X], ignore_index=True)\n",
    "elevators_y_df = pd.concat([Elevators_tgt_df_y, Elevators_source_df_y], ignore_index=True)\n",
    "\n",
    "elevators_np_train_X = elevators_X_df.to_numpy()\n",
    "elevators_np_train_y = elevators_y_df.to_numpy()\n",
    "\n",
    "elevators_np_test_X = Elevators_tgt_df_X.to_numpy()\n",
    "elevators_np_test_y = Elevators_tgt_df_y.to_numpy()\n",
    "\n",
    "elevators_np_train_y_list = elevators_np_train_y.ravel()\n",
    "elevators_np_test_y_list = elevators_np_test_y.ravel()\n",
    "\n",
    "src_size_elevators = len(Elevators_source_df_y)\n",
    "tgt_size_elevators = len(Elevators_tgt_df_y)\n",
    "\n",
    "print(\"---------------------------\")\n",
    "\n",
    "###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "RMSE of STrAdaboostR2: 0.0030410629395618218\n",
      "R^2 of STrAdaboostR2: 0.8194621213682728\n",
      "\n",
      "\n",
      "RMSE of STrAdaboostR2: [0.002994308060777424, 0.0031586784175449143, 0.0030899128906734154, 0.0029370613376077503, 0.002979925497627271, 0.003239176309770462, 0.00300647740216143, 0.003132132991311323, 0.0029844042782061837, 0.0031057849944312378, 0.002987418041278848, 0.002947721785694048, 0.0030862998031953364, 0.0028720097711430388, 0.002992840798409856, 0.002970085896208511, 0.003027284347944437, 0.003047229641552924, 0.0031942036399121177, 0.0030683028857859096]\n",
      "R^2 of STrAdaboostR2: [0.8199507310239796, 0.821083399390994, 0.8119872066950584, 0.8214132857989649, 0.8228260886680355, 0.8220660292097063, 0.8234383732919764, 0.8084758742208555, 0.8191318819350368, 0.8145416794310836, 0.8216833392530357, 0.8227030583806363, 0.824203445461412, 0.8291828711123767, 0.8173837047553305, 0.8222621822343771, 0.8177775776145914, 0.825013887026712, 0.7975790135171341, 0.8265387983441588]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#################################### STrAdaBoost.R2 Active Sampling Elevators ################################################################\n",
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "sample_size = [len(Elevators_tgt_df_X), len(Elevators_source_df_X)]\n",
    "n_estimators = 100\n",
    "steps = 30\n",
    "fold = 10\n",
    "random_state = np.random.RandomState(1)\n",
    "\n",
    "r2scorelist_stradaboost_elevators = []\n",
    "rmselist_stradaboost_elevators = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_stradaboost_elevators = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_elevators.fit(elevators_np_train_X, elevators_np_train_y_list)\n",
    "\n",
    "\n",
    "    y_pred_stradaboost_elevators = model_stradaboost_elevators.predict(elevators_np_test_X)\n",
    "\n",
    "    mse_stradaboost_elevators = sqrt(mean_squared_error(elevators_np_test_y, y_pred_stradaboost_elevators))\n",
    "    rmselist_stradaboost_elevators.append(mse_stradaboost_elevators)\n",
    "        \n",
    "    r2_score_stradaboost_elevators = pearsonr(elevators_np_test_y_list, y_pred_stradaboost_elevators)\n",
    "    r2_score_stradaboost_elevators = (r2_score_stradaboost_elevators[0])**2\n",
    "    r2scorelist_stradaboost_elevators.append(r2_score_stradaboost_elevators)\n",
    "\n",
    "\n",
    "print(\"RMSE of STrAdaboostR2:\", statistics.mean(rmselist_stradaboost_elevators))\n",
    "print(\"R^2 of STrAdaboostR2:\", statistics.mean(r2scorelist_stradaboost_elevators))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of STrAdaboostR2:\", rmselist_stradaboost_elevators)\n",
    "print(\"R^2 of STrAdaboostR2:\", r2scorelist_stradaboost_elevators)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Two-Stage TrAdaboost.R2\n",
      "-------------------------------------------\n",
      "RMSE of TrAdaboost.R2: 0.00425249625088312\n",
      "R^2 of TrAdaboostR2: 0.5665965475885255\n",
      "\n",
      "\n",
      "RMSE of TrAdaboost.R2: [0.004235960974722378, 0.0042532980185931105, 0.00424928425978152, 0.00423064506904706, 0.004229359136246281, 0.004229399458957712, 0.004257522157982159, 0.004272599373216897, 0.004229756431819963, 0.004391997594254764, 0.004213225304044787, 0.004251404880964993, 0.004237451430065547, 0.00423533847501916, 0.004224446056963165, 0.004262692696212362, 0.00427239590670089, 0.004253822044230903, 0.004252250988171132, 0.0042670747606676175]\n",
      "R^2 of TrAdaboostR2: [0.5695237913761826, 0.5664879912827565, 0.5668730999599445, 0.5705440879983508, 0.5704949220464953, 0.5704858668555313, 0.5651252515440068, 0.5634632387653482, 0.5704190193066773, 0.5432596988276073, 0.5735998024060381, 0.5669158394224875, 0.5689307309056442, 0.5695322523951819, 0.5714517808021735, 0.5653033524046835, 0.562530129718536, 0.5659629266770204, 0.5667259109208621, 0.5643012581549817]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### TwoStagetrAdaBoostR2 Elevators #######################################\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "############# Transfer Learning specifications #########################\n",
    "src_idx_elevators = np.arange(start=0, stop=(src_size_elevators - 1), step=1)\n",
    "tgt_idx_elevators = np.arange(start=src_size_elevators, stop=((src_size_elevators + tgt_size_elevators)-1), step=1)\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "\n",
    "\n",
    "print(\"Two-Stage TrAdaboost.R2\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_TwoTrAda_elevators = []\n",
    "rmselist_TwoTrAda_elevators = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_TwoTrAda_elevators = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 100) \n",
    "    model_TwoTrAda_elevators.fit(elevators_np_train_X, elevators_np_train_y_list, src_idx_elevators, tgt_idx_elevators)\n",
    "\n",
    "    y_pred_TwoTrAda_elevators = model_TwoTrAda_elevators.predict(elevators_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_elevators = sqrt(mean_squared_error(elevators_np_test_y, y_pred_TwoTrAda_elevators))\n",
    "    rmselist_TwoTrAda_elevators.append(mse_TwoTrAda_elevators)\n",
    "        \n",
    "    r2_score_TwoTrAda_elevators = pearsonr(elevators_np_test_y_list, y_pred_TwoTrAda_elevators)\n",
    "    r2_score_TwoTrAda_elevators = (r2_score_TwoTrAda_elevators[0])**2\n",
    "\n",
    "    r2scorelist_TwoTrAda_elevators.append(r2_score_TwoTrAda_elevators)\n",
    "\n",
    "\n",
    "print(\"RMSE of TrAdaboost.R2:\", statistics.mean(rmselist_TwoTrAda_elevators))\n",
    "print(\"R^2 of TrAdaboostR2:\", statistics.mean(r2scorelist_TwoTrAda_elevators))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of TrAdaboost.R2:\", rmselist_TwoTrAda_elevators)\n",
    "print(\"R^2 of TrAdaboostR2:\", r2scorelist_TwoTrAda_elevators)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE of GBRTL: 0.0026625453685619736\n",
      "R^2 of GBRTL: 0.8289294250562849\n",
      "\n",
      "\n",
      "RMSE of GBRTL: [0.0026595479194474105, 0.0026583236722272233, 0.0026589335344576965, 0.0026571863687977516, 0.002655751540759153, 0.00265766405344248, 0.0026586231496001185, 0.0026574318691075897, 0.0026595256699780663, 0.002659790631933799, 0.0026630375560211337, 0.0026947114611148076, 0.002659765325440391, 0.0026603059820020716, 0.00265621711218301, 0.0026631985916709634, 0.002657825846625134, 0.0026581598805280704, 0.0026964279225725075, 0.0026584792833300934]\n",
      "R^2 of GBRTL: [0.8293344383594842, 0.8294677575307577, 0.8293976798555139, 0.8296512683650047, 0.8298145696207326, 0.8295702189337696, 0.8294608315000559, 0.8295985313576577, 0.8293432990891416, 0.8293030714444692, 0.8288369466732859, 0.8247096781131288, 0.8292851312921292, 0.8292084726400372, 0.8297853027939636, 0.8288373567830716, 0.8295467033757878, 0.8295049144742852, 0.8244777066964467, 0.8294546222269742]\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################### Gradient Boosting Regression Transfer Learning Elevators #######################################\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "\n",
    "print(\"Gradient Boosting Regression Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_GBRTL_elevators = []\n",
    "rmselist_GBRTL_elevators = []\n",
    "\n",
    "for x in range(0, 20):\n",
    "\n",
    "    model_GBRTL_elevators = GradientBoostingRegressor(learning_rate = 0.1, max_depth = 6, n_estimators = 100)\n",
    "    model_GBRTL_elevators.fit(elevators_np_train_X, elevators_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_elevators = model_GBRTL_elevators.predict(elevators_np_test_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_elevators = sqrt(mean_squared_error(elevators_np_test_y, y_pred_GBRTL_elevators))\n",
    "    rmselist_GBRTL_elevators.append(mse_GBRTL_elevators)\n",
    "        \n",
    "    r2_score_GBRTL_elevators = pearsonr(elevators_np_test_y_list, y_pred_GBRTL_elevators)\n",
    "    r2_score_GBRTL_elevators = (r2_score_GBRTL_elevators[0])**2\n",
    "    r2scorelist_GBRTL_elevators.append(r2_score_GBRTL_elevators)\n",
    "\n",
    "\n",
    "print(\"RMSE of GBRTL:\", statistics.mean(rmselist_GBRTL_elevators))\n",
    "print(\"R^2 of GBRTL:\", statistics.mean(r2scorelist_GBRTL_elevators))\n",
    "print(\"\\n\")\n",
    "print(\"RMSE of GBRTL:\", rmselist_GBRTL_elevators)\n",
    "print(\"R^2 of GBRTL:\", r2scorelist_GBRTL_elevators)\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Abalone Data\n",
      "(4177, 9)\n",
      "Target Set:  (1241, 8)\n",
      "Source Set 1:  (1465, 8)\n",
      "Source Set 2:  (1471, 8)\n",
      "Final Source Set:  (2936, 8)\n"
     ]
    }
   ],
   "source": [
    "################################### Abalone ###########################################\n",
    "#### range: [0.0 - 1.130]\n",
    "#### Mid of correlation variable: Whole_weight\n",
    "#### [0, 0.12] [0.12, 0.15], [0.15, 1.130]\n",
    "#### Target variable: Rings\n",
    "#######################################################################################\n",
    "target_var_abalone = ['Rings']\n",
    "colnames_abalone = ['Sex', 'Length', 'Diameter', 'Height', 'Whole_weight', 'Shucked_weight', 'Viscera_weight', 'Shell_weight', 'Rings']\n",
    "AbaloneData_df = pd.read_csv('UCI_regression/Abalone/abalone.data', header = None, names = colnames_abalone)\n",
    "\n",
    "gender = {'M': 1,'F': 2, 'I': 3} \n",
    "AbaloneData_df.Sex = [gender[item] for item in AbaloneData_df.Sex] \n",
    "\n",
    "print(\"Abalone Data\")\n",
    "print(AbaloneData_df.shape)\n",
    "\n",
    "########## Corr Abalone ################\n",
    "# print(\"The correlation matrix is: \")\n",
    "# print(AbaloneData_df.corr()['Rings'])\n",
    "\n",
    "##################### Splitting in 3 equal parts #######################################\n",
    "# print(AbaloneData_df.sort_values('Whole_weight')['Whole_weight'])\n",
    "# print(len(AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] <= 0.5)]))\n",
    "# print(len(AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] > 0.5) & (AbaloneData_df['Whole_weight'] <= 1.0)]))\n",
    "# print(len(AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] > 1.0)]))\n",
    "\n",
    "#################### Splitting with small target set and large source and test set #############\n",
    "# AbaloneData_df_remain, AbaloneData_df_tgt = train_test_split(AbaloneData_df, test_size = 0.05) ## test_size = tgt size\n",
    "# AbaloneData_df_source, AbaloneData_df_test = train_test_split(AbaloneData_df_remain, test_size = 0.3) ## test_size = tgt size\n",
    "# print(AbaloneData_df_tgt.shape, AbaloneData_df_source.shape, AbaloneData_df_test.shape)\n",
    "\n",
    "\n",
    "drop_col_abalone = ['Whole_weight']\n",
    "\n",
    "abalone_tgt_df = AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] <= 0.5)]\n",
    "abalone_tgt_df = abalone_tgt_df.drop(drop_col_abalone, axis = 1)\n",
    "abalone_tgt_df = abalone_tgt_df.reset_index(drop=True)\n",
    "print(\"Target Set: \",abalone_tgt_df.shape)\n",
    "\n",
    "abalone_source1_df = AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] > 0.5) & (AbaloneData_df['Whole_weight'] <= 1.0)]\n",
    "abalone_source1_df = abalone_source1_df.drop(drop_col_abalone, axis = 1)\n",
    "abalone_source1_df = abalone_source1_df.reset_index(drop=True)\n",
    "print(\"Source Set 1: \",abalone_source1_df.shape)\n",
    "\n",
    "abalone_source2_df = AbaloneData_df.loc[(AbaloneData_df['Whole_weight'] > 1.0)]\n",
    "abalone_source2_df = abalone_source2_df.drop(drop_col_abalone, axis = 1)\n",
    "abalone_source2_df = abalone_source2_df.reset_index(drop=True)\n",
    "print(\"Source Set 2: \",abalone_source2_df.shape)\n",
    "\n",
    "\n",
    "################################# Standardization #################################\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "abalone_cols = abalone_tgt_df.columns.difference(['Rings'])\n",
    "\n",
    "ss = StandardScaler()\n",
    "abalone_tgt_df[abalone_cols] = ss.fit_transform(abalone_tgt_df[abalone_cols])\n",
    "abalone_source1_df[abalone_cols] = ss.fit_transform(abalone_source1_df[abalone_cols])\n",
    "abalone_source2_df[abalone_cols] = ss.fit_transform(abalone_source2_df[abalone_cols])\n",
    "\n",
    "\n",
    "################################# Concatenating the source datasets #################################\n",
    "abalone_source_df = pd.concat([abalone_source1_df, abalone_source2_df], ignore_index = True)\n",
    "abalone_source_df = abalone_source_df.reset_index(drop = True)\n",
    "print(\"Final Source Set: \",abalone_source_df.shape)\n",
    "\n",
    "\n",
    "# #################### Splitting into features and target ####################\n",
    "# target_abalone = ['Rings']\n",
    "\n",
    "# abalone_source_df_y = abalone_source_df[target_abalone]\n",
    "# abalone_source_df_X = abalone_source_df.drop(target_abalone, axis = 1)\n",
    "\n",
    "# features_abalone = abalone_source_df_X.columns\n",
    "# print(\"---------------------------\")\n",
    "# ##############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of the extracted data is: \n",
      "(60, 8)\n",
      "-------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################################## Variance Sampling: Phase 1 ###################################################\n",
    "kmeans = KMeans(n_clusters = 60, random_state=0).fit(abalone_tgt_df)\n",
    "\n",
    "abalone_alternate_df = abalone_tgt_df.copy()\n",
    "abalone_alternate_df_np = abalone_tgt_df.to_numpy()\n",
    "\n",
    "idxlist = []\n",
    "abalone_new_df_list = []\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in abalone_alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "#     print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", mindist)\n",
    "#     print(\"Matrix shape: \", abalone_alternate_df_np.shape)\n",
    "    abalone_new_df_list.append(rowval)\n",
    "    abalone_alternate_df = np.delete(abalone_alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "abalone_new_df = pd.DataFrame(np.vstack(abalone_new_df_list))\n",
    "\n",
    "print(\"The shape of the extracted data is: \")\n",
    "print(abalone_new_df.shape)\n",
    "print(\"-------------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of source before : (2936, 8)\n",
      "Shape of source after : (2876, 8)\n",
      "----------------------------------------------\n",
      "Shape of target before : (1241, 8)\n",
      "Shape of target after : (1301, 8)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "##################################################### Variance Sampling: Phase 2 ################################################\n",
    "\n",
    "abalone_alternate_source_df = abalone_source_df.copy()\n",
    "abalone_alternate_source_df_np = abalone_alternate_source_df.to_numpy()\n",
    "\n",
    "idxlist2 = []\n",
    "abalone_final_df_list = []\n",
    "\n",
    "for row_nm in abalone_new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in abalone_alternate_source_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "#     print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", min_dist)\n",
    "#     print(\"Matrix shape: \", abalone_alternate_source_df_np.shape)\n",
    "    abalone_final_df_list.append(row_val)\n",
    "    abalone_alternate_source_df_np = np.delete(abalone_alternate_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "abalone_final_df = pd.DataFrame(np.vstack(abalone_final_df_list), columns= abalone_alternate_source_df.columns)\n",
    "\n",
    "print(\"Shape of source before :\",abalone_source_df.shape)\n",
    "abalone_source_df = pd.DataFrame(np.vstack(abalone_alternate_source_df_np), columns= abalone_source_df.columns)\n",
    "print(\"Shape of source after :\", abalone_source_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "print(\"Shape of target before :\", abalone_tgt_df.shape)\n",
    "abalone_tgt_df = pd.concat([abalone_tgt_df, abalone_final_df], ignore_index=True)\n",
    "print(\"Shape of target after :\", abalone_tgt_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Set:  (1301, 8)\n",
      "Source Set:  (734, 8)\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "################################## Importance Sampling: Manhattan Distance ######################################################\n",
    "\n",
    "abalone_source_df[\"ManDis\"] = \"\"\n",
    "\n",
    "abalone_tgt_df_mean = []\n",
    "prow = abalone_tgt_df.mean()\n",
    "abalone_tgt_df_mean = [prow.Sex, prow.Length, prow.Diameter, prow.Height, prow.Shucked_weight,\n",
    "       prow.Viscera_weight, prow.Shell_weight, prow.Rings]\n",
    "\n",
    "rowidx = 0\n",
    "\n",
    "for row in abalone_source_df.itertuples():\n",
    "    row_list =[row.Sex, row.Length, row.Diameter, row.Height, row.Shucked_weight,\n",
    "       row.Viscera_weight, row.Shell_weight, row.Rings]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = abalone_tgt_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "\n",
    "    abalone_source_df.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "abalone_source_df = abalone_source_df.sort_values(by =['ManDis'])\n",
    "abalone_source_df = abalone_source_df.head(734) \n",
    "abalone_source_df = abalone_source_df.drop(['ManDis'], axis =1)\n",
    "abalone_source_df = abalone_source_df.reset_index(drop=True)\n",
    "\n",
    "print(\"Target Set: \", abalone_tgt_df.shape)\n",
    "print(\"Source Set: \", abalone_source_df.shape)\n",
    "\n",
    "#################### Splitting into features and target ####################\n",
    "target_abalone = ['Rings']\n",
    "\n",
    "abalone_source_df_y = abalone_source_df[target_abalone]\n",
    "abalone_source_df_X = abalone_source_df.drop(target_abalone, axis = 1)\n",
    "\n",
    "# features_abalone = abalone_source_df_X.columns\n",
    "print(\"---------------------------\")\n",
    "\n",
    "###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "RMSE List of STrAdaboostR2: [1.4998097152267589, 1.8162314223167797, 2.095896899220678, 2.782995910293012, 1.6713044531991006, 1.7085902984893293, 1.7711095331330846, 1.7775509731209713, 1.4719510937976386, 1.417697114433143, 1.8310863168676341, 2.3723851550253663, 1.4753089956416976, 1.4352865704743354, 1.585213323437924, 1.930806894951238, 1.6300976733946837, 1.9777275810505046, 1.598596882271576, 6.70593226693369]\n",
      "R^2 List of STrAdaboostR2: [0.5821170600076994, 0.5219044058329887, 0.5023839637970224, 0.3597438924831273, 0.5596693913022083, 0.4380588009843879, 0.26742413521105424, 0.3576282583797772, 0.29474534610066344, 0.4917738994081539, 0.39924006241949184, 0.47558570969461694, 0.6487533368448863, 0.564041002651409, 0.46050218505733187, 0.4965248961222965, 0.5217608543963609, 0.2776639604216498, 0.45192908936236353, 0.07337886613315295]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 2.027778953663957 1.15133882201889\n",
      "Mean, STDev of R^2: 0.43724145583053214 0.1347231365895204\n"
     ]
    }
   ],
   "source": [
    "#################################### STrAdaBoost.R2 Abalone ################################################################\n",
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "n_estimators = 100\n",
    "steps = 30\n",
    "fold = 10\n",
    "random_state = np.random.RandomState(1)\n",
    "\n",
    "r2scorelist_stradaboost_abalone = []\n",
    "rmselist_stradaboost_abalone = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(abalone_tgt_df):\n",
    "        \n",
    "    abalone_train_df_X = abalone_tgt_df.iloc[train_idx].loc[:, features_abalone]\n",
    "    abalone_test_df_X = abalone_tgt_df.iloc[test_idx][features_abalone]\n",
    "    abalone_train_df_y = abalone_tgt_df.iloc[train_idx].loc[:,target_abalone]\n",
    "    abalone_test_df_y = abalone_tgt_df.loc[test_idx][target_abalone] \n",
    "    \n",
    "    abalone_X_df = pd.concat([abalone_source_df_X, abalone_train_df_X], ignore_index=True)\n",
    "    abalone_y_df = pd.concat([abalone_source_df_y, abalone_train_df_y], ignore_index=True)\n",
    "\n",
    "    abalone_np_train_X = abalone_X_df.to_numpy()\n",
    "    abalone_np_train_y = abalone_y_df.to_numpy()\n",
    "\n",
    "    abalone_np_test_X = abalone_test_df_X.to_numpy()\n",
    "    abalone_np_test_y = abalone_test_df_y.to_numpy()\n",
    "\n",
    "    abalone_np_train_y_list = abalone_np_train_y.ravel()\n",
    "    abalone_np_test_y_list = abalone_np_test_y.ravel()\n",
    "    \n",
    "    sample_size = [len(abalone_source_df_X), len(abalone_train_df_X)]\n",
    "\n",
    "    model_stradaboost_abalone = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_abalone.fit(abalone_np_train_X, abalone_np_train_y_list)\n",
    "    y_pred_stradaboost_abalone = model_stradaboost_abalone.predict(abalone_np_test_X)\n",
    "\n",
    "    mse_stradaboost_abalone = sqrt(mean_squared_error(abalone_np_test_y, y_pred_stradaboost_abalone))\n",
    "    rmselist_stradaboost_abalone.append(mse_stradaboost_abalone)\n",
    "        \n",
    "    r2_score_stradaboost_abalone = pearsonr(abalone_np_test_y_list, y_pred_stradaboost_abalone)\n",
    "    r2_score_stradaboost_abalone = (r2_score_stradaboost_abalone[0])**2\n",
    "    r2scorelist_stradaboost_abalone.append(r2_score_stradaboost_abalone)\n",
    "\n",
    "print(\"RMSE List of STrAdaboostR2:\", rmselist_stradaboost_abalone)\n",
    "print(\"R^2 List of STrAdaboostR2:\", r2scorelist_stradaboost_abalone)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_stradaboost_abalone), statistics.stdev(rmselist_stradaboost_abalone))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_stradaboost_abalone), statistics.stdev(r2scorelist_stradaboost_abalone))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Two-Stage TrAdaboost.R2\n",
      "-------------------------------------------\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "RMSE List of TrAdaboost.R2: [1.4312725505338255, 1.890274886070674, 1.8914627820468175, 3.3259271834855673, 1.7941492032884396, 1.906524030341246, 1.619550054564094, 1.6265393700922177, 1.6495580663529212, 1.4715631610224822, 1.6362725730657035, 2.068976339943138, 2.3041492683264053, 1.2275142428012118, 1.4366394841434167, 1.6743631646627484, 2.1152435712989273, 1.4310508059715614, 1.9855702455915705, 2.2863439092520927]\n",
      "R^2 List of TrAdaboost.R2: [0.5835238559641075, 0.49239800242877724, 0.5541651834625402, 0.1984636090841293, 0.5437001816242056, 0.2487755330668291, 0.5435487223393739, 0.3171840748062478, 0.21472732237355036, 0.4249306076788884, 0.46997893301847804, 0.34057063400249266, 0.4944089512940969, 0.522364586287356, 0.48962080706230654, 0.4425073525410472, 0.4623260017037914, 0.6007155932269538, 0.3314683855470182, 0.1432776830902255]\n",
      "\n",
      "\n",
      "Mean, Stdev of RMSE: 1.838647244642753 0.45831143121197937\n",
      "Mean, Stdev of R^2: 0.42093280103012076 0.13778270188605737\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### TwoStagetrAdaBoostR2 Abalone #######################################\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "#####################################################################################################################\n",
    "\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "print(\"Two-Stage TrAdaboost.R2\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_TwoTrAda_abalone = []\n",
    "rmselist_TwoTrAda_abalone = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(abalone_tgt_df):\n",
    "        \n",
    "    abalone_train_df_X = abalone_tgt_df.iloc[train_idx].loc[:, features_abalone]\n",
    "    abalone_test_df_X = abalone_tgt_df.iloc[test_idx][features_abalone]\n",
    "    abalone_train_df_y = abalone_tgt_df.iloc[train_idx].loc[:,target_abalone]\n",
    "    abalone_test_df_y = abalone_tgt_df.loc[test_idx][target_abalone] \n",
    "    \n",
    "    abalone_X_df = pd.concat([abalone_source_df_X, abalone_train_df_X], ignore_index=True)\n",
    "    abalone_y_df = pd.concat([abalone_source_df_y, abalone_train_df_y], ignore_index=True)\n",
    "\n",
    "    abalone_np_train_X = abalone_X_df.to_numpy()\n",
    "    abalone_np_train_y = abalone_y_df.to_numpy()\n",
    "\n",
    "    abalone_np_test_X = abalone_test_df_X.to_numpy()\n",
    "    abalone_np_test_y = abalone_test_df_y.to_numpy()\n",
    "\n",
    "    abalone_np_train_y_list = abalone_np_train_y.ravel()\n",
    "    abalone_np_test_y_list = abalone_np_test_y.ravel()\n",
    "\n",
    "    src_size_abalone = len(abalone_source_df_y)\n",
    "    tgt_size_abalone = len(abalone_train_df_y)\n",
    "    \n",
    "    src_idx_abalone = np.arange(start = 0, stop = (src_size_abalone - 1), step = 1)\n",
    "    tgt_idx_abalone = np.arange(start = src_size_abalone, stop = ((src_size_abalone + tgt_size_abalone)-1), step=1)\n",
    "\n",
    "\n",
    "    model_TwoTrAda_abalone = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 100) \n",
    "    model_TwoTrAda_abalone.fit(abalone_np_train_X, abalone_np_train_y_list, src_idx_abalone, tgt_idx_abalone)\n",
    "\n",
    "    y_pred_TwoTrAda_abalone = model_TwoTrAda_abalone.predict(abalone_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_abalone = sqrt(mean_squared_error(abalone_np_test_y, y_pred_TwoTrAda_abalone))\n",
    "    rmselist_TwoTrAda_abalone.append(mse_TwoTrAda_abalone)\n",
    "        \n",
    "    r2_score_TwoTrAda_abalone = pearsonr(abalone_np_test_y_list, y_pred_TwoTrAda_abalone)\n",
    "    r2_score_TwoTrAda_abalone = (r2_score_TwoTrAda_abalone[0])**2\n",
    "    r2scorelist_TwoTrAda_abalone.append(r2_score_TwoTrAda_abalone)\n",
    "\n",
    "print(\"RMSE List of TrAdaboost.R2:\", rmselist_TwoTrAda_abalone)\n",
    "print(\"R^2 List of TrAdaboost.R2:\", r2scorelist_TwoTrAda_abalone)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, Stdev of RMSE:\", statistics.mean(rmselist_TwoTrAda_abalone), statistics.stdev(rmselist_TwoTrAda_abalone))\n",
    "print(\"Mean, Stdev of R^2:\", statistics.mean(r2scorelist_TwoTrAda_abalone), statistics.stdev(r2scorelist_TwoTrAda_abalone))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost.R2 Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of Adaboost.R2 TL: [2.5290666881904516, 2.1488972042538133, 1.9522801071877496, 2.7899746322239314, 1.7413380279463884, 2.0133871176693883, 2.0240787020592905, 1.9514312457592426, 1.9899748505045955, 1.8491238408094293, 2.2242333037816757, 2.3015161513978075, 2.151370914532238, 1.7900755092588247, 1.7550443224941028, 2.1101598855229144, 1.6544845365715772, 1.8516142281973458, 2.6138260111972738, 1.963993817921549]\n",
      "R^2 List of Adaboost.R2 TL: [0.5120724306222029, 0.5174869631221757, 0.5358036355505292, 0.23519673296447782, 0.665209044517986, 0.26882933303414985, 0.44135901227758384, 0.24395236410390966, 0.2579067876625059, 0.35962591323102827, 0.4523293379096005, 0.335312472879681, 0.5248312987330939, 0.5197905970538608, 0.5580212741347623, 0.46477686252952405, 0.6344677698487458, 0.6066738573088932, 0.27832951911332654, 0.3346181891137798]\n",
      "\n",
      "\n",
      "Mean, Stdev of RMSE: 2.0702935548739796 0.3010747424242606\n",
      "Mean, Stdev of R^2: 0.43732966978559085 0.13809028010043006\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### AdaBoostR2 Transfer Learning Abalone #####################################################\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "\n",
    "print(\"Adaboost.R2 Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_AdaTr_abalone = []\n",
    "rmselist_AdaTr_abalone = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(abalone_tgt_df):\n",
    "        \n",
    "    abalone_train_df_X = abalone_tgt_df.iloc[train_idx].loc[:, features_abalone]\n",
    "    abalone_test_df_X = abalone_tgt_df.iloc[test_idx][features_abalone]\n",
    "    abalone_train_df_y = abalone_tgt_df.iloc[train_idx].loc[:,target_abalone]\n",
    "    abalone_test_df_y = abalone_tgt_df.loc[test_idx][target_abalone] \n",
    "    \n",
    "    abalone_X_df = pd.concat([abalone_source_df_X, abalone_train_df_X], ignore_index=True)\n",
    "    abalone_y_df = pd.concat([abalone_source_df_y, abalone_train_df_y], ignore_index=True)\n",
    "\n",
    "    abalone_np_train_X = abalone_X_df.to_numpy()\n",
    "    abalone_np_train_y = abalone_y_df.to_numpy()\n",
    "\n",
    "    abalone_np_test_X = abalone_test_df_X.to_numpy()\n",
    "    abalone_np_test_y = abalone_test_df_y.to_numpy()\n",
    "\n",
    "    abalone_np_train_y_list = abalone_np_train_y.ravel()\n",
    "    abalone_np_test_y_list = abalone_np_test_y.ravel()\n",
    "\n",
    "\n",
    "    model_AdaTr_abalone = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 6), learning_rate = 0.1, n_estimators = 100) \n",
    "    model_AdaTr_abalone.fit(abalone_np_train_X, abalone_np_train_y_list)\n",
    "\n",
    "    y_pred_AdaTr_abalone = model_AdaTr_abalone.predict(abalone_np_test_X)\n",
    "\n",
    "    mse_AdaTr_abalone = sqrt(mean_squared_error(abalone_np_test_y, y_pred_AdaTr_abalone))\n",
    "    rmselist_AdaTr_abalone.append(mse_AdaTr_abalone)\n",
    "        \n",
    "    r2_score_AdaTr_abalone = pearsonr(abalone_np_test_y_list, y_pred_AdaTr_abalone)\n",
    "    r2_score_AdaTr_abalone = (r2_score_AdaTr_abalone[0])**2\n",
    "    r2scorelist_AdaTr_abalone.append(r2_score_AdaTr_abalone)\n",
    "\n",
    "print(\"RMSE List of Adaboost.R2 TL:\", rmselist_AdaTr_abalone)\n",
    "print(\"R^2 List of Adaboost.R2 TL:\", r2scorelist_AdaTr_abalone)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, Stdev of RMSE:\", statistics.mean(rmselist_AdaTr_abalone), statistics.stdev(rmselist_AdaTr_abalone))\n",
    "print(\"Mean, Stdev of R^2:\", statistics.mean(r2scorelist_AdaTr_abalone), statistics.stdev(r2scorelist_AdaTr_abalone))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of GBRTL: [1.8507269172605925, 1.9234750624963846, 1.9629408314503771, 3.0182828295704303, 1.7147400468350609, 1.2725147848561504, 1.4352807137678807, 1.6039601259191367, 1.3413723759961382, 1.373152135431503, 1.5618801721871551, 2.054574840884452, 2.401464579709429, 1.077651815137251, 1.224267394955141, 1.7761503824238791, 1.962032254723272, 1.1293788923527823, 2.204111610237548, 1.3999665011318236]\n",
      "R^2 List of GBRTL: [0.3865122435871249, 0.48465059147920453, 0.49053742668564926, 0.2832022011353206, 0.5808478221369245, 0.2460812779646555, 0.4051564968296789, 0.14887942682547967, 0.2778352932814985, 0.35991641418896725, 0.423663029929573, 0.3204231404617369, 0.4488662827169648, 0.45214477452854906, 0.515735897239244, 0.3911135195019772, 0.5451451505150051, 0.5747261380637243, 0.2621003981980692, 0.2637105933386203]\n",
      "\n",
      "\n",
      "Mean, Stdev of RMSE of GBRTL: 1.7143962133663193 0.4794698226914729\n",
      "Mean, Stdev of R^2 of GBRTL: 0.3930624059303984 0.121156782514187\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################### Gradient Boosting Regression Transfer Learning Abalone #######################################\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "\n",
    "print(\"Gradient Boosting Regression Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_GBRTL_abalone = []\n",
    "rmselist_GBRTL_abalone = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(abalone_tgt_df):\n",
    "        \n",
    "    abalone_train_df_X = abalone_tgt_df.iloc[train_idx].loc[:, features_abalone]\n",
    "    abalone_test_df_X = abalone_tgt_df.iloc[test_idx][features_abalone]\n",
    "    abalone_train_df_y = abalone_tgt_df.iloc[train_idx].loc[:,target_abalone]\n",
    "    abalone_test_df_y = abalone_tgt_df.loc[test_idx][target_abalone] \n",
    "    \n",
    "    abalone_X_df = pd.concat([abalone_source_df_X, abalone_train_df_X], ignore_index=True)\n",
    "    abalone_y_df = pd.concat([abalone_source_df_y, abalone_train_df_y], ignore_index=True)\n",
    "\n",
    "    abalone_np_train_X = abalone_X_df.to_numpy()\n",
    "    abalone_np_train_y = abalone_y_df.to_numpy()\n",
    "\n",
    "    abalone_np_test_X = abalone_test_df_X.to_numpy()\n",
    "    abalone_np_test_y = abalone_test_df_y.to_numpy()\n",
    "\n",
    "    abalone_np_train_y_list = abalone_np_train_y.ravel()\n",
    "    abalone_np_test_y_list = abalone_np_test_y.ravel()\n",
    "\n",
    "\n",
    "    model_GBRTL_abalone = GradientBoostingRegressor(learning_rate = 0.1, max_depth = 6, n_estimators = 100)\n",
    "    model_GBRTL_abalone.fit(abalone_np_train_X, abalone_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_abalone = model_GBRTL_abalone.predict(abalone_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_abalone = sqrt(mean_squared_error(abalone_np_test_y, y_pred_GBRTL_abalone))\n",
    "    rmselist_GBRTL_abalone.append(mse_GBRTL_abalone)\n",
    "        \n",
    "    r2_score_GBRTL_abalone = pearsonr(abalone_np_test_y_list, y_pred_GBRTL_abalone)\n",
    "    r2_score_GBRTL_abalone = (r2_score_GBRTL_abalone[0])**2\n",
    "    r2scorelist_GBRTL_abalone.append(r2_score_GBRTL_abalone)\n",
    "\n",
    "\n",
    "print(\"RMSE List of GBRTL:\", rmselist_GBRTL_abalone)\n",
    "print(\"R^2 List of GBRTL:\", r2scorelist_GBRTL_abalone)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, Stdev of RMSE of GBRTL:\", statistics.mean(rmselist_GBRTL_abalone), statistics.stdev(rmselist_GBRTL_abalone))\n",
    "print(\"Mean, Stdev of R^2 of GBRTL:\", statistics.mean(r2scorelist_GBRTL_abalone), statistics.stdev(r2scorelist_GBRTL_abalone))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4774158366799477 0.1778150997140501\n",
      "1.4702303687021976 0.17970051016641017\n",
      "0.180939186766893 0.8609152412759344\n",
      "\n",
      "\n",
      "-0.5852548329307646 0.5745091033401768\n",
      "-0.36585924469666253 0.7239555216280504\n",
      "0.9064702983841878 0.3911716398994435\n"
     ]
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "stats_StAd_rmse, pval_StAd_rmse = stats.ttest_ind(rmselist_stradaboost_abalone, rmselist_AdaTr_abalone)\n",
    "print(stats_StAd_rmse, pval_StAd_rmse)\n",
    "\n",
    "stats_StGb_rmse, pval_StGb_rmse = stats.ttest_ind(rmselist_stradaboost_abalone, rmselist_GBRTL_abalone)\n",
    "print(stats_StGb_rmse, pval_StGb_rmse)\n",
    "\n",
    "stats_StTr_rmse, pval_StTr_rmse = stats.ttest_ind(rmselist_stradaboost_abalone, rmselist_TwoTrAda_abalone)\n",
    "print(stats_StTr_rmse, pval_StTr_rmse)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "stats_StAd_r2, pval_StAd_r2 = stats.ttest_ind(r2scorelist_stradaboost_abalone, r2scorelist_AdaTr_abalone)\n",
    "print(stats_StAd_r2, pval_StAd_r2)\n",
    "\n",
    "stats_StGb_r2, pval_StGb_r2 = stats.ttest_ind(r2scorelist_stradaboost_abalone, r2scorelist_GBRTL_abalone)\n",
    "print(stats_StGb_r2, pval_StGb_r2)\n",
    "\n",
    "stats_StTr_r2, pval_StTr_r2 = stats.ttest_ind(r2scorelist_stradaboost_abalone, r2scorelist_TwoTrAda_abalone)\n",
    "print(stats_StTr_r2, pval_StTr_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3950437583222772 0.07475725214402565\n",
      "2.1242400051017136 0.10085945561990267\n",
      "0.32488683877032 0.7615486615910663\n",
      "\n",
      "\n",
      "-1.881842769362401 0.1329977130272984\n",
      "-1.2603856460659317 0.2760438976285486\n",
      "2.293616600098016 0.0835237480871644\n"
     ]
    }
   ],
   "source": [
    "#### Dependent(paired) t-test\n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "stats_StAd_rmse, pval_StAd_rmse = stats.ttest_rel(rmselist_stradaboost_abalone, rmselist_AdaTr_abalone)\n",
    "print(stats_StAd_rmse, pval_StAd_rmse)\n",
    "\n",
    "stats_StGb_rmse, pval_StGb_rmse = stats.ttest_rel(rmselist_stradaboost_abalone, rmselist_GBRTL_abalone)\n",
    "print(stats_StGb_rmse, pval_StGb_rmse)\n",
    "\n",
    "stats_StTr_rmse, pval_StTr_rmse = stats.ttest_rel(rmselist_stradaboost_abalone, rmselist_TwoTrAda_abalone)\n",
    "print(stats_StTr_rmse, pval_StTr_rmse)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "stats_StAd_r2, pval_StAd_r2 = stats.ttest_rel(r2scorelist_stradaboost_abalone, r2scorelist_AdaTr_abalone)\n",
    "print(stats_StAd_r2, pval_StAd_r2)\n",
    "\n",
    "stats_StGb_r2, pval_StGb_r2 = stats.ttest_rel(r2scorelist_stradaboost_abalone, r2scorelist_GBRTL_abalone)\n",
    "print(stats_StGb_r2, pval_StGb_r2)\n",
    "\n",
    "stats_StTr_r2, pval_StTr_r2 = stats.ttest_rel(r2scorelist_stradaboost_abalone, r2scorelist_TwoTrAda_abalone)\n",
    "print(stats_StTr_r2, pval_StTr_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kinematics Data\n",
      "(8192, 9)\n",
      "Target Set:  (2878, 8)\n",
      "Source Set 1:  (2396, 8)\n",
      "Source Set 2:  (2918, 8)\n",
      "Final Source Set:  (5314, 8)\n"
     ]
    }
   ],
   "source": [
    "################################################## Kinematics ######################################################################\n",
    "#### range: [0.04 - 1.45]\n",
    "#### Mid of correlation variable: theta7\n",
    "#### [0, 0.6] [0.6, 0.85], [0.6, 0.85]\n",
    "####################################################################################################################################\n",
    "target_var_Kinematics = ['y']\n",
    "colnames_Kinematics = ['theta1', 'theta2', 'theta3', 'theta4', 'theta5', 'theta6', 'theta7', 'theta8', 'y']\n",
    "KinematicsData_df = pd.read_csv('UCI_regression/Kinematics/kin8nm.data', header = None,  names = colnames_Kinematics)\n",
    "\n",
    "print(\"Kinematics Data\")\n",
    "print(KinematicsData_df.shape)\n",
    "\n",
    "############################################### Standardization ###############################################\n",
    "kinematics_cols = KinematicsData_df.columns.difference(['y'])\n",
    "\n",
    "ss = StandardScaler()\n",
    "KinematicsData_df[kinematics_cols] = ss.fit_transform(KinematicsData_df[kinematics_cols])\n",
    "\n",
    "\n",
    "########## Corr Kinematics ################\n",
    "# print(\"The correlation matrix is: \")\n",
    "# print(KinematicsData_df.corr().sort_values('y')['y'])\n",
    "\n",
    "##################### Splitting in 3 equal parts #######################################\n",
    "# print(KinematicsData_df.sort_values('theta7')['theta7'])\n",
    "\n",
    "drop_col_kinematics = ['theta7']\n",
    "\n",
    "kinematics_tgt_df = KinematicsData_df.loc[(KinematicsData_df['theta7'] <= -0.5)]\n",
    "kinematics_tgt_df = kinematics_tgt_df.drop(drop_col_kinematics, axis = 1)\n",
    "kinematics_tgt_df = kinematics_tgt_df.reset_index(drop=True)\n",
    "print(\"Target Set: \",kinematics_tgt_df.shape)\n",
    "\n",
    "\n",
    "kinematics_source1_df = KinematicsData_df.loc[(KinematicsData_df['theta7'] > -0.5) & (KinematicsData_df['theta7'] <= 0.5)]\n",
    "kinematics_source1_df = kinematics_source1_df.drop(drop_col_kinematics, axis = 1)\n",
    "kinematics_source1_df = kinematics_source1_df.reset_index(drop=True)\n",
    "print(\"Source Set 1: \",kinematics_source1_df.shape)\n",
    "\n",
    "\n",
    "kinematics_source2_df = KinematicsData_df.loc[(KinematicsData_df['theta7'] > 0.5)]\n",
    "kinematics_source2_df = kinematics_source2_df.drop(drop_col_kinematics, axis = 1)\n",
    "kinematics_source2_df = kinematics_source2_df.reset_index(drop=True)\n",
    "print(\"Source Set 2: \",kinematics_source2_df.shape)\n",
    "\n",
    "\n",
    "################################# Standardization #################################\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "kinematics_cols = kinematics_tgt_df.columns.difference(['y'])\n",
    "\n",
    "ss = StandardScaler()\n",
    "kinematics_tgt_df[kinematics_cols] = ss.fit_transform(kinematics_tgt_df[kinematics_cols])\n",
    "kinematics_source1_df[kinematics_cols] = ss.fit_transform(kinematics_source1_df[kinematics_cols])\n",
    "kinematics_source2_df[kinematics_cols] = ss.fit_transform(kinematics_source2_df[kinematics_cols])\n",
    "\n",
    "\n",
    "################################# Concatenating the source datasets #################################\n",
    "kinematics_source_df = pd.concat([kinematics_source1_df, kinematics_source2_df], ignore_index = True)\n",
    "kinematics_source_df = kinematics_source_df.reset_index(drop = True)\n",
    "print(\"Final Source Set: \",kinematics_source_df.shape)\n",
    "\n",
    "\n",
    "# #################### Splitting into features and target (Hide in STrAdaBoost.R2)####################\n",
    "# target_kinematics = ['y']\n",
    "\n",
    "# kinematics_source_df_y = kinematics_source_df[target_kinematics]\n",
    "# kinematics_source_df_X = kinematics_source_df.drop(target_kinematics, axis = 1)\n",
    "\n",
    "# features_kinematics = kinematics_source_df_X.columns\n",
    "# print(\"---------------------------\")\n",
    "# ##############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of dataset extracted: \n",
      "(140, 8)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################################## Variance Sampling: Phase 1 ###################################################\n",
    "kmeans = KMeans(n_clusters = 140, random_state=0).fit(kinematics_tgt_df)\n",
    "\n",
    "kinematics_alternate_df = kinematics_tgt_df.copy()\n",
    "kinematics_alternate_df_np = kinematics_tgt_df.to_numpy()\n",
    "\n",
    "idxlist = []\n",
    "kinematics_new_df_list = []\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in kinematics_alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "#     print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", mindist)\n",
    "#     print(\"Matrix shape: \", kinematics_alternate_df_np.shape)\n",
    "    kinematics_new_df_list.append(rowval)\n",
    "    kinematics_alternate_df = np.delete(kinematics_alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "kinematics_new_df = pd.DataFrame(np.vstack(kinematics_new_df_list))\n",
    "\n",
    "print(\"Shape of dataset extracted: \")\n",
    "print(kinematics_new_df.shape)\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Shape of source before : (5314, 8)\n",
      "Shape of source after : (5173, 8)\n",
      "----------------------------------------------\n",
      "Shape of target before : (2878, 8)\n",
      "Shape of target after : (3018, 8)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "##################################################### Variance Sampling: Phase 2 ################################################\n",
    "\n",
    "kinematics_alternate_source_df = kinematics_source_df[1:].copy()\n",
    "kinematics_alternate_source_df_np = kinematics_alternate_source_df.to_numpy()\n",
    "\n",
    "idxlist2 = []\n",
    "kinematics_final_df_list = []\n",
    "\n",
    "for row_nm in kinematics_new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in kinematics_alternate_source_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "#     print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", min_dist)\n",
    "#     print(\"Matrix shape: \", Elevators_alternate_source_df_np.shape)\n",
    "    kinematics_final_df_list.append(row_val)\n",
    "    kinematics_alternate_source_df_np = np.delete(kinematics_alternate_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "kinematics_final_df = pd.DataFrame(np.vstack(kinematics_final_df_list), columns= kinematics_source_df.columns)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "print(\"Shape of source before :\",kinematics_source_df.shape)\n",
    "kinematics_source_df = pd.DataFrame(np.vstack(kinematics_alternate_source_df_np), columns= kinematics_source_df.columns)\n",
    "print(\"Shape of source after :\", kinematics_source_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "print(\"Shape of target before :\", kinematics_tgt_df.shape)\n",
    "kinematics_tgt_df = pd.concat([kinematics_tgt_df, kinematics_final_df], ignore_index=True)\n",
    "print(\"Shape of target after :\", kinematics_tgt_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Set:  (3018, 8)\n",
      "Source Set:  (1330, 8)\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "################################## Importance Sampling: Manhattan Distance ######################################################\n",
    "\n",
    "kinematics_source_df[\"ManDis\"] = \"\"\n",
    "\n",
    "kinematics_tgt_df_mean = []\n",
    "prow = kinematics_tgt_df.mean()\n",
    "kinematics_tgt_df_mean = [prow.theta1, prow.theta2, prow.theta3, prow.theta4, prow.theta5, prow.theta6, prow.theta8, prow.y]\n",
    "\n",
    "rowidx = 0\n",
    "\n",
    "for row in kinematics_source_df.itertuples():\n",
    "    row_list =[row.theta1, row.theta2, row.theta3, row.theta4, row.theta5, row.theta6, row.theta8, row.y]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = kinematics_tgt_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "\n",
    "    kinematics_source_df.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "kinematics_source_df = kinematics_source_df.sort_values(by =['ManDis'])\n",
    "kinematics_source_df = kinematics_source_df.head(1330) \n",
    "kinematics_source_df = kinematics_source_df.drop(['ManDis'], axis =1)\n",
    "kinematics_source_df = kinematics_source_df.reset_index(drop=True)\n",
    "\n",
    "print(\"Target Set: \", kinematics_tgt_df.shape)\n",
    "print(\"Source Set: \", kinematics_source_df.shape)\n",
    "\n",
    "#################### Splitting into features and target ####################\n",
    "target_kinematics = ['y']\n",
    "\n",
    "kinematics_source_df_y = kinematics_source_df[target_kinematics]\n",
    "kinematics_source_df_X = kinematics_source_df.drop(target_kinematics, axis = 1)\n",
    "\n",
    "features_kinematics = kinematics_source_df_X.columns\n",
    "print(\"---------------------------\")\n",
    "###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "RMSE List of STrAdaboostR2: 0.18341542484176596\n",
      "R^2 List of STrAdaboostR2: 0.518757608436131\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 0.18341542484176596 0.017326708837706532\n",
      "Mean, STDev of R^2: 0.518757608436131 0.08719351766043475\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#################################### STrAdaBoost.R2 Active Learning Kinematics ################################################################\n",
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "n_estimators = 100\n",
    "steps = 30\n",
    "fold = 10\n",
    "random_state = np.random.RandomState(1)\n",
    "\n",
    "r2scorelist_stradaboost_kinematics = []\n",
    "rmselist_stradaboost_kinematics = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(kinematics_tgt_df):\n",
    "        \n",
    "    kinematics_train_df_X = kinematics_tgt_df.iloc[train_idx].loc[:, features_kinematics]\n",
    "    kinematics_test_df_X = kinematics_tgt_df.iloc[test_idx][features_kinematics]\n",
    "    kinematics_train_df_y = kinematics_tgt_df.iloc[train_idx].loc[:,target_kinematics]\n",
    "    kinematics_test_df_y = kinematics_tgt_df.loc[test_idx][target_kinematics] \n",
    "    \n",
    "    kinematics_X_df = pd.concat([kinematics_source_df_X, kinematics_train_df_X], ignore_index=True)\n",
    "    kinematics_y_df = pd.concat([kinematics_source_df_y, kinematics_train_df_y], ignore_index=True)\n",
    "\n",
    "    kinematics_np_train_X = kinematics_X_df.to_numpy()\n",
    "    kinematics_np_train_y = kinematics_y_df.to_numpy()\n",
    "\n",
    "    kinematics_np_test_X = kinematics_test_df_X.to_numpy()\n",
    "    kinematics_np_test_y = kinematics_test_df_y.to_numpy()\n",
    "\n",
    "    kinematics_np_train_y_list = kinematics_np_train_y.ravel()\n",
    "    kinematics_np_test_y_list = kinematics_np_test_y.ravel()\n",
    "    \n",
    "    \n",
    "    sample_size = [len(kinematics_source_df_X), len(kinematics_train_df_X)]\n",
    "\n",
    "    model_stradaboost_kinematics = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_kinematics.fit(kinematics_np_train_X, kinematics_np_train_y_list)\n",
    "    y_pred_stradaboost_kinematics = model_stradaboost_kinematics.predict(kinematics_np_test_X)\n",
    "\n",
    "\n",
    "    mse_stradaboost_kinematics = sqrt(mean_squared_error(kinematics_np_test_y, y_pred_stradaboost_kinematics))\n",
    "\n",
    "    rmselist_stradaboost_kinematics.append(mse_stradaboost_kinematics)\n",
    "        \n",
    "    r2_score_stradaboost_kinematics = pearsonr(kinematics_np_test_y_list, y_pred_stradaboost_kinematics)\n",
    "    r2_score_stradaboost_kinematics = (r2_score_stradaboost_kinematics[0])**2\n",
    "\n",
    "    r2scorelist_stradaboost_kinematics.append(r2_score_stradaboost_kinematics)\n",
    "\n",
    "    \n",
    "print(\"RMSE List of STrAdaboostR2:\", statistics.mean(rmselist_stradaboost_kinematics))\n",
    "print(\"R^2 List of STrAdaboostR2:\", statistics.mean(r2scorelist_stradaboost_kinematics))\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_stradaboost_kinematics), statistics.stdev(rmselist_stradaboost_kinematics))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_stradaboost_kinematics), statistics.stdev(r2scorelist_stradaboost_kinematics))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Two-Stage TrAdaboost.R2\n",
      "-------------------------------------------\n",
      "RMSE List of TrAdaboost.R2: [0.16386342963105477, 0.14973421150473964, 0.17020251283183155, 0.18005335812476686, 0.17339254515732253, 0.17299677656388374, 0.17406071202280785, 0.16457432173978623, 0.164975165574514, 0.18090395683438726, 0.15078632061289363, 0.15087890387057476, 0.15212834705955194, 0.17608881421299705, 0.16968458220646884, 0.16345390065529547, 0.17025474559355513, 0.18148089480471902, 0.17735815430175156, 0.17660698682517628]\n",
      "R^2 List of TrAdaboostR2: [0.5916336512581157, 0.7291927461307379, 0.5825380002381263, 0.494415277976975, 0.5462840981055812, 0.6317771931723815, 0.4630044081075447, 0.5449863245975171, 0.5605880727816586, 0.4857566904279455, 0.6333482556443955, 0.6502527590337912, 0.45860509263742566, 0.553658712623355, 0.453163305949092, 0.5115701723439828, 0.5771870253961963, 0.510004437022482, 0.5569683901266973, 0.4565209914769184]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 0.1681739320064039 0.010405503944168315\n",
      "Mean, STDev of R^2: 0.549572780252546 0.07393665620853554\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### TwoStagetrAdaBoostR2 Kinematics #######################################\n",
    "\n",
    "############# Transfer Learning specifications #########################\n",
    "\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "print(\"Two-Stage TrAdaboost.R2\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_TwoTrAda_kinematics = []\n",
    "rmselist_TwoTrAda_kinematics = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(kinematics_tgt_df):\n",
    "        \n",
    "    kinematics_train_df_X = kinematics_tgt_df.iloc[train_idx].loc[:, features_kinematics]\n",
    "    kinematics_test_df_X = kinematics_tgt_df.iloc[test_idx][features_kinematics]\n",
    "    kinematics_train_df_y = kinematics_tgt_df.iloc[train_idx].loc[:,target_kinematics]\n",
    "    kinematics_test_df_y = kinematics_tgt_df.loc[test_idx][target_kinematics] \n",
    "    \n",
    "    kinematics_X_df = pd.concat([kinematics_source_df_X, kinematics_train_df_X], ignore_index=True)\n",
    "    kinematics_y_df = pd.concat([kinematics_source_df_y, kinematics_train_df_y], ignore_index=True)\n",
    "\n",
    "    kinematics_np_train_X = kinematics_X_df.to_numpy()\n",
    "    kinematics_np_train_y = kinematics_y_df.to_numpy()\n",
    "\n",
    "    kinematics_np_test_X = kinematics_test_df_X.to_numpy()\n",
    "    kinematics_np_test_y = kinematics_test_df_y.to_numpy()\n",
    "\n",
    "    kinematics_np_train_y_list = kinematics_np_train_y.ravel()\n",
    "    kinematics_np_test_y_list = kinematics_np_test_y.ravel()\n",
    "\n",
    "    src_size_kinematics = len(kinematics_source_df_y)\n",
    "    tgt_size_kinematics = len(kinematics_train_df_y)\n",
    "\n",
    "    src_idx_kinematics = np.arange(start = 0, stop = (src_size_kinematics - 1), step = 1)\n",
    "    tgt_idx_kinematics = np.arange(start = src_size_kinematics, stop = ((src_size_kinematics + tgt_size_kinematics)-1), step = 1)\n",
    "\n",
    "\n",
    "    model_TwoTrAda_kinematics = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 100)\n",
    "    model_TwoTrAda_kinematics.fit(kinematics_np_train_X, kinematics_np_train_y_list, src_idx_kinematics, tgt_idx_kinematics)\n",
    "\n",
    "    y_pred_TwoTrAda_kinematics = model_TwoTrAda_kinematics.predict(kinematics_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_kinematics = sqrt(mean_squared_error(kinematics_np_test_y, y_pred_TwoTrAda_kinematics))\n",
    "    rmselist_TwoTrAda_kinematics.append(mse_TwoTrAda_kinematics)\n",
    "        \n",
    "    r2_score_TwoTrAda_kinematics = pearsonr(kinematics_np_test_y_list, y_pred_TwoTrAda_kinematics)\n",
    "    r2_score_TwoTrAda_kinematics = (r2_score_TwoTrAda_kinematics[0])**2\n",
    "    r2scorelist_TwoTrAda_kinematics.append(r2_score_TwoTrAda_kinematics)\n",
    "\n",
    "print(\"RMSE List of TrAdaboost.R2:\", rmselist_TwoTrAda_kinematics)\n",
    "print(\"R^2 List of TrAdaboostR2:\", r2scorelist_TwoTrAda_kinematics)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_TwoTrAda_kinematics), statistics.stdev(rmselist_TwoTrAda_kinematics))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_TwoTrAda_kinematics), statistics.stdev(r2scorelist_TwoTrAda_kinematics))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost.R2 Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of Adaboost.R2: [0.2148208351878945, 0.19732349316902073, 0.21147502169410173, 0.20196537476013698, 0.20749616206177746, 0.20408368839125243, 0.19269742025750516, 0.19966789592117412, 0.20324606499226852, 0.19313285193059057, 0.19648345714473303, 0.21317518049084488, 0.19250472494382503, 0.1977828386485602, 0.2094451178224044, 0.19659907998980244, 0.19739239030582825, 0.22518728884866462, 0.20860444922671703, 0.1898129771808412]\n",
      "R^2 List of Adaboost.R2: [0.3766517591469746, 0.5269464544745807, 0.381522079958307, 0.4137276440306096, 0.41411427765648967, 0.586414786935785, 0.39225666904325135, 0.3845864455631169, 0.3762215435213061, 0.4932299304753315, 0.4080687419379144, 0.34734250359843744, 0.20957975317968483, 0.4595525533495378, 0.24906941705898636, 0.3355342794702446, 0.5194457019216171, 0.25120567479796063, 0.4025592582079927, 0.44114664247360846]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 0.20264481564839718 0.00909972322666589\n",
      "Mean, STDev of R^2: 0.39845880584008686 0.09440510764063892\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### AdaBoostR2 Transfer Learning Kinematics #####################################################\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "\n",
    "\n",
    "print(\"Adaboost.R2 Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_AdaTL_kinematics = []\n",
    "rmselist_AdaTL_kinematics = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(kinematics_tgt_df):\n",
    "        \n",
    "    kinematics_train_df_X = kinematics_tgt_df.iloc[train_idx].loc[:, features_kinematics]\n",
    "    kinematics_test_df_X = kinematics_tgt_df.iloc[test_idx][features_kinematics]\n",
    "    kinematics_train_df_y = kinematics_tgt_df.iloc[train_idx].loc[:,target_kinematics]\n",
    "    kinematics_test_df_y = kinematics_tgt_df.loc[test_idx][target_kinematics] \n",
    "    \n",
    "    kinematics_X_df = pd.concat([kinematics_source_df_X, kinematics_train_df_X], ignore_index=True)\n",
    "    kinematics_y_df = pd.concat([kinematics_source_df_y, kinematics_train_df_y], ignore_index=True)\n",
    "\n",
    "    kinematics_np_train_X = kinematics_X_df.to_numpy()\n",
    "    kinematics_np_train_y = kinematics_y_df.to_numpy()\n",
    "\n",
    "    kinematics_np_test_X = kinematics_test_df_X.to_numpy()\n",
    "    kinematics_np_test_y = kinematics_test_df_y.to_numpy()\n",
    "\n",
    "    kinematics_np_train_y_list = kinematics_np_train_y.ravel()\n",
    "    kinematics_np_test_y_list = kinematics_np_test_y.ravel()\n",
    "\n",
    "    model_AdaTL_kinematics = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 6), learning_rate = 0.1, n_estimators = 100) \n",
    "    model_AdaTL_kinematics.fit(kinematics_np_train_X, kinematics_np_train_y_list)\n",
    "\n",
    "    y_pred_AdaTL_kinematics = model_AdaTL_kinematics.predict(kinematics_np_test_X)\n",
    "\n",
    "    mse_AdaTL_kinematics = sqrt(mean_squared_error(kinematics_np_test_y, y_pred_AdaTL_kinematics))\n",
    "    rmselist_AdaTL_kinematics.append(mse_AdaTL_kinematics)\n",
    "        \n",
    "    r2_score_AdaTL_kinematics = pearsonr(kinematics_np_test_y_list, y_pred_AdaTL_kinematics)\n",
    "    r2_score_AdaTL_kinematics = (r2_score_AdaTL_kinematics[0])**2\n",
    "    r2scorelist_AdaTL_kinematics.append(r2_score_AdaTL_kinematics)\n",
    "\n",
    "print(\"RMSE List of Adaboost.R2:\", rmselist_AdaTL_kinematics)\n",
    "print(\"R^2 List of Adaboost.R2:\", r2scorelist_AdaTL_kinematics)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_AdaTL_kinematics), statistics.stdev(rmselist_AdaTL_kinematics))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_AdaTL_kinematics), statistics.stdev(r2scorelist_AdaTL_kinematics))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of GBRTL: [0.2025103487478191, 0.1821500186364975, 0.19978936439458572, 0.18314394097706502, 0.20291731804279234, 0.1925074140880351, 0.1867931288538428, 0.1898943501047783, 0.1935201442578711, 0.17855342520859951, 0.19094907650744478, 0.2078290392589982, 0.195914645088323, 0.18324495813460193, 0.20174382726121048, 0.1969964554973966, 0.18178409008928115, 0.22140643669206533, 0.19643674713867923, 0.17130074655455774]\n",
      "R^2 List of GBRTL: [0.41638321580404286, 0.575895440504289, 0.4321412176280749, 0.4988215324327443, 0.4241015367265543, 0.5642100169274302, 0.4232077556092186, 0.4232695583936153, 0.41136587123675145, 0.5288424855155012, 0.4283360484181283, 0.3621443934674914, 0.2461373553468829, 0.5098368118804155, 0.2892080059984776, 0.35355024533524326, 0.5561657824109684, 0.2980232535040607, 0.4625407105031838, 0.5106745457574627]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 0.19296927377672224 0.011568349540849428\n",
      "Mean, STDev of R^2: 0.43574278917002685 0.09323551144865609\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################### Gradient Boosting Regression Transfer Learning Kinematics #######################################\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "\n",
    "print(\"Gradient Boosting Regression Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_GBRTL_kinematics = []\n",
    "rmselist_GBRTL_kinematics = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(kinematics_tgt_df):\n",
    "        \n",
    "    kinematics_train_df_X = kinematics_tgt_df.iloc[train_idx].loc[:, features_kinematics]\n",
    "    kinematics_test_df_X = kinematics_tgt_df.iloc[test_idx][features_kinematics]\n",
    "    kinematics_train_df_y = kinematics_tgt_df.iloc[train_idx].loc[:,target_kinematics]\n",
    "    kinematics_test_df_y = kinematics_tgt_df.loc[test_idx][target_kinematics] \n",
    "    \n",
    "    kinematics_X_df = pd.concat([kinematics_source_df_X, kinematics_train_df_X], ignore_index=True)\n",
    "    kinematics_y_df = pd.concat([kinematics_source_df_y, kinematics_train_df_y], ignore_index=True)\n",
    "\n",
    "    kinematics_np_train_X = kinematics_X_df.to_numpy()\n",
    "    kinematics_np_train_y = kinematics_y_df.to_numpy()\n",
    "\n",
    "    kinematics_np_test_X = kinematics_test_df_X.to_numpy()\n",
    "    kinematics_np_test_y = kinematics_test_df_y.to_numpy()\n",
    "\n",
    "    kinematics_np_train_y_list = kinematics_np_train_y.ravel()\n",
    "    kinematics_np_test_y_list = kinematics_np_test_y.ravel()\n",
    "\n",
    "\n",
    "    model_GBRTL_kinematics = GradientBoostingRegressor(learning_rate = 0.1, max_depth = 6, n_estimators = 100)\n",
    "    model_GBRTL_kinematics.fit(kinematics_np_train_X, kinematics_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_kinematics = model_GBRTL_kinematics.predict(kinematics_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_kinematics = sqrt(mean_squared_error(kinematics_np_test_y, y_pred_GBRTL_kinematics))\n",
    "    rmselist_GBRTL_kinematics.append(mse_GBRTL_kinematics)\n",
    "        \n",
    "    r2_score_GBRTL_kinematics = pearsonr(kinematics_np_test_y_list, y_pred_GBRTL_kinematics)\n",
    "    r2_score_GBRTL_kinematics = (r2_score_GBRTL_kinematics[0])**2\n",
    "\n",
    "    r2scorelist_GBRTL_kinematics.append(r2_score_GBRTL_kinematics)\n",
    "\n",
    "\n",
    "print(\"RMSE List of GBRTL:\", rmselist_GBRTL_kinematics)\n",
    "print(\"R^2 List of GBRTL:\", r2scorelist_GBRTL_kinematics)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "    \n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_GBRTL_kinematics), statistics.stdev(rmselist_GBRTL_kinematics))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_GBRTL_kinematics), statistics.stdev(r2scorelist_GBRTL_kinematics))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.239211166142098 0.05549370834515708\n",
      "5.710744979214197 0.00044878897195905153\n",
      "12.933901052446585 1.2086974283329565e-06\n",
      "\n",
      "\n",
      "0.15020428465586927 0.8843214418707841\n",
      "-0.580453091569201 0.5775889224254931\n",
      "-5.759859685898271 0.0004241772800097817\n"
     ]
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "stats_StAd_rmse, pval_StAd_rmse = stats.ttest_ind(rmselist_stradaboost_kinematics, rmselist_AdaTL_kinematics)\n",
    "print(stats_StAd_rmse, pval_StAd_rmse)\n",
    "\n",
    "stats_StGb_rmse, pval_StGb_rmse = stats.ttest_ind(rmselist_stradaboost_kinematics, rmselist_GBRTL_kinematics)\n",
    "print(stats_StGb_rmse, pval_StGb_rmse)\n",
    "\n",
    "stats_StTr_rmse, pval_StTr_rmse = stats.ttest_ind(rmselist_stradaboost_kinematics, rmselist_TwoTrAda_kinematics)\n",
    "print(stats_StTr_rmse, pval_StTr_rmse)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "stats_StAd_r2, pval_StAd_r2 = stats.ttest_ind(r2scorelist_stradaboost_kinematics, r2scorelist_AdaTL_kinematics)\n",
    "print(stats_StAd_r2, pval_StAd_r2)\n",
    "\n",
    "stats_StGb_r2, pval_StGb_r2 = stats.ttest_ind(r2scorelist_stradaboost_kinematics, r2scorelist_GBRTL_kinematics)\n",
    "print(stats_StGb_r2, pval_StGb_r2)\n",
    "\n",
    "stats_StTr_r2, pval_StTr_r2 = stats.ttest_ind(r2scorelist_stradaboost_kinematics, r2scorelist_TwoTrAda_kinematics)\n",
    "print(stats_StTr_r2, pval_StTr_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6296148561965667 0.02216745053260249\n",
      "6.519298221944031 0.0028583835971875094\n",
      "13.492049030039581 0.0001746222481559807\n",
      "\n",
      "\n",
      "0.7297095707949993 0.5060026819401974\n",
      "-4.064041932226505 0.015297728196215526\n",
      "-9.89590741109181 0.0005852307476539434\n"
     ]
    }
   ],
   "source": [
    "#### Dependent(paired) t-test\n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "\n",
    "stats_StAd_rmse, pval_StAd_rmse = stats.ttest_rel(rmselist_stradaboost_kinematics, rmselist_AdaTL_kinematics)\n",
    "print(stats_StAd_rmse, pval_StAd_rmse)\n",
    "\n",
    "stats_StGb_rmse, pval_StGb_rmse = stats.ttest_rel(rmselist_stradaboost_kinematics, rmselist_GBRTL_kinematics)\n",
    "print(stats_StGb_rmse, pval_StGb_rmse)\n",
    "\n",
    "stats_StTr_rmse, pval_StTr_rmse = stats.ttest_rel(rmselist_stradaboost_kinematics, rmselist_TwoTrAda_kinematics)\n",
    "print(stats_StTr_rmse, pval_StTr_rmse)\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "stats_StAd_r2, pval_StAd_r2 = stats.ttest_rel(r2scorelist_stradaboost_kinematics, r2scorelist_AdaTL_kinematics)\n",
    "print(stats_StAd_r2, pval_StAd_r2)\n",
    "\n",
    "stats_StGb_r2, pval_StGb_r2 = stats.ttest_rel(r2scorelist_stradaboost_kinematics, r2scorelist_GBRTL_kinematics)\n",
    "print(stats_StGb_r2, pval_StGb_r2)\n",
    "\n",
    "stats_StTr_r2, pval_StTr_r2 = stats.ttest_rel(r2scorelist_stradaboost_kinematics, r2scorelist_TwoTrAda_kinematics)\n",
    "print(stats_StTr_r2, pval_StTr_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computer Activity Data\n",
      "(8192, 22)\n",
      "Target Set:  (2766, 21)\n",
      "Source Set 1:  (2539, 21)\n",
      "Source Set 2:  (2887, 21)\n",
      "Final Source Set:  (5426, 21)\n"
     ]
    }
   ],
   "source": [
    "################################### Computer Activity ###########################################\n",
    "colnames_CompAct = ['lread', 'lwrite', 'scall', 'sread', 'swrite', 'fork', 'exec', 'rchar', 'wchar', 'pgout', 'ppgout', \n",
    "                    'pgfree', 'pgscan', 'atch', 'pgin', 'ppgin', 'pflt', 'vflt', 'runqsz', 'freemem', 'freeswap', 'usr' ]\n",
    "CompActData_df = pd.read_csv('UCI_regression/ComputerActivity/cpu_act.data', header = None, names = colnames_CompAct)\n",
    "print(\"Computer Activity Data\")\n",
    "print(CompActData_df.shape)\n",
    "\n",
    "##################### Corr Computer Activity #####################\n",
    "# print(\"The correlation matrix is: \")\n",
    "# print(CompActData_df.corr().sort_values('usr')['usr'])\n",
    "\n",
    "##################### Splitting in 3 equal parts #######################################\n",
    "# print(CompActData_df.sort_values('pgin')['pgin'])\n",
    "\n",
    "drop_col_compact = ['pgin']\n",
    "\n",
    "compact_tgt_df = CompActData_df.loc[(CompActData_df['pgin'] <= 1.0)]\n",
    "compact_tgt_df = compact_tgt_df.drop(drop_col_compact, axis = 1)\n",
    "compact_tgt_df = compact_tgt_df.reset_index(drop=True)\n",
    "print(\"Target Set: \",compact_tgt_df.shape)\n",
    "\n",
    "\n",
    "compact_source1_df = CompActData_df.loc[(CompActData_df['pgin'] > 1.0) & (CompActData_df['pgin'] <= 6.0)]\n",
    "compact_source1_df = compact_source1_df.drop(drop_col_compact, axis = 1)\n",
    "compact_source1_df = compact_source1_df.reset_index(drop = True)\n",
    "print(\"Source Set 1: \",compact_source1_df.shape)\n",
    "\n",
    "\n",
    "compact_source2_df = CompActData_df.loc[(CompActData_df['pgin'] > 6.0)]\n",
    "compact_source2_df = compact_source2_df.drop(drop_col_compact, axis = 1)\n",
    "compact_source2_df = compact_source2_df.reset_index(drop = True)\n",
    "print(\"Source Set 2: \",compact_source2_df.shape)\n",
    "\n",
    "\n",
    "################################# Standardization #################################\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "compact_cols = compact_tgt_df.columns.difference(['usr'])\n",
    "\n",
    "ss = StandardScaler()\n",
    "compact_tgt_df[compact_cols] = ss.fit_transform(compact_tgt_df[compact_cols])\n",
    "compact_source1_df[compact_cols] = ss.fit_transform(compact_source1_df[compact_cols])\n",
    "compact_source2_df[compact_cols] = ss.fit_transform(compact_source2_df[compact_cols])\n",
    "\n",
    "\n",
    "##################### Concatenating the source datasets #####################\n",
    "compact_source_df = pd.concat([compact_source1_df, compact_source2_df], ignore_index = True)\n",
    "compact_source_df = compact_source_df.reset_index(drop = True)\n",
    "print(\"Final Source Set: \",compact_source_df.shape)\n",
    "\n",
    "\n",
    "# #################### Splitting into features and target (Hide for STrAdaBoost.R2)####################\n",
    "# target_compact = ['usr']\n",
    "\n",
    "# compact_source_df_y = compact_source_df[target_compact]\n",
    "# compact_source_df_X = compact_source_df.drop(target_compact, axis = 1)\n",
    "\n",
    "# features_compact = compact_source_df_X.columns\n",
    "# print(\"---------------------------\")\n",
    "# ##############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset extracted is: \n",
      "(135, 21)\n",
      "------------------------------------------------------\n",
      "Shape of source before : (5426, 21)\n",
      "Shape of source after : (5291, 21)\n",
      "----------------------------------------------\n",
      "Shape of target before : (2766, 21)\n",
      "Shape of target after : (2766, 21)\n",
      "----------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################################## Variance Sampling: Phase 1 ###################################################\n",
    "kmeans = KMeans(n_clusters = 135, random_state=0).fit(compact_tgt_df)\n",
    "\n",
    "compact_alternate_df = compact_tgt_df.copy()\n",
    "compact_alternate_df_np = compact_tgt_df.to_numpy()\n",
    "\n",
    "idxlist = []\n",
    "compact_new_df_list = []\n",
    "\n",
    "for rowkm in kmeans.cluster_centers_:\n",
    "    mindist = -99\n",
    "    rowidx = 0\n",
    "    idx = 0\n",
    "    for row in compact_alternate_df_np:\n",
    "        dst = distance.euclidean(row, rowkm)\n",
    "\n",
    "        if(dst >= mindist):\n",
    "            mindist = dst\n",
    "            rowidx = idx\n",
    "            rowval = row\n",
    "\n",
    "        idx = idx + 1\n",
    "\n",
    "#     print(\"Row selected: \", rowidx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", mindist)\n",
    "#     print(\"Matrix shape: \", compact_alternate_df_np.shape)\n",
    "    compact_new_df_list.append(rowval)\n",
    "    compact_alternate_df = np.delete(compact_alternate_df_np, rowidx, 0)\n",
    "    idxlist.append(rowidx)\n",
    "\n",
    "\n",
    "compact_new_df = pd.DataFrame(np.vstack(compact_new_df_list))\n",
    "\n",
    "print(\"The dataset extracted is: \")\n",
    "print(compact_new_df.shape)\n",
    "print(\"------------------------------------------------------\")\n",
    "\n",
    "##################################################### Variance Sampling: Phase 2 ################################################\n",
    "\n",
    "compact_alternate_source_df = compact_source_df.copy()\n",
    "compact_alternate_source_df_np = compact_alternate_source_df.to_numpy()\n",
    "\n",
    "idxlist2 = []\n",
    "compact_final_df_list = []\n",
    "\n",
    "for row_nm in compact_new_df_list:\n",
    "    min_dist = -99\n",
    "    row_idx = 0\n",
    "    idx_val = 0\n",
    "    for row_alt in compact_alternate_source_df_np:\n",
    "        dst = distance.euclidean(row_alt, row_nm)\n",
    "        if(dst >= mindist):\n",
    "            min_dist = dst\n",
    "            row_idx = idx_val\n",
    "            row_val = row_alt\n",
    "\n",
    "        idx_val = idx_val + 1\n",
    "\n",
    "#     print(\"Row selected: \", row_idx) #Alternate_df.loc[rowidx,:]\\\n",
    "#     print(\"Min. distance: \", min_dist)\n",
    "#     print(\"Matrix shape: \", compact_alternate_source_df_np.shape)\n",
    "    compact_final_df_list.append(row_val)\n",
    "    compact_alternate_source_df_np = np.delete(compact_alternate_source_df_np, row_idx, 0)\n",
    "    idxlist2.append(row_idx)\n",
    "\n",
    "\n",
    "compact_final_df = pd.DataFrame(np.vstack(compact_final_df_list), columns= compact_alternate_source_df.columns)\n",
    "\n",
    "print(\"Shape of source before :\",compact_source_df.shape)\n",
    "compact_source_df = pd.DataFrame(np.vstack(compact_alternate_source_df_np), columns= compact_source_df.columns)\n",
    "print(\"Shape of source after :\", compact_source_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "\n",
    "print(\"Shape of target before :\", compact_tgt_df.shape)\n",
    "Elevators_tgt_df = pd.concat([compact_tgt_df, compact_final_df], ignore_index=True)\n",
    "print(\"Shape of target after :\", compact_tgt_df.shape)\n",
    "\n",
    "print(\"----------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target Set:  (2766, 21)\n",
      "Source Set:  (1356, 21)\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "################################## Importance Sampling: Manhattan Distance ######################################################\n",
    "compact_source_df[\"ManDis\"] = \"\"\n",
    "\n",
    "compact_tgt_df_mean = []\n",
    "prow = compact_tgt_df.mean()\n",
    "compact_tgt_df_mean = [prow.lread, prow.lwrite, prow.scall, prow.sread, prow.swrite, prow.fork, prow.exec, prow.rchar,\n",
    "       prow.wchar, prow.pgout, prow.ppgout, prow.pgfree, prow.pgscan, prow.atch, prow.ppgin, prow.pflt, prow.vflt, prow.runqsz,\n",
    "        prow.freemem, prow.freeswap, prow.usr]\n",
    "\n",
    "rowidx = 0\n",
    "\n",
    "for row in compact_source_df.itertuples():\n",
    "    row_list =[row.lread, row.lwrite, row.scall, row.sread, row.swrite, row.fork, row.exec, row.rchar,\n",
    "       row.wchar, row.pgout, row.ppgout, row.pgfree, row.pgscan, row.atch, row.ppgin, row.pflt, row.vflt, row.runqsz,\n",
    "        row.freemem, row.freeswap, row.usr]\n",
    "\n",
    "    man_dis = 0\n",
    "    for i in range(0, len(row_list)):\n",
    "        tempval = compact_tgt_df_mean[i] - row_list[i]\n",
    "        man_dis = man_dis + abs(tempval)\n",
    "\n",
    "    compact_source_df.loc[rowidx,\"ManDis\"] = man_dis\n",
    "    rowidx = rowidx + 1\n",
    "\n",
    "\n",
    "# kinematics_source_df = kinematics_source_df.sort_values(by =['ManDis'])\n",
    "# kinematics_source_df = kinematics_source_df.head(1330) \n",
    "# kinematics_source_df = kinematics_source_df.drop(['ManDis'], axis =1)\n",
    "# kinematics_source_df = kinematics_source_df.reset_index(drop=True)\n",
    "\n",
    "# print(\"Target Set: \", kinematics_tgt_df.shape)\n",
    "# print(\"Source Set: \", kinematics_source_df.shape)\n",
    "\n",
    "compact_source_df = compact_source_df.sort_values(by =['ManDis'])\n",
    "compact_source_df = compact_source_df.head(1356) #### For Computer Activity 650 instances\n",
    "compact_source_df = compact_source_df.drop(['ManDis'], axis =1)\n",
    "compact_source_df = compact_source_df.reset_index(drop=True)\n",
    "\n",
    "print(\"Target Set: \", compact_tgt_df.shape)\n",
    "print(\"Source Set: \", compact_source_df.shape)\n",
    "\n",
    "\n",
    "#################### Splitting into features and target ####################\n",
    "target_compact = ['usr']\n",
    "\n",
    "compact_source_df_y = compact_source_df[target_compact]\n",
    "compact_source_df_X = compact_source_df.drop(target_compact, axis = 1)\n",
    "\n",
    "features_compact = compact_source_df_X.columns\n",
    "print(\"---------------------------\")\n",
    "###############################################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STrAdaboost.R2 (50 iter)\n",
      "-------------------------------------------\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "Inside Updated Sampling TrAdaBoost.R2\n",
      "RMSE List of STrAdaboost.R2: [2.6186617707787057, 2.6159304095258746, 2.305655221463425, 2.286693451050622, 2.4407287618657896, 2.750041727772971, 2.4052096465625996, 2.375339909084825, 2.1071420933127882, 2.4710156796403653, 2.5004399589856425, 2.7265901630007545, 2.6652046927599313, 2.2437764671173466, 2.5094200241520523, 2.4770057257617544, 2.5692084860495776, 2.552654584603582, 2.6087811330529305, 2.4605433795977785]\n",
      "R^2 List of STrAdaboost.R2: [0.9697618214250323, 0.9471268238534477, 0.9537408874275689, 0.9811896394867949, 0.9863121196304444, 0.98029208142645, 0.8988257313662421, 0.9702058099595829, 0.9814023873846419, 0.9765031739023816, 0.8880733092788969, 0.9422785935234765, 0.9670349673979034, 0.9557847612158992, 0.8352212434957901, 0.9866359960823643, 0.9760553234882533, 0.8610307947012773, 0.9478487676618945, 0.9809250287427717]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 2.484502164306966 0.1650485498838983\n",
      "Mean, STDev of R^2: 0.9493124630725557 0.043900509409169425\n"
     ]
    }
   ],
   "source": [
    "#################################### STrAdaBoost.R2 compAct ################################################################\n",
    "from two_TrAdaBoostR2 import TwoStageTrAdaBoostR2\n",
    "\n",
    "n_estimators = 100\n",
    "steps = 30\n",
    "fold = 10\n",
    "random_state = np.random.RandomState(1)\n",
    "\n",
    "r2scorelist_stradaboost_compact = []\n",
    "rmselist_stradaboost_compact = []\n",
    "\n",
    "print(\"STrAdaboost.R2 (50 iter)\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(compact_tgt_df):\n",
    "        \n",
    "    compact_train_df_X = compact_tgt_df.iloc[train_idx].loc[:, features_compact]\n",
    "    compact_test_df_X = compact_tgt_df.iloc[test_idx][features_compact]\n",
    "    compact_train_df_y = compact_tgt_df.iloc[train_idx].loc[:,target_compact]\n",
    "    compact_test_df_y = compact_tgt_df.loc[test_idx][target_compact] \n",
    "    \n",
    "    compact_X_df = pd.concat([compact_source_df_X, compact_train_df_X], ignore_index=True)\n",
    "    compact_y_df = pd.concat([compact_source_df_y, compact_train_df_y], ignore_index=True)\n",
    "\n",
    "    compact_np_train_X = compact_X_df.to_numpy()\n",
    "    compact_np_train_y = compact_y_df.to_numpy()\n",
    "\n",
    "    compact_np_test_X = compact_test_df_X.to_numpy()\n",
    "    compact_np_test_y = compact_test_df_y.to_numpy()\n",
    "\n",
    "    compact_np_train_y_list = compact_np_train_y.ravel()\n",
    "    compact_np_test_y_list = compact_np_test_y.ravel()\n",
    "    \n",
    "    sample_size = [len(compact_source_df_X), len(compact_train_df_X)]\n",
    "\n",
    "    model_stradaboost_compact = TwoStageTrAdaBoostR2(DecisionTreeRegressor(max_depth=6),\n",
    "                        n_estimators = n_estimators, sample_size = sample_size,\n",
    "                        steps = steps, fold = fold, random_state = random_state)\n",
    "\n",
    "\n",
    "    model_stradaboost_compact.fit(compact_np_train_X, compact_np_train_y_list)\n",
    "    y_pred_stradaboost_compact = model_stradaboost_compact.predict(compact_np_test_X)\n",
    "    \n",
    "    mse_stradaboost_compact = sqrt(mean_squared_error(compact_np_test_y, y_pred_stradaboost_compact))\n",
    "    rmselist_stradaboost_compact.append(mse_stradaboost_compact)\n",
    "        \n",
    "    r2_score_stradaboost_compact = pearsonr(compact_np_test_y_list, y_pred_stradaboost_compact)\n",
    "    r2_score_stradaboost_compact = (r2_score_stradaboost_compact[0])**2\n",
    "    r2scorelist_stradaboost_compact.append(r2_score_stradaboost_compact)\n",
    "        \n",
    "\n",
    "print(\"RMSE List of STrAdaboost.R2:\", rmselist_stradaboost_compact)\n",
    "print(\"R^2 List of STrAdaboost.R2:\", r2scorelist_stradaboost_compact)\n",
    "\n",
    "print(\"\\n\")\n",
    "        \n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_stradaboost_compact), statistics.stdev(rmselist_stradaboost_compact))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_stradaboost_compact), statistics.stdev(r2scorelist_stradaboost_compact))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Two-Stage TrAdaboost.R2\n",
      "-------------------------------------------\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "Binary search has not converged. Set value to the current best.\n",
      "RMSE List of TrAdaboost.R2: [2.974364983450536, 2.8386225606024946, 2.5688433337336827, 2.9784953062109985, 2.4337502103105435, 2.881959812721326, 2.5195713591073314, 2.462417441887926, 2.6417051200666504, 2.3914609372935387, 2.834158919223588, 2.9518825572366145, 2.717115569404248, 2.3069658933152217, 2.7808900214306216, 2.8761605033937205, 2.7591701086596396, 2.587466284816961, 2.830020528477602, 2.4411657670458045]\n",
      "R^2 List of TrAdaboost.R2: [0.9576502114133822, 0.9288206518603569, 0.9355649075793303, 0.9643648274499126, 0.9827794480109617, 0.9763801288668895, 0.8753831550221396, 0.9664025228128864, 0.9675465134171953, 0.9755051151176428, 0.8373836512247603, 0.9266424777418726, 0.9586739574858707, 0.9519211842905118, 0.7785287106490205, 0.9794910300847643, 0.9663933561571955, 0.8501264757644221, 0.9293959712993723, 0.9790860823185291]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 2.6888093609194526 0.2117838693400038\n",
      "Mean, STDev of R^2: 0.9344020189283508 0.056061262818860785\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### TwoStagetrAdaBoostR2 compAct #######################################\n",
    "\n",
    "############# Transfer Learning specifications #########################\n",
    "\n",
    "def get_estimator(**kwargs):\n",
    "    return DecisionTreeRegressor(max_depth = 6)\n",
    "\n",
    "kwargs_TwoTrAda = {'steps': 30,\n",
    "                    'fold': 10,\n",
    "                  'learning_rate': 0.1}\n",
    "\n",
    "#####################################################################################################################\n",
    "\n",
    "from adapt.instance_based import TrAdaBoost, TrAdaBoostR2, TwoStageTrAdaBoostR2\n",
    "\n",
    "print(\"Two-Stage TrAdaboost.R2\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_TwoTrAda_compact = []\n",
    "rmselist_TwoTrAda_compact = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(compact_tgt_df):\n",
    "        \n",
    "    compact_train_df_X = compact_tgt_df.iloc[train_idx].loc[:, features_compact]\n",
    "    compact_test_df_X = compact_tgt_df.iloc[test_idx][features_compact]\n",
    "    compact_train_df_y = compact_tgt_df.iloc[train_idx].loc[:,target_compact]\n",
    "    compact_test_df_y = compact_tgt_df.loc[test_idx][target_compact] \n",
    "    \n",
    "    compact_X_df = pd.concat([compact_source_df_X, compact_train_df_X], ignore_index=True)\n",
    "    compact_y_df = pd.concat([compact_source_df_y, compact_train_df_y], ignore_index=True)\n",
    "\n",
    "    compact_np_train_X = compact_X_df.to_numpy()\n",
    "    compact_np_train_y = compact_y_df.to_numpy()\n",
    "\n",
    "    compact_np_test_X = compact_test_df_X.to_numpy()\n",
    "    compact_np_test_y = compact_test_df_y.to_numpy()\n",
    "\n",
    "    compact_np_train_y_list = compact_np_train_y.ravel()\n",
    "    compact_np_test_y_list = compact_np_test_y.ravel()\n",
    "\n",
    "    src_size_compact = len(compact_source_df_y)\n",
    "    tgt_size_compact = len(compact_train_df_y)\n",
    "    \n",
    "    src_idx_compact = np.arange(start=0, stop=(src_size_compact - 1), step=1)\n",
    "    tgt_idx_compact = np.arange(start=src_size_compact, stop=((src_size_compact + tgt_size_compact)-1), step=1)\n",
    "\n",
    "    model_TwoTrAda_compact = TwoStageTrAdaBoostR2(get_estimator = get_estimator, n_estimators = 100)\n",
    "    model_TwoTrAda_compact.fit(compact_np_train_X, compact_np_train_y_list, src_idx_compact, tgt_idx_compact)\n",
    "\n",
    "    y_pred_TwoTrAda_compact = model_TwoTrAda_compact.predict(compact_np_test_X)\n",
    "\n",
    "    mse_TwoTrAda_compact = sqrt(mean_squared_error(compact_np_test_y, y_pred_TwoTrAda_compact))\n",
    "\n",
    "    rmselist_TwoTrAda_compact.append(mse_TwoTrAda_compact)\n",
    "        \n",
    "    r2_score_TwoTrAda_compact = pearsonr(compact_np_test_y_list, y_pred_TwoTrAda_compact)\n",
    "    r2_score_TwoTrAda_compact = (r2_score_TwoTrAda_compact[0])**2\n",
    "\n",
    "    r2scorelist_TwoTrAda_compact.append(r2_score_TwoTrAda_compact)\n",
    "\n",
    "\n",
    "print(\"RMSE List of TrAdaboost.R2:\", rmselist_TwoTrAda_compact)\n",
    "print(\"R^2 List of TrAdaboost.R2:\", r2scorelist_TwoTrAda_compact)\n",
    "\n",
    "print(\"\\n\")\n",
    "        \n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_TwoTrAda_compact), statistics.stdev(rmselist_TwoTrAda_compact))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_TwoTrAda_compact), statistics.stdev(r2scorelist_TwoTrAda_compact))\n",
    "\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adaboost.R2 Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of Adaboost.R2: [2.7804522140826924, 2.59957444225543, 2.7363516917701385, 2.5935756353654735, 2.4499808252895963, 2.509217039650656, 2.506379329776235, 2.4485106345268526, 2.2902084470472617, 2.6426973077338114, 2.6034115752963807, 2.8386489234790244, 2.7864363659513387, 2.272562663536608, 2.816422221432482, 2.6680289281408167, 2.7630193666141594, 2.9085836115160517, 2.860128301730891, 2.679232119672356]\n",
      "R^2 List of Adaboost.R2: [0.9618893262786425, 0.9423354884109288, 0.930212591974738, 0.9729009978610867, 0.9842466192395385, 0.9832411912130833, 0.8730517416344266, 0.9676449060778342, 0.9759298178115617, 0.9711568141828248, 0.8741348307385571, 0.9356469706174296, 0.9579123832552064, 0.9533199344927383, 0.8078351505370172, 0.9820802040616357, 0.9692455041767469, 0.8127275521475399, 0.9338265751654369, 0.9746586811507773]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 2.6376710822434126 0.18198002532461902\n",
      "Mean, STDev of R^2: 0.9381998640513876 0.05395918274684536\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "########################### AdaBoostR2 Transfer Learning compAct #####################################################\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "#earlystopping = EarlyStopping(monitor=\"mean_squared_error\", patience=40, verbose=1, mode='auto')\n",
    "#epochs = 100, batch_size = 5, callbacks=[earlystopping])\n",
    "\n",
    "print(\"Adaboost.R2 Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_AdaTL_compact = []\n",
    "rmselist_AdaTL_compact = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(compact_tgt_df):\n",
    "        \n",
    "    compact_train_df_X = compact_tgt_df.iloc[train_idx].loc[:, features_compact]\n",
    "    compact_test_df_X = compact_tgt_df.iloc[test_idx][features_compact]\n",
    "    compact_train_df_y = compact_tgt_df.iloc[train_idx].loc[:,target_compact]\n",
    "    compact_test_df_y = compact_tgt_df.loc[test_idx][target_compact] \n",
    "    \n",
    "    compact_X_df = pd.concat([compact_source_df_X, compact_train_df_X], ignore_index=True)\n",
    "    compact_y_df = pd.concat([compact_source_df_y, compact_train_df_y], ignore_index=True)\n",
    "\n",
    "    compact_np_train_X = compact_X_df.to_numpy()\n",
    "    compact_np_train_y = compact_y_df.to_numpy()\n",
    "\n",
    "    compact_np_test_X = compact_test_df_X.to_numpy()\n",
    "    compact_np_test_y = compact_test_df_y.to_numpy()\n",
    "\n",
    "    compact_np_train_y_list = compact_np_train_y.ravel()\n",
    "    compact_np_test_y_list = compact_np_test_y.ravel()\n",
    "\n",
    "    model_AdaTL_compact = AdaBoostRegressor(DecisionTreeRegressor(max_depth = 6), learning_rate = 0.1, n_estimators = 100) \n",
    "    model_AdaTL_compact.fit(compact_np_train_X, compact_np_train_y_list) \n",
    "\n",
    "    y_pred_AdaTL_compact = model_AdaTL_compact.predict(compact_np_test_X)\n",
    "\n",
    "    mse_AdaTL_compact = sqrt(mean_squared_error(compact_np_test_y, y_pred_AdaTL_compact))\n",
    "    rmselist_AdaTL_compact.append(mse_AdaTL_compact)\n",
    "        \n",
    "    r2_score_AdaTL_compact = pearsonr(compact_np_test_y_list, y_pred_AdaTL_compact)\n",
    "    r2_score_AdaTL_compact = (r2_score_AdaTL_compact[0])**2\n",
    "    r2scorelist_AdaTL_compact.append(r2_score_AdaTL_compact)\n",
    "\n",
    "\n",
    "print(\"RMSE List of Adaboost.R2:\", rmselist_AdaTL_compact)\n",
    "print(\"R^2 List of Adaboost.R2:\", r2scorelist_AdaTL_compact)\n",
    "\n",
    "print(\"\\n\")\n",
    "        \n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_AdaTL_compact), statistics.stdev(rmselist_AdaTL_compact))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_AdaTL_compact), statistics.stdev(r2scorelist_AdaTL_compact))\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Regression Transfer Learning\n",
      "-------------------------------------------\n",
      "RMSE List of GBRTL: [1.899317021544848, 1.9184883482738562, 1.8845726749347893, 2.039165793153535, 1.8114062485489288, 1.7476573427474975, 1.8573902506838045, 2.0744618740930427, 1.5237154053558284, 2.1728125872796853, 2.1562541433778737, 2.100548622416806, 2.0223128810461324, 1.9187440647832126, 1.7983023937891427, 1.8278806725079348, 2.299259476430266, 2.0326359364094864, 2.2739487317345892, 1.9441087667195327]\n",
      "R^2 List of GBRTL: [0.9779281956699454, 0.9606879758958644, 0.9614236680966668, 0.9809021111129699, 0.9882793829721787, 0.9910777746201741, 0.9208983111751095, 0.9720479094976766, 0.985527492146757, 0.9795125771135973, 0.9004803813018428, 0.9594583844430099, 0.9735274111990455, 0.9631249993483756, 0.8933481319887063, 0.9898759161413495, 0.9763552720446795, 0.8876218837008548, 0.947976733437696, 0.982967023299449]\n",
      "\n",
      "\n",
      "Mean, STDev of RMSE: 1.9651491617915395 0.18777234173564544\n",
      "Mean, STDev of R^2: 0.9596510767602974 0.03283068957332542\n",
      "-------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "######################### Gradient Boosting Regression Transfer Learning compAct #######################################\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "\n",
    "print(\"Gradient Boosting Regression Transfer Learning\")\n",
    "print(\"-------------------------------------------\")\n",
    "\n",
    "r2scorelist_GBRTL_compact = []\n",
    "rmselist_GBRTL_compact = []\n",
    "\n",
    "kf = KFold(n_splits = 20)\n",
    "\n",
    "for train_idx, test_idx in kf.split(compact_tgt_df):\n",
    "        \n",
    "    compact_train_df_X = compact_tgt_df.iloc[train_idx].loc[:, features_compact]\n",
    "    compact_test_df_X = compact_tgt_df.iloc[test_idx][features_compact]\n",
    "    compact_train_df_y = compact_tgt_df.iloc[train_idx].loc[:,target_compact]\n",
    "    compact_test_df_y = compact_tgt_df.loc[test_idx][target_compact] \n",
    "    \n",
    "    compact_X_df = pd.concat([compact_source_df_X, compact_train_df_X], ignore_index=True)\n",
    "    compact_y_df = pd.concat([compact_source_df_y, compact_train_df_y], ignore_index=True)\n",
    "\n",
    "    compact_np_train_X = compact_X_df.to_numpy()\n",
    "    compact_np_train_y = compact_y_df.to_numpy()\n",
    "\n",
    "    compact_np_test_X = compact_test_df_X.to_numpy()\n",
    "    compact_np_test_y = compact_test_df_y.to_numpy()\n",
    "\n",
    "    compact_np_train_y_list = compact_np_train_y.ravel()\n",
    "    compact_np_test_y_list = compact_np_test_y.ravel()\n",
    "\n",
    "\n",
    "    model_GBRTL_compact = GradientBoostingRegressor(learning_rate = 0.1, max_depth = 6, n_estimators = 100) #, subsample=0.5)\n",
    "    model_GBRTL_compact.fit(compact_np_train_X, compact_np_train_y_list)\n",
    "\n",
    "    y_pred_GBRTL_compact = model_GBRTL_compact.predict(compact_test_df_X) ##Using dataframe instead of the numpy matrix\n",
    "\n",
    "    mse_GBRTL_compact = sqrt(mean_squared_error(compact_np_test_y, y_pred_GBRTL_compact))\n",
    "    rmselist_GBRTL_compact.append(mse_GBRTL_compact)\n",
    "        \n",
    "    r2_score_GBRTL_compact = pearsonr(compact_np_test_y_list, y_pred_GBRTL_compact)\n",
    "    r2_score_GBRTL_compact = (r2_score_GBRTL_compact[0])**2\n",
    "    r2scorelist_GBRTL_compact.append(r2_score_GBRTL_compact)\n",
    "\n",
    "\n",
    "print(\"RMSE List of GBRTL:\", rmselist_GBRTL_compact)\n",
    "print(\"R^2 List of GBRTL:\", r2scorelist_GBRTL_compact)\n",
    "\n",
    "print(\"\\n\")\n",
    "        \n",
    "print(\"Mean, STDev of RMSE:\", statistics.mean(rmselist_GBRTL_compact), statistics.stdev(rmselist_GBRTL_compact))\n",
    "print(\"Mean, STDev of R^2:\", statistics.mean(r2scorelist_GBRTL_compact), statistics.stdev(r2scorelist_GBRTL_compact))\n",
    "\n",
    "print(\"-------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}