running_pred_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run prediction pipeline for radio-detectable AGN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jupyter Notebook to run the prediction pipeline presented in Carvajal et al., 2023.\n",
    "\n",
    "Example code uses data from IR-detected sources in either HETDEX Spring field or Stripe 82."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "# Static plots\n",
    "import numpy as np\n",
    "from pycaret import classification as pyc\n",
    "from pycaret import regression as pyr\n",
    "from joblib import load\n",
    "import pandas as pd\n",
    "import global_variables as gv\n",
    "import global_functions as gf\n",
    "import os\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select the field to predict sources."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "used_field = 'HETDEX'  # 'HETDEX' or 'S82'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name_dict      = {'S82': gv.file_S82, 'HETDEX': gv.file_HETDEX}\n",
    "file_name           = gv.model_path + file_name_dict[used_field]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check that data files have been downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File predicted_rAGN_HETDEX.parquet has been already downloaded\n"
     ]
    }
   ],
   "source": [
    "gf.download_from_zenodo(file_name_dict[used_field], gv.model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this notebook, not all columns will be needed. Select those that will be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "used_cols = ['Z', 'band_num', 'class', 'W1mproPM', 'W2mproPM', 'gmag', 'rmag', 'imag', 'zmag', 'ymag', 'W3mag', 'W4mag', 'Jmag', 'Hmag', 'Kmag', 'LOFAR_detect', 'radio_AGN']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = pd.read_parquet(file_name, \n",
    "                          engine='fastparquet', \n",
    "                          columns=used_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create new columns with colours. Only create colours used by models in pipeline. They are listed in the article."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "colours_AGN   = ['g_r', 'r_i', 'r_J', 'i_z', 'i_y', 'z_y', 'z_W2', 'y_J', 'y_W1', 'y_W2', 'J_H', 'H_K', 'H_W3', 'W1_W2', 'W1_W3', 'W3_W4']\n",
    "colours_radio = ['g_r', 'g_i', 'r_i', 'r_z', 'i_z', 'z_y', 'z_W1', 'y_J', 'y_W1', 'J_H', 'H_K', 'K_W3', 'K_W4', 'W1_W2', 'W2_W3']\n",
    "colours_z     = ['g_r', 'g_W3', 'r_i', 'r_z', 'i_z', 'i_y', 'z_y', 'y_J', 'y_W1', 'J_H', 'H_K', 'K_W3', 'K_W4', 'W1_W2', 'W2_W3']\n",
    "    \n",
    "new_colours   = list(np.unique(colours_AGN + colours_radio + colours_z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mag_names = {'g': 'gmag', 'r': 'rmag', 'i':'imag', 'z': 'zmag', 'y': 'ymag', 'J': 'Jmag', 'H': 'Hmag', 'K': 'Kmag', 'W1': 'W1mproPM', 'W2': 'W2mproPM', 'W3': 'W3mag', 'W4': 'W4mag'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for colour in new_colours:\n",
    "    mag_a_str, mag_b_str = colour.split('_')\n",
    "    new_col = data_df.loc[:, mag_names[mag_a_str]] - data_df.loc[:, mag_names[mag_b_str]]\n",
    "    data_df[colour] = new_col"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check that model files have been downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File classification_AGN_galaxy.pkl has been already downloaded\n",
      "File cal_classification_AGN_galaxy.joblib has been already downloaded\n",
      "File classification_radio_detection.pkl has been already downloaded\n",
      "File cal_classification_radio_detection.joblib has been already downloaded\n",
      "File regression_redshift.pkl has been already downloaded\n"
     ]
    }
   ],
   "source": [
    "gf.download_from_zenodo(gv.AGN_gal_model + '.pkl', gv.model_path)\n",
    "gf.download_from_zenodo(gv.cal_AGN_gal_model, gv.model_path)\n",
    "gf.download_from_zenodo(gv.radio_model + '.pkl', gv.model_path)\n",
    "gf.download_from_zenodo(gv.cal_radio_model, gv.model_path)\n",
    "gf.download_from_zenodo(gv.full_z_model + '.pkl', gv.model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformation Pipeline and Model Successfully Loaded\n",
      "Transformation Pipeline and Model Successfully Loaded\n",
      "Transformation Pipeline and Model Successfully Loaded\n"
     ]
    }
   ],
   "source": [
    "AGN_gal_clf           = pyc.load_model(gv.model_path + gv.AGN_gal_model)  #\n",
    "cal_AGN_gal_clf       = load(gv.model_path + gv.cal_AGN_gal_model)  # calibrated model\n",
    "radio_det_AGN_clf     = pyc.load_model(gv.model_path + gv.radio_model)\n",
    "cal_radio_det_AGN_clf = load(gv.model_path + gv.cal_radio_model)  # calibrated model\n",
    "redshift_reg_rAGN     = pyr.load_model(gv.model_path + gv.full_z_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run prediction models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run models over all sources in dataset. The user can select, afterwards, sources that would be predicted as radio-detectable AGN (or any other combination of predictions).\n",
    "The following steps take some time to run."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify between AGN and galaxies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = gf.predict_AGN_gal(data_df, \n",
    "                             AGN_gal_clf,\n",
    "                             cal_AGN_gal_clf,\n",
    "                             gv.AGN_thresh,\n",
    "                             gv.cal_AGN_thresh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify between radio-detectable and non radio-detectable sources."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = gf.predict_radio_det(data_df,\n",
    "                               radio_det_AGN_clf,\n",
    "                               cal_radio_det_AGN_clf,\n",
    "                               gv.radio_thresh,\n",
    "                               gv.cal_radio_thresh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict photometric redshifts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = gf.predict_z(data_df, \n",
    "                       redshift_reg_rAGN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display ten first radio-detected AGN correctly predicted as such in data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_data = np.array(data_df.loc[:, 'class'] == 1) & np.array(data_df.loc[:, 'pred_class_cal'] == 1) & np.array(data_df.loc[:, 'LOFAR_detect'] == 1) & np.array(data_df.loc[:, 'pred_radio_cal_AGN'] == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Z</th>\n",
       "      <th>band_num</th>\n",
       "      <th>class</th>\n",
       "      <th>W1mproPM</th>\n",
       "      <th>W2mproPM</th>\n",
       "      <th>gmag</th>\n",
       "      <th>rmag</th>\n",
       "      <th>imag</th>\n",
       "      <th>zmag</th>\n",
       "      <th>ymag</th>\n",
       "      <th>W3mag</th>\n",
       "      <th>W4mag</th>\n",
       "      <th>Jmag</th>\n",
       "      <th>Hmag</th>\n",
       "      <th>Kmag</th>\n",
       "      <th>LOFAR_detect</th>\n",
       "      <th>radio_AGN</th>\n",
       "      <th>H_K</th>\n",
       "      <th>H_W3</th>\n",
       "      <th>J_H</th>\n",
       "      <th>K_W3</th>\n",
       "      <th>K_W4</th>\n",
       "      <th>W1_W2</th>\n",
       "      <th>W1_W3</th>\n",
       "      <th>W2_W3</th>\n",
       "      <th>W3_W4</th>\n",
       "      <th>g_W3</th>\n",
       "      <th>g_i</th>\n",
       "      <th>g_r</th>\n",
       "      <th>i_y</th>\n",
       "      <th>i_z</th>\n",
       "      <th>r_J</th>\n",
       "      <th>r_i</th>\n",
       "      <th>r_z</th>\n",
       "      <th>y_J</th>\n",
       "      <th>y_W1</th>\n",
       "      <th>y_W2</th>\n",
       "      <th>z_W1</th>\n",
       "      <th>z_W2</th>\n",
       "      <th>z_y</th>\n",
       "      <th>pred_class</th>\n",
       "      <th>Score_AGN</th>\n",
       "      <th>Prob_AGN</th>\n",
       "      <th>pred_class_cal</th>\n",
       "      <th>pred_radio_AGN</th>\n",
       "      <th>Score_radio_AGN</th>\n",
       "      <th>Prob_radio_AGN</th>\n",
       "      <th>pred_radio_cal_AGN</th>\n",
       "      <th>pred_Z_rAGN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16456</th>\n",
       "      <td>0.846</td>\n",
       "      <td>8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.337000</td>\n",
       "      <td>18.363001</td>\n",
       "      <td>23.299999</td>\n",
       "      <td>21.890100</td>\n",
       "      <td>21.287201</td>\n",
       "      <td>20.959600</td>\n",
       "      <td>20.519300</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>-0.026001</td>\n",
       "      <td>1.667000</td>\n",
       "      <td>1.693001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>6.629999</td>\n",
       "      <td>2.012798</td>\n",
       "      <td>1.409899</td>\n",
       "      <td>0.767900</td>\n",
       "      <td>0.327600</td>\n",
       "      <td>4.440100</td>\n",
       "      <td>0.602900</td>\n",
       "      <td>0.930500</td>\n",
       "      <td>3.069300</td>\n",
       "      <td>2.182301</td>\n",
       "      <td>2.156300</td>\n",
       "      <td>2.622601</td>\n",
       "      <td>2.596600</td>\n",
       "      <td>0.440300</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500016</td>\n",
       "      <td>0.657091</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.438512</td>\n",
       "      <td>0.410227</td>\n",
       "      <td>1</td>\n",
       "      <td>0.7586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27470</th>\n",
       "      <td>0.736</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.676001</td>\n",
       "      <td>17.408001</td>\n",
       "      <td>20.004499</td>\n",
       "      <td>19.639299</td>\n",
       "      <td>19.607401</td>\n",
       "      <td>19.268499</td>\n",
       "      <td>19.199200</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>0.268000</td>\n",
       "      <td>1.006001</td>\n",
       "      <td>0.738001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>3.334499</td>\n",
       "      <td>0.397099</td>\n",
       "      <td>0.365200</td>\n",
       "      <td>0.408201</td>\n",
       "      <td>0.338902</td>\n",
       "      <td>2.189299</td>\n",
       "      <td>0.031898</td>\n",
       "      <td>0.370800</td>\n",
       "      <td>1.749199</td>\n",
       "      <td>1.523199</td>\n",
       "      <td>1.791199</td>\n",
       "      <td>1.592499</td>\n",
       "      <td>1.860498</td>\n",
       "      <td>0.069300</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500121</td>\n",
       "      <td>0.988672</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.310507</td>\n",
       "      <td>0.312719</td>\n",
       "      <td>1</td>\n",
       "      <td>0.7748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40804</th>\n",
       "      <td>1.078</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.311001</td>\n",
       "      <td>16.582001</td>\n",
       "      <td>18.525999</td>\n",
       "      <td>18.507601</td>\n",
       "      <td>18.457300</td>\n",
       "      <td>18.362700</td>\n",
       "      <td>18.208500</td>\n",
       "      <td>15.373000</td>\n",
       "      <td>14.180</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>1.867</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>1.217000</td>\n",
       "      <td>2.410000</td>\n",
       "      <td>0.729000</td>\n",
       "      <td>1.938001</td>\n",
       "      <td>1.209001</td>\n",
       "      <td>1.193</td>\n",
       "      <td>3.152999</td>\n",
       "      <td>0.068699</td>\n",
       "      <td>0.018398</td>\n",
       "      <td>0.248800</td>\n",
       "      <td>0.094601</td>\n",
       "      <td>1.057600</td>\n",
       "      <td>0.050301</td>\n",
       "      <td>0.144901</td>\n",
       "      <td>0.758499</td>\n",
       "      <td>0.897499</td>\n",
       "      <td>1.626499</td>\n",
       "      <td>1.051699</td>\n",
       "      <td>1.780699</td>\n",
       "      <td>0.154200</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500122</td>\n",
       "      <td>0.989167</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.400925</td>\n",
       "      <td>0.382640</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44825</th>\n",
       "      <td>0.877</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.306000</td>\n",
       "      <td>18.175001</td>\n",
       "      <td>19.906601</td>\n",
       "      <td>19.838699</td>\n",
       "      <td>19.618500</td>\n",
       "      <td>19.524599</td>\n",
       "      <td>19.334000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>0.130999</td>\n",
       "      <td>1.636000</td>\n",
       "      <td>1.505001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>3.236601</td>\n",
       "      <td>0.288101</td>\n",
       "      <td>0.067902</td>\n",
       "      <td>0.284500</td>\n",
       "      <td>0.093901</td>\n",
       "      <td>2.388699</td>\n",
       "      <td>0.220200</td>\n",
       "      <td>0.314100</td>\n",
       "      <td>1.883999</td>\n",
       "      <td>1.028000</td>\n",
       "      <td>1.158998</td>\n",
       "      <td>1.218599</td>\n",
       "      <td>1.349598</td>\n",
       "      <td>0.190599</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500121</td>\n",
       "      <td>0.988471</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.501761</td>\n",
       "      <td>0.455192</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45506</th>\n",
       "      <td>0.643</td>\n",
       "      <td>12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.579000</td>\n",
       "      <td>15.040000</td>\n",
       "      <td>17.124001</td>\n",
       "      <td>17.214199</td>\n",
       "      <td>17.213301</td>\n",
       "      <td>17.390400</td>\n",
       "      <td>17.359301</td>\n",
       "      <td>14.250999</td>\n",
       "      <td>13.386</td>\n",
       "      <td>17.230000</td>\n",
       "      <td>17.105</td>\n",
       "      <td>16.527</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.577999</td>\n",
       "      <td>2.854</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>2.276001</td>\n",
       "      <td>3.141001</td>\n",
       "      <td>0.539001</td>\n",
       "      <td>1.328001</td>\n",
       "      <td>0.789001</td>\n",
       "      <td>0.865</td>\n",
       "      <td>2.873001</td>\n",
       "      <td>-0.089300</td>\n",
       "      <td>-0.090199</td>\n",
       "      <td>-0.146000</td>\n",
       "      <td>-0.177099</td>\n",
       "      <td>-0.015800</td>\n",
       "      <td>0.000898</td>\n",
       "      <td>-0.176201</td>\n",
       "      <td>0.129301</td>\n",
       "      <td>1.780300</td>\n",
       "      <td>2.319301</td>\n",
       "      <td>1.811399</td>\n",
       "      <td>2.350400</td>\n",
       "      <td>0.031099</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500122</td>\n",
       "      <td>0.988938</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.875211</td>\n",
       "      <td>0.726470</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53799</th>\n",
       "      <td>0.712</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.170000</td>\n",
       "      <td>18.677999</td>\n",
       "      <td>21.766199</td>\n",
       "      <td>21.311600</td>\n",
       "      <td>20.481701</td>\n",
       "      <td>19.868799</td>\n",
       "      <td>19.801800</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>-0.507999</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>2.007999</td>\n",
       "      <td>2.050</td>\n",
       "      <td>5.096199</td>\n",
       "      <td>1.284498</td>\n",
       "      <td>0.454599</td>\n",
       "      <td>0.679901</td>\n",
       "      <td>0.612902</td>\n",
       "      <td>3.861599</td>\n",
       "      <td>0.829899</td>\n",
       "      <td>1.442801</td>\n",
       "      <td>2.351799</td>\n",
       "      <td>1.631800</td>\n",
       "      <td>1.123800</td>\n",
       "      <td>1.698799</td>\n",
       "      <td>1.190800</td>\n",
       "      <td>0.066999</td>\n",
       "      <td>0</td>\n",
       "      <td>0.499981</td>\n",
       "      <td>0.349309</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.508161</td>\n",
       "      <td>0.459659</td>\n",
       "      <td>1</td>\n",
       "      <td>0.7047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58134</th>\n",
       "      <td>1.639</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.097000</td>\n",
       "      <td>18.287001</td>\n",
       "      <td>21.199100</td>\n",
       "      <td>21.056299</td>\n",
       "      <td>20.669300</td>\n",
       "      <td>20.693300</td>\n",
       "      <td>20.603001</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>0.809999</td>\n",
       "      <td>2.427000</td>\n",
       "      <td>1.617001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>4.529100</td>\n",
       "      <td>0.529800</td>\n",
       "      <td>0.142801</td>\n",
       "      <td>0.066299</td>\n",
       "      <td>-0.024000</td>\n",
       "      <td>3.606298</td>\n",
       "      <td>0.386999</td>\n",
       "      <td>0.362999</td>\n",
       "      <td>3.153000</td>\n",
       "      <td>1.506001</td>\n",
       "      <td>2.316000</td>\n",
       "      <td>1.596300</td>\n",
       "      <td>2.406300</td>\n",
       "      <td>0.090300</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500120</td>\n",
       "      <td>0.988100</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.453618</td>\n",
       "      <td>0.421117</td>\n",
       "      <td>1</td>\n",
       "      <td>1.6791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62617</th>\n",
       "      <td>3.193</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.868000</td>\n",
       "      <td>19.752001</td>\n",
       "      <td>21.003599</td>\n",
       "      <td>20.839701</td>\n",
       "      <td>20.900000</td>\n",
       "      <td>20.761200</td>\n",
       "      <td>20.644501</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>0.115999</td>\n",
       "      <td>3.198000</td>\n",
       "      <td>3.082001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>4.333599</td>\n",
       "      <td>0.103600</td>\n",
       "      <td>0.163898</td>\n",
       "      <td>0.255499</td>\n",
       "      <td>0.138800</td>\n",
       "      <td>3.389700</td>\n",
       "      <td>-0.060299</td>\n",
       "      <td>0.078501</td>\n",
       "      <td>3.194500</td>\n",
       "      <td>0.776501</td>\n",
       "      <td>0.892500</td>\n",
       "      <td>0.893200</td>\n",
       "      <td>1.009199</td>\n",
       "      <td>0.116699</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500119</td>\n",
       "      <td>0.987949</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.306487</td>\n",
       "      <td>0.309471</td>\n",
       "      <td>1</td>\n",
       "      <td>2.8276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63673</th>\n",
       "      <td>0.906</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.917000</td>\n",
       "      <td>18.932001</td>\n",
       "      <td>22.117001</td>\n",
       "      <td>22.021601</td>\n",
       "      <td>21.557800</td>\n",
       "      <td>20.837400</td>\n",
       "      <td>20.292400</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>14.620</td>\n",
       "      <td>17.450001</td>\n",
       "      <td>17.240</td>\n",
       "      <td>16.590</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.210001</td>\n",
       "      <td>-0.080000</td>\n",
       "      <td>1.970000</td>\n",
       "      <td>-0.015001</td>\n",
       "      <td>2.247000</td>\n",
       "      <td>2.262001</td>\n",
       "      <td>2.050</td>\n",
       "      <td>5.447001</td>\n",
       "      <td>0.559200</td>\n",
       "      <td>0.095400</td>\n",
       "      <td>1.265400</td>\n",
       "      <td>0.720400</td>\n",
       "      <td>4.571600</td>\n",
       "      <td>0.463800</td>\n",
       "      <td>1.184200</td>\n",
       "      <td>2.842400</td>\n",
       "      <td>1.375401</td>\n",
       "      <td>1.360399</td>\n",
       "      <td>1.920401</td>\n",
       "      <td>1.905399</td>\n",
       "      <td>0.545000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500105</td>\n",
       "      <td>0.979736</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.606100</td>\n",
       "      <td>0.526861</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69989</th>\n",
       "      <td>0.150</td>\n",
       "      <td>12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.379999</td>\n",
       "      <td>16.229000</td>\n",
       "      <td>18.221100</td>\n",
       "      <td>18.099800</td>\n",
       "      <td>17.578800</td>\n",
       "      <td>17.939301</td>\n",
       "      <td>17.516899</td>\n",
       "      <td>14.873000</td>\n",
       "      <td>13.917</td>\n",
       "      <td>17.027000</td>\n",
       "      <td>16.590</td>\n",
       "      <td>16.445</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.145000</td>\n",
       "      <td>1.717</td>\n",
       "      <td>0.437000</td>\n",
       "      <td>1.572000</td>\n",
       "      <td>2.528000</td>\n",
       "      <td>0.150999</td>\n",
       "      <td>1.506999</td>\n",
       "      <td>1.356000</td>\n",
       "      <td>0.956</td>\n",
       "      <td>3.348100</td>\n",
       "      <td>0.642300</td>\n",
       "      <td>0.121300</td>\n",
       "      <td>0.061901</td>\n",
       "      <td>-0.360500</td>\n",
       "      <td>1.072800</td>\n",
       "      <td>0.521000</td>\n",
       "      <td>0.160500</td>\n",
       "      <td>0.489899</td>\n",
       "      <td>1.136900</td>\n",
       "      <td>1.287899</td>\n",
       "      <td>1.559301</td>\n",
       "      <td>1.710300</td>\n",
       "      <td>0.422401</td>\n",
       "      <td>1</td>\n",
       "      <td>0.500117</td>\n",
       "      <td>0.986725</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.484179</td>\n",
       "      <td>0.442849</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1523</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Z  band_num  class   W1mproPM   W2mproPM       gmag       rmag  \\\n",
       "index                                                                       \n",
       "16456  0.846         8    1.0  18.337000  18.363001  23.299999  21.890100   \n",
       "27470  0.736         9    1.0  17.676001  17.408001  20.004499  19.639299   \n",
       "40804  1.078         9    1.0  17.311001  16.582001  18.525999  18.507601   \n",
       "44825  0.877         9    1.0  18.306000  18.175001  19.906601  19.838699   \n",
       "45506  0.643        12    1.0  15.579000  15.040000  17.124001  17.214199   \n",
       "53799  0.712         9    1.0  18.170000  18.677999  21.766199  21.311600   \n",
       "58134  1.639         9    1.0  19.097000  18.287001  21.199100  21.056299   \n",
       "62617  3.193         9    1.0  19.868000  19.752001  21.003599  20.839701   \n",
       "63673  0.906         9    1.0  18.917000  18.932001  22.117001  22.021601   \n",
       "69989  0.150        12    1.0  16.379999  16.229000  18.221100  18.099800   \n",
       "\n",
       "            imag       zmag       ymag      W3mag   W4mag       Jmag    Hmag  \\\n",
       "index                                                                          \n",
       "16456  21.287201  20.959600  20.519300  16.670000  14.620  17.450001  17.240   \n",
       "27470  19.607401  19.268499  19.199200  16.670000  14.620  17.450001  17.240   \n",
       "40804  18.457300  18.362700  18.208500  15.373000  14.180  17.450001  17.240   \n",
       "44825  19.618500  19.524599  19.334000  16.670000  14.620  17.450001  17.240   \n",
       "45506  17.213301  17.390400  17.359301  14.250999  13.386  17.230000  17.105   \n",
       "53799  20.481701  19.868799  19.801800  16.670000  14.620  17.450001  17.240   \n",
       "58134  20.669300  20.693300  20.603001  16.670000  14.620  17.450001  17.240   \n",
       "62617  20.900000  20.761200  20.644501  16.670000  14.620  17.450001  17.240   \n",
       "63673  21.557800  20.837400  20.292400  16.670000  14.620  17.450001  17.240   \n",
       "69989  17.578800  17.939301  17.516899  14.873000  13.917  17.027000  16.590   \n",
       "\n",
       "         Kmag  LOFAR_detect  radio_AGN       H_K   H_W3       J_H      K_W3  \\\n",
       "index                                                                         \n",
       "16456  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "27470  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "40804  16.590             1          1  0.650000  1.867  0.210001  1.217000   \n",
       "44825  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "45506  16.527             1          1  0.577999  2.854  0.125000  2.276001   \n",
       "53799  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "58134  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "62617  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "63673  16.590             1          1  0.650000  0.570  0.210001 -0.080000   \n",
       "69989  16.445             1          1  0.145000  1.717  0.437000  1.572000   \n",
       "\n",
       "           K_W4     W1_W2     W1_W3     W2_W3  W3_W4      g_W3       g_i  \\\n",
       "index                                                                      \n",
       "16456  1.970000 -0.026001  1.667000  1.693001  2.050  6.629999  2.012798   \n",
       "27470  1.970000  0.268000  1.006001  0.738001  2.050  3.334499  0.397099   \n",
       "40804  2.410000  0.729000  1.938001  1.209001  1.193  3.152999  0.068699   \n",
       "44825  1.970000  0.130999  1.636000  1.505001  2.050  3.236601  0.288101   \n",
       "45506  3.141001  0.539001  1.328001  0.789001  0.865  2.873001 -0.089300   \n",
       "53799  1.970000 -0.507999  1.500000  2.007999  2.050  5.096199  1.284498   \n",
       "58134  1.970000  0.809999  2.427000  1.617001  2.050  4.529100  0.529800   \n",
       "62617  1.970000  0.115999  3.198000  3.082001  2.050  4.333599  0.103600   \n",
       "63673  1.970000 -0.015001  2.247000  2.262001  2.050  5.447001  0.559200   \n",
       "69989  2.528000  0.150999  1.506999  1.356000  0.956  3.348100  0.642300   \n",
       "\n",
       "            g_r       i_y       i_z       r_J       r_i       r_z       y_J  \\\n",
       "index                                                                         \n",
       "16456  1.409899  0.767900  0.327600  4.440100  0.602900  0.930500  3.069300   \n",
       "27470  0.365200  0.408201  0.338902  2.189299  0.031898  0.370800  1.749199   \n",
       "40804  0.018398  0.248800  0.094601  1.057600  0.050301  0.144901  0.758499   \n",
       "44825  0.067902  0.284500  0.093901  2.388699  0.220200  0.314100  1.883999   \n",
       "45506 -0.090199 -0.146000 -0.177099 -0.015800  0.000898 -0.176201  0.129301   \n",
       "53799  0.454599  0.679901  0.612902  3.861599  0.829899  1.442801  2.351799   \n",
       "58134  0.142801  0.066299 -0.024000  3.606298  0.386999  0.362999  3.153000   \n",
       "62617  0.163898  0.255499  0.138800  3.389700 -0.060299  0.078501  3.194500   \n",
       "63673  0.095400  1.265400  0.720400  4.571600  0.463800  1.184200  2.842400   \n",
       "69989  0.121300  0.061901 -0.360500  1.072800  0.521000  0.160500  0.489899   \n",
       "\n",
       "           y_W1      y_W2      z_W1      z_W2       z_y  pred_class  \\\n",
       "index                                                                 \n",
       "16456  2.182301  2.156300  2.622601  2.596600  0.440300           1   \n",
       "27470  1.523199  1.791199  1.592499  1.860498  0.069300           1   \n",
       "40804  0.897499  1.626499  1.051699  1.780699  0.154200           1   \n",
       "44825  1.028000  1.158998  1.218599  1.349598  0.190599           1   \n",
       "45506  1.780300  2.319301  1.811399  2.350400  0.031099           1   \n",
       "53799  1.631800  1.123800  1.698799  1.190800  0.066999           0   \n",
       "58134  1.506001  2.316000  1.596300  2.406300  0.090300           1   \n",
       "62617  0.776501  0.892500  0.893200  1.009199  0.116699           1   \n",
       "63673  1.375401  1.360399  1.920401  1.905399  0.545000           1   \n",
       "69989  1.136900  1.287899  1.559301  1.710300  0.422401           1   \n",
       "\n",
       "       Score_AGN  Prob_AGN  pred_class_cal  pred_radio_AGN  Score_radio_AGN  \\\n",
       "index                                                                         \n",
       "16456   0.500016  0.657091               1               0         0.438512   \n",
       "27470   0.500121  0.988672               1               0         0.310507   \n",
       "40804   0.500122  0.989167               1               0         0.400925   \n",
       "44825   0.500121  0.988471               1               0         0.501761   \n",
       "45506   0.500122  0.988938               1               0         0.875211   \n",
       "53799   0.499981  0.349309               1               0         0.508161   \n",
       "58134   0.500120  0.988100               1               0         0.453618   \n",
       "62617   0.500119  0.987949               1               0         0.306487   \n",
       "63673   0.500105  0.979736               1               0         0.606100   \n",
       "69989   0.500117  0.986725               1               0         0.484179   \n",
       "\n",
       "       Prob_radio_AGN  pred_radio_cal_AGN  pred_Z_rAGN  \n",
       "index                                                   \n",
       "16456        0.410227                   1       0.7586  \n",
       "27470        0.312719                   1       0.7748  \n",
       "40804        0.382640                   1       1.0918  \n",
       "44825        0.455192                   1       0.9805  \n",
       "45506        0.726470                   1       0.6672  \n",
       "53799        0.459659                   1       0.7047  \n",
       "58134        0.421117                   1       1.6791  \n",
       "62617        0.309471                   1       2.8276  \n",
       "63673        0.526861                   1       0.9320  \n",
       "69989        0.442849                   1       0.1523  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(data_df.loc[filter_data][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}