notebooks/1 - pre-processing/a - Solar Data Pre-Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f51791ac-61ff-4bd3-9101-a4c9f36cd069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing dependencies\n",
    "\n",
    "import pandas as pd\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f116ec5-25f0-4dc2-86bd-cbf7110b9a15",
   "metadata": {},
   "source": [
    "# Data Collection\n",
    "\n",
    "<img src = 'https://raw.githubusercontent.com/nickmccarty/solar-energy-predictor/main/images/tesla-app-screenshot.jpg' width = 30%>\n",
    "\n",
    "Thanks, Tesla, for making me download a year's worth of solar data one week at a time using your mobile app -- said no one ever...nevertheless, we persist and read them all into a single data structure for processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a097c192-b1cd-4de4-8932-33e397b544f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Looking in the data folder for the raw solar data CSVs,\n",
    "# then concatenating them into a dataframe\n",
    "\n",
    "df = pd.concat(map(pd.read_csv, glob.glob(\"../../data/raw/solar/*\")))\n",
    "\n",
    "# Casting `Date time` column as `datetime` object\n",
    "\n",
    "df['Date time'] = pd.to_datetime(df['Date time'])\n",
    "\n",
    "# Instantiating sorted df\n",
    "\n",
    "df = df.sort_values(by='Date time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "619249cc-3139-46e2-b2d4-48595a953a36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date time</th>\n",
       "      <th>Solar Energy (kWh)</th>\n",
       "      <th>Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Energy (kWh)</th>\n",
       "      <th>Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Lifetime Cumulative Energy (kWh)</th>\n",
       "      <th>Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Energy (kWh)</th>\n",
       "      <th>Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Lifetime Cumulative Energy (kWh)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-11-01 01:00:00-08:00</td>\n",
       "      <td>21.4</td>\n",
       "      <td>10.3</td>\n",
       "      <td>40358.6</td>\n",
       "      <td>11.1</td>\n",
       "      <td>38080.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-11-02 01:00:00-08:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40358.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38080.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-11-03 01:00:00-08:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40358.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38080.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-11-04 01:00:00-08:00</td>\n",
       "      <td>27.0</td>\n",
       "      <td>12.9</td>\n",
       "      <td>40371.5</td>\n",
       "      <td>14.1</td>\n",
       "      <td>38094.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-11-05 01:00:00-08:00</td>\n",
       "      <td>26.7</td>\n",
       "      <td>12.7</td>\n",
       "      <td>40384.2</td>\n",
       "      <td>14.0</td>\n",
       "      <td>38108.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Date time  Solar Energy (kWh)  \\\n",
       "0 2021-11-01 01:00:00-08:00                21.4   \n",
       "1 2021-11-02 01:00:00-08:00                 0.0   \n",
       "2 2021-11-03 01:00:00-08:00                 0.0   \n",
       "3 2021-11-04 01:00:00-08:00                27.0   \n",
       "4 2021-11-05 01:00:00-08:00                26.7   \n",
       "\n",
       "   Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Energy (kWh)  \\\n",
       "0                                               10.3            \n",
       "1                                                0.0            \n",
       "2                                                0.0            \n",
       "3                                               12.9            \n",
       "4                                               12.7            \n",
       "\n",
       "   Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Lifetime Cumulative Energy (kWh)  \\\n",
       "0                                            40358.6                                \n",
       "1                                            40358.6                                \n",
       "2                                            40358.6                                \n",
       "3                                            40371.5                                \n",
       "4                                            40384.2                                \n",
       "\n",
       "   Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Energy (kWh)  \\\n",
       "0                                               11.1            \n",
       "1                                                0.0            \n",
       "2                                                0.0            \n",
       "3                                               14.1            \n",
       "4                                               14.0            \n",
       "\n",
       "   Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Lifetime Cumulative Energy (kWh)  \n",
       "0                                            38080.7                               \n",
       "1                                            38080.7                               \n",
       "2                                            38080.7                               \n",
       "3                                            38094.8                               \n",
       "4                                            38108.8                               "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Taking a peek\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "00504377-c89c-4032-8ab2-b336f7e0a52f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date time</th>\n",
       "      <th>Solar Energy (kWh)</th>\n",
       "      <th>Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Energy (kWh)</th>\n",
       "      <th>Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Lifetime Cumulative Energy (kWh)</th>\n",
       "      <th>Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Energy (kWh)</th>\n",
       "      <th>Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Lifetime Cumulative Energy (kWh)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-11-02 01:00:00-08:00</td>\n",
       "      <td>11.1</td>\n",
       "      <td>11.1</td>\n",
       "      <td>46646.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44068.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-11-03 01:00:00-08:00</td>\n",
       "      <td>15.5</td>\n",
       "      <td>15.5</td>\n",
       "      <td>46661.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44068.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-11-04 01:00:00-08:00</td>\n",
       "      <td>14.9</td>\n",
       "      <td>14.9</td>\n",
       "      <td>46676.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44068.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2022-11-05 01:00:00-08:00</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>46677.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44068.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2022-11-06 01:00:00-08:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>46677.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44068.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Date time  Solar Energy (kWh)  \\\n",
       "2 2022-11-02 01:00:00-08:00                11.1   \n",
       "3 2022-11-03 01:00:00-08:00                15.5   \n",
       "4 2022-11-04 01:00:00-08:00                14.9   \n",
       "5 2022-11-05 01:00:00-08:00                 1.5   \n",
       "6 2022-11-06 01:00:00-08:00                 0.0   \n",
       "\n",
       "   Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Energy (kWh)  \\\n",
       "2                                               11.1            \n",
       "3                                               15.5            \n",
       "4                                               14.9            \n",
       "5                                                1.5            \n",
       "6                                                0.0            \n",
       "\n",
       "   Inverter 5441d581-c9d9-4e00-a7c9-f8a9b45ec456 Lifetime Cumulative Energy (kWh)  \\\n",
       "2                                            46646.0                                \n",
       "3                                            46661.5                                \n",
       "4                                            46676.4                                \n",
       "5                                            46677.9                                \n",
       "6                                            46677.9                                \n",
       "\n",
       "   Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Energy (kWh)  \\\n",
       "2                                                0.0            \n",
       "3                                                0.0            \n",
       "4                                                0.0            \n",
       "5                                                0.0            \n",
       "6                                                0.0            \n",
       "\n",
       "   Inverter ea39caf1-d86f-4368-a7b9-a90a3502b4be Lifetime Cumulative Energy (kWh)  \n",
       "2                                            44068.8                               \n",
       "3                                            44068.8                               \n",
       "4                                            44068.8                               \n",
       "5                                            44068.8                               \n",
       "6                                            44068.8                               "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Last day of fully-generated solar energy was 11-4-2022\n",
    "\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fdb61917-72a4-4749-86ab-941f56173ead",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Exporting processed CSV\n",
    "\n",
    "df.to_csv('../../data/interim/combined-solar-data.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}