src/notebooks/preprocessing/test_data_noaa_pre.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 317,
   "id": "361b207f-cce6-44b9-b132-008096c290bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic setting for Jupyter_notebook to import utils\n",
    "import os\n",
    "import sys\n",
    "\n",
    "notebook_path = os.path.abspath(\"\")\n",
    "project_root = os.path.abspath(os.path.join(notebook_path, \"../../\"))\n",
    "\n",
    "sys.path.append(project_root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\"\"\"OLD VERSION\"\"\""
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b62a181f01fa7dfb"
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "5df02c9f-0444-467c-9cbd-9839f258e9d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import xarray as xr\n",
    "import gzip\n",
    "import pandas as pd\n",
    "import shutil\n",
    "from utils import folder_utils\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e676bf20-9bc5-4551-9556-e0dd1880e0af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example usage\n",
    "\n",
    "country = \"GB\"\n",
    "data_folder = \"data\"\n",
    "data_test_category = \"test_data\"\n",
    "data_read_category = \"raw_data\"\n",
    "data_save_category = \"processed_data\"\n",
    "# output_folder = \"ASOS_DATA\"\n",
    "# output_folder2 = \"ERA5_DATA\"\n",
    "output_folder3 = \"NOAA_DATA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "628134bb-fe42-4049-949f-5f6777e32f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = folder_utils.find_folder(country, data_folder, data_read_category, output_folder3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b2bcbf8-71d8-4b30-8651-520a31f7d823",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/raw_data/NOAA_DATA/GB_NOAA_DATA'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bfab054d-7123-4c4a-b30b-cbabf95f669c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/raw_data/NOAA_DATA/GB_NOAA_DATA/3419834.csv'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csv_name = \"3419834.csv\"\n",
    "raw_csv_path =  os.path.join(folder, csv_name)\n",
    "raw_csv_path "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2dfebb2c-d253-4f1b-b574-17557fd52428",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3381870930.py:1: DtypeWarning: Columns (6,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,63) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  raw_df = pd.read_csv(raw_csv_path)\n"
     ]
    }
   ],
   "source": [
    "raw_df = pd.read_csv(raw_csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8b999478-3724-4f28-9038-f2c37c9f5e52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>SOURCE</th>\n",
       "      <th>REPORT_TYPE</th>\n",
       "      <th>CALL_SIGN</th>\n",
       "      <th>QUALITY_CONTROL</th>\n",
       "      <th>...</th>\n",
       "      <th>OD1</th>\n",
       "      <th>OD2</th>\n",
       "      <th>QUALITY_CONTROL.1</th>\n",
       "      <th>REM</th>\n",
       "      <th>REPORT_TYPE.1</th>\n",
       "      <th>SLP</th>\n",
       "      <th>SOURCE.1</th>\n",
       "      <th>TMP</th>\n",
       "      <th>VIS</th>\n",
       "      <th>WND</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08803862 17257 /2010 10125 20119 30192 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10204,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0125,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>200,1,N,0051,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T01:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>4,99,0113,1,999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08803862 47257 /2012 10126 20119 30189 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10201,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0126,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>200,1,N,0062,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T02:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN07603862 47257 /1607 10124 20117 30188 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10200,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0124,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>160,1,N,0036,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T03:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN07603862 47259 /1809 10125 20118 30183 4019...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10195,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0125,1</td>\n",
       "      <td>009000,1,9,9</td>\n",
       "      <td>180,1,N,0046,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T04:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08203862 47259 /1913 10123 20117 30176 4018...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10188,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0123,1</td>\n",
       "      <td>009000,1,9,9</td>\n",
       "      <td>190,1,N,0067,1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      STATION             NAME  LATITUDE  LONGITUDE  ELEVATION  \\\n",
       "0  3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "1  3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "2  3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "3  3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "4  3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "\n",
       "                  DATE SOURCE REPORT_TYPE  CALL_SIGN QUALITY_CONTROL  ...  \\\n",
       "0  2022-01-01T00:00:00      4       FM-12      99999            V020  ...   \n",
       "1  2022-01-01T01:00:00      4       FM-12      99999            V020  ...   \n",
       "2  2022-01-01T02:00:00      4       FM-12      99999            V020  ...   \n",
       "3  2022-01-01T03:00:00      4       FM-12      99999            V020  ...   \n",
       "4  2022-01-01T04:00:00      4       FM-12      99999            V020  ...   \n",
       "\n",
       "               OD1  OD2 QUALITY_CONTROL.1  \\\n",
       "0              NaN  NaN              V020   \n",
       "1  4,99,0113,1,999  NaN              V020   \n",
       "2              NaN  NaN              V020   \n",
       "3              NaN  NaN              V020   \n",
       "4              NaN  NaN              V020   \n",
       "\n",
       "                                                 REM REPORT_TYPE.1      SLP  \\\n",
       "0  SYN08803862 17257 /2010 10125 20119 30192 4020...         FM-12  10204,1   \n",
       "1  SYN08803862 47257 /2012 10126 20119 30189 4020...         FM-12  10201,1   \n",
       "2  SYN07603862 47257 /1607 10124 20117 30188 4020...         FM-12  10200,1   \n",
       "3  SYN07603862 47259 /1809 10125 20118 30183 4019...         FM-12  10195,1   \n",
       "4  SYN08203862 47259 /1913 10123 20117 30176 4018...         FM-12  10188,1   \n",
       "\n",
       "  SOURCE.1      TMP           VIS             WND  \n",
       "0        4  +0125,1  007000,1,9,9  200,1,N,0051,1  \n",
       "1        4  +0126,1  007000,1,9,9  200,1,N,0062,1  \n",
       "2        4  +0124,1  007000,1,9,9  160,1,N,0036,1  \n",
       "3        4  +0125,1  009000,1,9,9  180,1,N,0046,1  \n",
       "4        4  +0123,1  009000,1,9,9  190,1,N,0067,1  \n",
       "\n",
       "[5 rows x 67 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5a63a5b3-5101-48bb-8134-5ed6fd63dbfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>SOURCE</th>\n",
       "      <th>REPORT_TYPE</th>\n",
       "      <th>CALL_SIGN</th>\n",
       "      <th>QUALITY_CONTROL</th>\n",
       "      <th>...</th>\n",
       "      <th>OD1</th>\n",
       "      <th>OD2</th>\n",
       "      <th>QUALITY_CONTROL.1</th>\n",
       "      <th>REM</th>\n",
       "      <th>REPORT_TYPE.1</th>\n",
       "      <th>SLP</th>\n",
       "      <th>SOURCE.1</th>\n",
       "      <th>TMP</th>\n",
       "      <th>VIS</th>\n",
       "      <th>WND</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08803862 17257 /2010 10125 20119 30192 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10204,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0125,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>200,1,N,0051,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T01:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>4,99,0113,1,999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08803862 47257 /2012 10126 20119 30189 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10201,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0126,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>200,1,N,0062,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T02:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN07603862 47257 /1607 10124 20117 30188 4020...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10200,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0124,1</td>\n",
       "      <td>007000,1,9,9</td>\n",
       "      <td>160,1,N,0036,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T03:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN07603862 47259 /1809 10125 20118 30183 4019...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10195,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0125,1</td>\n",
       "      <td>009000,1,9,9</td>\n",
       "      <td>180,1,N,0046,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH, UK</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>2022-01-01T04:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>SYN08203862 47259 /1913 10123 20117 30176 4018...</td>\n",
       "      <td>FM-12</td>\n",
       "      <td>10188,1</td>\n",
       "      <td>4</td>\n",
       "      <td>+0123,1</td>\n",
       "      <td>009000,1,9,9</td>\n",
       "      <td>190,1,N,0067,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857984</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>EDINBURGH, UK</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>2023-08-10T19:20:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>MET051METAR EGPH 101920Z 03002KT 9999 FEW034 1...</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999,9</td>\n",
       "      <td>4</td>\n",
       "      <td>+0190,1</td>\n",
       "      <td>009999,1,9,9</td>\n",
       "      <td>030,1,N,0010,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857985</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>EDINBURGH, UK</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>2023-08-10T19:50:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>MET053METAR EGPH 101950Z 01003KT 320V030 CAVOK...</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999,9</td>\n",
       "      <td>4</td>\n",
       "      <td>+0190,1</td>\n",
       "      <td>009900,5,9,9</td>\n",
       "      <td>010,1,V,0015,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857986</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>EDINBURGH, UK</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>2023-08-10T20:20:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>MET045METAR EGPH 102020Z 08004KT CAVOK 19/17 Q...</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999,9</td>\n",
       "      <td>4</td>\n",
       "      <td>+0190,1</td>\n",
       "      <td>009900,5,9,9</td>\n",
       "      <td>080,1,N,0021,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857987</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>EDINBURGH, UK</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>2023-08-10T20:50:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>MET053METAR EGPH 102050Z 07003KT 020V100 CAVOK...</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999,9</td>\n",
       "      <td>4</td>\n",
       "      <td>+0190,1</td>\n",
       "      <td>009900,5,9,9</td>\n",
       "      <td>070,1,V,0015,1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857988</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>EDINBURGH, UK</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>2023-08-10T21:20:00</td>\n",
       "      <td>4</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999</td>\n",
       "      <td>V020</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V020</td>\n",
       "      <td>MET051METAR EGPH 102120Z 04002KT 9999 SCT044 1...</td>\n",
       "      <td>FM-15</td>\n",
       "      <td>99999,9</td>\n",
       "      <td>4</td>\n",
       "      <td>+0180,1</td>\n",
       "      <td>009999,1,9,9</td>\n",
       "      <td>040,1,N,0010,1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2857989 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            STATION             NAME  LATITUDE  LONGITUDE  ELEVATION  \\\n",
       "0        3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "1        3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "2        3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "3        3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "4        3862099999  BOURNEMOUTH, UK     50.78    -1.8425      11.58   \n",
       "...             ...              ...       ...        ...        ...   \n",
       "2857984  3160099999    EDINBURGH, UK     55.95    -3.3725      41.14   \n",
       "2857985  3160099999    EDINBURGH, UK     55.95    -3.3725      41.14   \n",
       "2857986  3160099999    EDINBURGH, UK     55.95    -3.3725      41.14   \n",
       "2857987  3160099999    EDINBURGH, UK     55.95    -3.3725      41.14   \n",
       "2857988  3160099999    EDINBURGH, UK     55.95    -3.3725      41.14   \n",
       "\n",
       "                        DATE SOURCE REPORT_TYPE  CALL_SIGN QUALITY_CONTROL  \\\n",
       "0        2022-01-01T00:00:00      4       FM-12      99999            V020   \n",
       "1        2022-01-01T01:00:00      4       FM-12      99999            V020   \n",
       "2        2022-01-01T02:00:00      4       FM-12      99999            V020   \n",
       "3        2022-01-01T03:00:00      4       FM-12      99999            V020   \n",
       "4        2022-01-01T04:00:00      4       FM-12      99999            V020   \n",
       "...                      ...    ...         ...        ...             ...   \n",
       "2857984  2023-08-10T19:20:00      4       FM-15      99999            V020   \n",
       "2857985  2023-08-10T19:50:00      4       FM-15      99999            V020   \n",
       "2857986  2023-08-10T20:20:00      4       FM-15      99999            V020   \n",
       "2857987  2023-08-10T20:50:00      4       FM-15      99999            V020   \n",
       "2857988  2023-08-10T21:20:00      4       FM-15      99999            V020   \n",
       "\n",
       "         ...              OD1  OD2 QUALITY_CONTROL.1  \\\n",
       "0        ...              NaN  NaN              V020   \n",
       "1        ...  4,99,0113,1,999  NaN              V020   \n",
       "2        ...              NaN  NaN              V020   \n",
       "3        ...              NaN  NaN              V020   \n",
       "4        ...              NaN  NaN              V020   \n",
       "...      ...              ...  ...               ...   \n",
       "2857984  ...              NaN  NaN              V020   \n",
       "2857985  ...              NaN  NaN              V020   \n",
       "2857986  ...              NaN  NaN              V020   \n",
       "2857987  ...              NaN  NaN              V020   \n",
       "2857988  ...              NaN  NaN              V020   \n",
       "\n",
       "                                                       REM REPORT_TYPE.1  \\\n",
       "0        SYN08803862 17257 /2010 10125 20119 30192 4020...         FM-12   \n",
       "1        SYN08803862 47257 /2012 10126 20119 30189 4020...         FM-12   \n",
       "2        SYN07603862 47257 /1607 10124 20117 30188 4020...         FM-12   \n",
       "3        SYN07603862 47259 /1809 10125 20118 30183 4019...         FM-12   \n",
       "4        SYN08203862 47259 /1913 10123 20117 30176 4018...         FM-12   \n",
       "...                                                    ...           ...   \n",
       "2857984  MET051METAR EGPH 101920Z 03002KT 9999 FEW034 1...         FM-15   \n",
       "2857985  MET053METAR EGPH 101950Z 01003KT 320V030 CAVOK...         FM-15   \n",
       "2857986  MET045METAR EGPH 102020Z 08004KT CAVOK 19/17 Q...         FM-15   \n",
       "2857987  MET053METAR EGPH 102050Z 07003KT 020V100 CAVOK...         FM-15   \n",
       "2857988  MET051METAR EGPH 102120Z 04002KT 9999 SCT044 1...         FM-15   \n",
       "\n",
       "             SLP SOURCE.1      TMP           VIS             WND  \n",
       "0        10204,1        4  +0125,1  007000,1,9,9  200,1,N,0051,1  \n",
       "1        10201,1        4  +0126,1  007000,1,9,9  200,1,N,0062,1  \n",
       "2        10200,1        4  +0124,1  007000,1,9,9  160,1,N,0036,1  \n",
       "3        10195,1        4  +0125,1  009000,1,9,9  180,1,N,0046,1  \n",
       "4        10188,1        4  +0123,1  009000,1,9,9  190,1,N,0067,1  \n",
       "...          ...      ...      ...           ...             ...  \n",
       "2857984  99999,9        4  +0190,1  009999,1,9,9  030,1,N,0010,1  \n",
       "2857985  99999,9        4  +0190,1  009900,5,9,9  010,1,V,0015,1  \n",
       "2857986  99999,9        4  +0190,1  009900,5,9,9  080,1,N,0021,1  \n",
       "2857987  99999,9        4  +0190,1  009900,5,9,9  070,1,V,0015,1  \n",
       "2857988  99999,9        4  +0180,1  009999,1,9,9  040,1,N,0010,1  \n",
       "\n",
       "[2857989 rows x 67 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "15536321-5720-4a47-9f1d-23a7e80a0fc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/362160291.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  selected_df[\"NAME\"] = selected_df[\"NAME\"].str.replace(\", UK\", \"\").str.replace('\"', \"\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            STATION                   NAME   LATITUDE  LONGITUDE  ELEVATION\n",
      "0        3862099999            BOURNEMOUTH  50.780000  -1.842500      11.58\n",
      "32850    3882099999  HERSTMONCEUX WEST END  50.900000   0.316667      52.00\n",
      "46628    3158099999            CHARTERHALL  55.700000  -2.383333     111.00\n",
      "58169    3384099999         LECONFIELD AUT  53.866667  -0.433333       6.00\n",
      "80254    3148099999              GLEN OGLE  56.416667  -4.316667     564.00\n",
      "...             ...                    ...        ...        ...        ...\n",
      "2761498  3323399999              LIVERPOOL  53.333611  -2.849722      24.38\n",
      "2788938  3225099999                   SHAP  54.500000  -2.683333     255.00\n",
      "2802735  3230099999         REDESDALE CAMP  55.283333  -2.283333     212.00\n",
      "2816624  3155099999              DRUMALBIN  55.616667  -3.733333     245.00\n",
      "2830522  3160099999              EDINBURGH  55.950000  -3.372500      41.14\n",
      "\n",
      "[150 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "def bulid_noaa_station_network(raw_df):\n",
    "    # build station_network\n",
    "    # extract the specific columns\n",
    "    selected_columns = ['STATION', 'NAME', 'LATITUDE', 'LONGITUDE', 'ELEVATION']\n",
    "    selected_df = raw_df[selected_columns]\n",
    "    \n",
    "    # Remove \"UK\"\n",
    "    selected_df[\"NAME\"] = selected_df[\"NAME\"].str.replace(\", UK\", \"\").str.replace('\"', \"\")\n",
    "    \n",
    "    # 保留两位小数\n",
    "    unique_df[\"LATITUDE\"] = unique_df[\"LATITUDE\"].round(2)\n",
    "    unique_df[\"LONGITUDE\"] = unique_df[\"LONGITUDE\"].round(2)\n",
    "    unique_df[\"ELEVATION\"] = unique_df[\"ELEVATION\"].round(2)\n",
    "    \n",
    "    \n",
    "    # 去除重复的行\n",
    "    unique_df = selected_df.drop_duplicates()\n",
    "    \n",
    "    # 存储 station_network\n",
    "    \n",
    "    \n",
    "    # 打印结果\n",
    "    print(unique_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "4c215283-5862-4cb7-9182-0ab89a6b9429",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/processed_data/NOAA_DATA/GB_NOAA_DATA' created successfully.\n"
     ]
    }
   ],
   "source": [
    "folder_utils.create_folder(country, data_folder, data_save_category, output_folder3)\n",
    "save_folder = folder_utils.find_folder(country, data_folder, data_save_category, output_folder3)\n",
    "save_csv=\"noaa_station_network.csv\"\n",
    "save_path = os.path.join(save_folder, save_csv)\n",
    "unique_df.to_csv(save_path, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "05f14b9b-a885-4f1a-be21-1665f3b11dc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['STATION', 'NAME', 'LATITUDE', 'LONGITUDE', 'ELEVATION', 'DATE', 'SOURCE', 'REPORT_TYPE', 'CALL_SIGN', 'QUALITY_CONTROL', 'AA1', 'AA2', 'AA3', 'AJ1', 'AT1', 'AT2', 'AT3', 'AT4', 'AT5', 'AT6', 'AW1', 'AW2', 'AW3', 'AY1', 'AY2', 'AZ1', 'AZ2', 'CALL_SIGN.1', 'CIG', 'DEW', 'ED1', 'EQD', 'GA1', 'GA2', 'GA3', 'GA4', 'GA5', 'GA6', 'GD1', 'GD2', 'GD3', 'GD4', 'GE1', 'GF1', 'IA1', 'IA2', 'KA1', 'KA2', 'KA3', 'KA4', 'MA1', 'MD1', 'MW1', 'MW2', 'MW3', 'MW4', 'OC1', 'OD1', 'OD2', 'QUALITY_CONTROL.1', 'REM', 'REPORT_TYPE.1', 'SLP', 'SOURCE.1', 'TMP', 'VIS', 'WND']\n"
     ]
    }
   ],
   "source": [
    "# 提取列名\n",
    "column_names = raw_df.columns.tolist()\n",
    "print(column_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef2e7319-e526-4237-a8bc-d6ea2dd1a6ca",
   "metadata": {},
   "source": [
    "# function test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "35b627f7-913d-417d-adf6-113327bf1991",
   "metadata": {},
   "outputs": [],
   "source": [
    "def restore_decimal_format(encoded_value):\n",
    "    # Convert data to decimal format\n",
    "    if encoded_value in [\"9999\", \"99999\"]:\n",
    "        return None  # Handle special cases if needed\n",
    "    return float(encoded_value) / 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fe3f1422-a304-45a9-82f7-bbc7afea42e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_temperature(temperature_str):\n",
    "    parts = temperature_str.split(\",\")\n",
    "    symbol = parts[0][0]\n",
    "\n",
    "    temperature = restore_decimal_format(int(parts[0][1:]))\n",
    "    if symbol == \"-\":\n",
    "        temperature = -temperature\n",
    "    temperature_kelvin = temperature + 273.15\n",
    "    return temperature_kelvin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "bc9ea090-875b-476c-9ed4-34bfe8349b37",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_wind(wind_str):\n",
    "    wind_parts = wind_str.split(\",\")\n",
    "    wind_directory = wind_parts[0]\n",
    "    wind_speed = restore_decimal_format(wind_parts[3])\n",
    "    return pd.Series({\"WIND_DIRECTORY\": wind_directory, \"WIND_SPEED\": wind_speed})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "fd5ac0a0-f86d-4908-adde-48f675b7ff7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_aa(aa_str):\n",
    "    if aa_str.strip() == \"\" or aa_str.strip().lower() == \"nan\":\n",
    "        return None, None\n",
    "        \n",
    "    aa_parts = aa_str.split(\",\")\n",
    "    hour = int(aa_parts[0])\n",
    "    value = restore_decimal_format(aa_parts[1])\n",
    "    return hour, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "f19b77fa-bb6b-414a-8767-084e7522749b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_data(df):\n",
    "    # 根据time创建新列\n",
    "    times = [1, 3, 6, 12, 24]\n",
    "    for t in times:\n",
    "        df[f\"{t}_hour_tp\"] = np.nan  # 默认值设置为NaN\n",
    "\n",
    "    for col in [\"AA1\", \"AA2\", \"AA3\"]:\n",
    "        # 检查值是否为nan\n",
    "        mask = ~df[col].isna()\n",
    "        filtered_data = df.loc[mask, col]\n",
    "        filtered_data = filtered_data.dropna()  # 去除None和NaN值\n",
    "\n",
    "        # 拆分数据\n",
    "        if not filtered_data.empty:\n",
    "            splits = filtered_data.str.split(',', expand=True)\n",
    "            time_col = splits[0].astype(int)\n",
    "            value_col = splits[1].apply(restore_decimal_format)  # 使用.apply()来应用函数\n",
    "\n",
    "            for t in times:\n",
    "                # 仅更新时间匹配的行\n",
    "                idx_to_update = time_col[time_col == t].index\n",
    "                df.loc[idx_to_update, f\"{t}_hour_tp\"] = value_col.loc[idx_to_update]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "84fe74a2-36a0-4696-9045-8ed5a9c2e6aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def noaa_data_preprocess(raw_df):\n",
    "    # Reserve specified columns\n",
    "    processed_df = raw_df[[\"STATION\", \"LATITUDE\", \"LONGITUDE\", \"ELEVATION\"]]\n",
    "\n",
    "    # Remove \"UK\"\n",
    "    processed_df[\"NAME\"] = raw_df[\"NAME\"].str.replace(\", UK\", \"\").str.replace('\"', \"\")\n",
    "\n",
    "    # Split \"DATE\" to unified format\n",
    "    date_time = pd.to_datetime(raw_df[\"DATE\"])\n",
    "    processed_df[\"DATE\"] = date_time.dt.strftime(\"%Y%m%d\")\n",
    "    processed_df[\"TIME\"] = date_time.dt.strftime(\"%H:%M\")\n",
    "\n",
    "\n",
    "    # Convert \"TMP\" and \"DEW\" from Celsius to Kelvin to match ERA5\n",
    "\n",
    "    processed_df[\"TMP\"] = raw_df[\"TMP\"].apply(process_temperature)\n",
    "    processed_df[\"DEW\"] = raw_df[\"DEW\"].apply(process_temperature)\n",
    "\n",
    "\n",
    "    # Split \"WIND\"\n",
    "    wind_df = raw_df[\"WND\"].apply(process_wind)\n",
    "    processed_df = pd.concat([processed_df, wind_df], axis=1)\n",
    "\n",
    "\n",
    "    # Process AA1, AA2, AA3 as total precipitation\n",
    "    # test_df[[\"AA1\", \"AA2\", \"AA3\"]] = test_df[[\"AA1\", \"AA2\", \"AA3\"]].astype(str)\n",
    "    tp_df = transform_data(raw_df)[[ \"1_hour_tp\", \"3_hour_tp\", \"6_hour_tp\",\"12_hour_tp\",\"24_hour_tp\",]].copy()\n",
    "    processed_df = pd.concat([processed_df, tp_df], axis=1)\n",
    "\n",
    "\n",
    "    # # filter the specific column\n",
    "    # columns_to_keep = [\n",
    "    #     \"STATION\",\n",
    "    #     \"NAME\",\n",
    "    #     \"LATITUDE\",\n",
    "    #     \"LONGITUDE\",\n",
    "    #     \"ELEVATION\",\n",
    "    #     \"DATE\",\n",
    "    #     \"TIME\",\n",
    "    #     \"1_hour_tp\",\n",
    "    #     \"3_hour_tp\",\n",
    "    #     \"6_hour_tp\",\n",
    "    #     \"12_hour_tp\",\n",
    "    #     \"24_hour_tp\",\n",
    "    #     \"DEW\",\n",
    "    #     \"TMP\",\n",
    "    #     \"WIND_DIRECTORY\",\n",
    "    #     \"WIND_SPEED\",\n",
    "    # ]\n",
    "\n",
    "\n",
    "    # unique_columns_to_keep = list(set(columns_to_keep))\n",
    "\n",
    "    # processed_df = processed_df.filter(items=unique_columns_to_keep)\n",
    "\n",
    "    return processed_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "3bb73c0b-7f55-4699-9c5b-b8fc943f9329",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_noaa_processed_data(\n",
    "    processed_df, country, data_folder, data_category, output_folder\n",
    "):\n",
    "    output_directory = folder_utils.create_folder(\n",
    "        country, data_folder, data_category, output_folder\n",
    "    )\n",
    "    output_filename = f\"{country}_NOAA_processed_data.csv\"\n",
    "    output_filepath = os.path.join(output_directory, output_filename)\n",
    "    processed_df.to_csv(output_filepath, index=False, encoding=\"utf-8\")\n",
    "    print(f\"{output_filename} done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "fdb3c9a0-f2aa-4c22-825b-18e6385944f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def noaa_data_preprocess(raw_df):\n",
    "    \n",
    "    # filter the time_range\n",
    "    raw_df = time_select_noaa(raw_df, start_date, end_date)\n",
    "    \n",
    "    # Reserve specified columns\n",
    "    processed_df = raw_df[[\"STATION\", \"LATITUDE\", \"LONGITUDE\", \"ELEVATION\"]]\n",
    "\n",
    "    # Remove \"UK\"\n",
    "    processed_df[\"NAME\"] = raw_df[\"NAME\"].str.replace(\", UK\", \"\").str.replace('\"', \"\")\n",
    "\n",
    "    # Split \"DATE\" to unified format\n",
    "    date_time = pd.to_datetime(raw_df[\"DATE\"])\n",
    "    processed_df[\"DATE\"] = date_time.dt.strftime(\"%Y%m%d\")\n",
    "    processed_df[\"TIME\"] = date_time.dt.strftime(\"%H:%M\")\n",
    "\n",
    "    # Convert \"TMP\" and \"DEW\" from Celsius to Kelvin to match ERA5\n",
    "\n",
    "    tqdm.pandas()\n",
    "    processed_df[\"TMP\"] = raw_df[\"TMP\"].progress_apply(process_temperature)\n",
    "    processed_df[\"DEW\"] = raw_df[\"DEW\"].progress_apply(process_temperature)\n",
    "\n",
    "    # Split \"WIND\"\n",
    "    wind_df = raw_df[\"WND\"].progress_apply(process_wind)\n",
    "    processed_df = pd.concat([processed_df, wind_df], axis=1)\n",
    "\n",
    "    # Process AA1, AA2, AA3 as total precipitation\n",
    "    # test_df[[\"AA1\", \"AA2\", \"AA3\"]] = test_df[[\"AA1\", \"AA2\", \"AA3\"]].astype(str)\n",
    "    tp_df = transform_data(raw_df)[\n",
    "        [\n",
    "            \"1_hour_tp\",\n",
    "            \"3_hour_tp\",\n",
    "            \"6_hour_tp\",\n",
    "            \"12_hour_tp\",\n",
    "            \"24_hour_tp\",\n",
    "        ]\n",
    "    ].copy()\n",
    "    processed_df = pd.concat([processed_df, tp_df], axis=1)\n",
    "\n",
    "    # # filter the specific column\n",
    "    # columns_to_keep = [\n",
    "    #     \"STATION\",\n",
    "    #     \"NAME\",\n",
    "    #     \"LATITUDE\",\n",
    "    #     \"LONGITUDE\",\n",
    "    #     \"ELEVATION\",\n",
    "    #     \"DATE\",\n",
    "    #     \"TIME\",\n",
    "    #     \"1_hour_tp\",\n",
    "    #     \"3_hour_tp\",\n",
    "    #     \"6_hour_tp\",\n",
    "    #     \"12_hour_tp\",\n",
    "    #     \"24_hour_tp\",\n",
    "    #     \"DEW\",\n",
    "    #     \"TMP\",\n",
    "    #     \"WIND_DIRECTORY\",\n",
    "    #     \"WIND_SPEED\",\n",
    "    # ]\n",
    "    #\n",
    "    # # Save specific columns\n",
    "    # processed_df = processed_df.filter(items=columns_to_keep)\n",
    "\n",
    "    return processed_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "e088dd87-239f-4a0c-8003-cd32fc5acc61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3907326148.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  processed_df[\"NAME\"] = raw_df[\"NAME\"].str.replace(\", UK\", \"\").str.replace('\"', \"\")\n",
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3907326148.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  processed_df[\"DATE\"] = date_time.dt.strftime(\"%Y%m%d\")\n",
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3907326148.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  processed_df[\"TIME\"] = date_time.dt.strftime(\"%H:%M\")\n",
      "100%|████████████████████████████████| 2857989/2857989 [00:01<00:00, 1479982.99it/s]\n",
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3907326148.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  processed_df[\"TMP\"] = raw_df[\"TMP\"].progress_apply(process_temperature)\n",
      "100%|████████████████████████████████| 2857989/2857989 [00:01<00:00, 1435099.56it/s]\n",
      "/var/folders/tt/fs_l1by51m30s907pzxq6gf40000gp/T/ipykernel_60817/3907326148.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  processed_df[\"DEW\"] = raw_df[\"DEW\"].progress_apply(process_temperature)\n",
      "100%|███████████████████████████████████| 2857989/2857989 [06:55<00:00, 6884.53it/s]\n"
     ]
    }
   ],
   "source": [
    "p_df = noaa_data_preprocess(raw_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "8883ca8d-77ff-41c7-9ba6-79d13635b37d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>DATE</th>\n",
       "      <th>TIME</th>\n",
       "      <th>TMP</th>\n",
       "      <th>DEW</th>\n",
       "      <th>WIND_DIRECTORY</th>\n",
       "      <th>WIND_SPEED</th>\n",
       "      <th>1_hour_tp</th>\n",
       "      <th>3_hour_tp</th>\n",
       "      <th>6_hour_tp</th>\n",
       "      <th>12_hour_tp</th>\n",
       "      <th>24_hour_tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>00:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>5.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>01:00</td>\n",
       "      <td>285.75</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>6.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>02:00</td>\n",
       "      <td>285.55</td>\n",
       "      <td>284.85</td>\n",
       "      <td>160</td>\n",
       "      <td>3.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>03:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>284.95</td>\n",
       "      <td>180</td>\n",
       "      <td>4.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>04:00</td>\n",
       "      <td>285.45</td>\n",
       "      <td>284.85</td>\n",
       "      <td>190</td>\n",
       "      <td>6.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>05:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>284.45</td>\n",
       "      <td>180</td>\n",
       "      <td>6.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>283.75</td>\n",
       "      <td>180</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:20</td>\n",
       "      <td>286.15</td>\n",
       "      <td>283.15</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:50</td>\n",
       "      <td>286.15</td>\n",
       "      <td>283.15</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>07:00</td>\n",
       "      <td>286.05</td>\n",
       "      <td>283.05</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      STATION  LATITUDE  LONGITUDE  ELEVATION         NAME      DATE   TIME  \\\n",
       "0  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  00:00   \n",
       "1  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  01:00   \n",
       "2  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  02:00   \n",
       "3  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  03:00   \n",
       "4  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  04:00   \n",
       "5  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  05:00   \n",
       "6  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  06:00   \n",
       "7  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  06:20   \n",
       "8  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  06:50   \n",
       "9  3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101  07:00   \n",
       "\n",
       "      TMP     DEW WIND_DIRECTORY  WIND_SPEED  1_hour_tp  3_hour_tp  6_hour_tp  \\\n",
       "0  285.65  285.05            200         5.1        NaN        NaN        0.0   \n",
       "1  285.75  285.05            200         6.2        NaN        NaN        NaN   \n",
       "2  285.55  284.85            160         3.6        NaN        NaN        NaN   \n",
       "3  285.65  284.95            180         4.6        NaN        NaN        NaN   \n",
       "4  285.45  284.85            190         6.7        NaN        NaN        NaN   \n",
       "5  285.65  284.45            180         6.7        NaN        NaN        NaN   \n",
       "6  285.65  283.75            180         5.7        NaN        NaN        NaN   \n",
       "7  286.15  283.15            190         5.7        NaN        NaN        NaN   \n",
       "8  286.15  283.15            190         5.7        NaN        NaN        NaN   \n",
       "9  286.05  283.05            190         5.7        NaN        NaN        NaN   \n",
       "\n",
       "   12_hour_tp  24_hour_tp  \n",
       "0         NaN         NaN  \n",
       "1         NaN         NaN  \n",
       "2         NaN         NaN  \n",
       "3         NaN         NaN  \n",
       "4         NaN         NaN  \n",
       "5         NaN         NaN  \n",
       "6         0.0         3.2  \n",
       "7         NaN         NaN  \n",
       "8         NaN         NaN  \n",
       "9         NaN         NaN  "
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "3ebe323b-c23a-447b-9cbe-71dda23b613d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example usage\n",
    "\n",
    "country = \"GB\"\n",
    "data_folder = \"data\"\n",
    "data_read_category = \"raw_data\"\n",
    "data_save_category = \"processed_data\"\n",
    "output_folder = \"NOAA_DATA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "1f6e6955-8987-4136-b9c4-f1a34f7d9ff9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/processed_data/NOAA_DATA/GB_NOAA_DATA' created successfully.\n",
      "GB_NOAA_processed_data.csv done!\n"
     ]
    }
   ],
   "source": [
    "save_noaa_processed_data(\n",
    "    p_df, country, data_folder, data_save_category, output_folder\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "a41f3807-19c1-408a-9163-d34cc7221836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>DATE</th>\n",
       "      <th>TIME</th>\n",
       "      <th>TMP</th>\n",
       "      <th>DEW</th>\n",
       "      <th>WIND_DIRECTORY</th>\n",
       "      <th>WIND_SPEED</th>\n",
       "      <th>1_hour_tp</th>\n",
       "      <th>3_hour_tp</th>\n",
       "      <th>6_hour_tp</th>\n",
       "      <th>12_hour_tp</th>\n",
       "      <th>24_hour_tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>00:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>5.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>01:00</td>\n",
       "      <td>285.75</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>6.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>02:00</td>\n",
       "      <td>285.55</td>\n",
       "      <td>284.85</td>\n",
       "      <td>160</td>\n",
       "      <td>3.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>03:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>284.95</td>\n",
       "      <td>180</td>\n",
       "      <td>4.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>20220101</td>\n",
       "      <td>04:00</td>\n",
       "      <td>285.45</td>\n",
       "      <td>284.85</td>\n",
       "      <td>190</td>\n",
       "      <td>6.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857984</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>EDINBURGH</td>\n",
       "      <td>20230810</td>\n",
       "      <td>19:20</td>\n",
       "      <td>292.15</td>\n",
       "      <td>290.15</td>\n",
       "      <td>030</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857985</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>EDINBURGH</td>\n",
       "      <td>20230810</td>\n",
       "      <td>19:50</td>\n",
       "      <td>292.15</td>\n",
       "      <td>290.15</td>\n",
       "      <td>010</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857986</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>EDINBURGH</td>\n",
       "      <td>20230810</td>\n",
       "      <td>20:20</td>\n",
       "      <td>292.15</td>\n",
       "      <td>290.15</td>\n",
       "      <td>080</td>\n",
       "      <td>2.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857987</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>EDINBURGH</td>\n",
       "      <td>20230810</td>\n",
       "      <td>20:50</td>\n",
       "      <td>292.15</td>\n",
       "      <td>289.15</td>\n",
       "      <td>070</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857988</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>55.95</td>\n",
       "      <td>-3.3725</td>\n",
       "      <td>41.14</td>\n",
       "      <td>EDINBURGH</td>\n",
       "      <td>20230810</td>\n",
       "      <td>21:20</td>\n",
       "      <td>291.15</td>\n",
       "      <td>289.15</td>\n",
       "      <td>040</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2857989 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            STATION  LATITUDE  LONGITUDE  ELEVATION         NAME      DATE  \\\n",
       "0        3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101   \n",
       "1        3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101   \n",
       "2        3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101   \n",
       "3        3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101   \n",
       "4        3862099999     50.78    -1.8425      11.58  BOURNEMOUTH  20220101   \n",
       "...             ...       ...        ...        ...          ...       ...   \n",
       "2857984  3160099999     55.95    -3.3725      41.14    EDINBURGH  20230810   \n",
       "2857985  3160099999     55.95    -3.3725      41.14    EDINBURGH  20230810   \n",
       "2857986  3160099999     55.95    -3.3725      41.14    EDINBURGH  20230810   \n",
       "2857987  3160099999     55.95    -3.3725      41.14    EDINBURGH  20230810   \n",
       "2857988  3160099999     55.95    -3.3725      41.14    EDINBURGH  20230810   \n",
       "\n",
       "          TIME     TMP     DEW WIND_DIRECTORY  WIND_SPEED  1_hour_tp  \\\n",
       "0        00:00  285.65  285.05            200         5.1        NaN   \n",
       "1        01:00  285.75  285.05            200         6.2        NaN   \n",
       "2        02:00  285.55  284.85            160         3.6        NaN   \n",
       "3        03:00  285.65  284.95            180         4.6        NaN   \n",
       "4        04:00  285.45  284.85            190         6.7        NaN   \n",
       "...        ...     ...     ...            ...         ...        ...   \n",
       "2857984  19:20  292.15  290.15            030         1.0        NaN   \n",
       "2857985  19:50  292.15  290.15            010         1.5        NaN   \n",
       "2857986  20:20  292.15  290.15            080         2.1        NaN   \n",
       "2857987  20:50  292.15  289.15            070         1.5        NaN   \n",
       "2857988  21:20  291.15  289.15            040         1.0        NaN   \n",
       "\n",
       "         3_hour_tp  6_hour_tp  12_hour_tp  24_hour_tp  \n",
       "0              NaN        0.0         NaN         NaN  \n",
       "1              NaN        NaN         NaN         NaN  \n",
       "2              NaN        NaN         NaN         NaN  \n",
       "3              NaN        NaN         NaN         NaN  \n",
       "4              NaN        NaN         NaN         NaN  \n",
       "...            ...        ...         ...         ...  \n",
       "2857984        NaN        NaN         NaN         NaN  \n",
       "2857985        NaN        NaN         NaN         NaN  \n",
       "2857986        NaN        NaN         NaN         NaN  \n",
       "2857987        NaN        NaN         NaN         NaN  \n",
       "2857988        NaN        NaN         NaN         NaN  \n",
       "\n",
       "[2857989 rows x 16 columns]"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "cf807f65-42c8-4d7f-86cf-5ce1bbbadcb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            STATION      DATE   TIME     TMP     DEW WIND_DIRECTORY  \\\n",
      "0        3862099999  20220101  00:00  285.65  285.05            200   \n",
      "1        3862099999  20220101  01:00  285.75  285.05            200   \n",
      "2        3862099999  20220101  02:00  285.55  284.85            160   \n",
      "3        3862099999  20220101  03:00  285.65  284.95            180   \n",
      "4        3862099999  20220101  04:00  285.45  284.85            190   \n",
      "...             ...       ...    ...     ...     ...            ...   \n",
      "2857984  3160099999  20230810  19:20  292.15  290.15            030   \n",
      "2857985  3160099999  20230810  19:50  292.15  290.15            010   \n",
      "2857986  3160099999  20230810  20:20  292.15  290.15            080   \n",
      "2857987  3160099999  20230810  20:50  292.15  289.15            070   \n",
      "2857988  3160099999  20230810  21:20  291.15  289.15            040   \n",
      "\n",
      "         WIND_SPEED  1_hour_tp  3_hour_tp  6_hour_tp  12_hour_tp  24_hour_tp  \n",
      "0               5.1        NaN        NaN        0.0         NaN         NaN  \n",
      "1               6.2        NaN        NaN        NaN         NaN         NaN  \n",
      "2               3.6        NaN        NaN        NaN         NaN         NaN  \n",
      "3               4.6        NaN        NaN        NaN         NaN         NaN  \n",
      "4               6.7        NaN        NaN        NaN         NaN         NaN  \n",
      "...             ...        ...        ...        ...         ...         ...  \n",
      "2857984         1.0        NaN        NaN        NaN         NaN         NaN  \n",
      "2857985         1.5        NaN        NaN        NaN         NaN         NaN  \n",
      "2857986         2.1        NaN        NaN        NaN         NaN         NaN  \n",
      "2857987         1.5        NaN        NaN        NaN         NaN         NaN  \n",
      "2857988         1.0        NaN        NaN        NaN         NaN         NaN  \n",
      "\n",
      "[2857989 rows x 12 columns]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "columns_to_drop = [\"LATITUDE\",\"LONGITUDE\",\t\"ELEVATION\",\"NAME\"]\n",
    "\n",
    "p_df2 = p_df.drop(columns=columns_to_drop)\n",
    "\n",
    "# 打印结果\n",
    "print(p_df2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "9a4c49e3-272c-42b4-b1f4-3558fec3b358",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/processed_data/NOAA_DATA/GB_NOAA_DATA' created successfully.\n",
      "GB_NOAA_processed_data.csv done!\n"
     ]
    }
   ],
   "source": [
    "save_noaa_processed_data(\n",
    "    p_df2, country, data_folder, data_save_category, output_folder\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "60cce1b8-8dd2-4fc5-97e0-e11e8ea89d8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_csv_station(country, data_folder, data_category, output_folder):\n",
    "    # Find csvs\n",
    "    input_folder = folder_utils.find_folder(\n",
    "        country, data_folder, data_category, output_folder\n",
    "    )\n",
    "    station_network_csv = \"noaa_station_network.csv\"\n",
    "    noaa_data_csv = \"GB_NOAA_processed_data.csv\"\n",
    "    station_network_csv_path = os.path.join(input_folder, station_network_csv)\n",
    "    noaa_data_csv_path = os.path.join(input_folder, noaa_data_csv)\n",
    "\n",
    "    # Read csvs\n",
    "    station_id_df = pd.read_csv(station_network_csv_path)\n",
    "    station_info_df = pd.read_csv(noaa_data_csv_path)\n",
    "\n",
    "    # Merge by \"STATION\"\n",
    "    merged_df = pd.merge(station_id_df, station_info_df, on=\"STATION\", how=\"left\")\n",
    "    return merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "cfba17c1-edf1-48fc-90b3-aca5bb6dacfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_df = merge_csv_station(country, data_folder, data_save_category, output_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "b7c959fb-9bc5-4950-91e1-1193b9bba1a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>TIME</th>\n",
       "      <th>TMP</th>\n",
       "      <th>DEW</th>\n",
       "      <th>WIND_DIRECTORY</th>\n",
       "      <th>WIND_SPEED</th>\n",
       "      <th>1_hour_tp</th>\n",
       "      <th>3_hour_tp</th>\n",
       "      <th>6_hour_tp</th>\n",
       "      <th>12_hour_tp</th>\n",
       "      <th>24_hour_tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>00:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>5.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>01:00</td>\n",
       "      <td>285.75</td>\n",
       "      <td>285.05</td>\n",
       "      <td>200</td>\n",
       "      <td>6.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>02:00</td>\n",
       "      <td>285.55</td>\n",
       "      <td>284.85</td>\n",
       "      <td>160</td>\n",
       "      <td>3.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>03:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>284.95</td>\n",
       "      <td>180</td>\n",
       "      <td>4.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>04:00</td>\n",
       "      <td>285.45</td>\n",
       "      <td>284.85</td>\n",
       "      <td>190</td>\n",
       "      <td>6.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>05:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>284.45</td>\n",
       "      <td>180</td>\n",
       "      <td>6.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:00</td>\n",
       "      <td>285.65</td>\n",
       "      <td>283.75</td>\n",
       "      <td>180</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:20</td>\n",
       "      <td>286.15</td>\n",
       "      <td>283.15</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>06:50</td>\n",
       "      <td>286.15</td>\n",
       "      <td>283.15</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>BOURNEMOUTH</td>\n",
       "      <td>50.78</td>\n",
       "      <td>-1.8425</td>\n",
       "      <td>11.58</td>\n",
       "      <td>20220101</td>\n",
       "      <td>07:00</td>\n",
       "      <td>286.05</td>\n",
       "      <td>283.05</td>\n",
       "      <td>190</td>\n",
       "      <td>5.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      STATION         NAME  LATITUDE  LONGITUDE  ELEVATION      DATE   TIME  \\\n",
       "0  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  00:00   \n",
       "1  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  01:00   \n",
       "2  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  02:00   \n",
       "3  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  03:00   \n",
       "4  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  04:00   \n",
       "5  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  05:00   \n",
       "6  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  06:00   \n",
       "7  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  06:20   \n",
       "8  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  06:50   \n",
       "9  3862099999  BOURNEMOUTH     50.78    -1.8425      11.58  20220101  07:00   \n",
       "\n",
       "      TMP     DEW  WIND_DIRECTORY  WIND_SPEED  1_hour_tp  3_hour_tp  \\\n",
       "0  285.65  285.05             200         5.1        NaN        NaN   \n",
       "1  285.75  285.05             200         6.2        NaN        NaN   \n",
       "2  285.55  284.85             160         3.6        NaN        NaN   \n",
       "3  285.65  284.95             180         4.6        NaN        NaN   \n",
       "4  285.45  284.85             190         6.7        NaN        NaN   \n",
       "5  285.65  284.45             180         6.7        NaN        NaN   \n",
       "6  285.65  283.75             180         5.7        NaN        NaN   \n",
       "7  286.15  283.15             190         5.7        NaN        NaN   \n",
       "8  286.15  283.15             190         5.7        NaN        NaN   \n",
       "9  286.05  283.05             190         5.7        NaN        NaN   \n",
       "\n",
       "   6_hour_tp  12_hour_tp  24_hour_tp  \n",
       "0        0.0         NaN         NaN  \n",
       "1        NaN         NaN         NaN  \n",
       "2        NaN         NaN         NaN  \n",
       "3        NaN         NaN         NaN  \n",
       "4        NaN         NaN         NaN  \n",
       "5        NaN         NaN         NaN  \n",
       "6        NaN         0.0         3.2  \n",
       "7        NaN         NaN         NaN  \n",
       "8        NaN         NaN         NaN  \n",
       "9        NaN         NaN         NaN  "
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "32afd9ac-4813-4354-a4bc-980dfacaa8fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_select_noaa(processed_df,start_date, end_date):\n",
    "\n",
    "    processed_df[\"DATE\"] = pd.to_datetime(processed_df[\"DATE\"], format=\"%Y%m%d\")\n",
    "        \n",
    "    filtered_df = processed_df[(processed_df[\"DATE\"] >= start_date) & (processed_df[\"DATE\"] <= end_date)]\n",
    "    \n",
    "    return filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "0094c0a6-be73-4eff-b768-628c9b8fd494",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = pd.Timestamp(\"2022-08-01\")\n",
    "end_date = pd.Timestamp(\"2023-08-01\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "fb41b586-93d0-4e19-8fb4-eff1aded9a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_df3 = time_select_noaa(p_df2, start_date, end_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "33b3b8f3-b523-4418-98fd-346a72af0274",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>TIME</th>\n",
       "      <th>TMP</th>\n",
       "      <th>DEW</th>\n",
       "      <th>WIND_DIRECTORY</th>\n",
       "      <th>WIND_SPEED</th>\n",
       "      <th>1_hour_tp</th>\n",
       "      <th>3_hour_tp</th>\n",
       "      <th>6_hour_tp</th>\n",
       "      <th>12_hour_tp</th>\n",
       "      <th>24_hour_tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11853</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>00:00</td>\n",
       "      <td>290.05</td>\n",
       "      <td>287.35</td>\n",
       "      <td>300</td>\n",
       "      <td>3.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11854</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>01:00</td>\n",
       "      <td>288.75</td>\n",
       "      <td>287.25</td>\n",
       "      <td>290</td>\n",
       "      <td>2.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11855</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>02:00</td>\n",
       "      <td>289.35</td>\n",
       "      <td>288.15</td>\n",
       "      <td>310</td>\n",
       "      <td>2.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11856</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>03:00</td>\n",
       "      <td>288.35</td>\n",
       "      <td>287.35</td>\n",
       "      <td>310</td>\n",
       "      <td>2.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11857</th>\n",
       "      <td>3862099999</td>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>04:00</td>\n",
       "      <td>287.55</td>\n",
       "      <td>286.25</td>\n",
       "      <td>310</td>\n",
       "      <td>2.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857595</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>2023-08-01</td>\n",
       "      <td>21:50</td>\n",
       "      <td>287.15</td>\n",
       "      <td>286.15</td>\n",
       "      <td>080</td>\n",
       "      <td>3.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857596</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>2023-08-01</td>\n",
       "      <td>22:20</td>\n",
       "      <td>287.15</td>\n",
       "      <td>286.15</td>\n",
       "      <td>060</td>\n",
       "      <td>4.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857597</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>2023-08-01</td>\n",
       "      <td>22:50</td>\n",
       "      <td>287.15</td>\n",
       "      <td>286.15</td>\n",
       "      <td>060</td>\n",
       "      <td>4.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857598</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>2023-08-01</td>\n",
       "      <td>23:20</td>\n",
       "      <td>287.15</td>\n",
       "      <td>286.15</td>\n",
       "      <td>070</td>\n",
       "      <td>5.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857599</th>\n",
       "      <td>3160099999</td>\n",
       "      <td>2023-08-01</td>\n",
       "      <td>23:50</td>\n",
       "      <td>287.15</td>\n",
       "      <td>286.15</td>\n",
       "      <td>070</td>\n",
       "      <td>5.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1803343 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            STATION       DATE   TIME     TMP     DEW WIND_DIRECTORY  \\\n",
       "11853    3862099999 2022-08-01  00:00  290.05  287.35            300   \n",
       "11854    3862099999 2022-08-01  01:00  288.75  287.25            290   \n",
       "11855    3862099999 2022-08-01  02:00  289.35  288.15            310   \n",
       "11856    3862099999 2022-08-01  03:00  288.35  287.35            310   \n",
       "11857    3862099999 2022-08-01  04:00  287.55  286.25            310   \n",
       "...             ...        ...    ...     ...     ...            ...   \n",
       "2857595  3160099999 2023-08-01  21:50  287.15  286.15            080   \n",
       "2857596  3160099999 2023-08-01  22:20  287.15  286.15            060   \n",
       "2857597  3160099999 2023-08-01  22:50  287.15  286.15            060   \n",
       "2857598  3160099999 2023-08-01  23:20  287.15  286.15            070   \n",
       "2857599  3160099999 2023-08-01  23:50  287.15  286.15            070   \n",
       "\n",
       "         WIND_SPEED  1_hour_tp  3_hour_tp  6_hour_tp  12_hour_tp  24_hour_tp  \n",
       "11853           3.1        NaN        NaN        0.0         NaN         NaN  \n",
       "11854           2.6        NaN        NaN        NaN         NaN         NaN  \n",
       "11855           2.1        NaN        NaN        NaN         NaN         NaN  \n",
       "11856           2.6        NaN        NaN        NaN         NaN         NaN  \n",
       "11857           2.6        NaN        NaN        NaN         NaN         NaN  \n",
       "...             ...        ...        ...        ...         ...         ...  \n",
       "2857595         3.6        NaN        NaN        NaN         NaN         NaN  \n",
       "2857596         4.1        NaN        NaN        NaN         NaN         NaN  \n",
       "2857597         4.1        NaN        NaN        NaN         NaN         NaN  \n",
       "2857598         5.1        NaN        NaN        NaN         NaN         NaN  \n",
       "2857599         5.1        NaN        NaN        NaN         NaN         NaN  \n",
       "\n",
       "[1803343 rows x 12 columns]"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "b4fad5ad-8170-49e8-8454-6a221808b7a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/processed_data/NOAA_DATA/GB_NOAA_DATA' created successfully.\n",
      "GB_NOAA_processed_data.csv done!\n"
     ]
    }
   ],
   "source": [
    "save_noaa_processed_data(\n",
    "    p_df3, country, data_folder, data_save_category, output_folder\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ba09839-d97f-4400-b353-0ae5bf49f644",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}