src/notebooks/data_collection/data_asos_crwal_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4b60d6c2-4a50-4d1d-af52-0e14fb13f028",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The European Union has 27 member states\n",
    "# The site use GB to denote UK_ASOS\n",
    "# the eu_member_codes has 28 variables: UK + EU\n",
    "\n",
    "import os\n",
    "import requests\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import time\n",
    "import concurrent.futures\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf3cb0e2-171c-471d-93b9-d89313eee9cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "eu_member_codes = [\n",
    "    'AT', 'BE', 'BG', 'HR', 'CY', 'CZ', 'DK', 'EE', 'FI', 'FR',\n",
    "    'DE', 'GR', 'HU', 'IE', 'IT', 'LV', 'LT', 'LU', 'MT', 'NL',\n",
    "    'PL', 'PT', 'RO', 'SK', 'SI', 'ES', 'SE', 'GB',\n",
    "]\n",
    "\n",
    "# Extract data time range\n",
    "# startts = datetime.datetime(2022, 1, 1)\n",
    "# endts = datetime.datetime(2023, 8, 2)\n",
    "\n",
    "# Number of attempts to download data\n",
    "MAX_ATTEMPTS = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "118cf732-36a7-4a46-b102-ddad3f9990a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_network():\n",
    "    url = \"https://mesonet.agron.iastate.edu/request/download.phtml?network=FR__ASOS\"\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    select_element = soup.find(\"select\")\n",
    "    if select_element:\n",
    "        option_elements = select_element.find_all(\"option\")\n",
    "\n",
    "        option_values = [option[\"value\"] for option in option_elements if option.get(\"value\")]\n",
    "\n",
    "        print(\"Option values:\")\n",
    "        for value in option_values:\n",
    "            print(value)\n",
    "    else:\n",
    "        print(\"Select element not found on the page.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6eacbe5e-e23d-4142-bd38-f112d767d2da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_network_url(country_list):\n",
    "    valid_urls = []\n",
    "    invalid_urls = []\n",
    "\n",
    "    # for i in eu_member_codes:\n",
    "    for i in country_list:\n",
    "        url_network = f\"https://mesonet.agron.iastate.edu/request/download.phtml?network={i}__ASOS\"\n",
    "        response = requests.head(url_network)\n",
    "\n",
    "        if response.status_code == 200:\n",
    "            valid_urls.append(url_network)\n",
    "        else:\n",
    "            invalid_urls.append(f\"Invalid URL: {url_network}\")\n",
    "\n",
    "    return valid_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61980b90-e3fb-4e60-b4e3-399524cc2c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_current_directory():\n",
    "    if \"__file__\" in globals():\n",
    "        # Running in a Python file\n",
    "        return os.path.abspath(os.path.dirname(__file__))\n",
    "    else:\n",
    "        # Running in a Jupyter Notebook\n",
    "        return os.path.abspath(os.path.dirname(\"\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a87da83c-6c8c-4857-a1ed-300205e99136",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_folder(c):\n",
    "    current_directory = get_current_directory()\n",
    "    folder_name = f\"{c}_ASOS_DATA\"\n",
    "    folder_path = os.path.join(current_directory, folder_name)\n",
    "\n",
    "    try:\n",
    "        os.mkdir(folder_path)\n",
    "        print(f\"Folder '{folder_path}' created successfully.\")\n",
    "    except FileExistsError:\n",
    "        print(f\"Folder '{folder_path}' already exists.\")\n",
    "\n",
    "    return folder_path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff855d5c-44fc-45ec-9848-63325db50225",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_station_by_network(country_list):\n",
    "    # valid_urls = get_network_url()\n",
    "    for i in country_list:\n",
    "        url_station_geojson = (\n",
    "            f\"https://mesonet.agron.iastate.edu/geojson/network/{i}__ASOS.geojson\"\n",
    "        )\n",
    "        output_directory = create_folder(i)\n",
    "        output_filename = f\"{i}__asos_station_network.csv\"\n",
    "        output_filepath = os.path.join(output_directory, output_filename)\n",
    "\n",
    "        # Get GeoJSON data\n",
    "        response = requests.get(url_station_geojson)\n",
    "        geojson_data = response.json()\n",
    "        #  Create a list to store geojson\n",
    "        data = []\n",
    "\n",
    "        # Extract GeoJSON items\n",
    "        for feature in geojson_data[\"features\"]:\n",
    "            properties = feature[\"properties\"]\n",
    "            geometry = feature[\"geometry\"]\n",
    "            row = {\n",
    "                # \"Type\": feature.get(\"type\", None),\n",
    "                \"Name\": properties.get(\"sname\", None),\n",
    "                \"ID\": feature.get(\"id\", None),\n",
    "                \"Latitude\": geometry.get(\"coordinates\", None)[1],\n",
    "                \"Logitude\": geometry.get(\"coordinates\", None)[0],\n",
    "                \"Elevation\": properties.get(\"elevation\", None),\n",
    "                \"Country\": properties.get(\"country\", None),\n",
    "                \"Network\": properties.get(\"network\", None),\n",
    "                \"Archive_Begin\": properties.get(\"archive_begin\", None),\n",
    "                \"Archive_End\": properties.get(\"archive_end\", None),\n",
    "                \"Time_Domain\": properties.get(\"time_domain\", None),\n",
    "                # \"Properties\": properties\n",
    "            }\n",
    "            data.append(row)\n",
    "\n",
    "        # Transfer the list to Pandas DataFrame\n",
    "        df = pd.DataFrame(data)\n",
    "        df.to_csv(output_filepath, index=False, encoding=\"utf-8\")\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e31c7ba3-855e-4605-9067-d21b3aa330ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_url(df, startts, endts):\n",
    "    url_site_list = []\n",
    "    id_list = []\n",
    "    url_site_header = \"https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?\"\n",
    "    url_site_tail = f\"data=tmpc&\"  # add all data variables f\"data=tmpf&\"\n",
    "    url_site_tail += startts.strftime(\"year1=%Y&month1=%m&day1=%d&\")  # add start date\n",
    "    url_site_tail += endts.strftime(\"year2=%Y&month2=%m&day2=%d&\")  # add end date\n",
    "    url_site_tail += f\"tz=Etc%2FUTC&format=onlycomma&latlon=no&elev=no&missing=null&trace=T&direct=no&report_type=3&report_type=4\"  # add data format\n",
    "    for id in df[\"ID\"]:\n",
    "        url_site = f\"{url_site_header}station={id}&{url_site_tail}\"  # add all stations\n",
    "        url_site_list.append(url_site)\n",
    "        id_list.append(id)\n",
    "\n",
    "    return url_site_list, id_list  # return station list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "23838bdb-2aa1-4d5b-bb61-621bcba7e5e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_data(url_data):\n",
    "    attempt = 0\n",
    "    while attempt < MAX_ATTEMPTS:\n",
    "        try:\n",
    "            response = requests.get(url_data, timeout=300)\n",
    "            data = response.text\n",
    "            if data is not None and not data.startswith(\"ERROR\"):\n",
    "                return data\n",
    "        except Exception as e:\n",
    "            print(f\"download_data({url_data}) failed with {e}\")\n",
    "            time.sleep(5)\n",
    "        attempt += 1\n",
    "\n",
    "    print(\"Exhausted attempts to download, returning empty data\")\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5c845de6-a614-4509-87f8-e0f4b8f68216",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_data(url_site, country, station, startts, endts):\n",
    "    data = download_data(url_site)\n",
    "    # output_filename = f\"{country}_{startts:%Y%m%d%H%M}_{endts:%Y%m%d%H%M}.csv\"\n",
    "    output_directory = create_folder(country)\n",
    "    output_filename = f\"{country}_{station}_{startts:%Y%m%d}_{endts:%Y%m%d}.csv\"\n",
    "    output_filepath = os.path.join(output_directory, output_filename)\n",
    "    # Split the data into lines\n",
    "    lines = data.split(\"\\n\")\n",
    "\n",
    "    # Extract column names from the first line\n",
    "    column_names = lines[0].split(\",\")\n",
    "\n",
    "    # Initialize lists to store data\n",
    "    data_rows = []\n",
    "\n",
    "    # Iterate through the remaining lines (data rows)\n",
    "    for line in lines[1:]:\n",
    "        if line:  # Skip empty lines\n",
    "            data_row = line.split(\",\")\n",
    "            data_rows.append(data_row)\n",
    "\n",
    "    # Create a DataFrame from the data rows using the extracted column names\n",
    "    df = pd.DataFrame(data_rows, columns=column_names)\n",
    "\n",
    "    # Save the DataFrame to a CSV file\n",
    "    df.to_csv(output_filepath, index=False, encoding=\"utf-8\")\n",
    "    print(f'{output_filename} done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "08b2a858-baf8-4163-97a2-da3c9a2fb32c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_save_data(url_site, country, station, startts, endts):\n",
    "    start_time = time.time()  # Record start time\n",
    "    data = download_data(url_site)\n",
    "    end_time = time.time()  # Record end time\n",
    "    download_time = end_time - start_time\n",
    "\n",
    "    if data:\n",
    "        save_data(url_site, country, station, startts, endts)\n",
    "        print(f\"{station} - Download time: {download_time:.3f} s\")\n",
    "    else:\n",
    "        print(f\"{station} - Download failed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "162ddb21-92e2-4aa0-a76e-351cd1b9aa52",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_save_data_thread(args):\n",
    "    url_site, country, station_id, startts, endts = args\n",
    "    download_and_save_data(url_site, country, station_id, startts, endts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4c95e970-2a5d-49fe-ba53-36bcf6f876c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# UK example\n",
    "country = [\n",
    "    \"GB\",\n",
    "]\n",
    "\n",
    "data_folder = \"data\"\n",
    "data_category = \"raw_data\"\n",
    "output_folder = \"ASOS_DATA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9acf859-51e2-49d4-8796-24cf2a072316",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(1976,2023):\n",
    "#     start_date = datetime.datetime(i, 1, 1)\n",
    "#     end_date = datetime.datetime(i+1, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "eeee1604-25c4-4625-99bd-8457c90c969c",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = datetime.datetime(1976, 1, 1)\n",
    "end_date = datetime.datetime(2023, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "84b19c34-e7a4-4048-b090-114587bf34dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n"
     ]
    }
   ],
   "source": [
    "gb_df = get_all_station_by_network(country)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5fc302b2-b5a5-4e18-8012-9c72be1bda99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>ID</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Logitude</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Country</th>\n",
       "      <th>Network</th>\n",
       "      <th>Archive_Begin</th>\n",
       "      <th>Archive_End</th>\n",
       "      <th>Time_Domain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Aberdeen</td>\n",
       "      <td>EGPD</td>\n",
       "      <td>57.2049</td>\n",
       "      <td>-2.2053</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1928-01-01</td>\n",
       "      <td>None</td>\n",
       "      <td>(1928-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alderney</td>\n",
       "      <td>EGJA</td>\n",
       "      <td>49.7061</td>\n",
       "      <td>-2.2147</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1987-06-29</td>\n",
       "      <td>None</td>\n",
       "      <td>(1987-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Angelsey</td>\n",
       "      <td>EGOV</td>\n",
       "      <td>53.2527</td>\n",
       "      <td>-4.5365</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1928-01-01</td>\n",
       "      <td>None</td>\n",
       "      <td>(1928-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Barkston Heath</td>\n",
       "      <td>EGYE</td>\n",
       "      <td>52.9622</td>\n",
       "      <td>-0.5616</td>\n",
       "      <td>112.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1995-11-29</td>\n",
       "      <td>None</td>\n",
       "      <td>(1995-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Belfast</td>\n",
       "      <td>EGAC</td>\n",
       "      <td>54.6135</td>\n",
       "      <td>-5.8735</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1973-01-01</td>\n",
       "      <td>None</td>\n",
       "      <td>(1973-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>Wittering</td>\n",
       "      <td>EGXT</td>\n",
       "      <td>52.6114</td>\n",
       "      <td>-0.4612</td>\n",
       "      <td>84.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1973-02-28</td>\n",
       "      <td>None</td>\n",
       "      <td>(1973-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>Woodvale</td>\n",
       "      <td>EGOW</td>\n",
       "      <td>53.5816</td>\n",
       "      <td>-3.0555</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>2005-06-20</td>\n",
       "      <td>None</td>\n",
       "      <td>(2005-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>Wyton</td>\n",
       "      <td>EGUY</td>\n",
       "      <td>52.3572</td>\n",
       "      <td>-0.1078</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1950-10-02</td>\n",
       "      <td>2019-11-18</td>\n",
       "      <td>(1950-2019)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>Yeovilton</td>\n",
       "      <td>EGDY</td>\n",
       "      <td>51.0063</td>\n",
       "      <td>-2.6428</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>1972-12-31</td>\n",
       "      <td>None</td>\n",
       "      <td>(1972-Now)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>Yorkshire and Humberside</td>\n",
       "      <td>EGSY</td>\n",
       "      <td>53.3943</td>\n",
       "      <td>-1.3885</td>\n",
       "      <td>68.423706</td>\n",
       "      <td>GB</td>\n",
       "      <td>GB__ASOS</td>\n",
       "      <td>2020-07-21</td>\n",
       "      <td>None</td>\n",
       "      <td>(2020-Now)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>112 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Name    ID  Latitude  Logitude   Elevation Country  \\\n",
       "0                    Aberdeen  EGPD   57.2049   -2.2053   65.000000      GB   \n",
       "1                    Alderney  EGJA   49.7061   -2.2147   71.000000      GB   \n",
       "2                    Angelsey  EGOV   53.2527   -4.5365   11.000000      GB   \n",
       "3              Barkston Heath  EGYE   52.9622   -0.5616  112.000000      GB   \n",
       "4                     Belfast  EGAC   54.6135   -5.8735    5.000000      GB   \n",
       "..                        ...   ...       ...       ...         ...     ...   \n",
       "107                 Wittering  EGXT   52.6114   -0.4612   84.000000      GB   \n",
       "108                  Woodvale  EGOW   53.5816   -3.0555   12.000000      GB   \n",
       "109                     Wyton  EGUY   52.3572   -0.1078   41.000000      GB   \n",
       "110                 Yeovilton  EGDY   51.0063   -2.6428   23.000000      GB   \n",
       "111  Yorkshire and Humberside  EGSY   53.3943   -1.3885   68.423706      GB   \n",
       "\n",
       "      Network Archive_Begin Archive_End  Time_Domain  \n",
       "0    GB__ASOS    1928-01-01        None   (1928-Now)  \n",
       "1    GB__ASOS    1987-06-29        None   (1987-Now)  \n",
       "2    GB__ASOS    1928-01-01        None   (1928-Now)  \n",
       "3    GB__ASOS    1995-11-29        None   (1995-Now)  \n",
       "4    GB__ASOS    1973-01-01        None   (1973-Now)  \n",
       "..        ...           ...         ...          ...  \n",
       "107  GB__ASOS    1973-02-28        None   (1973-Now)  \n",
       "108  GB__ASOS    2005-06-20        None   (2005-Now)  \n",
       "109  GB__ASOS    1950-10-02  2019-11-18  (1950-2019)  \n",
       "110  GB__ASOS    1972-12-31        None   (1972-Now)  \n",
       "111  GB__ASOS    2020-07-21        None   (2020-Now)  \n",
       "\n",
       "[112 rows x 10 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fc6f6be9-c19e-4477-9190-1f1fd0d75cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "url_site_list, id_list = get_data_url(gb_df, start_date, end_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "966fbc1e-4d4b-4203-bba9-61617093c618",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGJA_19760101_20230101.csv done!\n",
      "EGJA - Download time: 10.705 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGOV_19760101_20230101.csv done!\n",
      "EGOV - Download time: 9.162 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGYE_19760101_20230101.csv done!\n",
      "EGYE - Download time: 2.118 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGAC_19760101_20230101.csv done!\n",
      "EGAC - Download time: 6.284 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPD_19760101_20230101.csv done!\n",
      "EGPD - Download time: 61.817 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPL_19760101_20230101.csv done!\n",
      "EGPL - Download time: 9.092 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGAA_19760101_20230101.csv done!\n",
      "EGAA - Download time: 34.521 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUB_19760101_20230101.csv done!\n",
      "EGUB - Download time: 15.477 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGKB_19760101_20230101.csv done!\n",
      "EGKB - Download time: 14.089 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGBB_19760101_20230101.csv done!\n",
      "EGBB - Download time: 17.049 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNH_19760101_20230101.csv done!\n",
      "EGNH - Download time: 10.930 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDM_19760101_20230101.csv done!\n",
      "EGDM - Download time: 17.566 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGQM_19760101_20230101.csv done!\n",
      "EGQM - Download time: 13.957 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHH_19760101_20230101.csv done!\n",
      "EGHH - Download time: 11.627 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGTG_19760101_20230101.csv done!\n",
      "EGTG - Download time: 1.146 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGGD_19760101_20230101.csv done!\n",
      "EGGD - Download time: 13.700 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGCK_19760101_20230101.csv done!\n",
      "EGCK - Download time: 5.440 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGSC_19760101_20230101.csv done!\n",
      "EGSC - Download time: 4.234 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGEC_19760101_20230101.csv done!\n",
      "EGEC - Download time: 2.249 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGVN_19760101_20230101.csv done!\n",
      "EGVN - Download time: 10.625 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNC_19760101_20230101.csv done!\n",
      "EGNC - Download time: 7.389 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXG_19760101_20230101.csv done!\n",
      "EGXG - Download time: 2.948 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUO_19760101_20230101.csv done!\n",
      "EGUO - Download time: 0.705 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGFF_19760101_20230101.csv done!\n",
      "EGFF - Download time: 24.848 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGWC_19760101_20230101.csv done!\n",
      "EGWC - Download time: 1.163 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXC_19760101_20230101.csv done!\n",
      "EGXC - Download time: 8.819 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGBE_19760101_20230101.csv done!\n",
      "EGBE - Download time: 9.194 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGTC_19760101_20230101.csv done!\n",
      "EGTC - Download time: 2.636 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGYD_19760101_20230101.csv done!\n",
      "EGYD - Download time: 16.229 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDR_19760101_20230101.csv done!\n",
      "EGDR - Download time: 8.675 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXD_19760101_20230101.csv done!\n",
      "EGXD - Download time: 4.502 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXS_19760101_20230101.csv done!\n",
      "EGXS - Download time: 4.887 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGCN_19760101_20230101.csv done!\n",
      "EGCN - Download time: 9.345 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPN_19760101_20230101.csv done!\n",
      "EGPN - Download time: 6.270 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNX_19760101_20230101.csv done!\n",
      "EGNX - Download time: 21.885 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGTE_19760101_20230101.csv done!\n",
      "EGTE - Download time: 8.806 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPH_19760101_20230101.csv done!\n",
      "EGPH - Download time: 40.355 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGVA_19760101_20230101.csv done!\n",
      "EGVA - Download time: 8.867 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGLF_19760101_20230101.csv done!\n",
      "EGLF - Download time: 7.487 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGBJ_19760101_20230101.csv done!\n",
      "EGBJ - Download time: 5.072 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPF_19760101_20230101.csv done!\n",
      "EGPF - Download time: 16.240 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGJB_19760101_20230101.csv done!\n",
      "EGJB - Download time: 25.597 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNR_19760101_20230101.csv done!\n",
      "EGNR - Download time: 11.399 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGYH_19760101_20230101.csv done!\n",
      "EGYH - Download time: 12.643 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNJ_19760101_20230101.csv done!\n",
      "EGNJ - Download time: 7.468 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPI_19760101_20230101.csv done!\n",
      "EGPI - Download time: 6.417 s\n",
      "GB_EGPE_19760101_20230101.csv done!\n",
      "EGPE - Download time: 13.085 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNS_19760101_20230101.csv done!\n",
      "EGNS - Download time: 11.960 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGJJ_19760101_20230101.csv done!\n",
      "EGJJ - Download time: 12.131 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPA_19760101_20230101.csv done!\n",
      "EGPA - Download time: 24.013 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGQK_19760101_20230101.csv done!\n",
      "EGQK - Download time: 15.370 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHC_19760101_20230101.csv done!\n",
      "EGHC - Download time: 4.432 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUL_19760101_20230101.csv done!\n",
      "EGUL - Download time: 10.591 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXV_19760101_20230101.csv done!\n",
      "EGXV - Download time: 5.074 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXE_19760101_20230101.csv done!\n",
      "EGXE - Download time: 12.996 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNM_19760101_20230101.csv done!\n",
      "EGNM - Download time: 16.972 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXU_19760101_20230101.csv done!\n",
      "EGXU - Download time: 12.921 s\n",
      "GB_EGQL_19760101_20230101.csv done!\n",
      "EGQL - Download time: 13.014 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGGP_19760101_20230101.csv done!\n",
      "EGGP - Download time: 7.885 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGSS_19760101_20230101.csv done!\n",
      "EGSS - Download time: 15.411 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGGW_19760101_20230101.csv done!\n",
      "EGGW - Download time: 8.359 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGLC_19760101_20230101.csv done!\n",
      "EGLC - Download time: 11.363 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGLL_19760101_20230101.csv done!\n",
      "EGLL - Download time: 14.931 s\n",
      "GB_EGKK_19760101_20230101.csv done!\n",
      "EGKK - Download time: 29.462 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGAE_19760101_20230101.csv done!\n",
      "EGAE - Download time: 7.546 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGMD_19760101_20230101.csv done!\n",
      "EGMD - Download time: 4.582 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGQS_19760101_20230101.csv done!\n",
      "EGQS - Download time: 22.604 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDL_19760101_20230101.csv done!\n",
      "EGDL - Download time: 10.625 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGMH_19760101_20230101.csv done!\n",
      "EGMH - Download time: 6.940 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGYM_19760101_20230101.csv done!\n",
      "EGYM - Download time: 8.686 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGCC_19760101_20230101.csv done!\n",
      "EGCC - Download time: 28.355 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGVP_19760101_20230101.csv done!\n",
      "EGVP - Download time: 19.611 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUN_19760101_20230101.csv done!\n",
      "EGUN - Download time: 28.265 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHQ_19760101_20230101.csv done!\n",
      "EGHQ - Download time: 2.645 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNT_19760101_20230101.csv done!\n",
      "EGNT - Download time: 8.306 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGSD_19760101_20230101.csv done!\n",
      "EGSD - Download time: 1.103 s\n",
      "GB_EGEO_19760101_20230101.csv done!\n",
      "EGEO - Download time: 2.949 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGWU_19760101_20230101.csv done!\n",
      "EGWU - Download time: 8.580 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGSH_19760101_20230101.csv done!\n",
      "EGSH - Download time: 10.895 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGTK_19760101_20230101.csv done!\n",
      "EGTK - Download time: 9.477 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGVO_19760101_20230101.csv done!\n",
      "EGVO - Download time: 9.421 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHK_19760101_20230101.csv done!\n",
      "EGHK - Download time: 0.527 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHD_19760101_20230101.csv done!\n",
      "EGHD - Download time: 1.489 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGOP_19760101_20230101.csv done!\n",
      "EGOP - Download time: 21.528 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGYC_19760101_20230101.csv done!\n",
      "EGYC - Download time: 0.571 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDG_19760101_20230101.csv done!\n",
      "EGDG - Download time: 3.249 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXP_19760101_20230101.csv done!\n",
      "EGXP - Download time: 5.979 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPK_19760101_20230101.csv done!\n",
      "EGPK - Download time: 31.034 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPM_19760101_20230101.csv done!\n",
      "EGPM - Download time: 5.718 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHE_19760101_20230101.csv done!\n",
      "EGHE - Download time: 7.242 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGOS_19760101_20230101.csv done!\n",
      "EGOS - Download time: 14.322 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGKA_19760101_20230101.csv done!\n",
      "EGKA - Download time: 12.441 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGMC_19760101_20230101.csv done!\n",
      "EGMC - Download time: 7.702 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGHI_19760101_20230101.csv done!\n",
      "EGHI - Download time: 8.970 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDX_19760101_20230101.csv done!\n",
      "EGDX - Download time: 3.021 s\n",
      "GB_EGOM_19760101_20230101.csv done!\n",
      "EGOM - Download time: 4.889 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPO_19760101_20230101.csv done!\n",
      "EGPO - Download time: 11.290 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPB_19760101_20230101.csv done!\n",
      "EGPB - Download time: 10.554 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGQA_19760101_20230101.csv done!\n",
      "EGQA - Download time: 7.266 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNV_19760101_20230101.csv done!\n",
      "EGNV - Download time: 24.804 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPU_19760101_20230101.csv done!\n",
      "EGPU - Download time: 11.253 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGMW_19760101_20230101.csv done!\n",
      "EGMW - Download time: 0.547 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXZ_19760101_20230101.csv done!\n",
      "EGXZ - Download time: 2.957 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGNO_19760101_20230101.csv done!\n",
      "EGNO - Download time: 3.690 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXW_19760101_20230101.csv done!\n",
      "EGXW - Download time: 9.806 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUW_19760101_20230101.csv done!\n",
      "EGUW - Download time: 10.978 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGPC_19760101_20230101.csv done!\n",
      "EGPC - Download time: 11.494 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGOW_19760101_20230101.csv done!\n",
      "EGOW - Download time: 0.893 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGXT_19760101_20230101.csv done!\n",
      "EGXT - Download time: 12.772 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGUY_19760101_20230101.csv done!\n",
      "EGUY - Download time: 4.511 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGSY_19760101_20230101.csv done!\n",
      "EGSY - Download time: 1.395 s\n",
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/src/notebooks/GB_ASOS_DATA' already exists.\n",
      "GB_EGDY_19760101_20230101.csv done!\n",
      "EGDY - Download time: 8.954 s\n"
     ]
    }
   ],
   "source": [
    "args_list = [\n",
    "    (url_site, \"GB\", station_id, start_date, end_date)\n",
    "    for url_site, station_id in zip(url_site_list, id_list)\n",
    "]\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:\n",
    "    executor.map(download_and_save_data_thread, args_list)  # fast\n",
    "    time.sleep(10)  # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "f716c2e4-a046-4cbc-a9b1-f96589eeec5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:19, 19.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPD_20220101_20230802.csv done!\n",
      "Download time: 19.910 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:30, 14.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGJA_20220101_20230802.csv done!\n",
      "Download time: 10.869 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:43, 13.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGOV_20220101_20230802.csv done!\n",
      "Download time: 12.410 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4it [00:58, 14.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGYE_20220101_20230802.csv done!\n",
      "Download time: 15.216 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [01:19, 16.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGAC_20220101_20230802.csv done!\n",
      "Download time: 20.836 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [01:43, 19.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGAA_20220101_20230802.csv done!\n",
      "Download time: 24.716 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7it [02:08, 21.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPL_20220101_20230802.csv done!\n",
      "Download time: 24.431 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8it [02:32, 21.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUB_20220101_20230802.csv done!\n",
      "Download time: 23.652 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9it [02:51, 21.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGKB_20220101_20230802.csv done!\n",
      "Download time: 19.695 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [03:26, 25.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGBB_20220101_20230802.csv done!\n",
      "Download time: 34.389 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11it [03:35, 20.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNH_20220101_20230802.csv done!\n",
      "Download time: 9.551 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12it [03:49, 18.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDM_20220101_20230802.csv done!\n",
      "Download time: 14.204 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13it [03:59, 15.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGQM_20220101_20230802.csv done!\n",
      "Download time: 10.054 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14it [04:09, 14.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHH_20220101_20230802.csv done!\n",
      "Download time: 9.920 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15it [04:18, 12.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGGD_20220101_20230802.csv done!\n",
      "Download time: 8.510 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [04:18,  8.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGTG_20220101_20230802.csv done!\n",
      "Download time: 0.480 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17it [04:45, 14.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGVN_20220101_20230802.csv done!\n",
      "Download time: 26.334 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18it [05:09, 17.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGCK_20220101_20230802.csv done!\n",
      "Download time: 24.650 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19it [05:14, 13.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGSC_20220101_20230802.csv done!\n",
      "Download time: 4.893 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20it [05:20, 11.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGEC_20220101_20230802.csv done!\n",
      "Download time: 6.221 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21it [05:48, 16.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGFF_20220101_20230802.csv done!\n",
      "Download time: 27.069 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22it [05:48, 11.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNC_20220101_20230802.csv done!\n",
      "Download time: 0.476 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23it [05:49,  8.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXG_20220101_20230802.csv done!\n",
      "Download time: 0.564 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24it [05:49,  5.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUO_20220101_20230802.csv done!\n",
      "Download time: 0.560 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25it [06:05,  8.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXC_20220101_20230802.csv done!\n",
      "Download time: 15.658 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "26it [06:09,  7.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGWC_20220101_20230802.csv done!\n",
      "Download time: 4.331 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "27it [06:10,  5.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGBE_20220101_20230802.csv done!\n",
      "Download time: 0.480 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "28it [06:19,  6.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGTC_20220101_20230802.csv done!\n",
      "Download time: 9.553 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "29it [06:32,  8.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGYD_20220101_20230802.csv done!\n",
      "Download time: 12.813 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [06:45,  9.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDR_20220101_20230802.csv done!\n",
      "Download time: 12.952 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "31it [06:45,  7.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXD_20220101_20230802.csv done!\n",
      "Download time: 0.463 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "32it [06:50,  6.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGCN_20220101_20230802.csv done!\n",
      "Download time: 5.089 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "33it [07:09, 10.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXS_20220101_20230802.csv done!\n",
      "Download time: 18.914 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "34it [07:20, 10.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPN_20220101_20230802.csv done!\n",
      "Download time: 10.417 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [07:39, 13.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNX_20220101_20230802.csv done!\n",
      "Download time: 19.679 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36it [07:52, 13.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPH_20220101_20230802.csv done!\n",
      "Download time: 12.893 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "37it [08:07, 13.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGTE_20220101_20230802.csv done!\n",
      "Download time: 14.488 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38it [08:27, 15.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGVA_20220101_20230802.csv done!\n",
      "Download time: 20.256 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39it [08:51, 18.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGLF_20220101_20230802.csv done!\n",
      "Download time: 24.191 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "40it [09:10, 18.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPF_20220101_20230802.csv done!\n",
      "Download time: 18.206 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "41it [09:20, 15.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGBJ_20220101_20230802.csv done!\n",
      "Download time: 10.345 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42it [09:46, 18.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGJB_20220101_20230802.csv done!\n",
      "Download time: 26.277 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "43it [09:55, 15.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNR_20220101_20230802.csv done!\n",
      "Download time: 8.897 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "44it [10:21, 18.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGYH_20220101_20230802.csv done!\n",
      "Download time: 25.619 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45it [10:36, 17.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNJ_20220101_20230802.csv done!\n",
      "Download time: 15.690 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46it [11:00, 19.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPE_20220101_20230802.csv done!\n",
      "Download time: 24.054 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "47it [11:14, 17.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPI_20220101_20230802.csv done!\n",
      "Download time: 13.385 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "48it [11:36, 19.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNS_20220101_20230802.csv done!\n",
      "Download time: 22.406 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "49it [12:01, 20.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGJJ_20220101_20230802.csv done!\n",
      "Download time: 24.639 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "50it [12:21, 20.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGQK_20220101_20230802.csv done!\n",
      "Download time: 20.379 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51it [12:35, 18.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPA_20220101_20230802.csv done!\n",
      "Download time: 14.213 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "52it [13:00, 20.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUL_20220101_20230802.csv done!\n",
      "Download time: 24.753 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "53it [13:08, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHC_20220101_20230802.csv done!\n",
      "Download time: 8.047 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "54it [13:28, 17.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXV_20220101_20230802.csv done!\n",
      "Download time: 19.293 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "55it [13:59, 21.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNM_20220101_20230802.csv done!\n",
      "Download time: 31.852 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "56it [14:23, 22.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXE_20220101_20230802.csv done!\n",
      "Download time: 23.933 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "57it [14:38, 20.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGQL_20220101_20230802.csv done!\n",
      "Download time: 15.131 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "58it [14:39, 14.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXU_20220101_20230802.csv done!\n",
      "Download time: 0.505 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59it [14:58, 15.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGGP_20220101_20230802.csv done!\n",
      "Download time: 19.404 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "60it [15:23, 18.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGSS_20220101_20230802.csv done!\n",
      "Download time: 24.890 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "61it [15:40, 17.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGGW_20220101_20230802.csv done!\n",
      "Download time: 16.549 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "62it [15:55, 17.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGLC_20220101_20230802.csv done!\n",
      "Download time: 15.417 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "63it [16:08, 15.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGKK_20220101_20230802.csv done!\n",
      "Download time: 12.447 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "64it [16:26, 16.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGLL_20220101_20230802.csv done!\n",
      "Download time: 18.627 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [16:37, 14.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGAE_20220101_20230802.csv done!\n",
      "Download time: 10.702 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "66it [16:49, 14.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGQS_20220101_20230802.csv done!\n",
      "Download time: 12.290 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "67it [17:14, 17.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGMD_20220101_20230802.csv done!\n",
      "Download time: 25.069 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [17:15, 12.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDL_20220101_20230802.csv done!\n",
      "Download time: 0.489 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "69it [17:23, 10.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGCC_20220101_20230802.csv done!\n",
      "Download time: 7.793 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "70it [17:23,  7.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGMH_20220101_20230802.csv done!\n",
      "Download time: 0.489 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "71it [17:39, 10.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGYM_20220101_20230802.csv done!\n",
      "Download time: 15.648 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "72it [17:52, 11.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGVP_20220101_20230802.csv done!\n",
      "Download time: 13.356 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73it [18:10, 13.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUN_20220101_20230802.csv done!\n",
      "Download time: 17.669 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "74it [18:21, 12.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNT_20220101_20230802.csv done!\n",
      "Download time: 11.011 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75it [18:30, 11.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHQ_20220101_20230802.csv done!\n",
      "Download time: 9.305 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "76it [18:38, 10.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGEO_20220101_20230802.csv done!\n",
      "Download time: 8.081 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "77it [18:39,  7.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGSD_20220101_20230802.csv done!\n",
      "Download time: 0.476 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "78it [18:50,  8.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGWU_20220101_20230802.csv done!\n",
      "Download time: 11.015 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "79it [19:07, 11.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGSH_20220101_20230802.csv done!\n",
      "Download time: 16.920 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "80it [19:19, 11.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGVO_20220101_20230802.csv done!\n",
      "Download time: 12.459 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "81it [19:33, 12.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGTK_20220101_20230802.csv done!\n",
      "Download time: 13.849 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "82it [19:43, 11.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGOP_20220101_20230802.csv done!\n",
      "Download time: 10.300 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "83it [19:44,  8.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHK_20220101_20230802.csv done!\n",
      "Download time: 0.739 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "84it [19:44,  6.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHD_20220101_20230802.csv done!\n",
      "Download time: 0.543 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "85it [20:00,  8.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPK_20220101_20230802.csv done!\n",
      "Download time: 15.391 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "86it [20:00,  6.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGYC_20220101_20230802.csv done!\n",
      "Download time: 0.604 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "87it [20:01,  4.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDG_20220101_20230802.csv done!\n",
      "Download time: 0.490 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88it [20:07,  4.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXP_20220101_20230802.csv done!\n",
      "Download time: 5.714 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "89it [20:07,  3.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPM_20220101_20230802.csv done!\n",
      "Download time: 0.551 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "90it [20:11,  3.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHE_20220101_20230802.csv done!\n",
      "Download time: 3.980 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "91it [20:33,  9.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGOS_20220101_20230802.csv done!\n",
      "Download time: 21.594 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "92it [20:41,  8.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGKA_20220101_20230802.csv done!\n",
      "Download time: 8.154 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "93it [20:56, 10.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGHI_20220101_20230802.csv done!\n",
      "Download time: 14.977 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "94it [21:16, 13.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGMC_20220101_20230802.csv done!\n",
      "Download time: 19.701 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "95it [21:38, 16.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGOM_20220101_20230802.csv done!\n",
      "Download time: 22.599 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "96it [21:39, 11.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDX_20220101_20230802.csv done!\n",
      "Download time: 0.499 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "97it [22:08, 16.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPO_20220101_20230802.csv done!\n",
      "Download time: 29.036 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "98it [22:30, 18.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPB_20220101_20230802.csv done!\n",
      "Download time: 22.055 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "99it [22:37, 14.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGQA_20220101_20230802.csv done!\n",
      "Download time: 6.691 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [22:48, 13.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNV_20220101_20230802.csv done!\n",
      "Download time: 11.733 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "101it [22:56, 11.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPU_20220101_20230802.csv done!\n",
      "Download time: 7.284 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "102it [23:07, 11.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXZ_20220101_20230802.csv done!\n",
      "Download time: 11.315 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "103it [23:08,  8.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGMW_20220101_20230802.csv done!\n",
      "Download time: 0.708 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "104it [23:21,  9.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXW_20220101_20230802.csv done!\n",
      "Download time: 13.511 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "105it [23:31,  9.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGNO_20220101_20230802.csv done!\n",
      "Download time: 10.047 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "106it [23:36,  8.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUW_20220101_20230802.csv done!\n",
      "Download time: 5.036 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "107it [23:48,  9.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGPC_20220101_20230802.csv done!\n",
      "Download time: 12.010 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "108it [24:06, 11.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGXT_20220101_20230802.csv done!\n",
      "Download time: 17.695 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "109it [24:10,  9.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGOW_20220101_20230802.csv done!\n",
      "Download time: 4.541 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "110it [24:11,  6.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGUY_20220101_20230802.csv done!\n",
      "Download time: 0.454 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "111it [24:33, 11.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGDY_20220101_20230802.csv done!\n",
      "Download time: 21.984 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "112it [24:52, 13.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GB_EGSY_20220101_20230802.csv done!\n",
      "Download time: 19.512 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\"\"\"OLD_VERSION\"\"\"\n",
    "\n",
    "\n",
    "country = [\"GB\",]\n",
    "url_site_list=[]\n",
    "id_list=[] \n",
    "start_date =  datetime.datetime(2022,1,1)\n",
    "end_date =  datetime.datetime(2023,8, 2)\n",
    "\n",
    "gb_df = get_all_station_by_network(country)\n",
    "url_site_list,id_list = get_data_url(gb_df,start_date,end_date)\n",
    "for url_site, station_id in tqdm(zip(url_site_list, id_list)):\n",
    "    start_time = time.time()  # Record start time\n",
    "    save_data(url_site, \"GB\",station_id, start_date, end_date)\n",
    "    end_time = time.time()  # Record end time\n",
    "    download_time = end_time - start_time\n",
    "    print(f'Download time: {download_time:.3f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7559f03f-06e5-43bf-aca6-0af582f4b4f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_save_data(url_site, country, station, startts, endts):\n",
    "    start_time = time.time()  # Record start time\n",
    "    data = download_data(url_site)\n",
    "    end_time = time.time()  # Record end time\n",
    "    download_time = end_time - start_time\n",
    "\n",
    "    if data:\n",
    "        save_data(url_site, country, station, startts, endts)\n",
    "        print(f'{station} - Download time: {download_time:.3f} s')\n",
    "    else:\n",
    "        print(f'{station} - Download failed')\n",
    "\n",
    "def download_and_save_data_thread(args):\n",
    "    url_site, country, station_id, startts, endts = args\n",
    "    download_and_save_data(url_site, country, station_id, startts, endts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c2048d06-856f-4d0d-8533-9cfc6eb6ca05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54fc1d52-33b8-4dd6-a267-a14c12468be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# UK example\n",
    "country = [\n",
    "    \"GB\",\n",
    "]\n",
    "\n",
    "start_date = datetime.datetime(2022, 1, 1)\n",
    "end_date = datetime.datetime(2023, 8, 2)\n",
    "\n",
    "gb_df = get_all_station_by_network(country)\n",
    "url_site_list, id_list = get_data_url(gb_df, start_date, end_date)\n",
    "args_list = [\n",
    "    (url_site, \"GB\", station_id, start_date, end_date)\n",
    "    for url_site, station_id in zip(url_site_list, id_list)\n",
    "]\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor() as executor:\n",
    "    executor.map(download_and_save_data_thread, args_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7f6762e0-04f4-4555-88ec-725c96f8a6ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder '/Users/ww721/JupyterNotebookPath/IRP_20220602/irp_ww721_bakcup/data/GB_ASOS_DATA' already exists.\n"
     ]
    }
   ],
   "source": [
    "# UK example\n",
    "country = [\n",
    "    \"GB\",\n",
    "]\n",
    "\n",
    "start_date = datetime.datetime(2022, 1, 1)\n",
    "end_date = datetime.datetime(2023, 8, 2)\n",
    "\n",
    "gb_df = get_all_station_by_network(country)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c76ffc-bfd7-43f3-aa19-1a914808eb34",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}