{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "cdsapi_public_demo.ipynb", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "markdown", "source": [ "## Connect to Google Drive" ], "metadata": { "id": "hKGgIcGzvX9f" } }, { "cell_type": "code", "source": [ "from google.colab import drive\n", "drive.mount('/content/drive/')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "NBpZ6LAboiqh", "outputId": "4a4447aa-fab3-4a13-95cf-e2cc286ad28f" }, "execution_count": 28, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n" ] } ] }, { "cell_type": "markdown", "source": [ "## Authorization" ], "metadata": { "id": "5WlIW3-LvbCw" } }, { "cell_type": "code", "source": [ "# make sure\n", "# 1) Login to Copernicus website\n", "# 2) Accept the license terms for the account\n", "# 3) Accept the termsof 'CMIP6 - Data Access - Terms of Use' at https://cds.climate.copernicus.eu/cdsapp/#!/terms/cmip6-wps\n", "# 4) Use UID and API Key, not login details\n", "\n", "url = 'url: https://cds.climate.copernicus.eu/api/v2'\n", "key = 'key: your_key'\n", "\n", "with open('/root/.cdsapirc', 'w') as f:\n", " f.write('\\n'.join([url, key]))\n", "\n", "with open('/root/.cdsapirc') as f:\n", " print(f.read())" ], "metadata": { "id": "hExkzZH6olj4" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "## Install Required Libraries" ], "metadata": { "id": "XlLcIU1zvk7L" } }, { "cell_type": "code", "source": [ "! pip install rasterio\n", "! pip install rioxarray\n", "! pip install geopandas\n", "! pip install cartopy\n", "! pip install shapely\n", "! pip install rasterstats\n", "! pip install tqdm\n", "! pip install cdsapi\n", "\n", "# restart runtime\n", "# os.kill(os.getpid(), 9)" ], "metadata": { "id": "ho4MwxHEvGjN" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "## Import Libraries" ], "metadata": { "id": "LRJ3iCA-vqKZ" } }, { "cell_type": "code", "source": [ "import datetime\n", "from pathlib import Path \n", "import geopandas as gpd\n", "import pandas as pd\n", "import numpy as np\n", "import urllib\n", "import requests\n", "import sys\n", "import time\n", "from rasterstats import zonal_stats\n", "import rasterio\n", "from tqdm import tqdm \n", "import zipfile\n", "import io \n", "from shapely.geometry.polygon import Polygon \n", "from shapely.geometry.multipolygon import MultiPolygon\n", "import cdsapi\n", "import os\n", "import shutil\n", "import xarray as xr\n", "import rioxarray\n", "from shapely.geometry import mapping\n", "import cartopy.crs as ccrs\n", "import cftime\n", "import cartopy.feature as cfeature\n", "from shapely import wkt\n", "import matplotlib.pyplot as plt\n", "import warnings\n", "warnings.simplefilter(action='ignore')" ], "metadata": { "id": "0F9X6FqPot_k" }, "execution_count": 5, "outputs": [] }, { "cell_type": "markdown", "source": [ "## Define a Python class to extract CMIP climate variables via cdsapi" ], "metadata": { "id": "cpScWby3vwCD" } }, { "cell_type": "code", "execution_count": 34, "metadata": { "id": "gfMOxJn6m4Uj" }, "outputs": [], "source": [ "# Import cdsapi and other geospatial libraries\n", "# Note: to run cdsapi properly, make sure the following actions have been performed:\n", "# 1) Login to Copernicus website\n", "# 2) Accept the license terms for the account\n", "# 3) Accept the termsof 'CMIP6 - Data Access - Terms of Use' at https://cds.climate.copernicus.eu/cdsapp/#!/terms/cmip6-wps\n", "# 4) Input UID and API Key for data access. More details: https://cds.climate.copernicus.eu/api-how-to\n", "\n", "class CmipExtractor():\n", " variable: str\n", " experiment: str\n", " model: str\n", " roi: gpd.GeoDataFrame\n", " start: str\n", " end: str\n", " \n", " def __init__(self,variable,experiment,start,end,bounds,resolution,folder,model=None):\n", " self.variable = variable\n", " self.experiment = experiment\n", " self.bounds = bounds\n", " self.date = f\"{start}/{end}\"\n", " self.model = model\n", " self.download_folder = folder\n", " self.resolution = resolution\n", " self.nc_path = None\n", " \n", " def __str__(self):\n", " return f\"Variable: {self.variable}\\nExperiment: {self.experiment}\\nGCM Model: {self.model}\"\n", "\n", " @property\n", " def var_id(self) -> str:\n", " \"\"\"\n", " Get CMIP6 Variable ID\n", " Description\n", " ----------\n", " Extract variable id for variable names\n", " Parameters\n", " ----------\n", " variable: str\n", " variable name\n", " Returns\n", " -------\n", " id: str\n", " If variable exist in the dict of common variables, id will be derived. Otherwise, the input will be assumed to be id itself.\n", " \"\"\"\n", " # dictionary for common climate variables (cmip6)\n", " # https://confluence.ecmwf.int/display/CKB/CMIP6%3A+Global+climate+projections#CMIP6:Globalclimateprojections-Parameterlistings\n", "\n", " var_dict = {\n", " \"near_surface_air_temperature\": \"tas\",\n", " \"daily_maximum_near_surface_air_temperature\": \"tasmax\",\n", " \"daily_minimum_near_surface_air_temperature\": \"tasmin\",\n", " \"surface_temperature\": \"ts\",\n", " \"sea_level_pressure\":\t\"psl\",\n", " \"surface_air_pressure\":\t\"ps\",\n", " \"eastward_near_surface_wind\":\t\"uas\",\n", " \"northward_near_surface_wind\": \"vas\",\n", " \"precipitation\": \"pr\",\n", " \"evaporation_including_sublimation_and_transpiration\": \"evspsbl\",\n", " \"air_temperature\": \"ta\"\n", " }\n", "\n", " # transform to id\n", " var = self.variable\n", " id = var_dict[var] if var in var_dict else var\n", " return id\n", "\n", " def set_model(self, model):\n", " \"\"\"\n", " Set GCM after init\n", " \"\"\"\n", " self.model = model\n", " return self\n", "\n", " def unzipFile(self, path_to_zipfile: str, unzip_directory: str) -> None:\n", " \"\"\"\n", " Unzip netCDF file\n", " Description\n", " ----------\n", " Extract zip file in the target directory\n", " Parameters\n", " ----------\n", " path_to_zipfile: str\n", " Relative folder path of the zip file\n", " unzip_directory: str\n", " Relative folder path for the unzipped output\n", " Returns\n", " -------\n", " An unzipped file will be extracted at the target location\n", " \"\"\"\n", " extensions = ('.nc')\n", " # get file\n", " zip_file = zipfile.ZipFile(path_to_zipfile, 'r')\n", " for file in zip_file.namelist():\n", " # grab and unzip\n", " if file.endswith(extensions):\n", " zip_file.extract(file, unzip_directory)\n", " zip_file.close() \n", "\n", " def cds_retrieval(self) -> None:\n", " \"\"\"\n", " Retrieval CMIP6 data\n", " Description\n", " ----------\n", " Retrieval CMIP6 data from CDS using Python API for a single variable, experiment, and model\n", " Parameters\n", " ----------\n", " climate_folder: path\n", " base folder for raw file to be downloaded\n", " variable: str\n", " variable name\n", " experiment: str\n", " CMIP6 scenario\n", " model: str\n", " name of the climate model\n", " bounds: list\n", " miny, maxx, maxy, minx (in this order)\n", " Returns\n", " -------\n", " netcdf_fn: str\n", " Path for downloaded netCDF file\n", " \"\"\"\n", " model = self.model\n", " experiment = self.experiment\n", " variable = self.variable\n", " date = self.date\n", " bounds = self.bounds\n", " resolution = self.resolution\n", "\n", " # output file path\n", " netcdf_zip = os.path.join(self.download_folder, variable, experiment, f'{variable}_{experiment}_{model}.zip')\n", " \n", " # Download climate data (single variable) using cdsapi API\n", " if not os.path.exists(netcdf_zip):\n", " # cdsapi\n", " try:\n", " c = cdsapi.Client()\n", " c.retrieve(\n", " 'projections-cmip6',\n", " {\n", " 'temporal_resolution': resolution,\n", " 'experiment': experiment,\n", " 'level': 'single_levels',\n", " 'variable': variable,\n", " 'model': model,\n", " 'area': bounds,\n", " 'date': date,\n", " 'format': 'zip'\n", " },\n", " netcdf_zip\n", " ) \n", " except Exception as e: \n", " print('Error: {}. Continue to the next retrieval.'.format(e))\n", " \n", " # Unzip downloaded file\n", " unzip_folder = os.path.join('intermediate', 'Climate', variable, experiment, f'{experiment}_{model}')\n", " if os.path.exists(netcdf_zip) and not os.path.exists(unzip_folder):\n", " print('unzip folder')\n", " self.unzipFile(netcdf_zip, unzip_folder)\n", "\n", " # Return NetCDF file\n", " netcdf_fn = list(Path(unzip_folder).glob('*.nc'))\n", " assert len(netcdf_fn) != 0, \"No netCDF file retrieved!\"\n", " \n", " self.nc_path = netcdf_fn[0]\n", "\n", " def __call__(self):\n", " CmipExtractor.cds_retrieval(self)\n", " ds = xr.open_dataset(self.nc_path)\n", " return ds" ] }, { "cell_type": "markdown", "source": [ "## Data Retrieval" ], "metadata": { "id": "d8sl4qfYwTSJ" } }, { "cell_type": "code", "source": [ "variable = 'precipitation'\n", "experiment = 'historical'\n", "bounds = [42.51, -124.92, 32.03, -113.63]\n", "res = 'monthly'\n", "start = '2014-01-01'\n", "end = '2015-01-01'\n", "\n", "extractor = CmipExtractor(variable=variable, experiment=experiment, start=start, end=end, bounds=bounds, resolution=res, folder='raw/Climate')" ], "metadata": { "id": "K563bqmSo3jH" }, "execution_count": 35, "outputs": [] }, { "cell_type": "code", "source": [ "print(extractor)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "bASWPJcJqmC_", "outputId": "8b5f4717-1450-48c3-fd93-a765dcd6d5b0" }, "execution_count": 17, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Variable: precipitation\n", "Experiment: historical\n", "GCM Model: None\n" ] } ] }, { "cell_type": "code", "source": [ "extractor.set_model('cams_csm1_0')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Q9t13ag4qzEN", "outputId": "97877cf8-58b3-43bc-d2f2-61e7e6886dc1" }, "execution_count": 36, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "<__main__.CmipExtractor at 0x7fc0afe58bd0>" ] }, "metadata": {}, "execution_count": 36 } ] }, { "cell_type": "code", "source": [ "print(extractor)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "En-ZmkAzrJTw", "outputId": "79a04757-6f69-4572-f76c-09bd4524d420" }, "execution_count": 22, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Variable: precipitation\n", "Experiment: historical\n", "GCM Model: cams_csm1_0\n" ] } ] }, { "cell_type": "markdown", "source": [ "## Retrieved Data" ], "metadata": { "id": "qU_kL0u3wY9K" } }, { "cell_type": "code", "source": [ "ds = extractor()\n", "ds.pr" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 763 }, "id": "ECo_a4Laq6c-", "outputId": "14bbafa3-b284-40b1-916e-6e4dbbd628a9" }, "execution_count": 38, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "\n", "array([[[3.441843e-05, 3.819044e-05, ..., 4.129862e-05, 3.687123e-05],\n", " [3.062375e-05, 4.721568e-05, ..., 5.041794e-05, 4.803744e-05],\n", " ...,\n", " [7.147809e-05, 7.341129e-05, ..., 4.763632e-05, 2.715065e-05],\n", " [8.370028e-05, 8.313973e-05, ..., 3.947216e-05, 3.388934e-05]],\n", "\n", " [[1.518748e-05, 1.760686e-05, ..., 1.640480e-05, 1.483301e-05],\n", " [1.965561e-05, 2.135964e-05, ..., 1.515514e-05, 1.549916e-05],\n", " ...,\n", " [1.321277e-04, 1.458548e-04, ..., 4.551357e-05, 2.671114e-05],\n", " [1.324733e-04, 1.496662e-04, ..., 3.936305e-05, 1.935098e-05]],\n", "\n", " ...,\n", "\n", " [[5.346413e-06, 5.933125e-06, ..., 4.637337e-06, 4.768269e-06],\n", " [8.292531e-06, 9.363459e-06, ..., 4.073476e-06, 4.468129e-06],\n", " ...,\n", " [3.761025e-05, 2.845018e-05, ..., 2.050767e-05, 1.003549e-05],\n", " [5.221746e-05, 4.273639e-05, ..., 2.362386e-05, 1.198931e-05]],\n", "\n", " [[1.322935e-05, 1.168174e-05, ..., 8.636327e-09, 3.209383e-07],\n", " [1.230150e-05, 1.112628e-05, ..., 2.934464e-07, 4.028364e-07],\n", " ...,\n", " [2.066193e-05, 1.679383e-05, ..., 1.104106e-05, 7.818568e-06],\n", " [2.695961e-05, 1.870549e-05, ..., 1.132469e-05, 8.904548e-06]]],\n", " dtype=float32)\n", "Coordinates:\n", " * time (time) object 1981-01-16 12:00:00 ... 2014-12-16 12:00:00\n", " * lat (lat) float64 33.08 34.21 35.33 36.45 37.57 38.69 39.81 40.93 42.06\n", " * lon (lon) float64 235.1 236.2 237.4 238.5 ... 241.9 243.0 244.1 245.2\n", "Attributes:\n", " standard_name: precipitation_flux\n", " long_name: Precipitation\n", " comment: includes both liquid and solid phases\n", " units: kg m-2 s-1\n", " original_name: precip\n", " cell_methods: area: time: mean\n", " cell_measures: area: areacella\n", " history: 2019-07-01T12:19:04Z altered by CMOR: Inverted axis: lat." ], "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'pr' (time: 408, lat: 9, lon: 10)>\n",
              "array([[[3.441843e-05, 3.819044e-05, ..., 4.129862e-05, 3.687123e-05],\n",
              "        [3.062375e-05, 4.721568e-05, ..., 5.041794e-05, 4.803744e-05],\n",
              "        ...,\n",
              "        [7.147809e-05, 7.341129e-05, ..., 4.763632e-05, 2.715065e-05],\n",
              "        [8.370028e-05, 8.313973e-05, ..., 3.947216e-05, 3.388934e-05]],\n",
              "\n",
              "       [[1.518748e-05, 1.760686e-05, ..., 1.640480e-05, 1.483301e-05],\n",
              "        [1.965561e-05, 2.135964e-05, ..., 1.515514e-05, 1.549916e-05],\n",
              "        ...,\n",
              "        [1.321277e-04, 1.458548e-04, ..., 4.551357e-05, 2.671114e-05],\n",
              "        [1.324733e-04, 1.496662e-04, ..., 3.936305e-05, 1.935098e-05]],\n",
              "\n",
              "       ...,\n",
              "\n",
              "       [[5.346413e-06, 5.933125e-06, ..., 4.637337e-06, 4.768269e-06],\n",
              "        [8.292531e-06, 9.363459e-06, ..., 4.073476e-06, 4.468129e-06],\n",
              "        ...,\n",
              "        [3.761025e-05, 2.845018e-05, ..., 2.050767e-05, 1.003549e-05],\n",
              "        [5.221746e-05, 4.273639e-05, ..., 2.362386e-05, 1.198931e-05]],\n",
              "\n",
              "       [[1.322935e-05, 1.168174e-05, ..., 8.636327e-09, 3.209383e-07],\n",
              "        [1.230150e-05, 1.112628e-05, ..., 2.934464e-07, 4.028364e-07],\n",
              "        ...,\n",
              "        [2.066193e-05, 1.679383e-05, ..., 1.104106e-05, 7.818568e-06],\n",
              "        [2.695961e-05, 1.870549e-05, ..., 1.132469e-05, 8.904548e-06]]],\n",
              "      dtype=float32)\n",
              "Coordinates:\n",
              "  * time     (time) object 1981-01-16 12:00:00 ... 2014-12-16 12:00:00\n",
              "  * lat      (lat) float64 33.08 34.21 35.33 36.45 37.57 38.69 39.81 40.93 42.06\n",
              "  * lon      (lon) float64 235.1 236.2 237.4 238.5 ... 241.9 243.0 244.1 245.2\n",
              "Attributes:\n",
              "    standard_name:  precipitation_flux\n",
              "    long_name:      Precipitation\n",
              "    comment:        includes both liquid and solid phases\n",
              "    units:          kg m-2 s-1\n",
              "    original_name:  precip\n",
              "    cell_methods:   area: time: mean\n",
              "    cell_measures:  area: areacella\n",
              "    history:        2019-07-01T12:19:04Z altered by CMOR: Inverted axis: lat.
" ] }, "metadata": {}, "execution_count": 38 } ] } ] }