merge request for release 0.3.1

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/test-data"]
+ path = tests/test-data
+ url = https://github.com/TUW-GEO/pytesmo-test-data.git

CHANGES.md

@@ -1,3 +1,8 @@
+# v0.3.1, 2015-07-09
+* added validation framework and example on how to use it
+* fix bug (issue #51) in temporal matching
+* added test data as git submodule
+
 # v0.3.0, 2015-05-26
 * added calculation of pearson R confidence intervals based on fisher z transform
 * ISMN reader can now get the data coverage for stations and networks

docs/data_preparation_ASCAT_ISMN.ipynb

@@ -0,0 +1,113 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Example data preparation ASCAT - ISMN"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The validation framework allows users to prepare their data before the validation is performed by using the **DataPreparation** class which must contain two methods:\n",
+ "* ***prep_reference***, has at least one parameter - the reference dataframe\n",
+ "* ***prep_other***, has at least two parameters - the other dataframe and the name of the dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "class DataPreparation(object):\n",
+ " \"\"\"\n",
+ " Class for preparing the data before validation.\n",
+ " \"\"\"\n",
+ " @staticmethod\n",
+ " def prep_reference(reference):\n",
+ " \"\"\"\n",
+ " Static method used to prepare the reference dataset (ISMN).\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " reference : pandas.DataFrame\n",
+ " ISMN data.\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " reference : pandas.DataFrame\n",
+ " Masked reference.\n",
+ " \"\"\"\n",
+ " return reference\n",
+ "\n",
+ " @staticmethod\n",
+ " def prep_other(other, other_name, mask_snow=80, mask_frozen=80, mask_ssf=[0, 1]):\n",
+ " \"\"\"\n",
+ " Static method used to prepare the other datasets (ASCAT).\n",
+ "\n",
+ " Parameters\n",
+ " ----------\n",
+ " other : pandas.DataFrame\n",
+ " Containing at least the fields: sm, frozen_prob, snow_prob, ssf.\n",
+ " other_name : string\n",
+ " ASCAT.\n",
+ " mask_snow : int, optional\n",
+ " If set, all the observations with snow probability > mask_snow\n",
+ " are removed from the result. Default: 80.\n",
+ " mask_frozen : int, optional\n",
+ " If set, all the observations with frozen probability > mask_frozen\n",
+ " are removed from the result. Default: 80.\n",
+ " mask_ssf : list, optional\n",
+ " If set, all the observations with ssf != mask_ssf are removed from\n",
+ " the result. Default: [0, 1].\n",
+ "\n",
+ " Returns\n",
+ " -------\n",
+ " reference : pandas.DataFrame\n",
+ " Masked reference.\n",
+ " \"\"\"\n",
+ " if other_name == 'ASCAT':\n",
+ "\n",
+ " # mask frozen\n",
+ " if mask_frozen is not None:\n",
+ " other = other[other['frozen_prob'] < mask_frozen]\n",
+ "\n",
+ " # mask snow\n",
+ " if mask_snow is not None:\n",
+ " other = other[other['snow_prob'] < mask_snow]\n",
+ "\n",
+ " # mask ssf\n",
+ " if mask_ssf is not None:\n",
+ " other = other[(other['ssf'] == mask_ssf[0]) |\n",
+ " (other['ssf'] == mask_ssf[1])]\n",
+ " return other"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

docs/data_preparation_ASCAT_ISMN.rst

@@ -0,0 +1,74 @@
+
+Example data preparation ASCAT - ISMN
+=====================================
+
+The validation framework allows users to prepare their data before the
+validation is performed by using the **DataPreparation** class which
+must contain two methods: \* ***prep\_reference***, has at least one
+parameter - the reference dataframe \* ***prep\_other***, has at least
+two parameters - the other dataframe and the name of the dataset
+
+.. code:: python
+
+ class DataPreparation(object):
+ """
+ Class for preparing the data before validation.
+ """
+ @staticmethod
+ def prep_reference(reference):
+ """
+ Static method used to prepare the reference dataset (ISMN).
+
+ Parameters
+ ----------
+ reference : pandas.DataFrame
+ ISMN data.
+
+ Returns
+ -------
+ reference : pandas.DataFrame
+ Masked reference.
+ """
+ return reference
+
+ @staticmethod
+ def prep_other(other, other_name, mask_snow=80, mask_frozen=80, mask_ssf=[0, 1]):
+ """
+ Static method used to prepare the other datasets (ASCAT).
+
+ Parameters
+ ----------
+ other : pandas.DataFrame
+ Containing at least the fields: sm, frozen_prob, snow_prob, ssf.
+ other_name : string
+ ASCAT.
+ mask_snow : int, optional
+ If set, all the observations with snow probability > mask_snow
+ are removed from the result. Default: 80.
+ mask_frozen : int, optional
+ If set, all the observations with frozen probability > mask_frozen
+ are removed from the result. Default: 80.
+ mask_ssf : list, optional
+ If set, all the observations with ssf != mask_ssf are removed from
+ the result. Default: [0, 1].
+
+ Returns
+ -------
+ reference : pandas.DataFrame
+ Masked reference.
+ """
+ if other_name == 'ASCAT':
+
+ # mask frozen
+ if mask_frozen is not None:
+ other = other[other['frozen_prob'] < mask_frozen]
+
+ # mask snow
+ if mask_snow is not None:
+ other = other[other['snow_prob'] < mask_snow]
+
+ # mask ssf
+ if mask_ssf is not None:
+ other = other[(other['ssf'] == mask_ssf[0]) |
+ (other['ssf'] == mask_ssf[1])]
+ return other

docs/examples.rst

@@ -79,6 +79,23 @@ It can be found in the /examples folder of the pytesmo package under the name co
 .. include::
  compare_ASCAT_ISMN.rst
 
+Comparison of ASCAT and ISMN using the pytesmo validation framework
+===================================================================
+
+The pytesmo validation framework uses IPython to do a comparison of two or more
+datasets using parallel processing. The validation frameworks also stores the
+results as a CF compliant netCDF file.
+
+.. include::
+ setup_validation_ASCAT_ISMN.rst
+
+If custom data preparation e.g. special masking is necessary before the data
+comparison then a custom DataPreparation class can be used. The following shows
+an example of such a class:
+
+.. include::
+ data_preparation_ASCAT_ISMN.rst
+
 Triple collocation and triple collocation based scaling
 =======================================================
 

docs/introduction.rst

@@ -164,8 +164,7 @@ Windows
 
 A relatively easy way to install everything but matplotlib-basemap and netCDF4
 is to install `WinPython <https://winpython.github.io/>`_ and then
-download basemap, netCDF4 and, if you want to read the H14 product, the pygrib
-installer from http:https://www.lfd.uci.edu/~gohlke/pythonlibs/. Add them to your
+download basemap and netCDF4 from http:https://www.lfd.uci.edu/~gohlke/pythonlibs/. Add them to your
 winpython installation using the winpython Control Panel.
 
 Just make sure that you download both for the same architecture (32/64 bit) and