ISSUE-114 Extend Podaacpy to OceanColor Data

README.rst

@@ -22,7 +22,7 @@ What does podaacpy offer?
 -------------------------
 
 The library provides a Python toolkit for interacting with all
-[PO.DAAC Web Services v3.2.2 API's](http:https://podaac.jpl.nasa.gov/ws), namely
+[PO.DAAC Web Services v3.2.2 APIs](http:https://podaac.jpl.nasa.gov/ws), namely
 
 - `PO.DAAC Web Services <https://podaac.jpl.nasa.gov/ws/>`__: services
  include
@@ -62,6 +62,13 @@ The library provides a Python toolkit for interacting with all
  <https://podaac-tools.jpl.nasa.gov/hitide/>`__: allows users to subset 
  and download popular PO.DAAC level 2 (swath) datasets.
 
+Additionally, Podaacpy provides the following ocean-related data services 
+
+- `NASA OceanColor Web <https://oceancolor.gsfc.nasa.gov>`_:
+
+- `File Search <https://oceandata.sci.gsfc.nasa.gov/api/file_search>`_ - locate publically available files within the NASA Ocean Data Processing System (ODPS)
+- `Bulk data downloads via HTTP <https://oceancolor.gsfc.nasa.gov/forum/oceancolor/topic_show.pl?pid=12520>`_ - mimic FTP bulk data downloads using the `HTTP-based data distribution server <https://oceandata.sci.gsfc.nasa.gov>`_.
+
 Installation
 ------------
 
@@ -153,10 +160,8 @@ Copyright and Export Classification
 
 .. |license| image:: https://img.shields.io/github/license/nasa/podaacpy.svg?maxAge=2592000
  :target: http:https://www.apache.org/licenses/LICENSE-2.0
-.. |Python3| image:: https://img.shields.io/badge/python-3-blue.svg
- :target: https://www.python.org/downloads/
-.. |Python2| image:: https://img.shields.io/badge/python-2-blue.svg
- :target: https://www.python.org/downloads/
+.. |pyversion| image:: https://img.shields.io/pypi/pyversions/podaacpy.svg
+ :target: https://pypi.python.org/pypi/podaacpy
 .. |PyPI| image:: https://img.shields.io/pypi/v/podaacpy.svg?maxAge=2592000?style=plastic
  :target: https://pypi.python.org/pypi/podaacpy
 .. |documentation| image:: https://readthedocs.org/projects/podaacpy/badge/?version=latest

docs/source/index.rst

@@ -27,6 +27,7 @@ Contents:
  webservices
  mcc
  l2ss
+ oceancolor
 
 
 

docs/source/introduction.rst

@@ -42,4 +42,10 @@ The library provides a Python toolkit for interacting with all of PO.DAACs API's
  * `Granule Subset <http:https://podaac.jpl.nasa.gov/ws/subset/granule/index.html>`_ - Subset Granule service allows users to submit subset jobs
  * `Subset Status <http:https://podaac.jpl.nasa.gov/ws/subset/status/index.html>`_ - Subset Granule Status service allows users to check the status of submitted subset job
 
-* `Metadata Compliance Checker <http:https://podaac-uat.jpl.nasa.gov/mcc>`_: an online tool and web service designed to check and validate the contents of netCDF and HDF granules for the Climate and Forecast (CF) and Attribute Convention for Dataset Discovery (ACDD) metadata conventions.
+* `Metadata Compliance Checker <http:https://podaac-uat.jpl.nasa.gov/mcc>`_: an online tool and web service designed to check and validate the contents of netCDF and HDF granules for the Climate and Forecast (CF) and Attribute Convention for Dataset Discovery (ACDD) metadata conventions.
+
+Additionally, Podaacpy provides the following ocean-related data services 
+* `NASA OceanColor Web <https://oceancolor.gsfc.nasa.gov>`_:
+
+ * `File Search <https://oceandata.sci.gsfc.nasa.gov/api/file_search>`_ - locate publically available files within the NASA Ocean Data Processing System (ODPS)
+ * `Bulk data downloads via HTTP <https://oceancolor.gsfc.nasa.gov/forum/oceancolor/topic_show.pl?pid=12520>`_ - mimic FTP bulk data downloads using the `HTTP-based data distribution server <https://oceandata.sci.gsfc.nasa.gov>`_.

docs/source/oceancolor.rst

@@ -0,0 +1,20 @@
+.. # encoding: utf-8
+ # Copyright 2016 California Institute of Technology.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http:https://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ 
+NASA OceanColor API
+*******************
+
+.. autoclass:: oceancolor.OceanColor
+ :members:

podaac/oceancolor.py

@@ -0,0 +1,151 @@
+# Copyright 2016 California Institute of Technology.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+from future.moves.urllib.request import urlretrieve
+from future.moves.urllib.parse import urlparse
+import os
+
+SEARCH_URL = 'https://oceandata.sci.gsfc.nasa.gov/api/file_search?'
+#GET_URL = 'https://oceandata.sci.gsfc.nasa.gov/cgi/getfile/'
+HEADERS = {
+ 'User-Agent': 'Podaacpy Python Library'
+}
+
+class OceanColor:
+
+ def __init__(self):
+ self.SEARCH_URL = 'https://oceandata.sci.gsfc.nasa.gov/api/file_search?'
+ #self.GET_URL = 'https://oceandata.sci.gsfc.nasa.gov/cgi/getfile/'
+
+ def file_search(self, sensor='', sdate='', edate='', dtype='', add_url='1', results_as_file='1',
+ search='', sub_id='', std_only='1', cksum='', output_format='json'):
+ '''File search service retrieves publically available files within the \
+ NASA Ocean Data Processing System.
+
+ :param sensor: mission name. valid options include: aquarius, seawifs, \
+ aqua, terra, meris, octs, czcs, hico, viirs
+ :type sensor: :mod:`string`
+
+ :param sdate: start date for a search
+ :type sdate: :mod:`string`
+
+ :param edate: end date for a search
+ :type edate: :mod:`string`
+
+ :param dtype: data type (i.e. level). valid options: L0, L1, L2, L3b \
+ (for binned data), L3m (for mapped data), MET (for ancillary \
+ data), misc (for sundry products)
+ :type dtype: :mod:`string`
+
+ :param add_url: include full url in search result (boolean, \
+ 1=yes, 0=no)
+ :type add_url: :mod:`string`
+
+ :param results_as_file: return results as a test file \
+ listing (boolean, 1=yes, 0=no, thus returns and HTML page)
+ :type results_as_file: :mod:`string`
+
+ :param search: text string search
+ :type search: :mod:`string`
+
+ :param sub_id: non-extracted subscription ID to search
+ :type sub_id: :mod:`string`
+
+ :param std_only: restrict results to standard products \
+ (i.e. ignore extracts, regional processings, etc.; boolean)
+ :type std_only: :mod:`string`
+
+ :param cksum: return a checksum file for search results \
+ (boolean; sha1sums except for Aquarius soil moisture \
+ products which are md5sums; forces results_as_file; ignores addurl)
+ :type cksum: :mod:`string`
+
+ :param output_format: valid options are: 'json', 'txt' and 'html'
+ :type output_format: :mod:`string`
+
+ :returns: by default a json response based on the requested 'output_format'. Options \
+ are 'json, 'txt' and 'html'.
+
+ '''
+ try:
+ url = SEARCH_URL
+ if sensor:
+ url = url + 'sensor=' + sensor
+ else:
+ raise Exception("'sensor' parameter is required!")
+ if sdate:
+ url = url + '&sdate=' + sdate
+ if edate:
+ url = url + '&edate=' + edate
+ if dtype:
+ url = url + '&dtype=' + dtype
+ url = url + '&addurl=' + str(add_url)
+ url = url + '&results_as_file=' + str(results_as_file)
+ if search:
+ url = url + '&search=' + search
+ elif sub_id:
+ url = url + '&subID=' + sub_id
+ else:
+ raise Exception("Either 'search' or 'sub_id' parameter is required!")
+ url = url + '&std_only=' + str(std_only)
+ if cksum:
+ url = url + '&cksum=' + cksum
+ url = url + '&format=' + output_format
+
+ response = requests.post(url, headers=HEADERS)
+ status_codes = [404, 400, 503, 408]
+ if response.status_code in status_codes:
+ response.raise_for_status()
+
+ except requests.exceptions.HTTPError as error:
+ print(error)
+ raise
+
+ return response.text
+
+ def get_file(self, url='', path=''):
+ '''It is possible to mimic FTP bulk data downloads using the \
+ HTTP-based data distribution server at https://oceandata.sci.gsfc.nasa.gov.
+
+ :param url: a single file name which can be obtained by calling #file_search() \
+ an example would be \
+ https://oceandata.sci.gsfc.nasa.gov/cgi/getfile/O1997001.L3b_DAY_CHL.nc
+ :type url: :mod:`string`
+
+ :param path: Destination directory into which the granule \
+ needs to be downloaded.
+ :type path: :mod:`string`
+
+ :returns: a file object downloaded from the \
+ HTTP-based data distribution server at https://oceandata.sci.gsfc.nasa.gov.
+
+ '''
+ try:
+ #url = GET_URL
+ if url:
+ url = url
+ else:
+ raise Exception("'file' parameter is required!")
+ file = os.path.basename(urlparse(url).path)
+ if path == '':
+ path = os.path.join(os.path.dirname(__file__), file)
+ else:
+ path = path + '/' + file
+ urlretrieve(url, path)
+ print("Downloaded '%s' to '%s'" % (file, path))
+ return file
+
+ except Exception:
+ raise

podaac/tests/oceancolor_test.py

@@ -0,0 +1,51 @@
+# Copyright 2016 California Institute of Technology.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http:https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..oceancolor import OceanColor
+import os
+import requests
+import xml.etree.ElementTree as ET
+from nose.tools import assert_raises
+import unittest
+from future.moves.urllib.error import HTTPError
+
+
+class TestOceanColor(unittest.TestCase):
+
+ @classmethod
+ def setUp(self):
+ self.oceancolor = OceanColor()
+
+ # test case for the function file_search()
+ def test_file_search(self):
+ data = self.oceancolor.file_search(sensor='octs', sdate='1996-11-01', edate='1997-01-01',
+ dtype='L3b', add_url='1', results_as_file='1', search='*DAY_CHL*')
+
+ assert data != None
+ print(data)
+ assert type(data) is str
+ assert len(data) != 0
+
+ # test case for the function get_file(()
+ def test_get_file(self):
+ url = 'https://oceandata.sci.gsfc.nasa.gov/cgi/getfile/O1996307.L3b_DAY_CHL.nc'
+ path = os.path.dirname(os.path.abspath(__file__))
+ granule_name = self.oceancolor.get_file(url, path)
+
+ assert granule_name != None
+ assert_raises(Exception, self.oceancolor.get_file,
+ url='ABCDEF')
+
+ path = os.path.join(os.path.dirname(__file__), granule_name)
+ os.remove(path)