using requests to check RTOFS urls (fixing redirect issue)

hyo2/soundspeed/__init__.py

@@ -12,7 +12,7 @@
 logger.addHandler(logging.NullHandler())
 
 name = "Sound Speed"
-__version__ = '2020.0.8'
+__version__ = '2020.0.9'
 __copyright__ = 'Copyright 2020 University of New Hampshire, Center for Coastal and Ocean Mapping'
 
 lib_info = LibInfo()

hyo2/soundspeed/atlas/rtofs.py

@@ -1,12 +1,10 @@
 from datetime import datetime as dt, date, timedelta
-from http import client
-from urllib import parse
-import socket
 import logging
 from typing import Optional, Union
 
 from netCDF4 import Dataset
 import numpy as np
+import requests
 
 from hyo2.abc.lib.progress.cli_progress import CliProgress
 
@@ -320,29 +318,29 @@ def __repr__(self) -> str:
  @staticmethod
  def _check_url(url: str) -> bool:
  try:
- p = parse.urlparse(url)
- conn = client.HTTPConnection(p.netloc)
- conn.request('HEAD', p.path)
- resp = conn.getresponse()
- conn.close()
- # logger.debug("passed url: %s -> %s" % (url, resp.status))
-
- except socket.error as e:
+ resp = requests.get(url, allow_redirects=True, stream=True)
+ logger.debug("passed url: %s -> %s" % (url, resp.status_code))
+ if resp.status_code == 200:
+  return True
+ else:
+  return False
+
+ except Exception as e:
  logger.warning("while checking %s, %s" % (url, e))
  return False
 
- return resp.status < 400
-
  @staticmethod
  def _build_check_urls(input_date: date) -> tuple:
  """make up the url to use for salinity and temperature"""
  # Primary server: http:https://nomads.ncep.noaa.gov/pub/data/nccf/com/rtofs/prod/rtofs.20160410/
  url_temp = 'https://nomads.ncep.noaa.gov/pub/data/nccf/com/rtofs/prod/rtofs.%s/' \
  'rtofs_glo_3dz_n024_daily_3ztio.nc' \
  % input_date.strftime("%Y%m%d")
+ logger.debug("check temp: %s" % url_temp)
  url_sal = 'https://nomads.ncep.noaa.gov/pub/data/nccf/com/rtofs/prod/rtofs.%s/' \
  'rtofs_glo_3dz_n024_daily_3zsio.nc' \
  % input_date.strftime("%Y%m%d")
+ logger.debug("check sal: %s" % url_sal)
  return url_temp, url_sal
 
  @staticmethod
@@ -384,6 +382,7 @@ def _download_files(self, datestamp: dt, server_mode: bool = False):
  url_ck_temp, url_ck_sal = self._build_check_urls(datestamp)
  if not self._check_url(url_ck_temp) or not self._check_url(url_ck_sal):
 
+ logger.info('issue with %s -> trying with the previous day' % datestamp)
  datestamp -= timedelta(days=1)
  url_ck_temp, url_ck_sal = self._build_check_urls(datestamp)
 
@@ -405,8 +404,8 @@ def _download_files(self, datestamp: dt, server_mode: bool = False):
  progress.update(80)
  self._day_idx = 2 # usually 3 1-day steps
 
- except (RuntimeError, IOError):
- logger.warning("unable to access data: %s" % datestamp.strftime("%Y%m%d"))
+ except (RuntimeError, IOError) as e:
+ logger.warning("unable to access data: %s -> %s" % (datestamp.strftime("%Y%m%d"), e))
  self.clear_data()
  progress.end()
  return False

hyo2/soundspeedmanager/__init__.py

@@ -11,7 +11,7 @@
 logger.addHandler(logging.NullHandler())
 
 name = "Sound Speed Manager"
-__version__ = "2020.0.8"
+__version__ = "2020.0.9"
 __copyright__ = "Copyright 2020 University of New Hampshire, Center for Coastal and Ocean Mapping"
 
 app_info = AppInfo()

hyo2/soundspeedsettings/__init__.py

@@ -11,7 +11,7 @@
 logger.addHandler(logging.NullHandler())
 
 name = "Sound Speed Settings"
-__version__ = '2020.0.8'
+__version__ = '2020.0.9'
 __copyright__ = 'Copyright 2020 University of New Hampshire, Center for Coastal and Ocean Mapping'
 
 app_info = AppInfo()

setup.py

@@ -60,6 +60,7 @@ def find_version(*file_paths):
  "netCDF4",
  "pillow",
  "pyserial",
+ "requests",
  "scipy"
  ],
  python_requires='>=3.5',