Read dates from input dataset and loop over those instead of using it…

…erator.

Iterator did stop when exception occurs.

CHANGES.rst

@@ -2,6 +2,11 @@
 Changelog
 =========
 
+Version 0.3
+===========
+
+- Enable image to timeseries conversion if missing images are encountered.
+
 Version 0.2
 ===========
 

repurpose/img2ts.py

@@ -391,23 +391,18 @@ def img_bulk(self):
  # image counter
  read_images = 0
 
- # call the img_iterator in a infinite loop so that we are able
- # to catch exceptions properly
- # if an exception bubbles up the iterator stops and
- # we must read the next image.
- img_generator = self.imgin.iter_images(
- self.startdate, self.enddate, **self.input_kwargs)
- img_iterator = iter(img_generator)
- while True:
+ dates = self.imgin.tstamps_for_daterange(self.startdate,
+ self.enddate)
+ for date in dates:
  try:
  (input_img, metadata,
  image_datetime, lon,
- lat, time_arr) = next(img_iterator)
+ lat, time_arr) = self.imgin.read(date, **self.input_kwargs)
  except IOError as e:
- logging.log(logging.INFO, e.message)
+ msg = "I/O error({0}): {1}".format(e.errno,
+ e.strerror)
+ logging.log(logging.INFO, msg)
  continue
- except StopIteration:
- break
  read_images += 1
  logging.log(logging.INFO, "read" + image_datetime.isoformat())
  if self.resample:

tests/test_img2ts.py

@@ -50,6 +50,8 @@ class TestImageDataset(ImageBase):
 
  def read(self, timestamp=None, additional_kw=None):
 
+ if timestamp == datetime(2016, 1, 1):
+ raise IOError("no data for day")
  # 2x2 pixels around zero lat, lon
  return Image(np.array([0.5, 0.5,
  -0.5, -0.5]),
@@ -133,3 +135,33 @@ def test_img2ts_daily_no_resampling():
  assert dates_should == list(ts['time'])
  nptest.assert_allclose(ds.dataset.variables['location_id'][:],
  np.array([0, 1, 2, 3]))
+
+
+def test_img2ts_daily_no_resampling_missing_day():
+ """
+ Test resampling over missing day 2016-01-01 (see reader above)
+ """
+ input_grid = BasicGrid(np.array([0.5, 0.5, -0.5, -0.5]),
+ np.array([1, -1, 1, -1]), )
+
+ outputpath = tempfile.mkdtemp()
+ start = datetime(2015, 12, 5)
+ end = datetime(2016, 1, 10)
+
+ ds_in = TestMultiTemporalImageDatasetDaily()
+ img2ts = Img2Ts(ds_in,
+ outputpath, start, end, imgbuffer=15,
+ input_grid=input_grid)
+
+ ts_should = np.concatenate([np.arange(5, 32, dtype=np.float),
+ np.arange(2, 11, dtype=np.float)])
+ dates_should = ds_in.tstamps_for_daterange(start, end)
+ dates_should.remove(datetime(2016, 1, 1))
+ img2ts.calc()
+ ts_file = os.path.join(outputpath, '0000.nc')
+ with OrthoMultiTs(ts_file) as ds:
+ ts = ds.read_ts('var1', 0)
+ nptest.assert_allclose(ts['var1'], ts_should)
+ assert dates_should == list(ts['time'])
+ nptest.assert_allclose(ds.dataset.variables['location_id'][:],
+ np.array([0, 1, 2, 3]))