Update logging

src/repurpose/ts2img.py

@@ -130,6 +130,9 @@ class Ts2Img:
  afterwards. Then convert time series time stamps to deltas (>0) from
  the image time stamps and store them in a new image variable
  'timedelta_seconds'.
+ loglevel: str, optional (default: 'WARNING')
+ Logging level.
+ Must be one of 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'.
  """
 
  # Some variables are generated internally and cannot be used.
@@ -140,7 +143,7 @@ class Ts2Img:
 
  def __init__(self, ts_reader, img_grid, timestamps,
  variables=None, read_function='read',
- max_dist=18000, time_collocation=True):
+ max_dist=18000, time_collocation=True, loglevel="WARNING"):
 
  self.ts_reader = ts_reader
  self.img_grid: CellGrid = Regular3dimImageStack._eval_grid(img_grid)
@@ -155,6 +158,7 @@ def __init__(self, ts_reader, img_grid, timestamps,
  self.max_dist = max_dist
  self.time_collocation = time_collocation
 
+ self.loglevel = loglevel
  self.stack = None
 
 
@@ -191,7 +195,7 @@ def _read_nn(self, lon: float, lat: float) -> Union[pd.DataFrame, None]:
  logging.error(f"Error reading Time series data at "
  f"lon: {lon}, lat: {lat}: {e}")
  return None
- if self.variables is not None:
+ if (self.variables is not None) and (ts is not None):
  ts = ts.rename(columns=self.variables)[self.variables.values()]
  return ts
 
@@ -201,8 +205,8 @@ def _calc_chunk(self, timestamps, log_path=None, n_proc=1):
  See: self.calc
  """
  self.timestamps = timestamps
- logging.debug(f"Processing chunk from {timestamps[0]} to "
-  f"{timestamps[-1]}")
+ logging.info(f"Processing chunk from {timestamps[0]} to "
+ f"{timestamps[-1]}")
 
  # Transfer time series to images, parallel for cells
  STATIC_KWARGS = {'converter': self}
@@ -218,7 +222,7 @@ def _calc_chunk(self, timestamps, log_path=None, n_proc=1):
  stack = parallel_process_async(
  _convert, ITER_KWARGS, STATIC_KWARGS, n_proc=n_proc,
  show_progress_bars=True, log_path=log_path,
- debug_mode=False)
+ verbose=False, ignore_errors=True)
 
  stack = xr.combine_by_coords(stack)
 
@@ -445,5 +449,7 @@ def store_netcdf_images(self, path_out, fn_template=f"{datetime}.nc",
  'encoding': encoding},
  n_proc=n_proc,
  show_progress_bars=True,
+ verbose=False,
+ loglevel=self.loglevel,
  ignore_errors=True,
  )

src/repurpose/utils.py

@@ -53,7 +53,8 @@ def parallel_process_async(
  show_progress_bars=True,
  ignore_errors=False,
  log_path=None,
- debug_mode=False,
+ loglevel="WARNING",
+ verbose=False,
 ):
  """
  Applies the passed function to all elements of the passed iterables.
@@ -84,8 +85,11 @@ def parallel_process_async(
  Show how many iterables were processed already.
  log_path: str, optional (default: None)
  If provided, a log file is created in the passed directory.
- debug_mode: float, optional (default: False)
- Print logging messages to stdout, useful for debugging.
+ loglevel: str, optional (default: "WARNING")
+ Log level to use for logging. Must be one of
+ ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"].
+ verbose: float, optional (default: False)
+ Print all logging messages to stdout, useful for debugging.
 
  Returns
  -------
@@ -101,7 +105,7 @@ def parallel_process_async(
  if STATIC_KWARGS is None:
  STATIC_KWARGS = dict()
 
- if debug_mode:
+ if verbose:
  logger.setLevel('DEBUG')
  logger.addHandler(streamHandler)
 
@@ -116,7 +120,7 @@ def parallel_process_async(
  os.makedirs(os.path.dirname(log_file), exist_ok=True)
  logging.basicConfig(
  filename=log_file,
- level=logging.INFO,
+ level=loglevel.upper(),
  format="%(levelname)s %(asctime)s %(message)s",
  datefmt="%Y-%m-%d %H:%M:%S",
  )
@@ -187,7 +191,7 @@ def error(e) -> None:
  if pbar is not None:
  pbar.close()
 
- if debug_mode:
+ if verbose:
  logger.handlers.clear()
 
  handlers = logger.handlers[:]

tests/test_utils.py

@@ -1,5 +1,3 @@
-import logging
-import os
 import numpy as np
 import pandas as pd
 import time
@@ -31,21 +29,7 @@ def test_apply_to_elements():
  static_kwargs = {'p': 2}
  with tempfile.TemporaryDirectory() as log_path:
  res = parallel_process_async(
- func, iter_kwargs, static_kwargs, n_proc=2,
- show_progress_bars=False, debug_mode=True,
+ func, iter_kwargs, static_kwargs, n_proc=1,
+ show_progress_bars=False, verbose=False, loglevel="DEBUG",
  ignore_errors=True, log_path=log_path)
  assert sorted(res) == [1, 4, 9, 16]
-
- # no log files created with pytest, probably because pytest logs
- # everything already. Keeping this for possible later inclusion
-
- # logfile = os.listdir(log_path)
- # assert len(logfile) == 1
- # logfile = logfile[0]
- # with open(os.path.join(log_path, logfile), 'r') as file:
- # cont = file.read().replace('\n', '')
- # assert "x=2, p=2" in cont
- # assert "INFO" in cont
-
-if __name__ == '__main__':
- test_apply_to_elements()