Merge branch 'kinesis' of https://github.com/ColumbiaDVMM/ColumbiaIma…

…geSearch into kinesis

cufacesearch/cufacesearch/ingester/kinesis_ingester.py

@@ -262,7 +262,7 @@ def get_msg_json(self):
  print(msg.format(self.pp, len(records)))
  if self.verbose > 4:
  lag_ms = rec_response['MillisBehindLatest']
- msg = "[{}: log] Lagging by {.3f}s"
+ msg = "[{}: log] Lagging by {:.2f}s"
  print(msg.format(self.pp, lag_ms/1000.0))
  sleep_count = 0
  for rec in records:
@@ -322,4 +322,4 @@ def get_msg_json(self):
  time.sleep(1)
  except Exception as inst:
  msg = "[{}: ERROR] get_msg_json failed with error {}"
- print(msg.format(self.pp, inst))
+ print(msg.format(self.pp, inst))

cufacesearch/cufacesearch/updater/extraction_checker.py

@@ -323,7 +323,12 @@ def run(self, daemon=False):
 
  if list_check_sha1s:
  # Check which images have not been processed (or pushed in an update) yet
+ # This seems slow
+ start_check = time.time()
  unprocessed_rows = self.get_unprocessed_rows(list_check_sha1s)
+ msg = "[{}: log] Found {}/{} unprocessed images in {:.2f}s"
+ print(msg.format(self.pp, len(unprocessed_rows), len(list_check_sha1s), time.time() - start_check))
+ #unprocessed_rows = self.get_unprocessed_rows(list_check_sha1s)
  self.nb_imgs_check += len(list_check_sha1s)
  push_delay = (time.time() - self.last_push) > self.max_delay / 60
  if push_delay and self.nb_imgs_unproc_lastprint != self.nb_imgs_unproc: