remove duplicate in list_check_sha1s

ColumbiaDVMM · Oct 9, 2019 · 1228a19 · 1228a19
1 parent 45f45be
commit 1228a19
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/cufacesearch/cufacesearch/updater/extraction_checker.py b/cufacesearch/cufacesearch/updater/extraction_checker.py
@@ -303,6 +303,11 @@ def run(self, daemon=False):
  raise ValueError('Unknown keys in msg: {}'.format(msg.keys()))
  # This is dangerous, as it assumes the self.ingester.get_msg_json() generator
  # would restart from the next point... Is this the case for Kafka?
+ prev_len = len(list_check_sha1s)
+ list_check_sha1s = list(set(list_check_sha1s))
+ if len(list_check_sha1s) < prev_len:
+ msg = "[{}: log] Removed {} duplicate from `list_check_sha1s`"
+ print(msg.format(self.pp, prev_len - len(list_check_sha1s)))
  if len(list_check_sha1s) >= self.indexer.batch_update_size:
  break
  except Exception as inst: