Refactor with a working false positive test

benchmark.py

@@ -3,7 +3,7 @@
 import multiprocessing as mp
 import numpy as np
 import datetime
-from frigate.edgetpu import ObjectDetector, EdgeTPUProcess, RemoteObjectDetector, load_labels
+from frigate.edgetpu import LocalObjectDetector, EdgeTPUProcess, RemoteObjectDetector, load_labels
 
 my_frame = np.expand_dims(np.full((300,300,3), 1, np.uint8), axis=0)
 labels = load_labels('/labelmap.txt')

detect_objects.py

@@ -262,7 +262,7 @@ def on_connect(client, userdata, flags, rc):
         camera_process = mp.Process(target=track_camera, args=(name, config, GLOBAL_OBJECT_CONFIG, frame_queue, frame_shape,
             tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['process_fps'], 
             camera_processes[name]['detection_fps'], 
-            camera_processes[name]['read_start'], camera_processes[name]['detection_frame']))
+            camera_processes[name]['read_start'], camera_processes[name]['detection_frame'], stop_event))
         camera_process.daemon = True
         camera_processes[name]['process'] = camera_process
 

frigate/edgetpu.py

@@ -2,6 +2,7 @@
 import datetime
 import hashlib
 import multiprocessing as mp
+from abc import ABC, abstractmethod
 import numpy as np
 import pyarrow.plasma as plasma
 import tflite_runtime.interpreter as tflite
@@ -27,8 +28,18 @@ def load_labels(path, encoding='utf-8'):
     else:
         return {index: line.strip() for index, line in enumerate(lines)}
 
-class ObjectDetector():
-    def __init__(self):
+class ObjectDetector(ABC):
+    @abstractmethod
+    def detect(self, tensor_input, threshold = .4):
+        pass
+
+class LocalObjectDetector(ObjectDetector):
+    def __init__(self, labels=None):
+        if labels is None:
+            self.labels = {}
+        else:
+            self.labels = load_labels(labels)
+
         edge_tpu_delegate = None
         try:
             edge_tpu_delegate = load_delegate('libedgetpu.so.1.0', {"device": "usb"})
@@ -53,6 +64,21 @@ def __init__(self):
         self.tensor_input_details = self.interpreter.get_input_details()
         self.tensor_output_details = self.interpreter.get_output_details()
 
+    def detect(self, tensor_input, threshold=.4):
+        detections = []
+
+        raw_detections = self.detect_raw(tensor_input)
+
+        for d in raw_detections:
+            if d[1] < threshold:
+                break
+            detections.append((
+                self.labels[int(d[0])],
+                float(d[1]),
+                (d[2], d[3], d[4], d[5])
+            ))
+        return detections
+
     def detect_raw(self, tensor_input):
         self.interpreter.set_tensor(self.tensor_input_details[0]['index'], tensor_input)
         self.interpreter.invoke()
@@ -70,7 +96,7 @@ def run_detector(detection_queue, avg_speed, start):
     print(f"Starting detection process: {os.getpid()}")
     listen()
     plasma_client = plasma.connect("/tmp/plasma")
-    object_detector = ObjectDetector()
+    object_detector = LocalObjectDetector()
 
     while True:
         object_id_str = detection_queue.get()

frigate/object_processing.py

@@ -11,7 +11,7 @@
 import itertools
 import pyarrow.plasma as plasma
 import matplotlib.pyplot as plt
-from frigate.util import draw_box_with_label, PlasmaManager
+from frigate.util import draw_box_with_label, PlasmaFrameManager
 from frigate.edgetpu import load_labels
 
 PATH_TO_LABELS = '/labelmap.txt'
@@ -91,7 +91,7 @@ def __init__(self, camera_config, zone_config, client, topic_prefix, tracked_obj
         for i, zone in enumerate(self.zone_data.values()):
             zone['color'] = tuple(int(round(255 * c)) for c in colors(i)[:3])
 
-        self.plasma_client = PlasmaManager(self.stop_event)
+        self.plasma_client = PlasmaFrameManager(self.stop_event)
 
     def get_best(self, camera, label):
         if label in self.camera_data[camera]['best_objects']:

frigate/test/test_false_positives.py

@@ -0,0 +1,71 @@
+import datetime
+from unittest import TestCase, main
+from frigate.video import process_frames, start_or_restart_ffmpeg, capture_frames
+from frigate.util import DictFrameManager, EventsPerSecond, draw_box_with_label
+from frigate.motion import MotionDetector
+from frigate.edgetpu import LocalObjectDetector
+from frigate.objects import ObjectTracker
+import multiprocessing as mp
+import numpy as np
+import cv2
+from frigate.object_processing import COLOR_MAP
+
+class FalsePositiveTests(TestCase):
+
+    def test_back_1594395958_675351_0(self):
+        ### load in frames
+        frame_shape = (1080,1920,3)
+        frame_manager = DictFrameManager()
+        frame_queue = mp.Queue()
+        fps = EventsPerSecond()
+        skipped_fps = EventsPerSecond()
+        stop_event = mp.Event()
+        detection_frame = mp.Value('d', datetime.datetime.now().timestamp()+100000)
+        ffmpeg_cmd = "ffmpeg -hide_banner -loglevel panic -i /debug/false_positives/back-1595647759.228381-0.mp4 -f rawvideo -pix_fmt rgb24 pipe:".split(" ")
+        ffmpeg_process = start_or_restart_ffmpeg(ffmpeg_cmd, frame_shape[0]*frame_shape[1]*frame_shape[2])
+        capture_frames(ffmpeg_process, "back", frame_shape, frame_manager, frame_queue, 1, fps, skipped_fps, stop_event, detection_frame)
+        ffmpeg_process.wait()
+        ffmpeg_process.communicate()
+        assert(frame_queue.qsize() > 0)
+
+        ### process frames
+        mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
+        mask[:] = 255
+        motion_detector = MotionDetector(frame_shape, mask)
+
+        object_detector = LocalObjectDetector(labels='/labelmap.txt')
+        object_tracker = ObjectTracker(10)
+        detected_objects_queue = mp.Queue()
+        process_fps = EventsPerSecond()
+        current_frame = mp.Value('d', 0.0)
+
+        process_frames("back", frame_queue, frame_shape, frame_manager, motion_detector, object_detector, object_tracker, detected_objects_queue, 
+            process_fps, current_frame, ['person'], {}, mask, stop_event, exit_on_empty=True)
+        assert(detected_objects_queue.qsize() > 0)
+
+        ### check result
+        while(not detected_objects_queue.empty()):
+            camera_name, frame_time, current_tracked_objects = detected_objects_queue.get()
+
+            current_frame = frame_manager.get(f"{camera_name}{frame_time}")
+            # draw the bounding boxes on the frame
+            for obj in current_tracked_objects.values():
+                thickness = 2
+                color = COLOR_MAP[obj['label']]
+
+                if obj['frame_time'] != frame_time:
+                    thickness = 1
+                    color = (255,0,0)
+
+                # draw the bounding boxes on the frame
+                box = obj['box']
+                draw_box_with_label(current_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
+                # draw the regions on the frame
+                region = obj['region']
+                draw_box_with_label(current_frame, region[0], region[1], region[2], region[3], 'region', f"{region[2]-region[0]}", thickness=1, color=(0,255,0))
+
+            cv2.imwrite(f"/debug/frames/{int(frame_time*1000000)}.jpg", cv2.cvtColor(current_frame, cv2.COLOR_RGB2BGR))
+
+
+if __name__ == '__main__':
+    main()

frigate/util.py

@@ -1,3 +1,4 @@
+from abc import ABC, abstractmethod
 import datetime
 import time
 import signal
@@ -139,7 +140,33 @@ def print_stack(sig, frame):
 def listen():
     signal.signal(signal.SIGUSR1, print_stack)
 
-class PlasmaManager:
+class FrameManager(ABC):
+    @abstractmethod
+    def get(self, name, timeout_ms=0):
+        pass
+
+    @abstractmethod
+    def put(self, name, frame):
+        pass
+
+    @abstractmethod
+    def delete(self, name):
+        pass
+
+class DictFrameManager(FrameManager):
+    def __init__(self):
+        self.frames = {}
+
+    def get(self, name, timeout_ms=0):
+        return self.frames.get(name)
+
+    def put(self, name, frame):
+        self.frames[name] = frame
+
+    def delete(self, name):
+        del self.frames[name]
+
+class PlasmaFrameManager(FrameManager):
     def __init__(self, stop_event=None):
         self.stop_event = stop_event
         self.connect()
@@ -161,18 +188,21 @@ def get(self, name, timeout_ms=0):
             if self.stop_event != None and self.stop_event.is_set():
                 return
             try:
-                return self.plasma_client.get(object_id, timeout_ms=timeout_ms)
+                frame = self.plasma_client.get(object_id, timeout_ms=timeout_ms)
+                if frame is plasma.ObjectNotAvailable:
+                    return None
+                return frame
             except:
                 self.connect()
                 time.sleep(1)
 
-    def put(self, name, obj):
+    def put(self, name, frame):
         object_id = plasma.ObjectID(hashlib.sha1(str.encode(name)).digest())
         while True:
             if self.stop_event != None and self.stop_event.is_set():
                 return
             try:
-                self.plasma_client.put(obj, object_id)
+                self.plasma_client.put(frame, object_id)
                 return
             except Exception as e:
                 print(f"Failed to put in plasma: {e}")