From ae0aba44dc4584d6f21fee25ca1d289ef3ba155a Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Wed, 31 May 2023 10:12:43 -0400 Subject: [PATCH] Improve tracking (#6516) --- .devcontainer/devcontainer.json | 7 +- frigate/log.py | 2 + frigate/track/__init__.py | 13 + .../{objects.py => track/centroid_tracker.py} | 17 +- frigate/track/norfair_tracker.py | 285 ++++++++++++++++++ frigate/video.py | 16 +- process_clip.py | 10 +- requirements-wheels.txt | 1 + 8 files changed, 332 insertions(+), 19 deletions(-) create mode 100644 frigate/track/__init__.py rename frigate/{objects.py => track/centroid_tracker.py} (95%) create mode 100644 frigate/track/norfair_tracker.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 60065a9650..a8264a2274 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -53,7 +53,8 @@ "csstools.postcss", "blanu.vscode-styled-jsx", "bradlc.vscode-tailwindcss", - "ms-python.isort" + "ms-python.isort", + "charliermarsh.ruff" ], "settings": { "remote.autoForwardPorts": false, @@ -69,9 +70,7 @@ "python.testing.unittestArgs": ["-v", "-s", "./frigate/test"], "files.trimTrailingWhitespace": true, "eslint.workingDirectories": ["./web"], - "isort.args": [ - "--settings-path=./pyproject.toml" - ], + "isort.args": ["--settings-path=./pyproject.toml"], "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true diff --git a/frigate/log.py b/frigate/log.py index 4a44545adf..5dbf4eed04 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -62,6 +62,8 @@ def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None: if stop_event.is_set(): break continue + if record.msg.startswith("You are using a scalar distance function"): + continue logger = logging.getLogger(record.name) logger.handle(record) diff --git a/frigate/track/__init__.py b/frigate/track/__init__.py new file mode 100644 index 0000000000..421a968dac --- /dev/null +++ b/frigate/track/__init__.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod + +from frigate.config import DetectConfig + + +class ObjectTracker(ABC): + @abstractmethod + def __init__(self, config: DetectConfig): + pass + + @abstractmethod + def match_and_update(self, detections): + pass diff --git a/frigate/objects.py b/frigate/track/centroid_tracker.py similarity index 95% rename from frigate/objects.py rename to frigate/track/centroid_tracker.py index d11aff2938..fd4a293f91 100644 --- a/frigate/objects.py +++ b/frigate/track/centroid_tracker.py @@ -6,10 +6,11 @@ from scipy.spatial import distance as dist from frigate.config import DetectConfig +from frigate.track import ObjectTracker from frigate.util import intersection_over_union -class ObjectTracker: +class CentroidTracker(ObjectTracker): def __init__(self, config: DetectConfig): self.tracked_objects = {} self.disappeared = {} @@ -134,11 +135,11 @@ def update_frame_times(self, frame_time): if self.is_expired(id): self.deregister(id) - def match_and_update(self, frame_time, new_objects): + def match_and_update(self, frame_time, detections): # group by name - new_object_groups = defaultdict(lambda: []) - for obj in new_objects: - new_object_groups[obj[0]].append( + detection_groups = defaultdict(lambda: []) + for obj in detections: + detection_groups[obj[0]].append( { "label": obj[0], "score": obj[1], @@ -153,17 +154,17 @@ def match_and_update(self, frame_time, new_objects): # update any tracked objects with labels that are not # seen in the current objects and deregister if needed for obj in list(self.tracked_objects.values()): - if obj["label"] not in new_object_groups: + if obj["label"] not in detection_groups: if self.disappeared[obj["id"]] >= self.max_disappeared: self.deregister(obj["id"]) else: self.disappeared[obj["id"]] += 1 - if len(new_objects) == 0: + if len(detections) == 0: return # track objects for each label type - for label, group in new_object_groups.items(): + for label, group in detection_groups.items(): current_objects = [ o for o in self.tracked_objects.values() if o["label"] == label ] diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py new file mode 100644 index 0000000000..76f50fd73a --- /dev/null +++ b/frigate/track/norfair_tracker.py @@ -0,0 +1,285 @@ +import random +import string + +import numpy as np +from norfair import Detection, Drawable, Tracker, draw_boxes +from norfair.drawing.drawer import Drawer + +from frigate.config import DetectConfig +from frigate.track import ObjectTracker +from frigate.util import intersection_over_union + + +# Normalizes distance from estimate relative to object size +# Other ideas: +# - if estimates are inaccurate for first N detections, compare with last_detection (may be fine) +# - could be variable based on time since last_detection +# - include estimated velocity in the distance (car driving by of a parked car) +# - include some visual similarity factor in the distance for occlusions +def distance(detection: np.array, estimate: np.array) -> float: + # ultimately, this should try and estimate distance in 3-dimensional space + # consider change in location, width, and height + + estimate_dim = np.diff(estimate, axis=0).flatten() + detection_dim = np.diff(detection, axis=0).flatten() + + # get bottom center positions + detection_position = np.array( + [np.average(detection[:, 0]), np.max(detection[:, 1])] + ) + estimate_position = np.array([np.average(estimate[:, 0]), np.max(estimate[:, 1])]) + + distance = (detection_position - estimate_position).astype(float) + # change in x relative to w + distance[0] /= estimate_dim[0] + # change in y relative to h + distance[1] /= estimate_dim[1] + + # get ratio of widths and heights + # normalize to 1 + widths = np.sort([estimate_dim[0], detection_dim[0]]) + heights = np.sort([estimate_dim[1], detection_dim[1]]) + width_ratio = widths[1] / widths[0] - 1.0 + height_ratio = heights[1] / heights[0] - 1.0 + + # change vector is relative x,y change and w,h ratio + change = np.append(distance, np.array([width_ratio, height_ratio])) + + # calculate euclidean distance of the change vector + return np.linalg.norm(change) + + +def frigate_distance(detection: Detection, tracked_object) -> float: + return distance(detection.points, tracked_object.estimate) + + +class NorfairTracker(ObjectTracker): + def __init__(self, config: DetectConfig): + self.tracked_objects = {} + self.disappeared = {} + self.positions = {} + self.max_disappeared = config.max_disappeared + self.detect_config = config + self.track_id_map = {} + # TODO: could also initialize a tracker per object class if there + # was a good reason to have different distance calculations + self.tracker = Tracker( + distance_function=frigate_distance, + distance_threshold=2.5, + initialization_delay=0, + hit_counter_max=self.max_disappeared, + ) + + def register(self, track_id, obj): + rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + id = f"{obj['frame_time']}-{rand_id}" + self.track_id_map[track_id] = id + obj["id"] = id + obj["start_time"] = obj["frame_time"] + obj["motionless_count"] = 0 + obj["position_changes"] = 0 + self.tracked_objects[id] = obj + self.disappeared[id] = 0 + self.positions[id] = { + "xmins": [], + "ymins": [], + "xmaxs": [], + "ymaxs": [], + "xmin": 0, + "ymin": 0, + "xmax": self.detect_config.width, + "ymax": self.detect_config.height, + } + + def deregister(self, id): + del self.tracked_objects[id] + del self.disappeared[id] + + # tracks the current position of the object based on the last N bounding boxes + # returns False if the object has moved outside its previous position + def update_position(self, id, box): + position = self.positions[id] + position_box = ( + position["xmin"], + position["ymin"], + position["xmax"], + position["ymax"], + ) + + xmin, ymin, xmax, ymax = box + + iou = intersection_over_union(position_box, box) + + # if the iou drops below the threshold + # assume the object has moved to a new position and reset the computed box + if iou < 0.6: + self.positions[id] = { + "xmins": [xmin], + "ymins": [ymin], + "xmaxs": [xmax], + "ymaxs": [ymax], + "xmin": xmin, + "ymin": ymin, + "xmax": xmax, + "ymax": ymax, + } + return False + + # if there are less than 10 entries for the position, add the bounding box + # and recompute the position box + if len(position["xmins"]) < 10: + position["xmins"].append(xmin) + position["ymins"].append(ymin) + position["xmaxs"].append(xmax) + position["ymaxs"].append(ymax) + # by using percentiles here, we hopefully remove outliers + position["xmin"] = np.percentile(position["xmins"], 15) + position["ymin"] = np.percentile(position["ymins"], 15) + position["xmax"] = np.percentile(position["xmaxs"], 85) + position["ymax"] = np.percentile(position["ymaxs"], 85) + + return True + + def is_expired(self, id): + obj = self.tracked_objects[id] + # get the max frames for this label type or the default + max_frames = self.detect_config.stationary.max_frames.objects.get( + obj["label"], self.detect_config.stationary.max_frames.default + ) + + # if there is no max_frames for this label type, continue + if max_frames is None: + return False + + # if the object has exceeded the max_frames setting, deregister + if ( + obj["motionless_count"] - self.detect_config.stationary.threshold + > max_frames + ): + return True + + return False + + def update(self, track_id, obj): + id = self.track_id_map[track_id] + self.disappeared[id] = 0 + # update the motionless count if the object has not moved to a new position + if self.update_position(id, obj["box"]): + self.tracked_objects[id]["motionless_count"] += 1 + if self.is_expired(id): + self.deregister(id) + return + else: + # register the first position change and then only increment if + # the object was previously stationary + if ( + self.tracked_objects[id]["position_changes"] == 0 + or self.tracked_objects[id]["motionless_count"] + >= self.detect_config.stationary.threshold + ): + self.tracked_objects[id]["position_changes"] += 1 + self.tracked_objects[id]["motionless_count"] = 0 + + self.tracked_objects[id].update(obj) + + def update_frame_times(self, frame_time): + # if the object was there in the last frame, assume it's still there + detections = [ + ( + obj["label"], + obj["score"], + obj["box"], + obj["area"], + obj["ratio"], + obj["region"], + ) + for id, obj in self.tracked_objects.items() + if self.disappeared[id] == 0 + ] + self.match_and_update(frame_time, detections=detections) + + def match_and_update(self, frame_time, detections): + norfair_detections = [] + + for obj in detections: + # centroid is used for other things downstream + centroid_x = int((obj[2][0] + obj[2][2]) / 2.0) + centroid_y = int((obj[2][1] + obj[2][3]) / 2.0) + + # track based on top,left and bottom,right corners instead of centroid + points = np.array([[obj[2][0], obj[2][1]], [obj[2][2], obj[2][3]]]) + + norfair_detections.append( + Detection( + points=points, + label=obj[0], + data={ + "label": obj[0], + "score": obj[1], + "box": obj[2], + "area": obj[3], + "ratio": obj[4], + "region": obj[5], + "frame_time": frame_time, + "centroid": (centroid_x, centroid_y), + }, + ) + ) + + tracked_objects = self.tracker.update(detections=norfair_detections) + + # update or create new tracks + active_ids = [] + for t in tracked_objects: + active_ids.append(t.global_id) + if t.global_id not in self.track_id_map: + self.register(t.global_id, t.last_detection.data) + # if there wasn't a detection in this frame, increment disappeared + elif t.last_detection.data["frame_time"] != frame_time: + id = self.track_id_map[t.global_id] + self.disappeared[id] += 1 + # else update it + else: + self.update(t.global_id, t.last_detection.data) + + # clear expired tracks + expired_ids = [k for k in self.track_id_map.keys() if k not in active_ids] + for e_id in expired_ids: + self.deregister(self.track_id_map[e_id]) + del self.track_id_map[e_id] + + def debug_draw(self, frame, frame_time): + active_detections = [ + Drawable(id=obj.id, points=obj.last_detection.points, label=obj.label) + for obj in self.tracker.tracked_objects + if obj.last_detection.data["frame_time"] == frame_time + ] + missing_detections = [ + Drawable(id=obj.id, points=obj.last_detection.points, label=obj.label) + for obj in self.tracker.tracked_objects + if obj.last_detection.data["frame_time"] != frame_time + ] + # draw the estimated bounding box + draw_boxes(frame, self.tracker.tracked_objects, color="green", draw_ids=True) + # draw the detections that were detected in the current frame + draw_boxes(frame, active_detections, color="blue", draw_ids=True) + # draw the detections that are missing in the current frame + draw_boxes(frame, missing_detections, color="red", draw_ids=True) + + # draw the distance calculation for the last detection + # estimate vs detection + for obj in self.tracker.tracked_objects: + ld = obj.last_detection + # bottom right + text_anchor = ( + ld.points[1, 0], + ld.points[1, 1], + ) + frame = Drawer.text( + frame, + f"{obj.id}: {str(obj.last_distance)}", + position=text_anchor, + size=None, + color=(255, 0, 0), + thickness=None, + ) diff --git a/frigate/video.py b/frigate/video.py index f7ec08e995..08808038e9 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -19,7 +19,8 @@ from frigate.log import LogPipe from frigate.motion import MotionDetector from frigate.object_detection import RemoteObjectDetector -from frigate.objects import ObjectTracker +from frigate.track import ObjectTracker +from frigate.track.norfair_tracker import NorfairTracker from frigate.util import ( EventsPerSecond, FrameManager, @@ -472,7 +473,7 @@ def receiveSignal(signalNumber, frame): name, labelmap, detection_queue, result_connection, model_config, stop_event ) - object_tracker = ObjectTracker(config.detect) + object_tracker = NorfairTracker(config.detect) frame_manager = SharedMemoryFrameManager() @@ -847,6 +848,17 @@ def process_frames( else: object_tracker.update_frame_times(frame_time) + # debug tracking by writing frames + if False: + bgr_frame = cv2.cvtColor( + frame, + cv2.COLOR_YUV2BGR_I420, + ) + object_tracker.debug_draw(bgr_frame, frame_time) + cv2.imwrite( + f"debug/frames/track-{'{:.6f}'.format(frame_time)}.jpg", bgr_frame + ) + # add to the queue if not full if detected_objects_queue.full(): frame_manager.delete(f"{camera_name}{frame_time}") diff --git a/process_clip.py b/process_clip.py index 9eb6f324b6..16bf7afa4c 100644 --- a/process_clip.py +++ b/process_clip.py @@ -10,13 +10,13 @@ import cv2 import numpy as np -sys.path.append("/lab/frigate") +sys.path.append("/workspace/frigate") from frigate.config import FrigateConfig # noqa: E402 from frigate.motion import MotionDetector # noqa: E402 from frigate.object_detection import LocalObjectDetector # noqa: E402 from frigate.object_processing import CameraState # noqa: E402 -from frigate.objects import ObjectTracker # noqa: E402 +from frigate.track.centroid_tracker import CentroidTracker # noqa: E402 from frigate.util import ( # noqa: E402 EventsPerSecond, SharedMemoryFrameManager, @@ -108,7 +108,7 @@ def process_frames( motion_detector = MotionDetector(self.frame_shape, self.camera_config.motion) motion_detector.save_images = False - object_tracker = ObjectTracker(self.camera_config.detect) + object_tracker = CentroidTracker(self.camera_config.detect) process_info = { "process_fps": mp.Value("d", 0.0), "detection_fps": mp.Value("d", 0.0), @@ -248,7 +248,7 @@ def process(path, label, output, debug_path): clips.append(path) json_config = { - "mqtt": {"host": "mqtt"}, + "mqtt": {"enabled": False}, "detectors": {"coral": {"type": "edgetpu", "device": "usb"}}, "cameras": { "camera": { @@ -282,7 +282,7 @@ def process(path, label, output, debug_path): json_config["cameras"]["camera"]["ffmpeg"]["inputs"][0]["path"] = c frigate_config = FrigateConfig(**json_config) - runtime_config = frigate_config.runtime_config + runtime_config = frigate_config.runtime_config() runtime_config.cameras["camera"].create_ffmpeg_cmds() process_clip = ProcessClip(c, frame_shape, runtime_config) diff --git a/requirements-wheels.txt b/requirements-wheels.txt index d785df5d6b..f1f2282916 100644 --- a/requirements-wheels.txt +++ b/requirements-wheels.txt @@ -19,6 +19,7 @@ types-PyYAML == 6.0.* requests == 2.30.* types-requests == 2.28.* scipy == 1.10.* +norfair == 2.2.* setproctitle == 1.3.* ws4py == 0.5.* # Openvino Library - Custom built with MYRIAD support