Ongoing review segments (blakeblackshear#10924)

* Update review maintainer to save events when ongoing

* Handle previews for in progress review items

* Reset DB items in app

* Handle in progress review items

* Scroll back down to selected event item

* Handle undefined end time

* Formatting

* remove unused

* Make export handles have full resolution

* reduce preview thumbnail props

* fix missing return

Co-authored-by: Josh Hawkins <[email protected]>

---------

Co-authored-by: Josh Hawkins <[email protected]>

frigate/api/media.py

@@ -1333,7 +1333,9 @@ def review_preview(id: str):
 
  padding = 8
  start_ts = review.start_time - padding
- end_ts = review.end_time + padding
+ end_ts = (
+ review.end_time + padding if review.end_time else datetime.now().timestamp()
+ )
  return preview_gif(review.camera, start_ts, end_ts)
 
 
@@ -1344,8 +1346,15 @@ def preview_thumbnail(file_name: str):
  safe_file_name_current = secure_filename(file_name)
  preview_dir = os.path.join(CACHE_DIR, "preview_frames")
 
- with open(os.path.join(preview_dir, safe_file_name_current), "rb") as image_file:
- jpg_bytes = image_file.read()
+ try:
+ with open(
+ os.path.join(preview_dir, safe_file_name_current), "rb"
+ ) as image_file:
+ jpg_bytes = image_file.read()
+ except FileNotFoundError:
+ return make_response(
+ jsonify({"success": False, "message": "Image file not found"}), 404
+ )
 
  response = make_response(jpg_bytes)
  response.headers["Content-Type"] = "image/jpeg"

frigate/api/review.py

@@ -27,10 +27,18 @@ def review():
 
  before = request.args.get("before", type=float, default=datetime.now().timestamp())
  after = request.args.get(
- "after", type=float, default=(datetime.now() - timedelta(hours=18)).timestamp()
+ "after", type=float, default=(datetime.now() - timedelta(hours=24)).timestamp()
  )
 
- clauses = [((ReviewSegment.start_time > after) & (ReviewSegment.end_time < before))]
+ clauses = [
+ (
+ (ReviewSegment.start_time > after)
+ & (
+ (ReviewSegment.end_time.is_null(True))
+ | (ReviewSegment.end_time < before)
+ )
+ )
+ ]
 
  if cameras != "all":
  camera_list = cameras.split(",")
@@ -45,6 +53,7 @@ def review():
  for label in filtered_labels:
  label_clauses.append(
  (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+ | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
  )
 
  label_clause = reduce(operator.or_, label_clauses)
@@ -94,6 +103,7 @@ def review_summary():
  for label in filtered_labels:
  label_clauses.append(
  (ReviewSegment.data["objects"].cast("text") % f'*"{label}"*')
+ | (ReviewSegment.data["audio"].cast("text") % f'*"{label}"*')
  )
 
  label_clause = reduce(operator.or_, label_clauses)

frigate/app.py

@@ -667,6 +667,14 @@ def stop(self) -> None:
  logger.info("Stopping...")
  self.stop_event.set()
 
+ # set an end_time on entries without an end_time before exiting
+ Event.update(end_time=datetime.datetime.now().timestamp()).where(
+ Event.end_time == None
+ ).execute()
+ ReviewSegment.update(end_time=datetime.datetime.now().timestamp()).where(
+ ReviewSegment.end_time == None
+ ).execute()
+
  # Stop Communicators
  self.inter_process_communicator.stop()
  self.inter_config_updater.stop()

frigate/events/maintainer.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 import threading
 from multiprocessing import Queue
@@ -112,10 +111,6 @@ def run(self) -> None:
 
  self.handle_external_detection(event_type, event_data)
 
- # set an end_time on events without an end_time before exiting
- Event.update(end_time=datetime.datetime.now().timestamp()).where(
- Event.end_time == None
- ).execute()
  self.event_receiver.stop()
  self.event_end_publisher.stop()
  logger.info("Exiting event processor...")

frigate/review/maintainer.py

@@ -66,6 +66,7 @@ def __init__(
  # thumbnail
  self.frame = np.zeros((THUMB_HEIGHT * 3 // 2, THUMB_WIDTH), np.uint8)
  self.frame_active_count = 0
+ self.frame_path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
 
  def update_frame(
  self, camera_config: CameraConfig, frame, objects: list[TrackedObject]
@@ -98,19 +99,19 @@ def update_frame(
  color_frame, dsize=(width, THUMB_HEIGHT), interpolation=cv2.INTER_AREA
  )
 
- def end(self) -> dict:
- path = os.path.join(CLIPS_DIR, f"thumb-{self.camera}-{self.id}.jpg")
-
  if self.frame is not None:
- cv2.imwrite(path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60])
+ cv2.imwrite(
+ self.frame_path, self.frame, [int(cv2.IMWRITE_WEBP_QUALITY), 60]
+ )
 
+ def get_data(self, ended: bool) -> dict:
  return {
  ReviewSegment.id: self.id,
  ReviewSegment.camera: self.camera,
  ReviewSegment.start_time: self.start_time,
- ReviewSegment.end_time: self.last_update,
+ ReviewSegment.end_time: self.last_update if ended else None,
  ReviewSegment.severity: self.severity.value,
- ReviewSegment.thumb_path: path,
+ ReviewSegment.thumb_path: self.frame_path,
  ReviewSegment.data: {
  "detections": list(set(self.detections.keys())),
  "objects": list(set(self.detections.values())),
@@ -141,9 +142,20 @@ def __init__(self, config: FrigateConfig, stop_event: MpEvent):
 
  self.stop_event = stop_event
 
+ def update_segment(self, segment: PendingReviewSegment) -> None:
+ """Update segment."""
+ seg_data = segment.get_data(ended=False)
+ self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
+ self.requestor.send_data(
+ "reviews",
+ json.dumps(
+ {"type": "update", "review": {k.name: v for k, v in seg_data.items()}}
+ ),
+ )
+
  def end_segment(self, segment: PendingReviewSegment) -> None:
  """End segment."""
- seg_data = segment.end()
+ seg_data = segment.get_data(ended=True)
  self.requestor.send_data(UPSERT_REVIEW_SEGMENT, seg_data)
  self.requestor.send_data(
  "reviews",
@@ -179,6 +191,7 @@ def update_existing_segment(
  )
  segment.update_frame(camera_config, yuv_frame, active_objects)
  self.frame_manager.close(frame_id)
+ self.update_segment(segment)
 
  for object in active_objects:
  if not object["sub_label"]:
@@ -263,6 +276,7 @@ def check_if_new_segment(
  camera_config, yuv_frame, active_objects
  )
  self.frame_manager.close(frame_id)
+ self.update_segment(self.active_review_segments[camera])
  elif len(motion) >= 20:
  self.active_review_segments[camera] = PendingReviewSegment(
  camera,
@@ -398,6 +412,11 @@ def run(self) -> None:
  "end_time"
  ]
 
+ self.config_subscriber.stop()
+ self.requestor.stop()
+ self.detection_subscriber.stop()
+ logger.info("Exiting review maintainer...")
+
 
 def get_active_objects(
  frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject]

web/src/components/card/ReviewCard.tsx

@@ -27,7 +27,9 @@ export default function ReviewCard({
  config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p",
  );
  const isSelected = useMemo(
- () => event.start_time <= currentTime && event.end_time >= currentTime,
+ () =>
+ event.start_time <= currentTime &&
+ (event.end_time ?? Date.now() / 1000) >= currentTime,
  [event, currentTime],
  );