Make automatic sync limited ot last 36 hours

frigate/record/cleanup.py

@@ -11,7 +11,7 @@
 from frigate.const import CACHE_DIR, RECORD_DIR
 from frigate.models import Event, Recordings
 from frigate.record.util import remove_empty_directories, sync_recordings
-from frigate.util.builtin import get_next_sunday_at_3
+from frigate.util.builtin import get_tomorrow_at_time
 
 logger = logging.getLogger(__name__)
 
@@ -181,9 +181,9 @@ def expire_recordings(self) -> None:
  def run(self) -> None:
  # on startup sync recordings with disk if enabled
  if self.config.record.sync_on_startup:
- sync_recordings()
+ sync_recordings(limited=False)
 
- next_sync = get_next_sunday_at_3()
+ next_sync = get_tomorrow_at_time(3)
 
  # Expire tmp clips every minute, recordings and clean directories every hour.
  for counter in itertools.cycle(range(self.config.record.expire_interval)):
@@ -194,8 +194,8 @@ def run(self) -> None:
  self.clean_tmp_clips()
 
  if datetime.datetime.now().astimezone(datetime.timezone.utc) > next_sync:
- sync_recordings()
- next_sync = get_next_sunday_at_3()
+ sync_recordings(limited=True)
+ next_sync = get_tomorrow_at_time(3)
 
  if counter == 0:
  self.expire_recordings()

frigate/record/util.py

@@ -1,5 +1,6 @@
 """Recordings Utilities."""
 
+import datetime
 import logging
 import os
 
@@ -27,13 +28,21 @@ def remove_empty_directories(directory: str) -> None:
  os.rmdir(path)
 
 
-def sync_recordings() -> None:
+def sync_recordings(limited: bool) -> None:
  """Check the db for stale recordings entries that don't exist in the filesystem."""
 
  def delete_db_entries_without_file(files_on_disk: list[str]) -> bool:
  """Delete db entries where file was deleted outside of frigate."""
- # get all recordings in the db
- recordings = Recordings.select(Recordings.id, Recordings.path)
+
+ if limited:
+ recordings = Recordings.select(Recordings.id, Recordings.path).where(
+ Recordings.start_time
+ >= (datetime.datetime.now() - datetime.timedelta(hours=36)).timestamp()
+ )
+ else:
+ # get all recordings in the db
+ recordings = Recordings.select(Recordings.id, Recordings.path)
+
  # Use pagination to process records in chunks
  page_size = 1000
  num_pages = (recordings.count() + page_size - 1) // page_size
@@ -97,12 +106,26 @@ def delete_files_without_db_entry(files_on_disk: list[str]):
 
  logger.debug("Start sync recordings.")
 
- # get all recordings files on disk and put them in a set
- files_on_disk = {
- os.path.join(root, file)
- for root, _, files in os.walk(RECORD_DIR)
- for file in files
- }
+ if limited:
+ # get recording files from last 36 hours
+ hour_check = (
+ datetime.datetime.now().astimezone(datetime.timezone.utc)
+ - datetime.timedelta(hours=36)
+ ).strftime("%Y-%m-%d/%H")
+ files_on_disk = {
+ os.path.join(root, file)
+ for root, _, files in os.walk(RECORD_DIR)
+ for file in files
+ if file > hour_check
+ }
+ else:
+ # get all recordings files on disk and put them in a set
+ files_on_disk = {
+ os.path.join(root, file)
+ for root, _, files in os.walk(RECORD_DIR)
+ for file in files
+ }
+
  db_success = delete_db_entries_without_file(files_on_disk)
 
  # only try to cleanup files if db cleanup was successful

frigate/util/builtin.py

@@ -263,18 +263,9 @@ def find_by_key(dictionary, target_key):
  return None
 
 
-def get_tomorrow_at_2() -> datetime.datetime:
+def get_tomorrow_at_time(hour: int) -> datetime.datetime:
  """Returns the datetime of the following day at 2am."""
  tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1)
- return tomorrow.replace(hour=2, minute=0, second=0).astimezone(
+ return tomorrow.replace(hour=hour, minute=0, second=0).astimezone(
  datetime.timezone.utc
  )
-
-
-def get_next_sunday_at_3() -> datetime.datetime:
- """Returns the datetime of the next Sunday at 3am."""
- # adapted from https://stackoverflow.com/a/16770463
- now = datetime.datetime.now(get_localzone())
- diff = datetime.timedelta((13 - now.weekday()) % 7)
- sunday = now + diff
- return sunday.replace(hour=3, minute=0, second=0).astimezone(datetime.timezone.utc)

frigate/video.py

@@ -26,7 +26,7 @@
 from frigate.track import ObjectTracker
 from frigate.track.norfair_tracker import NorfairTracker
 from frigate.types import PTZMetricsTypes
-from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2
+from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_time
 from frigate.util.image import (
  FrameManager,
  SharedMemoryFrameManager,
@@ -528,7 +528,7 @@ def process_frames(
  fps = process_info["process_fps"]
  detection_fps = process_info["detection_fps"]
  current_frame_time = process_info["detection_frame"]
- next_region_update = get_tomorrow_at_2()
+ next_region_update = get_tomorrow_at_time(2)
 
  fps_tracker = EventsPerSecond()
  fps_tracker.start()
@@ -550,7 +550,7 @@ def process_frames(
  except queue.Empty:
  logger.error(f"Unable to get updated region grid for {camera_name}")
 
- next_region_update = get_tomorrow_at_2()
+ next_region_update = get_tomorrow_at_time(2)
 
  try:
  if exit_on_empty: