Write recording segments to cache with timezone info (blakeblackshear…

…#8468)

* Store recording segments with timezone info

* Don't use _

* Use different separator due to timezone

frigate/config.py

@@ -17,6 +17,7 @@
  ALL_ATTRIBUTE_LABELS,
  AUDIO_MIN_CONFIDENCE,
  CACHE_DIR,
+ CACHE_SEGMENT_FORMAT,
  DEFAULT_DB_PATH,
  REGEX_CAMERA_NAME,
  YAML_EXT,
@@ -865,7 +866,7 @@ def _get_ffmpeg_cmd(self, ffmpeg_input: CameraInput):
 
  ffmpeg_output_args = (
  record_args
- + [f"{os.path.join(CACHE_DIR, self.name)}-%Y%m%d%H%M%S.mp4"]
+ + [f"{os.path.join(CACHE_DIR, self.name)}@{CACHE_SEGMENT_FORMAT}.mp4"]
  + ffmpeg_output_args
  )
 

frigate/const.py

@@ -50,6 +50,7 @@
 
 # Record Values
 
+CACHE_SEGMENT_FORMAT = "%Y%m%d%H%M%S%z"
 MAX_SEGMENT_DURATION = 600
 MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to account for cameras with inconsistent segment times
 

frigate/record/maintainer.py

@@ -20,6 +20,7 @@
 from frigate.config import FrigateConfig, RetainModeEnum
 from frigate.const import (
  CACHE_DIR,
+ CACHE_SEGMENT_FORMAT,
  INSERT_MANY_RECORDINGS,
  MAX_SEGMENT_DURATION,
  RECORD_DIR,
@@ -74,15 +75,13 @@ def __init__(
  self.end_time_cache: dict[str, Tuple[datetime.datetime, float]] = {}
 
  async def move_files(self) -> None:
- cache_files = sorted(
- [
- d
- for d in os.listdir(CACHE_DIR)
- if os.path.isfile(os.path.join(CACHE_DIR, d))
- and d.endswith(".mp4")
- and not d.startswith("clip_")
- ]
- )
+ cache_files = [
+ d
+ for d in os.listdir(CACHE_DIR)
+ if os.path.isfile(os.path.join(CACHE_DIR, d))
+ and d.endswith(".mp4")
+ and not d.startswith("clip_")
+ ]
 
  files_in_use = []
  for process in psutil.process_iter():
@@ -106,8 +105,12 @@ async def move_files(self) -> None:
 
  cache_path = os.path.join(CACHE_DIR, cache)
  basename = os.path.splitext(cache)[0]
- camera, date = basename.rsplit("-", maxsplit=1)
- start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S")
+ camera, date = basename.rsplit("@", maxsplit=1)
+
+ # important that start_time is utc because recordings are stored and compared in utc
+ start_time = datetime.datetime.strptime(
+ date, CACHE_SEGMENT_FORMAT
+ ).astimezone(datetime.timezone.utc)
 
  grouped_recordings[camera].append(
  {
@@ -119,6 +122,11 @@ async def move_files(self) -> None:
  # delete all cached files past the most recent 5
  keep_count = 5
  for camera in grouped_recordings.keys():
+ # sort based on start time
+ grouped_recordings[camera] = sorted(
+ grouped_recordings[camera], key=lambda s: s["start_time"]
+ )
+
  segment_count = len(grouped_recordings[camera])
  if segment_count > keep_count:
  logger.warning(
@@ -218,7 +226,7 @@ async def validate_and_move_segment(
  # if cached file's start_time is earlier than the retain days for the camera
  if start_time <= (
  (
- datetime.datetime.now()
+ datetime.datetime.now().astimezone(datetime.timezone.utc)
  - datetime.timedelta(
  days=self.config.cameras[camera].record.retain.days
  )
@@ -262,8 +270,8 @@ async def validate_and_move_segment(
  )
  retain_cutoff = datetime.datetime.fromtimestamp(
  most_recently_processed_frame_time - pre_capture
- ).astimezone(datetime.timezone.utc)
- if end_time.astimezone(datetime.timezone.utc) < retain_cutoff:
+ )
+ if end_time < retain_cutoff:
  Path(cache_path).unlink(missing_ok=True)
  self.end_time_cache.pop(cache_path, None)
  # else retain days includes this segment
@@ -275,10 +283,11 @@ async def validate_and_move_segment(
  )
 
  # ensure delayed segment info does not lead to lost segments
- if datetime.datetime.fromtimestamp(
- most_recently_processed_frame_time
- ).astimezone(datetime.timezone.utc) >= end_time.astimezone(
- datetime.timezone.utc
+ if (
+ datetime.datetime.fromtimestamp(
+ most_recently_processed_frame_time
+ ).astimezone(datetime.timezone.utc)
+ >= end_time
  ):
  record_mode = self.config.cameras[camera].record.retain.mode
  return await self.move_segment(
@@ -345,18 +354,18 @@ async def move_segment(
  self.end_time_cache.pop(cache_path, None)
  return
 
+ # directory will be in utc due to start_time being in utc
  directory = os.path.join(
  RECORD_DIR,
- start_time.astimezone(tz=datetime.timezone.utc).strftime("%Y-%m-%d/%H"),
+ start_time.strftime("%Y-%m-%d/%H"),
  camera,
  )
 
  if not os.path.exists(directory):
  os.makedirs(directory)
 
- file_name = (
- f"{start_time.replace(tzinfo=datetime.timezone.utc).strftime('%M.%S.mp4')}"
- )
+ # file will be in utc due to start_time being in utc
+ file_name = f"{start_time.strftime('%M.%S.mp4')}"
  file_path = os.path.join(directory, file_name)
 
  try:

frigate/video.py

@@ -16,6 +16,7 @@
  ALL_ATTRIBUTE_LABELS,
  ATTRIBUTE_LABEL_MAP,
  CACHE_DIR,
+ CACHE_SEGMENT_FORMAT,
  REQUEST_REGION_GRID,
 )
 from frigate.log import LogPipe
@@ -300,19 +301,19 @@ def get_latest_segment_datetime(self, latest_segment: datetime.datetime) -> int:
  and not d.startswith("clip_")
  ]
  )
- newest_segment_timestamp = latest_segment
+ newest_segment_time = latest_segment
 
  for file in cache_files:
  if self.camera_name in file:
  basename = os.path.splitext(file)[0]
- _, date = basename.rsplit("-", maxsplit=1)
- ts = datetime.datetime.strptime(date, "%Y%m%d%H%M%S").astimezone(
- datetime.timezone.utc
- )
- if ts > newest_segment_timestamp:
- newest_segment_timestamp = ts
-
- return newest_segment_timestamp
+ _, date = basename.rsplit("@", maxsplit=1)
+ segment_time = datetime.datetime.strptime(
+ date, CACHE_SEGMENT_FORMAT
+ ).astimezone(datetime.timezone.utc)
+ if segment_time > newest_segment_time:
+ newest_segment_time = segment_time
+
+ return newest_segment_time
 
 
 class CameraCapture(threading.Thread):