Performance increase with lots of recordings (blakeblackshear#8525)

* Performance increase with lots of recordings

* Formatting + review

* Update 020_update_index_recordings.py

Formatting

* Feedback + check fix

* Update 020_update_index_recordings.py

frigate/http.py

@@ -115,7 +115,7 @@ def is_healthy():
 @bp.route("/events/summary")
 def events_summary():
  tz_name = request.args.get("timezone", default="utc", type=str)
- hour_modifier, minute_modifier = get_tz_modifiers(tz_name)
+ hour_modifier, minute_modifier, seconds_offset = get_tz_modifiers(tz_name)
  has_clip = request.args.get("has_clip", type=int)
  has_snapshot = request.args.get("has_snapshot", type=int)
 
@@ -149,12 +149,7 @@ def events_summary():
  Event.camera,
  Event.label,
  Event.sub_label,
- fn.strftime(
- "%Y-%m-%d",
- fn.datetime(
- Event.start_time, "unixepoch", hour_modifier, minute_modifier
- ),
- ),
+ (Event.start_time + seconds_offset).cast("int") / (3600 * 24),
  Event.zones,
  )
  )
@@ -995,7 +990,7 @@ def events():
  if time_range != DEFAULT_TIME_RANGE:
  # get timezone arg to ensure browser times are used
  tz_name = request.args.get("timezone", default="utc", type=str)
- hour_modifier, minute_modifier = get_tz_modifiers(tz_name)
+ hour_modifier, minute_modifier, _ = get_tz_modifiers(tz_name)
 
  times = time_range.split(",")
  time_after = times[0]
@@ -1569,7 +1564,7 @@ def get_recordings_storage_usage():
 @bp.route("/<camera_name>/recordings/summary")
 def recordings_summary(camera_name):
  tz_name = request.args.get("timezone", default="utc", type=str)
- hour_modifier, minute_modifier = get_tz_modifiers(tz_name)
+ hour_modifier, minute_modifier, seconds_offset = get_tz_modifiers(tz_name)
  recording_groups = (
  Recordings.select(
  fn.strftime(
@@ -1583,22 +1578,8 @@ def recordings_summary(camera_name):
  fn.SUM(Recordings.objects).alias("objects"),
  )
  .where(Recordings.camera == camera_name)
- .group_by(
- fn.strftime(
- "%Y-%m-%d %H",
- fn.datetime(
- Recordings.start_time, "unixepoch", hour_modifier, minute_modifier
- ),
- )
- )
- .order_by(
- fn.strftime(
- "%Y-%m-%d H",
- fn.datetime(
- Recordings.start_time, "unixepoch", hour_modifier, minute_modifier
- ),
- ).desc()
- )
+ .group_by((Recordings.start_time + seconds_offset).cast("int") / 3600)
+ .order_by(Recordings.start_time.desc())
  .namedtuples()
  )
 
@@ -1613,14 +1594,7 @@ def recordings_summary(camera_name):
  fn.COUNT(Event.id).alias("count"),
  )
  .where(Event.camera == camera_name, Event.has_clip)
- .group_by(
- fn.strftime(
- "%Y-%m-%d %H",
- fn.datetime(
- Event.start_time, "unixepoch", hour_modifier, minute_modifier
- ),
- ),
- )
+ .group_by((Event.start_time + seconds_offset).cast("int") / 3600)
  .namedtuples()
  )
 

frigate/storage.py

@@ -35,7 +35,7 @@ def calculate_camera_bandwidth(self) -> None:
  if self.camera_storage_stats.get(camera, {}).get("needs_refresh", True):
  self.camera_storage_stats[camera] = {
  "needs_refresh": (
- Recordings.select(fn.COUNT(Recordings.id))
+ Recordings.select(fn.COUNT("*"))
  .where(Recordings.camera == camera, Recordings.segment_size > 0)
  .scalar()
  < 50

frigate/util/builtin.py

@@ -156,15 +156,15 @@ def load_labels(path, encoding="utf-8", prefill=91):
  return labels
 
 
-def get_tz_modifiers(tz_name: str) -> Tuple[str, str]:
+def get_tz_modifiers(tz_name: str) -> Tuple[str, str, int]:
  seconds_offset = (
  datetime.datetime.now(pytz.timezone(tz_name)).utcoffset().total_seconds()
  )
  hours_offset = int(seconds_offset / 60 / 60)
  minutes_offset = int(seconds_offset / 60 - hours_offset * 60)
  hour_modifier = f"{hours_offset} hour"
  minute_modifier = f"{minutes_offset} minute"
- return hour_modifier, minute_modifier
+ return hour_modifier, minute_modifier, seconds_offset
 
 
 def to_relative_box(

migrations/020_update_index_recordings.py

@@ -0,0 +1,40 @@
+"""Peewee migrations -- 020_update_index_recordings.py.
+
+Some examples (model - class or model name)::
+
+ > Model = migrator.orm['model_name'] # Return model in current state by name
+
+ > migrator.sql(sql) # Run custom SQL
+ > migrator.python(func, *args, **kwargs) # Run python code
+ > migrator.create_model(Model) # Create a model (could be used as decorator)
+ > migrator.remove_model(model, cascade=True) # Remove a model
+ > migrator.add_fields(model, **fields) # Add fields to a model
+ > migrator.change_fields(model, **fields) # Change fields
+ > migrator.remove_fields(model, *field_names, cascade=True)
+ > migrator.rename_field(model, old_field_name, new_field_name)
+ > migrator.rename_table(model, new_table_name)
+ > migrator.add_index(model, *col_names, unique=False)
+ > migrator.drop_index(model, *col_names)
+ > migrator.add_not_null(model, *field_names)
+ > migrator.drop_not_null(model, *field_names)
+ > migrator.add_default(model, field_name, default)
+
+"""
+import peewee as pw
+
+SQL = pw.SQL
+
+
+def migrate(migrator, database, fake=False, **kwargs):
+ migrator.sql("DROP INDEX recordings_end_time_start_time")
+ migrator.sql(
+ 'CREATE INDEX "recordings_camera_start_time_end_time" ON "recordings" ("camera", "start_time" DESC, "end_time" DESC)'
+ )
+ migrator.sql(
+ 'CREATE INDEX "recordings_api_recordings_summary" ON "recordings" ("camera", "start_time" DESC, "duration", "motion", "objects")'
+ )
+ migrator.sql('CREATE INDEX "recordings_start_time" ON "recordings" ("start_time")')
+
+
+def rollback(migrator, database, fake=False, **kwargs):
+ pass