From e32bd4ab15360b00ed1f1e5c24588def1bd9c060 Mon Sep 17 00:00:00 2001
From: tpjanssen <25168870+tpjanssen@users.noreply.github.com>
Date: Fri, 13 Oct 2023 13:17:41 +0200
Subject: [PATCH] Added audio sensors to camera metrics and API stats (#8109)

* Added audio sensor to camera metrics and API stats

* Update types.py

* Update app.py
---
 frigate/app.py          | 3 +++
 frigate/events/audio.py | 9 ++++++++-
 frigate/stats.py        | 2 ++
 frigate/types.py        | 2 ++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/frigate/app.py b/frigate/app.py
index 3a5da0e808..a2e3005268 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -163,6 +163,8 @@ def init_config(self) -> None:
                 "frame_queue": mp.Queue(maxsize=2),
                 "capture_process": None,
                 "process": None,
+                "audio_rms": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
+                "audio_dBFS": mp.Value("d", 0.0),  # type: ignore[typeddict-item]
             }
             self.ptz_metrics[camera_name] = {
                 "ptz_autotracker_enabled": mp.Value(  # type: ignore[typeddict-item]
@@ -500,6 +502,7 @@ def start_audio_processors(self) -> None:
                 args=(
                     self.config,
                     self.audio_recordings_info_queue,
+                    self.camera_metrics,
                     self.feature_metrics,
                     self.inter_process_communicator,
                 ),
diff --git a/frigate/events/audio.py b/frigate/events/audio.py
index f06b593c1d..50e1d9ab0a 100644
--- a/frigate/events/audio.py
+++ b/frigate/events/audio.py
@@ -26,7 +26,7 @@
 from frigate.ffmpeg_presets import parse_preset_input
 from frigate.log import LogPipe
 from frigate.object_detection import load_labels
-from frigate.types import FeatureMetricsTypes
+from frigate.types import CameraMetricsTypes, FeatureMetricsTypes
 from frigate.util.builtin import get_ffmpeg_arg_list
 from frigate.util.services import listen
 from frigate.video import start_or_restart_ffmpeg, stop_ffmpeg
@@ -52,6 +52,7 @@ def get_ffmpeg_command(input_args: list[str], input_path: str) -> list[str]:
 def listen_to_audio(
     config: FrigateConfig,
     recordings_info_queue: mp.Queue,
+    camera_metrics: dict[str, CameraMetricsTypes],
     process_info: dict[str, FeatureMetricsTypes],
     inter_process_communicator: InterProcessCommunicator,
 ) -> None:
@@ -80,6 +81,7 @@ def receiveSignal(signalNumber: int, frame: Optional[FrameType]) -> None:
             audio = AudioEventMaintainer(
                 camera,
                 recordings_info_queue,
+                camera_metrics,
                 process_info,
                 stop_event,
                 inter_process_communicator,
@@ -153,6 +155,7 @@ def __init__(
         self,
         camera: CameraConfig,
         recordings_info_queue: mp.Queue,
+        camera_metrics: dict[str, CameraMetricsTypes],
         feature_metrics: dict[str, FeatureMetricsTypes],
         stop_event: mp.Event,
         inter_process_communicator: InterProcessCommunicator,
@@ -161,6 +164,7 @@ def __init__(
         self.name = f"{camera.name}_audio_event_processor"
         self.config = camera
         self.recordings_info_queue = recordings_info_queue
+        self.camera_metrics = camera_metrics
         self.feature_metrics = feature_metrics
         self.inter_process_communicator = inter_process_communicator
         self.detections: dict[dict[str, any]] = {}
@@ -184,6 +188,9 @@ def detect_audio(self, audio) -> None:
         audio_as_float = audio.astype(np.float32)
         rms, dBFS = self.calculate_audio_levels(audio_as_float)
 
+        self.camera_metrics[self.config.name]["audio_rms"].value = rms
+        self.camera_metrics[self.config.name]["audio_dBFS"].value = dBFS
+
         # only run audio detection when volume is above min_volume
         if rms >= self.config.audio.min_volume:
             # add audio info to recordings queue
diff --git a/frigate/stats.py b/frigate/stats.py
index da5eb358e7..41ba2c367b 100644
--- a/frigate/stats.py
+++ b/frigate/stats.py
@@ -266,6 +266,8 @@ def stats_snapshot(
             "pid": pid,
             "capture_pid": cpid,
             "ffmpeg_pid": ffmpeg_pid,
+            "audio_rms": round(camera_stats["audio_rms"].value, 4),
+            "audio_dBFS": round(camera_stats["audio_dBFS"].value, 4),
         }
 
     stats["detectors"] = {}
diff --git a/frigate/types.py b/frigate/types.py
index 60e8d5a148..d97c0e53b9 100644
--- a/frigate/types.py
+++ b/frigate/types.py
@@ -23,6 +23,8 @@ class CameraMetricsTypes(TypedDict):
     process_fps: Synchronized
     read_start: Synchronized
     skipped_fps: Synchronized
+    audio_rms: Synchronized
+    audio_dBFS: Synchronized
 
 
 class PTZMetricsTypes(TypedDict):