Add healthcheck for go2rtc service (blakeblackshear#5545)

* Upgrade s6-overlay from 3.1.3.0 to 3.1.4.0

* Add go2rtc healthcheck service

Also don't make go2rtc exits cause the container to fail.

* Reword healthcheck message

Co-authored-by: Nicolas Mowen <[email protected]>

* Add timeout to go2rtc healthcheck

* Update healthcheck message

Co-authored-by: Nicolas Mowen <[email protected]>

* Give additional time for go2rtc start/restart

* Fix typo

* Avoid creating go2rtc config multiple times

* Fix healthcheck not starting

* Fix sleep

* Fix more hidden logs

* Decrease time window and use curl's timeout flag

---------

Co-authored-by: Nicolas Mowen <[email protected]>

Dockerfile

@@ -207,6 +207,10 @@ FROM deps AS devcontainer
 # But start a fake service for simulating the logs
 COPY docker/fake_frigate_run /etc/s6-overlay/s6-rc.d/frigate/run
 
+# Create symbolic link to the frigate source code, as go2rtc's create_config.sh uses it
+RUN mkdir -p /opt/frigate \
+ && ln -svf /workspace/frigate/frigate /opt/frigate/frigate
+
 # Install Node 16
 RUN apt-get update \
  && apt-get install wget -y \

docker/install_s6_overlay.sh

@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-s6_version="3.1.3.0"
+s6_version="3.1.4.0"
 
 if [[ "${TARGETARCH}" == "amd64" ]]; then
  s6_arch="x86_64"

docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/finish

@@ -4,27 +4,25 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
 readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="Frigate"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
  if [[ "${exit_code_container}" -eq 0 ]]; then
- echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+ echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
  fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
  if [[ "${exit_code_container}" -eq 0 ]]; then
- echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+ echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
  fi
-else
- # Exit code 0 is expected when Frigate is restarted by the user. In this case,
- # we create a signal for the go2rtc finish script to tolerate the restart.
- touch /dev/shm/restarting-frigate
 fi
 
 exec /run/s6/basedir/bin/halt

docker/rootfs/etc/s6-overlay/s6-rc.d/frigate/run

@@ -4,12 +4,14 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 # Tell S6-Overlay not to restart this service
 s6-svc -O .
 
-echo "[INFO] Starting Frigate..." >&2
+echo "[INFO] Starting Frigate..."
 
-cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate" >&2
+cd /opt/frigate || echo "[ERROR] Failed to change working directory to /opt/frigate"
 
 # Replace the bash process with the Frigate process, redirecting stderr to stdout
 exec 2>&1

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/finish

@@ -0,0 +1,12 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+readonly exit_code_service="${1}"
+readonly exit_code_signal="${2}"
+readonly service="go2rtc-healthcheck"
+
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/producer-for

@@ -0,0 +1 @@
+go2rtc-log

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/run

@@ -0,0 +1,22 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+# Start the go2rtc-healthcheck service
+
+set -o errexit -o nounset -o pipefail
+
+# Logs should be sent to stdout so that s6 can collect them
+
+# Give some additional time for go2rtc to start before start pinging
+sleep 10s
+echo "[INFO] Starting go2rtc healthcheck service..."
+
+while sleep 30s; do
+ # Check if the service is running
+ if ! curl --connect-timeout 10 --fail --silent --show-error --output /dev/null https://127.0.0.1:1984/api/streams 2>&1; then
+ echo "[ERROR] The go2rtc service is not responding to ping, restarting..."
+ # We can also use -r instead of -t to send kill signal rather than term
+ s6-svc -t /var/run/service/go2rtc 2>&1
+ # Give some additional time to go2rtc to restart before start pinging again
+ sleep 10s
+ fi
+done

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/timeout-kill

@@ -0,0 +1 @@
+5000

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-healthcheck/type

@@ -0,0 +1 @@
+longrun

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc-log/consumer-for

@@ -1 +1,2 @@
 go2rtc
+go2rtc-healthcheck

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/finish

@@ -1,32 +1,12 @@
 #!/command/with-contenv bash
 # shellcheck shell=bash
-# Take down the S6 supervision tree when the service exits
 
 set -o errexit -o nounset -o pipefail
 
-declare exit_code_container
-exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
-readonly exit_code_container
+# Logs should be sent to stdout so that s6 can collect them
+
 readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="go2rtc"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
-
-if [[ "${exit_code_service}" -eq 256 ]]; then
- if [[ "${exit_code_container}" -eq 0 ]]; then
- echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
- fi
-elif [[ "${exit_code_service}" -ne 0 ]]; then
- if [[ "${exit_code_container}" -eq 0 ]]; then
- echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
- fi
-else
- # go2rtc is not supposed to exit, so even when it exits with 0 we make the
- # container with 1. We only tolerate it when Frigate is restarting.
- if [[ "${exit_code_container}" -eq 0 && ! -f /dev/shm/restarting-frigate ]]; then
- echo "1" > /run/s6-linux-init-container-results/exitcode
- fi
-fi
-
-exec /run/s6/basedir/bin/halt
+echo "[INFO] The ${service} service exited with code ${exit_code_service} (by signal ${exit_code_signal})"

docker/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run

@@ -4,8 +4,7 @@
 
 set -o errexit -o nounset -o pipefail
 
-# Tell S6-Overlay not to restart this service
-s6-svc -O .
+# Logs should be sent to stdout so that s6 can collect them
 
 function get_ip_and_port_from_supervisor() {
  local ip_address
@@ -19,9 +18,9 @@ function get_ip_and_port_from_supervisor() {
  jq --exit-status --raw-output '.data.ipv4.address[0]'
  ) && [[ "${ip_address}" =~ ${ip_regex} ]]; then
  ip_address="${BASH_REMATCH[1]}"
- echo "[INFO] Got IP address from supervisor: ${ip_address}" >&2
+ echo "[INFO] Got IP address from supervisor: ${ip_address}"
  else
- echo "[WARN] Failed to get IP address from supervisor" >&2
+ echo "[WARN] Failed to get IP address from supervisor"
  return 0
  fi
 
@@ -35,26 +34,28 @@ function get_ip_and_port_from_supervisor() {
  jq --exit-status --raw-output '.data.network["8555/tcp"]'
  ) && [[ "${webrtc_port}" =~ ${port_regex} ]]; then
  webrtc_port="${BASH_REMATCH[1]}"
- echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}" >&2
+ echo "[INFO] Got WebRTC port from supervisor: ${webrtc_port}"
  else
- echo "[WARN] Failed to get WebRTC port from supervisor" >&2
+ echo "[WARN] Failed to get WebRTC port from supervisor"
  return 0
  fi
 
  export FRIGATE_GO2RTC_WEBRTC_CANDIDATE_INTERNAL="${ip_address}:${webrtc_port}"
 }
 
-echo "[INFO] Preparing go2rtc config..." >&2
+if [[ ! -f "/dev/shm/go2rtc.yaml" ]]; then
+ echo "[INFO] Preparing go2rtc config..."
 
-if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
- # Running as a Home Assistant add-on, infer the IP address and port
- get_ip_and_port_from_supervisor
-fi
+ if [[ -n "${SUPERVISOR_TOKEN:-}" ]]; then
+  # Running as a Home Assistant add-on, infer the IP address and port
+  get_ip_and_port_from_supervisor
+ fi
 
-raw_config=$(python3 /usr/local/go2rtc/create_config.py)
+ python3 /usr/local/go2rtc/create_config.py
+fi
 
-echo "[INFO] Starting go2rtc..." >&2
+echo "[INFO] Starting go2rtc..."
 
 # Replace the bash process with the go2rtc process, redirecting stderr to stdout
 exec 2>&1
-exec go2rtc -config="${raw_config}"
+exec go2rtc -config=/dev/shm/go2rtc.yaml

docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/finish

@@ -4,25 +4,27 @@
 
 set -o errexit -o nounset -o pipefail
 
+# Logs should be sent to stdout so that s6 can collect them
+
 declare exit_code_container
 exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
 readonly exit_code_container
 readonly exit_code_service="${1}"
 readonly exit_code_signal="${2}"
 readonly service="NGINX"
 
-echo "Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" >&2
+echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
 
 if [[ "${exit_code_service}" -eq 256 ]]; then
  if [[ "${exit_code_container}" -eq 0 ]]; then
- echo $((128 + exit_code_signal)) > /run/s6-linux-init-container-results/exitcode
+ echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
  fi
  if [[ "${exit_code_signal}" -eq 15 ]]; then
  exec /run/s6/basedir/bin/halt
  fi
 elif [[ "${exit_code_service}" -ne 0 ]]; then
  if [[ "${exit_code_container}" -eq 0 ]]; then
- echo "${exit_code_service}" > /run/s6-linux-init-container-results/exitcode
+ echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
  fi
  exec /run/s6/basedir/bin/halt
 fi

docker/rootfs/etc/s6-overlay/s6-rc.d/nginx/run

@@ -4,7 +4,9 @@
 
 set -o errexit -o nounset -o pipefail
 
-echo "[INFO] Starting NGINX..." >&2
+# Logs should be sent to stdout so that s6 can collect them
+
+echo "[INFO] Starting NGINX..."
 
 # Replace the bash process with the NGINX process, redirecting stderr to stdout
 exec 2>&1

docker/rootfs/usr/local/go2rtc/create_config.py

@@ -8,6 +8,7 @@
 sys.path.insert(0, "/opt/frigate")
 from frigate.const import BIRDSEYE_PIPE, BTBN_PATH
 from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode
+
 sys.path.remove("/opt/frigate")
 
 
@@ -50,7 +51,6 @@
 else:
  print(
  "[INFO] Not injecting WebRTC candidates into go2rtc config as it has been set manually",
- file=sys.stderr,
  )
 
 # sets default RTSP response to be equivalent to ?video=h264,h265&audio=aac
@@ -95,5 +95,6 @@
  else:
  go2rtc_config["streams"] = {"birdseye": ffmpeg_cmd}
 
-
-print(json.dumps(go2rtc_config))
+# Write go2rtc_config to /dev/shm/go2rtc.yaml
+with open("/dev/shm/go2rtc.yaml", "w") as f:
+ yaml.dump(go2rtc_config, f)