Handle error when live view stalls (#11665)

* Handle error when live view stalls

* Manually calculate buffer timeout

* Formatting

web/src/components/player/LivePlayer.tsx

@@ -8,7 +8,11 @@ import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
-import { LivePlayerMode, VideoResolutionType } from "@/types/live";
+import {
+ LivePlayerError,
+ LivePlayerMode,
+ VideoResolutionType,
+} from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
@@ -30,6 +34,7 @@ type LivePlayerProps = {
  autoLive?: boolean;
  onClick?: () => void;
  setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
+ onError?: (error: LivePlayerError) => void;
 };
 
 export default function LivePlayer({
@@ -47,6 +52,7 @@ export default function LivePlayer({
  autoLive = true,
  onClick,
  setFullResolution,
+ onError,
 }: LivePlayerProps) {
  // camera activity
 
@@ -145,6 +151,7 @@ export default function LivePlayer({
  onPlaying={() => setLiveReady(true)}
  pip={pip}
  setFullResolution={setFullResolution}
+ onError={onError}
  />
  );
  } else {

web/src/components/player/MsePlayer.tsx

@@ -1,5 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
-import { VideoResolutionType } from "@/types/live";
+import { LivePlayerError, VideoResolutionType } from "@/types/live";
 import {
  SetStateAction,
  useCallback,
@@ -17,6 +17,7 @@ type MSEPlayerProps = {
  pip?: boolean;
  onPlaying?: () => void;
  setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
+ onError?: (error: LivePlayerError) => void;
 };
 
 function MSEPlayer({
@@ -27,6 +28,7 @@ function MSEPlayer({
  pip = false,
  onPlaying,
  setFullResolution,
+ onError,
 }: MSEPlayerProps) {
  const RECONNECT_TIMEOUT: number = 30000;
 
@@ -45,6 +47,7 @@ function MSEPlayer({
 
  const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
  const [connectTS, setConnectTS] = useState<number>(0);
+ const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
 
  const videoRef = useRef<HTMLVideoElement>(null);
  const wsRef = useRef<WebSocket | null>(null);
@@ -308,7 +311,34 @@ function MSEPlayer({
  onPlaying?.();
  }}
  muted={!audioEnabled}
- onError={() => {
+ onProgress={
+ onError != undefined
+ ? () => {
+ if (videoRef.current?.paused) {
+ return;
+ }
+
+ if (bufferTimeout) {
+ clearTimeout(bufferTimeout);
+ setBufferTimeout(undefined);
+ }
+
+ setBufferTimeout(
+ setTimeout(() => {
+ onError("stalled");
+ }, 3000),
+ );
+ }
+ : undefined
+ }
+ onError={(e) => {
+ if (
+ // @ts-expect-error code does exist
+ e.target.error.code == MediaError.MEDIA_ERR_NETWORK
+ ) {
+ onError?.("startup");
+ }
+
  if (wsRef.current) {
  wsRef.current.close();
  wsRef.current = null;

web/src/components/player/WebRTCPlayer.tsx

@@ -1,4 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
+import { LivePlayerError } from "@/types/live";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
 type WebRtcPlayerProps = {
@@ -10,6 +11,7 @@ type WebRtcPlayerProps = {
  iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element
  pip?: boolean;
  onPlaying?: () => void;
+ onError?: (error: LivePlayerError) => void;
 };
 
 export default function WebRtcPlayer({
@@ -21,6 +23,7 @@ export default function WebRtcPlayer({
  iOSCompatFullScreen = false,
  pip = false,
  onPlaying,
+ onError,
 }: WebRtcPlayerProps) {
  // metadata
 
@@ -32,6 +35,7 @@ export default function WebRtcPlayer({
 
  const pcRef = useRef<RTCPeerConnection | undefined>();
  const videoRef = useRef<HTMLVideoElement | null>(null);
+ const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
 
  const PeerConnection = useCallback(
  async (media: string) => {
@@ -198,11 +202,39 @@ export default function WebRtcPlayer({
  playsInline
  muted={!audioEnabled}
  onLoadedData={onPlaying}
+ onProgress={
+ onError != undefined
+ ? () => {
+ if (videoRef.current?.paused) {
+ return;
+ }
+
+ if (bufferTimeout) {
+ clearTimeout(bufferTimeout);
+ setBufferTimeout(undefined);
+ }
+
+ setBufferTimeout(
+ setTimeout(() => {
+ onError("stalled");
+ }, 3000),
+ );
+ }
+ : undefined
+ }
  onClick={
  iOSCompatFullScreen
  ? () => setiOSCompatControls(!iOSCompatControls)
  : undefined
  }
+ onError={(e) => {
+ if (
+ // @ts-expect-error code does exist
+ e.target.error.code == MediaError.MEDIA_ERR_NETWORK
+ ) {
+ onError?.("startup");
+ }
+ }}
  />
  );
 }

web/src/types/live.ts

@@ -30,3 +30,5 @@ export type LiveStreamMetadata = {
  producers: LiveProducerMetadata[];
  consumers: LiveConsumerMetadata[];
 };
+
+export type LivePlayerError = "stalled" | "startup";

web/src/views/live/LiveCameraView.tsx

@@ -191,6 +191,7 @@ export default function LiveCameraView({
  const [audio, setAudio] = useState(false);
  const [mic, setMic] = useState(false);
  const [pip, setPip] = useState(false);
+ const [lowBandwidth, setLowBandwidth] = useState(false);
 
  const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
  width: 0,
@@ -202,8 +203,14 @@ export default function LiveCameraView({
  return "webrtc";
  }
 
+ if (lowBandwidth) {
+ return "jsmpeg";
+ }
+
  return "mse";
- }, [mic]);
+ }, [lowBandwidth, mic]);
+
+ // layout state
 
  const windowAspectRatio = useMemo(() => {
  return windowWidth / windowHeight;
@@ -419,6 +426,7 @@ export default function LiveCameraView({
  pip={pip}
  setFullResolution={setFullResolution}
  containerRef={containerRef}
+ onError={() => setLowBandwidth(true)}
  />
  </div>
  {camera.onvif.host != "" && (