Use full resolution aspect ratio when available (blakeblackshear#11173)

* base recordings and live views off of actual video resolution * don't set for jsmpeg * reset when changing main cam * rename * Only use resolution for main camera * fix lint --------- Co-authored-by: Nicolas Mowen <[email protected]>
guxiaoqi2007 · Apr 30, 2024 · 11ff7cb · 11ff7cb
1 parent 1c9626e
commit 11ff7cb
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 13 deletions.
diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx
@@ -1,8 +1,15 @@
-import { MutableRefObject, useEffect, useRef, useState } from "react";
+import {
+ MutableRefObject,
+ useCallback,
+ useEffect,
+ useRef,
+ useState,
+} from "react";
 import Hls from "hls.js";
 import { isAndroid, isDesktop, isMobile } from "react-device-detect";
 import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
 import VideoControls from "./VideoControls";
+import { VideoResolutionType } from "@/types/live";
 
 // Android native hls does not seek correctly
 const USE_NATIVE_HLS = !isAndroid;
@@ -21,6 +28,7 @@ type HlsVideoPlayerProps = {
  onPlayerLoaded?: () => void;
  onTimeUpdate?: (time: number) => void;
  onPlaying?: () => void;
+ setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 export default function HlsVideoPlayer({
  videoRef,
@@ -31,13 +39,26 @@ export default function HlsVideoPlayer({
  onPlayerLoaded,
  onTimeUpdate,
  onPlaying,
+ setFullResolution,
 }: HlsVideoPlayerProps) {
  // playback
 
  const hlsRef = useRef<Hls>();
  const [useHlsCompat, setUseHlsCompat] = useState(false);
  const [loadedMetadata, setLoadedMetadata] = useState(false);
 
+ const handleLoadedMetadata = useCallback(() => {
+ setLoadedMetadata(true);
+ if (videoRef.current) {
+ if (setFullResolution) {
+ setFullResolution({
+ width: videoRef.current.videoWidth,
+ height: videoRef.current.videoHeight,
+ });
+ }
+ }
+ }, [videoRef, setFullResolution]);
+
  useEffect(() => {
  if (!videoRef.current) {
  return;
@@ -193,7 +214,7 @@ export default function HlsVideoPlayer({
  : undefined
  }
  onLoadedData={onPlayerLoaded}
- onLoadedMetadata={() => setLoadedMetadata(true)}
+ onLoadedMetadata={handleLoadedMetadata}
  onEnded={onClipEnded}
  onError={(e) => {
  if (

diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx
@@ -8,7 +8,7 @@ import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
-import { LivePlayerMode } from "@/types/live";
+import { LivePlayerMode, VideoResolutionType } from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
@@ -27,6 +27,7 @@ type LivePlayerProps = {
  iOSCompatFullScreen?: boolean;
  pip?: boolean;
  onClick?: () => void;
+ setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 
 export default function LivePlayer({
@@ -41,6 +42,7 @@ export default function LivePlayer({
  iOSCompatFullScreen = false,
  pip,
  onClick,
+ setFullResolution,
 }: LivePlayerProps) {
  const [cameraHovered, setCameraHovered] = useState(false);
 
@@ -123,6 +125,7 @@ export default function LivePlayer({
  audioEnabled={playAudio}
  onPlaying={() => setLiveReady(true)}
  pip={pip}
+ setFullResolution={setFullResolution}
  />
  );
  } else {

diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx
@@ -1,5 +1,13 @@
 import { baseUrl } from "@/api/baseUrl";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { VideoResolutionType } from "@/types/live";
+import {
+ SetStateAction,
+ useCallback,
+ useEffect,
+ useMemo,
+ useRef,
+ useState,
+} from "react";
 
 type MSEPlayerProps = {
  camera: string;
@@ -8,6 +16,7 @@ type MSEPlayerProps = {
  audioEnabled?: boolean;
  pip?: boolean;
  onPlaying?: () => void;
+ setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
 };
 
 function MSEPlayer({
@@ -17,6 +26,7 @@ function MSEPlayer({
  audioEnabled = false,
  pip = false,
  onPlaying,
+ setFullResolution,
 }: MSEPlayerProps) {
  let connectTS: number = 0;
 
@@ -50,6 +60,15 @@ function MSEPlayer({
  return `${baseUrl.replace(/^http/, "ws")}live/mse/api/ws?src=${camera}`;
  }, [camera]);
 
+ const handleLoadedMetadata = useCallback(() => {
+ if (videoRef.current && setFullResolution) {
+ setFullResolution({
+ width: videoRef.current.videoWidth,
+ height: videoRef.current.videoHeight,
+ });
+ }
+ }, [setFullResolution]);
+
  const play = () => {
  const currentVideo = videoRef.current;
 
@@ -286,6 +305,7 @@ function MSEPlayer({
  playsInline
  preload="auto"
  onLoadedData={onPlaying}
+ onLoadedMetadata={handleLoadedMetadata}
  muted={!audioEnabled}
  />
  );

diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx
@@ -9,6 +9,7 @@ import { DynamicVideoController } from "./DynamicVideoController";
 import HlsVideoPlayer from "../HlsVideoPlayer";
 import { TimeRange } from "@/types/timeline";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
+import { VideoResolutionType } from "@/types/live";
 
 /**
  * Dynamically switches between video playback and scrubbing preview player.
@@ -24,6 +25,7 @@ type DynamicVideoPlayerProps = {
  onControllerReady: (controller: DynamicVideoController) => void;
  onTimestampUpdate?: (timestamp: number) => void;
  onClipEnded?: () => void;
+ setFullResolution: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
 };
 export default function DynamicVideoPlayer({
  className,
@@ -36,6 +38,7 @@ export default function DynamicVideoPlayer({
  onControllerReady,
  onTimestampUpdate,
  onClipEnded,
+ setFullResolution,
 }: DynamicVideoPlayerProps) {
  const apiHost = useApiHost();
  const { data: config } = useSWR<FrigateConfig>("config");
@@ -182,6 +185,7 @@ export default function DynamicVideoPlayer({
  setIsLoading(false);
  setNoRecording(false);
  }}
+ setFullResolution={setFullResolution}
  />
  <PreviewPlayer
  className={`${isScrubbing || isLoading ? "visible" : "hidden"} ${className}`}

diff --git a/web/src/types/live.ts b/web/src/types/live.ts
@@ -1 +1,5 @@
 export type LivePlayerMode = "webrtc" | "mse" | "jsmpeg" | "debug";
+export type VideoResolutionType = {
+ width: number;
+ height: number;
+};
diff --git a/web/src/types/record.ts b/web/src/types/record.ts
@@ -38,3 +38,6 @@ export type RecordingStartingPoint = {
  startTime: number;
  severity: ReviewSeverity;
 };
+
+export const ASPECT_VERTICAL_LAYOUT = 1.5;
+export const ASPECT_WIDE_LAYOUT = 2;
diff --git a/web/src/views/events/RecordingView.tsx b/web/src/views/events/RecordingView.tsx
@@ -41,6 +41,8 @@ import MobileReviewSettingsDrawer from "@/components/overlay/MobileReviewSetting
 import Logo from "@/components/Logo";
 import { Skeleton } from "@/components/ui/skeleton";
 import { FaVideo } from "react-icons/fa";
+import { VideoResolutionType } from "@/types/live";
+import { ASPECT_VERTICAL_LAYOUT, ASPECT_WIDE_LAYOUT } from "@/types/record";
 
 const SEGMENT_DURATION = 30;
 
@@ -189,9 +191,18 @@ export function RecordingView({
  // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [currentTime, scrubbing]);
 
+ const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
+ width: 0,
+ height: 0,
+ });
+
  const onSelectCamera = useCallback(
  (newCam: string) => {
  setMainCamera(newCam);
+ setFullResolution({
+ width: 0,
+ height: 0,
+ });
  setPlaybackStart(currentTime);
  },
  [currentTime],
@@ -205,6 +216,10 @@ export function RecordingView({
  return undefined;
  }
 
+ if (cam == mainCamera && fullResolution.width && fullResolution.height) {
+ return fullResolution.width / fullResolution.height;
+ }
+
  const camera = config.cameras[cam];
 
  if (!camera) {
@@ -213,17 +228,17 @@ export function RecordingView({
 
  return camera.detect.width / camera.detect.height;
  },
- [config],
+ [config, fullResolution, mainCamera],
  );
 
  const mainCameraAspect = useMemo(() => {
  const aspectRatio = getCameraAspect(mainCamera);
 
  if (!aspectRatio) {
  return "normal";
- } else if (aspectRatio > 2) {
+ } else if (aspectRatio > ASPECT_WIDE_LAYOUT) {
  return "wide";
- } else if (aspectRatio < 16 / 9) {
+ } else if (aspectRatio < ASPECT_VERTICAL_LAYOUT) {
  return "tall";
  } else {
  return "normal";
@@ -397,6 +412,7 @@ export function RecordingView({
  mainControllerRef.current = controller;
  }}
  isScrubbing={scrubbing || exportMode == "timeline"}
+ setFullResolution={setFullResolution}
  />
  </div>
  {isDesktop && (

diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx
@@ -20,6 +20,7 @@ import { TooltipProvider } from "@/components/ui/tooltip";
 import { useResizeObserver } from "@/hooks/resize-observer";
 import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { CameraConfig } from "@/types/frigateConfig";
+import { VideoResolutionType } from "@/types/live";
 import { CameraPtzInfo } from "@/types/ptz";
 import { RecordingStartingPoint } from "@/types/record";
 import React, {
@@ -149,14 +150,24 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
  const [fullscreen, setFullscreen] = useState(false);
  const [pip, setPip] = useState(false);
 
+ const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
+ width: 0,
+ height: 0,
+ });
+
  const growClassName = useMemo(() => {
- const aspect = camera.detect.width / camera.detect.height;
+ let aspect;
+ if (fullResolution.width && fullResolution.height) {
+ aspect = fullResolution.width / fullResolution.height;
+ } else {
+ aspect = camera.detect.width / camera.detect.height;
+ }
 
  if (isMobile) {
  if (isPortrait) {
  return "absolute left-2 right-2 top-[50%] -translate-y-[50%]";
  } else {
- if (aspect > 16 / 9) {
+ if (aspect > 1.5) {
  return "p-2 absolute left-0 top-[50%] -translate-y-[50%]";
  } else {
  return "p-2 absolute top-2 bottom-2 left-[50%] -translate-x-[50%]";
@@ -165,15 +176,15 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
  }
 
  if (fullscreen) {
- if (aspect > 16 / 9) {
+ if (aspect > 1.5) {
  return "absolute inset-x-2 top-[50%] -translate-y-[50%]";
  } else {
  return "absolute inset-y-2 left-[50%] -translate-x-[50%]";
  }
  } else {
  return "absolute top-2 bottom-2 left-[50%] -translate-x-[50%]";
  }
- }, [camera, fullscreen, isPortrait]);
+ }, [camera, fullscreen, isPortrait, fullResolution]);
 
  const preferredLiveMode = useMemo(() => {
  if (isSafari || mic) {
@@ -188,8 +199,12 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
  }, [windowWidth, windowHeight]);
 
  const cameraAspectRatio = useMemo(() => {
- return camera.detect.width / camera.detect.height;
- }, [camera]);
+ if (fullResolution.width && fullResolution.height) {
+ return fullResolution.width / fullResolution.height;
+ } else {
+ return camera.detect.width / camera.detect.height;
+ }
+ }, [camera, fullResolution]);
 
  const aspectRatio = useMemo<number>(() => {
  if (isMobile || fullscreen) {
@@ -347,6 +362,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) {
  iOSCompatFullScreen={isIOS}
  preferredLiveMode={preferredLiveMode}
  pip={pip}
+ setFullResolution={setFullResolution}
  />
  </div>
  {camera.onvif.host != "" && (