add plus integration for models (blakeblackshear#6328)

docs/docs/integrations/plus.md

@@ -5,13 +5,11 @@ title: Frigate+
 
 :::info
 
-Frigate+ is under active development and currently only offers the ability to submit your examples with annotations. Models will be available after enough examples are submitted to train a robust model. It is free to create an account and upload your examples.
+Frigate+ is under active development. Models are available as a part of an invitation only beta. It is free to create an account and upload/annotate your examples.
 
 :::
 
-Frigate+ offers models trained from scratch and specifically designed for the way Frigate NVR analyzes video footage. They offer higher accuracy with less resources. By uploading your own labeled examples, your model can be uniquely tuned for accuracy in your specific conditions. After tuning, performance is evaluated against a broad dataset and real world examples submitted by other Frigate+ users to prevent overfitting.
-
-Custom models also include a more relevant set of objects for security cameras such as person, face, car, license plate, delivery truck, package, dog, cat, deer, and more. Interested in detecting an object unique to you? Upload examples to incorporate your own objects without worrying that you are reducing the accuracy of other object types in the model.
+Frigate+ offers models trained from scratch and specifically designed for the way Frigate NVR analyzes video footage. They offer higher accuracy with less resources and include a more relevant set of objects for security cameras. By uploading your own labeled examples, your model can be uniquely tuned for accuracy in your specific conditions. After tuning, performance is evaluated against a broad dataset and real world examples submitted by other Frigate+ users to prevent overfitting.
 
 ## Setup
 
@@ -35,7 +33,7 @@ You cannot use the `environment_vars` section of your configuration file to set
 
 :::
 
-### Submit examples
+## Submit examples
 
 Once your API key is configured, you can submit examples directly from the events page in Frigate using the `SEND TO FRIGATE+` button.
 
@@ -52,3 +50,25 @@ Snapshots must be enabled to be able to submit examples to Frigate+
 You can view all of your submitted images at [https://plus.frigate.video](https://plus.frigate.video). Annotations can be added by clicking an image.
 
 ![Annotate](/img/annotate.png)
+
+## Use Models
+
+Models available in Frigate+ can be used with a special model path. No other information needs to be configured for Frigate+ models because it fetches the remaining config from Frigate+ automatically.
+
+```yaml
+model:
+ path: plus:https://e63b7345cc83a84ed79dedfc99c16616
+```
+
+Models are downloaded into the `/config/model_cache` folder and only downloaded if needed.
+
+You can override the labelmap for Frigate+ models like this:
+
+```yaml
+model:
+ path: plus:https://e63b7345cc83a84ed79dedfc99c16616
+ labelmap:
+ 3: animal
+ 4: animal
+ 5: animal
+```

frigate/app.py

@@ -18,7 +18,14 @@
 from frigate.comms.mqtt import MqttClient
 from frigate.comms.ws import WebSocketClient
 from frigate.config import FrigateConfig
-from frigate.const import CACHE_DIR, CLIPS_DIR, CONFIG_DIR, DEFAULT_DB_PATH, RECORD_DIR
+from frigate.const import (
+ CACHE_DIR,
+ CLIPS_DIR,
+ CONFIG_DIR,
+ DEFAULT_DB_PATH,
+ MODEL_CACHE_DIR,
+ RECORD_DIR,
+)
 from frigate.object_detection import ObjectDetectProcess
 from frigate.events import EventCleanup, EventProcessor
 from frigate.http import create_app
@@ -57,7 +64,7 @@ def set_environment_vars(self) -> None:
  os.environ[key] = value
 
  def ensure_dirs(self) -> None:
- for d in [CONFIG_DIR, RECORD_DIR, CLIPS_DIR, CACHE_DIR]:
+ for d in [CONFIG_DIR, RECORD_DIR, CLIPS_DIR, CACHE_DIR, MODEL_CACHE_DIR]:
  if not os.path.exists(d) and not os.path.islink(d):
  logger.info(f"Creating directory: {d}")
  os.makedirs(d)
@@ -81,7 +88,7 @@ def init_config(self) -> None:
  config_file = config_file_yaml
 
  user_config = FrigateConfig.parse_file(config_file)
- self.config = user_config.runtime_config
+ self.config = user_config.runtime_config(self.plus_api)
 
  for camera_name in self.config.cameras.keys():
  # create camera_metrics
@@ -379,6 +386,7 @@ def start(self) -> None:
  self.init_logger()
  logger.info(f"Starting Frigate ({VERSION})")
  try:
+ self.ensure_dirs()
  try:
  self.init_config()
  except Exception as e:
@@ -399,7 +407,6 @@ def start(self) -> None:
  self.log_process.terminate()
  sys.exit(1)
  self.set_environment_vars()
- self.ensure_dirs()
  self.set_log_levels()
  self.init_queues()
  self.init_database()

frigate/config.py

@@ -19,6 +19,7 @@
  YAML_EXT,
 )
 from frigate.detectors.detector_config import BaseDetectorConfig
+from frigate.plus import PlusApi
 from frigate.util import (
  create_mask,
  deep_merge,
@@ -906,8 +907,7 @@ class FrigateConfig(FrigateBaseModel):
  title="Global timestamp style configuration.",
  )
 
- @property
- def runtime_config(self) -> FrigateConfig:
+ def runtime_config(self, plus_api: PlusApi = None) -> FrigateConfig:
  """Merge camera config with globals."""
  config = self.copy(deep=True)
 
@@ -1031,6 +1031,7 @@ def runtime_config(self) -> FrigateConfig:
  enabled_labels.update(camera.objects.track)
 
  config.model.create_colormap(sorted(enabled_labels))
+ config.model.check_and_load_plus_model(plus_api)
 
  for key, detector in config.detectors.items():
  detector_config: DetectorConfig = parse_obj_as(DetectorConfig, detector)
@@ -1063,6 +1064,9 @@ def runtime_config(self) -> FrigateConfig:
  merged_model["path"] = "/edgetpu_model.tflite"
 
  detector_config.model = ModelConfig.parse_obj(merged_model)
+ detector_config.model.check_and_load_plus_model(
+ plus_api, detector_config.type
+ )
  detector_config.model.compute_model_hash()
  config.detectors[key] = detector_config
 

frigate/const.py

@@ -1,5 +1,6 @@
 CONFIG_DIR = "/config"
 DEFAULT_DB_PATH = f"{CONFIG_DIR}/frigate.db"
+MODEL_CACHE_DIR = f"{CONFIG_DIR}/model_cache"
 BASE_DIR = "/media/frigate"
 CLIPS_DIR = f"{BASE_DIR}/clips"
 RECORD_DIR = f"{BASE_DIR}/recordings"

frigate/detectors/detector_config.py

@@ -1,11 +1,16 @@
 import hashlib
+import json
 import logging
 from enum import Enum
+import os
 from typing import Dict, List, Optional, Tuple, Union, Literal
 
+
+import requests
 import matplotlib.pyplot as plt
 from pydantic import BaseModel, Extra, Field, validator
 from pydantic.fields import PrivateAttr
+from frigate.plus import PlusApi
 
 from frigate.util import load_labels
 
@@ -73,6 +78,45 @@ def __init__(self, **config):
  }
  self._colormap = {}
 
+ def check_and_load_plus_model(
+ self, plus_api: PlusApi, detector: str = None
+ ) -> None:
+ if not self.path or not self.path.startswith("plus:https://"):
+ return
+
+ model_id = self.path[7:]
+ self.path = f"/config/model_cache/{model_id}"
+ model_info_path = f"{self.path}.json"
+
+ # download the model if it doesn't exist
+ if not os.path.isfile(self.path):
+ download_url = plus_api.get_model_download_url(model_id)
+ r = requests.get(download_url)
+ with open(self.path, "wb") as f:
+ f.write(r.content)
+
+ # download the model info if it doesn't exist
+ if not os.path.isfile(model_info_path):
+ model_info = plus_api.get_model_info(model_id)
+ with open(model_info_path, "w") as f:
+ json.dump(model_info, f)
+ else:
+ with open(model_info_path, "r") as f:
+ model_info = json.load(f)
+
+ if detector and detector not in model_info["supportedDetectors"]:
+ raise ValueError(f"Model does not support detector type of {detector}")
+
+ self.width = model_info["width"]
+ self.height = model_info["height"]
+ self.input_tensor = model_info["inputShape"]
+ self.input_pixel_format = model_info["pixelFormat"]
+ self.model_type = model_info["type"]
+ self._merged_labelmap = {
+ **{int(key): val for key, val in model_info["labelMap"].items()},
+ **self.labelmap,
+ }
+
  def compute_model_hash(self) -> None:
  with open(self.path, "rb") as f:
  file_hash = hashlib.md5()

frigate/http.py

@@ -866,6 +866,11 @@ def config():
 
  config["plus"] = {"enabled": current_app.plus_api.is_active()}
 
+ for detector, detector_config in config["detectors"].items():
+ detector_config["model"]["labelmap"] = current_app.frigate_config.detectors[
+ detector
+ ].model.merged_labelmap
+
  return jsonify(config)
 
 

frigate/plus.py

@@ -3,7 +3,7 @@
 import logging
 import os
 import re
-from typing import List
+from typing import Any, Dict, List
 import requests
 from frigate.const import PLUS_ENV_VAR, PLUS_API_HOST
 from requests.models import Response
@@ -187,3 +187,24 @@ def add_annotation(
 
  if not r.ok:
  raise Exception(r.text)
+
+ def get_model_download_url(
+ self,
+ model_id: str,
+ ) -> str:
+ r = self._get(f"model/{model_id}/signed_url")
+
+ if not r.ok:
+ raise Exception(r.text)
+
+ presigned_url = r.json()
+
+ return str(presigned_url.get("url"))
+
+ def get_model_info(self, model_id: str) -> Any:
+ r = self._get(f"model/{model_id}")
+
+ if not r.ok:
+ raise Exception(r.text)
+
+ return r.json()