Add OpenVino Detector (blakeblackshear#3768)

* Initial work for adding OpenVino detector.  Not functional

* Load model and submit for inference.

Sucessfully load model and initialize OpenVino engine with either CPU or GPU as device.
Does not parse results for objects.

* Detection working with ssdlite_mobilenetv2 FP16 model

* Add OpenVIno support and model to docker image

* Add documentation for OpenVino detector configuration

* Adds support for ARM32/ARM64 and the Myriad X hardware

-  Use custom-built openvino wheel for all platforms
-  Add libusb build without udev for NCS2 support

* Add documentation around Intel CPU requirements and NCS2 setup

* Print all available output tensors

* Update documentation for config parameters

Dockerfile

@@ -5,6 +5,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 FROM debian:11 AS base
 
+FROM --platform=linux/amd64 debian:11 AS base_amd64
+
 FROM debian:11-slim AS slim-base
 
 FROM blakeblackshear/frigate-nginx:1.0.2 AS nginx
@@ -24,13 +26,62 @@ WORKDIR /rootfs/usr/local/go2rtc/bin
 RUN wget -qO go2rtc "https://github.com/AlexxIT/go2rtc/releases/download/v0.1-rc.3/go2rtc_linux_${TARGETARCH}" \
  && chmod +x go2rtc
 
+# Download and Convert OpenVino model
+FROM base_amd64 AS ov-converter
+ARG DEBIAN_FRONTEND
+
+# Install OpenVino Runtime and Dev library
+COPY requirements-ov.txt /requirements-ov.txt
+RUN apt-get -qq update \
+ && apt-get -qq install -y wget python3 python3-distutils \
+ && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && python3 get-pip.py "pip" \
+ && pip install -r /requirements-ov.txt
+
+# Get OpenVino Model
+RUN mkdir /models \
+ && cd /models && omz_downloader --name ssdlite_mobilenet_v2 \
+ && cd /models && omz_converter --name ssdlite_mobilenet_v2 --precision FP16
+
+
+# libUSB - No Udev
+FROM wget as libusb-build
+ARG TARGETARCH
+ARG DEBIAN_FRONTEND
+
+# Build libUSB without udev. Needed for Openvino NCS2 support
+WORKDIR /opt
+RUN apt-get update && apt-get install -y unzip build-essential automake libtool
+RUN wget -q https://github.com/libusb/libusb/archive/v1.0.25.zip -O v1.0.25.zip && \
+ unzip v1.0.25.zip && cd libusb-1.0.25 && \
+ ./bootstrap.sh && \
+ ./configure --disable-udev --enable-shared && \
+ make -j $(nproc --all)
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends libusb-1.0-0-dev && \
+ rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/libusb-1.0.25/libusb
+RUN /bin/mkdir -p '/usr/local/lib' && \
+ /bin/bash ../libtool --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \
+ /bin/mkdir -p '/usr/local/include/libusb-1.0' && \
+ /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \
+ /bin/mkdir -p '/usr/local/lib/pkgconfig' && \
+ cd /opt/libusb-1.0.25/ && \
+ /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \
+ ldconfig
+
+
 
 FROM wget AS models
 
 # Get model and labels
 RUN wget -qO edgetpu_model.tflite https://github.com/google-coral/test_data/raw/release-frogfish/ssdlite_mobiledet_coco_qat_postprocess_edgetpu.tflite
 RUN wget -qO cpu_model.tflite https://github.com/google-coral/test_data/raw/release-frogfish/ssdlite_mobiledet_coco_qat_postprocess.tflite
 COPY labelmap.txt .
+# Copy OpenVino model
+COPY --from=ov-converter /models/public/ssdlite_mobilenet_v2/FP16 openvino-model
+RUN wget -q https://github.com/openvinotoolkit/open_model_zoo/raw/master/data/dataset_classes/coco_91cl_bkgr.txt -O openvino-model/coco_91cl_bkgr.txt
+
 
 
 FROM wget AS s6-overlay
@@ -85,6 +136,7 @@ RUN pip3 wheel --wheel-dir=/wheels -r requirements-wheels.txt
 FROM scratch AS deps-rootfs
 COPY --from=nginx /usr/local/nginx/ /usr/local/nginx/
 COPY --from=go2rtc /rootfs/ /
+COPY --from=libusb-build /usr/local/lib /usr/local/lib
 COPY --from=s6-overlay /rootfs/ /
 COPY --from=models /rootfs/ /
 COPY docker/rootfs/ /
@@ -112,6 +164,8 @@ RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
 
 COPY --from=deps-rootfs / /
 
+RUN ldconfig
+
 EXPOSE 5000
 EXPOSE 1935
 EXPOSE 8554

docker/install_deps.sh

@@ -52,7 +52,9 @@ if [[ "${TARGETARCH}" == "amd64" ]]; then
  # Use debian testing repo only for hwaccel packages
  echo 'deb http:https://deb.debian.org/debian testing main non-free' >/etc/apt/sources.list.d/debian-testing.list
  apt-get -qq update
+ # intel-opencl-icd specifically for GPU support in OpenVino
  apt-get -qq install --no-install-recommends --no-install-suggests -y \
+ intel-opencl-icd \
  mesa-va-drivers libva-drm2 intel-media-va-driver-non-free i965-va-driver libmfx1 radeontop intel-gpu-tools
  rm -f /etc/apt/sources.list.d/debian-testing.list
 fi

docs/docs/configuration/detectors.md

@@ -77,3 +77,64 @@ detectors:
 ```
 
 When using CPU detectors, you can add a CPU detector per camera. Adding more detectors than the number of cameras should not improve performance.
+
+## OpenVINO
+
+The OpenVINO detector allows Frigate to run an OpenVINO IR model on Intel CPU, GPU and VPU hardware.
+
+### OpenVINO Devices
+
+The OpenVINO detector supports the Intel-supplied device plugins and can specify one or more devices in the configuration. See OpenVINO's device naming conventions in the [Device Documentation](https://docs.openvino.ai/latest/openvino_docs_OV_UG_Working_with_devices.html) for more detail. Other supported devices could be `AUTO`, `CPU`, `GPU`, `MYRIAD`, etc.
+
+```yaml
+detectors:
+ ov_detector:
+ type: openvino
+ device: GPU
+```
+
+OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. A supported Intel platform is required to use the GPU device with OpenVINO. The `MYRIAD` device may be run on any platform, including Arm devices. For detailed system requirements, see [OpenVINO System Requirements](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/system-requirements.html)
+
+#### Intel NCS2 VPU and Myriad X Setup
+
+Intel produces a neural net inference accelleration chip called Myriad X. This chip was sold in their Neural Compute Stick 2 (NCS2) which has been discontinued. If intending to use the MYRIAD device for accelleration, additional setup is required to pass through the USB device. The host needs a udev rule installed to handle the NCS2 device. 
+
+```bash
+sudo usermod -a -G users "$(whoami)"
+cat <<EOF > 97-myriad-usbboot.rules
+SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"
+SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"
+EOF
+sudo cp 97-myriad-usbboot.rules /etc/udev/rules.d/
+sudo udevadm control --reload-rules
+sudo udevadm trigger
+```
+
+Additionally, the Frigate docker container needs to run with the following configuration:
+
+```bash
+--device-cgroup-rule='c 189:\* rmw' -v /dev/bus/usb:/dev/bus/usb
+```
+or in your compose file:
+
+```yml
+device_cgroup_rules:
+ - 'c 189:* rmw'
+volumes:
+ - /dev/bus/usb:/dev/bus/usb
+```
+
+### OpenVINO Models
+
+The included model for an OpenVINO detector comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. Use the model configuration shown below when using the OpenVINO detector.
+
+```yaml
+model:
+ path: /openvino-model/ssdlite_mobilenet_v2.xml
+ width: 300
+ height: 300
+ input_tensor: nhwc
+ input_pixel_format: bgr
+ labelmap_path: /openvino-model/coco_91cl_bkgr.txt
+
+```

docs/docs/configuration/index.md

@@ -76,9 +76,9 @@ detectors:
  # Required: name of the detector
  coral:
  # Required: type of the detector
- # Valid values are 'edgetpu' (requires device property below) and 'cpu'.
+ # Valid values are 'edgetpu' (requires device property below)  `openvino` (see Detectors documentation), and 'cpu'.
  type: edgetpu
- # Optional: device name as defined here: https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api
+ # Optional: Edgetpu or OpenVino device name
  device: usb
  # Optional: num_threads value passed to the tflite.Interpreter (default: shown below)
  # This value is only used for CPU types
@@ -104,7 +104,7 @@ model:
  input_pixel_format: rgb
  # Optional: Object detection model input tensor format
  # Valid values are nhwc or nchw (default: shown below)
- input_tensor: "nhwc"
+ input_tensor: nhwc
  # Optional: Label name modifications. These are merged into the standard labelmap.
  labelmap:
  2: vehicle

frigate/config.py

@@ -54,6 +54,7 @@ class Config:
 
 class DetectorTypeEnum(str, Enum):
  edgetpu = "edgetpu"
+ openvino = "openvino"
  cpu = "cpu"
 
 

frigate/detectors/openvino.py

@@ -0,0 +1,54 @@
+import logging
+import numpy as np
+import openvino.runtime as ov
+
+from frigate.detectors.detection_api import DetectionApi
+
+
+logger = logging.getLogger(__name__)
+
+
+class OvDetector(DetectionApi):
+ def __init__(self, det_device=None, model_config=None, num_threads=1):
+ self.ov_core = ov.Core()
+ self.ov_model = self.ov_core.read_model(model_config.path)
+
+ self.interpreter = self.ov_core.compile_model(
+ model=self.ov_model, device_name=det_device
+ )
+ logger.info(f"Model Input Shape: {self.interpreter.input(0).shape}")
+ self.output_indexes = 0
+ while True:
+ try:
+ tensor_shape = self.interpreter.output(self.output_indexes).shape
+ logger.info(f"Model Output-{self.output_indexes} Shape: {tensor_shape}")
+ self.output_indexes += 1
+ except:
+ logger.info(f"Model has {self.output_indexes} Output Tensors")
+ break
+
+ def detect_raw(self, tensor_input):
+
+ infer_request = self.interpreter.create_infer_request()
+ infer_request.infer([tensor_input])
+
+ results = infer_request.get_output_tensor()
+
+ detections = np.zeros((20, 6), np.float32)
+ i = 0
+ for object_detected in results.data[0, 0, :]:
+ if object_detected[0] != -1:
+ logger.debug(object_detected)
+ if object_detected[2] < 0.1 or i == 20:
+ break
+ detections[i] = [
+ object_detected[1], # Label ID
+ float(object_detected[2]), # Confidence
+ object_detected[4], # y_min
+ object_detected[3], # x_min
+ object_detected[6], # y_max
+ object_detected[5], # x_max
+ ]
+ i += 1
+
+ return detections

frigate/object_detection.py

@@ -12,6 +12,7 @@
 
 from frigate.config import DetectorTypeEnum, InputTensorEnum
 from frigate.detectors.edgetpu_tfl import EdgeTpuTfl
+from frigate.detectors.openvino import OvDetector
 from frigate.detectors.cpu_tfl import CpuTfl
 
 from frigate.util import EventsPerSecond, SharedMemoryFrameManager, listen, load_labels
@@ -57,6 +58,10 @@ def __init__(
  self.detect_api = EdgeTpuTfl(
  det_device=det_device, model_config=model_config
  )
+ elif det_type == DetectorTypeEnum.openvino:
+ self.detect_api = OvDetector(
+ det_device=det_device, model_config=model_config
+ )
  else:
  logger.warning(
  "CPU detectors are not recommended and should only be used for testing or for trial purposes."

requirements-ov.txt

@@ -0,0 +1,3 @@
+numpy == 1.19.*
+openvino == 2022.*
+openvino-dev[tensorflow2] == 2022.*

requirements-wheels.txt

@@ -18,3 +18,7 @@ scipy == 1.8.*
 setproctitle == 1.2.*
 ws4py == 0.5.*
 zeroconf == 0.39.4
+# Openvino Library - Custom built with MYRIAD support
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-manylinux_2_31_x86_64.whl; platform_machine == 'x86_64'
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-linux_aarch64.whl; platform_machine == 'aarch64'
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-linux_armv7l.whl; platform_machine == 'armv7l'