Merge pull request #16 from occ-ai/roy.overhaul_detection_vad_models

Update whisper-utils.h and whisper-utils.cpp to use update_whisper_mo…

.github/scripts/.Aptfile

@@ -5,3 +5,4 @@ package 'jq'
 package 'ninja-build', bin: 'ninja'
 package 'pkg-config'
 package 'libopenblas-dev'
+package 'libavformat-dev'

CMakeLists.txt

@@ -34,6 +34,55 @@ if(ENABLE_QT)
  AUTORCC ON)
 endif()
 
+function(find_libav)
+ if(UNIX AND NOT APPLE)
+ find_package(PkgConfig REQUIRED)
+ pkg_check_modules(
+ FFMPEG
+ REQUIRED
+ IMPORTED_TARGET
+ libavformat
+ libavcodec
+ libavutil
+ libswresample)
+ if(FFMPEG_FOUND)
+ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE PkgConfig::FFMPEG)
+ else()
+ message(FATAL_ERROR "FFMPEG not found!")
+ endif()
+ return()
+ endif()
+
+ if(NOT buildspec)
+ file(READ "${CMAKE_CURRENT_SOURCE_DIR}/buildspec.json" buildspec)
+ endif()
+ string(
+ JSON
+ version
+ GET
+ ${buildspec}
+ dependencies
+ prebuilt
+ version)
+
+ if(MSVC)
+ set(arch ${CMAKE_GENERATOR_PLATFORM})
+ elseif(APPLE)
+ set(arch universal)
+ endif()
+ set(deps_root "${CMAKE_CURRENT_SOURCE_DIR}/.deps/obs-deps-${version}-${arch}")
+
+ target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE "${deps_root}/include")
+ target_link_libraries(
+ ${CMAKE_PROJECT_NAME}
+ PRIVATE "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avcodec${CMAKE_STATIC_LIBRARY_SUFFIX}"
+ "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avformat${CMAKE_STATIC_LIBRARY_SUFFIX}"
+ "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}avutil${CMAKE_STATIC_LIBRARY_SUFFIX}"
+ "${deps_root}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}swresample${CMAKE_STATIC_LIBRARY_SUFFIX}")
+endfunction(find_libav)
+
+find_libav()
+
 set(USE_SYSTEM_CURL
  OFF
  CACHE STRING "Use system cURL")
@@ -50,14 +99,44 @@ endif()
 include(cmake/BuildWhispercpp.cmake)
 target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE Whispercpp)
 
+set(USE_SYSTEM_ONNXRUNTIME
+ OFF
+ CACHE STRING "Use system ONNX Runtime")
+
+set(DISABLE_ONNXRUNTIME_GPU
+ OFF
+ CACHE STRING "Disables GPU support of ONNX Runtime (Only valid on Linux)")
+
+if(DISABLE_ONNXRUNTIME_GPU)
+ target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE DISABLE_ONNXRUNTIME_GPU)
+endif()
+
+if(USE_SYSTEM_ONNXRUNTIME)
+ if(OS_LINUX)
+ find_package(Onnxruntime 1.16.3 REQUIRED)
+ set(Onnxruntime_INCLUDE_PATH
+ ${Onnxruntime_INCLUDE_DIR} ${Onnxruntime_INCLUDE_DIR}/onnxruntime
+ ${Onnxruntime_INCLUDE_DIR}/onnxruntime/core/session ${Onnxruntime_INCLUDE_DIR}/onnxruntime/core/providers/cpu)
+ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIBRARIES}")
+ target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${Onnxruntime_INCLUDE_PATH}")
+ else()
+ message(FATAL_ERROR "System ONNX Runtime is only supported on Linux!")
+ endif()
+else()
+ include(cmake/FetchOnnxruntime.cmake)
+endif()
+
 target_sources(
  ${CMAKE_PROJECT_NAME}
  PRIVATE src/plugin-main.c
  src/cleanstream-filter.cpp
  src/cleanstream-filter.c
+ src/audio-utils/read-audio-file.cpp
  src/model-utils/model-downloader.cpp
  src/model-utils/model-downloader-ui.cpp
+ src/model-utils/model-infos.cpp
  src/whisper-utils/whisper-utils.cpp
- src/whisper-utils/whisper-processing.cpp)
+ src/whisper-utils/whisper-processing.cpp
+ src/whisper-utils/silero-vad-onnx.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})

buildspec.json

@@ -45,7 +45,7 @@
  }
  },
  "name": "obs-cleanstream",
- "version": "0.0.5",
+ "version": "0.0.6",
  "author": "Roy Shilkrot",
  "website": "https://github.com/occ-ai/obs-cleanstream/",
  "email": "[email protected]",

cmake/FetchOnnxruntime.cmake

@@ -0,0 +1,97 @@
+include(FetchContent)
+
+set(CUSTOM_ONNXRUNTIME_URL
+ ""
+ CACHE STRING "URL of a downloaded ONNX Runtime tarball")
+
+set(CUSTOM_ONNXRUNTIME_HASH
+ ""
+ CACHE STRING "Hash of a downloaded ONNX Runtime tarball")
+
+set(Onnxruntime_VERSION "1.17.1")
+
+if(CUSTOM_ONNXRUNTIME_URL STREQUAL "")
+ set(USE_PREDEFINED_ONNXRUNTIME ON)
+else()
+ if(CUSTOM_ONNXRUNTIME_HASH STREQUAL "")
+ message(FATAL_ERROR "Both of CUSTOM_ONNXRUNTIME_URL and CUSTOM_ONNXRUNTIME_HASH must be present!")
+ else()
+ set(USE_PREDEFINED_ONNXRUNTIME OFF)
+ endif()
+endif()
+
+if(USE_PREDEFINED_ONNXRUNTIME)
+ set(Onnxruntime_BASEURL "https://github.com/microsoft/onnxruntime/releases/download/v${Onnxruntime_VERSION}")
+
+ if(APPLE)
+ set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-osx-universal2-${Onnxruntime_VERSION}.tgz")
+ set(Onnxruntime_HASH SHA256=9FA57FA6F202A373599377EF75064AE568FDA8DA838632B26A86024C7378D306)
+ elseif(MSVC)
+ set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-win-x64-${Onnxruntime_VERSION}.zip")
+ set(OOnnxruntime_HASH SHA256=4802AF9598DB02153D7DA39432A48823FF69B2FB4B59155461937F20782AA91C)
+ else()
+ if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+ set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-linux-aarch64-${Onnxruntime_VERSION}.tgz")
+ set(Onnxruntime_HASH SHA256=70B6F536BB7AB5961D128E9DBD192368AC1513BFFB74FE92F97AAC342FBD0AC1)
+ else()
+ set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-linux-x64-gpu-${Onnxruntime_VERSION}.tgz")
+ set(Onnxruntime_HASH SHA256=613C53745EA4960ED368F6B3AB673558BB8561C84A8FA781B4EA7FB4A4340BE4)
+ endif()
+ endif()
+else()
+ set(Onnxruntime_URL "${CUSTOM_ONNXRUNTIME_URL}")
+ set(Onnxruntime_HASH "${CUSTOM_ONNXRUNTIME_HASH}")
+endif()
+
+FetchContent_Declare(
+ onnxruntime
+ URL ${Onnxruntime_URL}
+ URL_HASH ${Onnxruntime_HASH})
+FetchContent_MakeAvailable(onnxruntime)
+
+if(APPLE)
+ set(Onnxruntime_LIB "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.${Onnxruntime_VERSION}.dylib")
+ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIB}")
+ target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${onnxruntime_SOURCE_DIR}/include")
+ target_sources(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIB}")
+ set_property(SOURCE "${Onnxruntime_LIB}" PROPERTY MACOSX_PACKAGE_LOCATION Frameworks)
+ source_group("Frameworks" FILES "${Onnxruntime_LIB}")
+ # add a codesigning step
+ add_custom_command(
+ TARGET "${CMAKE_PROJECT_NAME}"
+ PRE_BUILD VERBATIM
+ COMMAND /usr/bin/codesign --force --verify --verbose --sign "${CODESIGN_IDENTITY}" "${Onnxruntime_LIB}")
+ add_custom_command(
+ TARGET "${CMAKE_PROJECT_NAME}"
+ POST_BUILD
+ COMMAND
+ ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libonnxruntime.${Onnxruntime_VERSION}.dylib"
+ "@loader_path/../Frameworks/libonnxruntime.${Onnxruntime_VERSION}.dylib" $<TARGET_FILE:${CMAKE_PROJECT_NAME}>)
+elseif(MSVC)
+ add_library(Ort INTERFACE)
+ set(Onnxruntime_LIB_NAMES onnxruntime;onnxruntime_providers_shared)
+ foreach(lib_name IN LISTS Onnxruntime_LIB_NAMES)
+ add_library(Ort::${lib_name} SHARED IMPORTED)
+ set_target_properties(Ort::${lib_name} PROPERTIES IMPORTED_IMPLIB ${onnxruntime_SOURCE_DIR}/lib/${lib_name}.lib)
+ set_target_properties(Ort::${lib_name} PROPERTIES IMPORTED_LOCATION ${onnxruntime_SOURCE_DIR}/lib/${lib_name}.dll)
+ set_target_properties(Ort::${lib_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${onnxruntime_SOURCE_DIR}/include)
+ target_link_libraries(Ort INTERFACE Ort::${lib_name})
+ install(FILES ${onnxruntime_SOURCE_DIR}/lib/${lib_name}.dll DESTINATION "obs-plugins/64bit")
+ endforeach()
+
+ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE Ort)
+
+else()
+ if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+ set(Onnxruntime_LINK_LIBS "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.so.${Onnxruntime_VERSION}")
+ set(Onnxruntime_INSTALL_LIBS ${Onnxruntime_LINK_LIBS})
+ else()
+ set(Onnxruntime_LINK_LIBS "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.so.${Onnxruntime_VERSION}")
+ set(Onnxruntime_INSTALL_LIBS ${Onnxruntime_LINK_LIBS}
+ "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime_providers_shared.so")
+ endif()
+ target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ${Onnxruntime_LINK_LIBS})
+ target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${onnxruntime_SOURCE_DIR}/include")
+ install(FILES ${Onnxruntime_INSTALL_LIBS} DESTINATION "${CMAKE_INSTALL_LIBDIR}/obs-plugins/${CMAKE_PROJECT_NAME}")
+ set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/${CMAKE_PROJECT_NAME}")
+endif()

data/locale/en-US.ini

@@ -1,2 +1,17 @@
 CleanStreamAudioFilter="Clean stream audio filter"
 CleanStreamFilterPlugin="Clean stream filter plugin"
+detect_regex="Detect regex"
+advanced_settings="Advanced settings"
+filler_p_threshold="Filler p threshold"
+do_silence="Do silence"
+vad_enabled="VAD enabled"
+log_level="Log level"
+log_words="Log words"
+whisper_model="Whisper model"
+Whisper_Parameters="Whisper Parameters"
+whisper_sampling_method="Whisper sampling method"
+n_threads="Number of threads"
+n_max_text_ctx="Number of max text context"
+no_context="No context"
+replace_sound_path="Replace Sound Path"
+replace_sound="Replace Sound"

src/audio-utils/read-audio-file.cpp

@@ -0,0 +1,137 @@
+#if defined(_WIN32) || defined(__APPLE__)
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libavutil/frame.h>
+#include <libavutil/mem.h>
+#include <libavutil/opt.h>
+#include <libswresample/swresample.h>
+}
+
+#include <iostream>
+#include <vector>
+
+#include "read-audio-file.h"
+#include "obs.h"
+#include "plugin-support.h"
+
+AudioDataFloat read_audio_file(const char *filename, int targetSampleRate)
+{
+ AVFormatContext *formatContext = nullptr;
+ if (avformat_open_input(&formatContext, filename, nullptr, nullptr) != 0) {
+ obs_log(LOG_ERROR, "Error opening file");
+ return {};
+ }
+
+ if (avformat_find_stream_info(formatContext, nullptr) < 0) {
+ obs_log(LOG_ERROR, "Error finding stream information");
+ return {};
+ }
+
+ int audioStreamIndex = -1;
+ for (unsigned int i = 0; i < formatContext->nb_streams; i++) {
+ if (formatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+ audioStreamIndex = i;
+ break;
+ }
+ }
+
+ if (audioStreamIndex == -1) {
+ obs_log(LOG_ERROR, "No audio stream found");
+ return {};
+ }
+
+ AVCodecParameters *codecParams = formatContext->streams[audioStreamIndex]->codecpar;
+ const AVCodec *codec = avcodec_find_decoder(codecParams->codec_id);
+ if (!codec) {
+ obs_log(LOG_ERROR, "Decoder not found");
+ return {};
+ }
+
+ AVCodecContext *codecContext = avcodec_alloc_context3(codec);
+ if (!codecContext) {
+ obs_log(LOG_ERROR, "Failed to allocate codec context");
+ return {};
+ }
+
+ if (avcodec_parameters_to_context(codecContext, codecParams) < 0) {
+ obs_log(LOG_ERROR, "Failed to copy codec parameters to codec context");
+ return {};
+ }
+
+ if (avcodec_open2(codecContext, codec, nullptr) < 0) {
+ obs_log(LOG_ERROR, "Failed to open codec");
+ return {};
+ }
+
+ AVFrame *frame = av_frame_alloc();
+ AVPacket packet;
+
+ // set up swresample
+ AVChannelLayout ch_layout;
+ av_channel_layout_from_string(&ch_layout, "mono");
+ SwrContext *swr = nullptr;
+ int ret;
+ ret = swr_alloc_set_opts2(&swr, &ch_layout, AV_SAMPLE_FMT_FLT, targetSampleRate,
+ &(codecContext->ch_layout), codecContext->sample_fmt,
+ codecContext->sample_rate, 0, nullptr);
+ if (ret < 0) {
+ char errbuf[AV_ERROR_MAX_STRING_SIZE];
+ av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE);
+ obs_log(LOG_ERROR, "Failed to set up swr context: %s", errbuf);
+ return {};
+ }
+ // init swr
+ ret = swr_init(swr);
+ if (ret < 0) {
+ char errbuf[AV_ERROR_MAX_STRING_SIZE];
+ av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE);
+ obs_log(LOG_ERROR, "Failed to initialize swr context: %s", errbuf);
+ return {};
+ }
+
+ AudioDataFloat audioFrames;
+
+ float *convertBuffer[1];
+ convertBuffer[0] = (float *)av_malloc(4096 * sizeof(float));
+ while (av_read_frame(formatContext, &packet) >= 0) {
+ if (packet.stream_index == audioStreamIndex) {
+ if (avcodec_send_packet(codecContext, &packet) == 0) {
+ while (avcodec_receive_frame(codecContext, frame) == 0) {
+ int ret = swr_convert(swr, (uint8_t **)convertBuffer, 4096,
+ (const uint8_t **)frame->data,
+ frame->nb_samples);
+ if (ret < 0) {
+ char errbuf[AV_ERROR_MAX_STRING_SIZE];
+ av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE);
+ obs_log(LOG_ERROR,
+ "Failed to convert audio frame: %s",
+ errbuf);
+ return {};
+ }
+ audioFrames.insert(audioFrames.end(), convertBuffer[0],
+ convertBuffer[0] + ret);
+ }
+ }
+ }
+ av_packet_unref(&packet);
+ }
+ av_free(convertBuffer[0]);
+
+ obs_log(LOG_INFO,
+ "Converted %lu frames of audio data (orig: %d, %s sample format, %d channels, %s)",
+ audioFrames.size(), codecContext->sample_rate,
+ av_get_sample_fmt_name(codecContext->sample_fmt),
+ codecContext->ch_layout.nb_channels,
+ av_sample_fmt_is_planar(codecContext->sample_fmt) ? "planar" : "packed");
+
+ swr_free(&swr);
+ av_frame_free(&frame);
+ avcodec_free_context(&codecContext);
+ avformat_close_input(&formatContext);
+
+ return audioFrames;
+}
+
+#endif