[WIP, don't merge] unity.cpp -> ggml master

CMakeLists.txt

@@ -4,6 +4,7 @@ project(ggml VERSION 0.1.0)
 set(CMAKE_EXPORT_COMPILE_COMMANDS "on")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
  set(GGML_STANDALONE ON)
@@ -167,11 +168,19 @@ endif()
 # dependencies
 
 set(CMAKE_C_STANDARD 11)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 
 find_package(Threads REQUIRED)
 
 # main
+# message(STATUS "Current directory is: ${CMAKE_CURRENT_SOURCE_DIR}/examples/kaldi-native-fbank/csrc/")
+file(GLOB KALDI_NATIVE_FBANK_SOURCES
+ "${CMAKE_CURRENT_SOURCE_DIR}/examples/kaldi-native-fbank/csrc/*"
+)
+add_library(kaldi-native-fbank STATIC ${KALDI_NATIVE_FBANK_SOURCES})
+target_include_directories(kaldi-native-fbank PUBLIC
+ ${CMAKE_CURRENT_SOURCE_DIR}/examples/kaldi-native-fbank/csrc
+)
 
 if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)

examples/CMakeLists.txt

@@ -29,3 +29,4 @@ add_subdirectory(mpt)
 add_subdirectory(starcoder)
 add_subdirectory(sam)
 add_subdirectory(yolo)
+add_subdirectory(unity)

examples/kaldi-native-fbank/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_subdirectory(csrc)
+
+if(KALDI_NATIVE_FBANK_BUILD_PYTHON)
+ message(STATUS "Building Python")
+ add_subdirectory(python)
+else()
+ message(STATUS "Disable building Python")
+endif()

examples/kaldi-native-fbank/csrc/CMakeLists.txt

@@ -0,0 +1,93 @@
+
+include_directories(${PROJECT_SOURCE_DIR})
+set(sources
+ feature-fbank.cc
+ feature-functions.cc
+ feature-window.cc
+ fftsg.c
+ mel-computations.cc
+ online-feature.cc
+ rfft.cc
+)
+
+if(KALDI_NATIVE_FBANK_ENABLE_CHECK)
+ list(APPEND sources log.cc)
+endif()
+
+add_library(kaldi-native-fbank-core ${sources})
+if(KALDI_NATIVE_FBANK_ENABLE_CHECK)
+ target_compile_definitions(kaldi-native-fbank-core PUBLIC KNF_ENABLE_CHECK=1)
+
+ if(KNF_HAVE_EXECINFO_H)
+ target_compile_definitions(kaldi-native-fbank-core PRIVATE KNF_HAVE_EXECINFO_H=1)
+ endif()
+
+ if(KNF_HAVE_CXXABI_H)
+ target_compile_definitions(kaldi-native-fbank-core PRIVATE KNF_HAVE_CXXABI_H=1)
+ endif()
+endif()
+
+# We are using std::call_once() in log.h,which requires us to link with -pthread
+if(NOT WIN32 AND KALDI_NATIVE_FBANK_ENABLE_CHECK)
+ target_link_libraries(kaldi-native-fbank-core -pthread)
+endif()
+
+if(KALDI_NATIVE_FBANK_BUILD_TESTS)
+ add_executable(test-online-fbank test-online-fbank.cc)
+ target_link_libraries(test-online-fbank kaldi-native-fbank-core)
+endif()
+
+function(kaldi_native_fbank_add_test source)
+ get_filename_component(name ${source} NAME_WE)
+ add_executable(${name} "${source}")
+ target_link_libraries(${name}
+ PRIVATE
+ kaldi-native-fbank-core
+ gtest
+ gtest_main
+ )
+
+ add_test(NAME "Test.${name}"
+ COMMAND
+ $<TARGET_FILE:${name}>
+ )
+endfunction()
+
+# please sort the source files alphabetically
+set(test_srcs
+ # test-online-feature.cc
+ test-log.cc
+ test-rfft.cc
+)
+
+if(KALDI_NATIVE_FBANK_BUILD_TESTS)
+ foreach(source IN LISTS test_srcs)
+ kaldi_native_fbank_add_test(${source})
+ endforeach()
+endif()
+
+install(TARGETS kaldi-native-fbank-core
+ DESTINATION lib
+)
+
+if(KALDI_NATIVE_FBANK_BUILD_TESTS)
+ install(TARGETS test-online-fbank
+ DESTINATION bin
+ )
+endif()
+
+file(MAKE_DIRECTORY
+ DESTINATION
+ ${PROJECT_BINARY_DIR}/include/kaldi-native-fbank/csrc
+)
+file(GLOB_RECURSE all_headers *.h)
+
+file(COPY
+ ${all_headers}
+ DESTINATION
+ ${PROJECT_BINARY_DIR}/include/kaldi-native-fbank/csrc
+)
+
+install(FILES ${all_headers}
+ DESTINATION include/kaldi-native-fbank/csrc
+)

examples/kaldi-native-fbank/csrc/feature-fbank.cc

@@ -0,0 +1,120 @@
+/**
+ * Copyright (c) 2022 Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file is copied/modified from kaldi/src/feat/feature-fbank.cc
+//
+#include "feature-fbank.h"
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <vector>
+
+#include "feature-functions.h"
+
+namespace knf {
+
+static void Sqrt(float *in_out, int32_t n) {
+ for (int32_t i = 0; i != n; ++i) {
+ in_out[i] = std::sqrt(in_out[i]);
+ }
+}
+
+std::ostream &operator<<(std::ostream &os, const FbankOptions &opts) {
+ os << opts.ToString();
+ return os;
+}
+
+FbankComputer::FbankComputer(const FbankOptions &opts)
+ : opts_(opts), rfft_(opts.frame_opts.PaddedWindowSize()) {
+ if (opts.energy_floor > 0.0f) {
+ log_energy_floor_ = logf(opts.energy_floor);
+ }
+
+ // We'll definitely need the filterbanks info for VTLN warping factor 1.0.
+ // [note: this call caches it.]
+ GetMelBanks(1.0f);
+}
+
+FbankComputer::~FbankComputer() {
+ for (auto iter = mel_banks_.begin(); iter != mel_banks_.end(); ++iter)
+ delete iter->second;
+}
+
+const MelBanks *FbankComputer::GetMelBanks(float vtln_warp) {
+ MelBanks *this_mel_banks = nullptr;
+
+ // std::map<float, MelBanks *>::iterator iter = mel_banks_.find(vtln_warp);
+ auto iter = mel_banks_.find(vtln_warp);
+ if (iter == mel_banks_.end()) {
+ this_mel_banks = new MelBanks(opts_.mel_opts, opts_.frame_opts, vtln_warp);
+ mel_banks_[vtln_warp] = this_mel_banks;
+ } else {
+ this_mel_banks = iter->second;
+ }
+ return this_mel_banks;
+}
+
+void FbankComputer::Compute(float signal_raw_log_energy, float vtln_warp,
+ std::vector<float> *signal_frame, float *feature) {
+ const MelBanks &mel_banks = *(GetMelBanks(vtln_warp));
+
+ KNF_CHECK_EQ(signal_frame->size(), opts_.frame_opts.PaddedWindowSize());
+
+ // Compute energy after window function (not the raw one).
+ if (opts_.use_energy && !opts_.raw_energy) {
+ signal_raw_log_energy = std::log(
+ std::max<float>(InnerProduct(signal_frame->data(), signal_frame->data(),
+ signal_frame->size()),
+ std::numeric_limits<float>::epsilon()));
+ }
+ rfft_.Compute(signal_frame->data()); // signal_frame is modified in-place
+ ComputePowerSpectrum(signal_frame);
+
+ // Use magnitude instead of power if requested.
+ if (!opts_.use_power) {
+ Sqrt(signal_frame->data(), signal_frame->size() / 2 + 1);
+ }
+
+ int32_t mel_offset = ((opts_.use_energy && !opts_.htk_compat) ? 1 : 0);
+
+ // Its length is opts_.mel_opts.num_bins
+ float *mel_energies = feature + mel_offset;
+
+ // Sum with mel filter banks over the power spectrum
+ mel_banks.Compute(signal_frame->data(), mel_energies);
+
+ if (opts_.use_log_fbank) {
+ // Avoid log of zero (which should be prevented anyway by dithering).
+ for (int32_t i = 0; i != opts_.mel_opts.num_bins; ++i) {
+ auto t = std::max(mel_energies[i], std::numeric_limits<float>::epsilon());
+ mel_energies[i] = std::log(t);
+ }
+ }
+
+ // Copy energy as first value (or the last, if htk_compat == true).
+ if (opts_.use_energy) {
+ if (opts_.energy_floor > 0.0 && signal_raw_log_energy < log_energy_floor_) {
+ signal_raw_log_energy = log_energy_floor_;
+ }
+ int32_t energy_index = opts_.htk_compat ? opts_.mel_opts.num_bins : 0;
+ feature[energy_index] = signal_raw_log_energy;
+ }
+}
+
+} // namespace knf

examples/kaldi-native-fbank/csrc/feature-fbank.h

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2022 Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http:https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file is copied/modified from kaldi/src/feat/feature-fbank.h
+
+#ifndef KALDI_NATIVE_FBANK_CSRC_FEATURE_FBANK_H_
+#define KALDI_NATIVE_FBANK_CSRC_FEATURE_FBANK_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "feature-window.h"
+#include "mel-computations.h"
+#include "rfft.h"
+
+namespace knf {
+
+struct FbankOptions {
+ FrameExtractionOptions frame_opts;
+ MelBanksOptions mel_opts;
+ // append an extra dimension with energy to the filter banks
+ bool use_energy = false;
+ float energy_floor = 0.0f; // active iff use_energy==true
+
+ // If true, compute log_energy before preemphasis and windowing
+ // If false, compute log_energy after preemphasis ans windowing
+ bool raw_energy = true; // active iff use_energy==true
+
+ // If true, put energy last (if using energy)
+ // If false, put energy first
+ bool htk_compat = false; // active iff use_energy==true
+
+ // if true (default), produce log-filterbank, else linear
+ bool use_log_fbank = true;
+
+ // if true (default), use power in filterbank
+ // analysis, else magnitude.
+ bool use_power = true;
+
+ FbankOptions() { mel_opts.num_bins = 23; }
+
+ std::string ToString() const {
+ std::ostringstream os;
+ os << "frame_opts: \n";
+ os << frame_opts << "\n";
+ os << "\n";
+
+ os << "mel_opts: \n";
+ os << mel_opts << "\n";
+
+ os << "use_energy: " << use_energy << "\n";
+ os << "energy_floor: " << energy_floor << "\n";
+ os << "raw_energy: " << raw_energy << "\n";
+ os << "htk_compat: " << htk_compat << "\n";
+ os << "use_log_fbank: " << use_log_fbank << "\n";
+ os << "use_power: " << use_power << "\n";
+ return os.str();
+ }
+};
+
+std::ostream &operator<<(std::ostream &os, const FbankOptions &opts);
+
+class FbankComputer {
+ public:
+ using Options = FbankOptions;
+
+ explicit FbankComputer(const FbankOptions &opts);
+ ~FbankComputer();
+
+ int32_t Dim() const {
+ return opts_.mel_opts.num_bins + (opts_.use_energy ? 1 : 0);
+ }
+
+ // if true, compute log_energy_pre_window but after dithering and dc removal
+ bool NeedRawLogEnergy() const { return opts_.use_energy && opts_.raw_energy; }
+
+ const FrameExtractionOptions &GetFrameOptions() const {
+ return opts_.frame_opts;
+ }
+
+ const FbankOptions &GetOptions() const { return opts_; }
+
+ /**
+ Function that computes one frame of features from
+ one frame of signal.
+
+ @param [in] signal_raw_log_energy The log-energy of the frame of the signal
+ prior to windowing and pre-emphasis, or
+ log(numeric_limits<float>::min()), whichever is greater. Must be
+ ignored by this function if this class returns false from
+ this->NeedsRawLogEnergy().
+ @param [in] vtln_warp The VTLN warping factor that the user wants
+ to be applied when computing features for this utterance. Will
+ normally be 1.0, meaning no warping is to be done. The value will
+ be ignored for feature types that don't support VLTN, such as
+ spectrogram features.
+ @param [in] signal_frame One frame of the signal,
+ as extracted using the function ExtractWindow() using the options
+ returned by this->GetFrameOptions(). The function will use the
+ vector as a workspace, which is why it's a non-const pointer.
+ @param [out] feature Pointer to a vector of size this->Dim(), to which
+ the computed feature will be written. It should be pre-allocated.
+ */
+ void Compute(float signal_raw_log_energy, float vtln_warp,
+ std::vector<float> *signal_frame, float *feature);
+
+ private:
+ const MelBanks *GetMelBanks(float vtln_warp);
+
+ FbankOptions opts_;
+ float log_energy_floor_;
+ std::map<float, MelBanks *> mel_banks_; // float is VTLN coefficient.
+ Rfft rfft_;
+};
+
+} // namespace knf
+
+#endif // KALDI_NATIVE_FBANK_CSRC_FEATURE_FBANK_H_