ggml : add coverage measurement for Clang, increase test coverage, F1…

…6 ggml_sum (#377) * First shot at adding clang/llvm coverage analysis * Fix for compiler dependency * Reducing dimensions in test-opt * cmake : try to fix test coverage build + CI * cmake : fix CMAKE option + CI * Adding some tests for half precision floating point tests * Adding missing tests for unary operations * Some more tests for unary operations * Fix syntax error. * Fix bug in relu derivative computation * Revert testing change * ggml : style fixes --------- Co-authored-by: Georgi Gerganov <[email protected]>
ggerganov · Jul 23, 2023 · 7b55e12 · 7b55e12
1 parent 28b314b
commit 7b55e12
Show file tree

Hide file tree

Showing 7 changed files with 531 additions and 97 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,6 +22,12 @@ jobs:
  steps:
  - uses: actions/checkout@v2
 
+ - name: Dependencies for Ubuntu
+ if: matrix.os == 'ubuntu-latest'
+ run: |
+ sudo apt-get update
+ sudo apt-get install llvm
+
  - name: Set GGML_N_THREADS for Ubuntu
  run: echo "GGML_N_THREADS=2" >> $GITHUB_ENV
  if: matrix.os == 'ubuntu-latest'
@@ -35,7 +41,7 @@ jobs:
 
  - name: Configure CMake
  working-directory: ./build
- run: cmake ..
+ run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_TEST_COVERAGE=ON ..
 
  - name: Build
  working-directory: ./build
@@ -44,3 +50,19 @@ jobs:
  - name: Test
  working-directory: ./build
  run: ctest --verbose --timeout 900
+
+ - name: Test Coverage for Ubuntu
+ if: matrix.os == 'ubuntu-latest'
+ working-directory: ./build
+ run: |
+ llvm-profdata merge -sparse tests/*.profraw -o ggml.profdata
+ llvm-cov report ./bin/test-grad0 -instr-profile=ggml.profdata
+ llvm-cov report ./bin/test-opt -instr-profile=ggml.profdata
+
+ - name: Test Coverage for MacOS
+ if: matrix.os == 'macos-latest'
+ working-directory: ./build
+ run: |
+ xcrun llvm-profdata merge -sparse tests/*.profraw -o ggml.profdata
+ xcrun llvm-cov report ./bin/test-grad0 -instr-profile=ggml.profdata
+ xcrun llvm-cov report ./bin/test-opt -instr-profile=ggml.profdata
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -25,6 +25,8 @@ option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF)
 option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE})
 option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
 
+option(GGML_TEST_COVERAGE "ggml: enable test coverage" OFF)
+
 option(GGML_PERF "ggml: enable perf timings" OFF)
 option(GGML_NO_ACCELERATE "ggml: disable Accelerate framework" OFF)
 option(GGML_OPENBLAS "ggml: use OpenBLAS" OFF)
@@ -67,6 +69,17 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "RelWithDebInfo")
 endif ()
 
+if (GGML_BUILD_TESTS)
+ if (GGML_TEST_COVERAGE)
+ if (CMAKE_C_COMPILER_ID MATCHES "Clang")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+ else()
+ message(WARNING "Test coverage is only supported for Clang")
+ endif()
+ endif()
+endif()
+
 add_subdirectory(src)
 
 if (GGML_BUILD_TESTS)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -118,7 +118,6 @@ else()
  endif()
 endif()
 
-
 # ggml
 
 set(TARGET ggml)
@@ -183,6 +182,7 @@ if (GGML_CLBLAST)
  message(WARNING "clBLAST not found")
  endif()
 endif()
+
 if (GGML_CUBLAS)
  cmake_minimum_required(VERSION 3.17)
 

diff --git a/src/ggml.c b/src/ggml.c
@@ -3605,14 +3605,22 @@ inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {
 #endif
 }
 
-inline static void ggml_vec_sum_ggf(const int n, ggml_float * s, const float * x) {
+inline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {
  ggml_float sum = 0.0;
  for (int i = 0; i < n; ++i) {
  sum += (ggml_float)x[i];
  }
  *s = sum;
 }
 
+inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; ++i) {
+ sum += GGML_FP16_TO_FP32(x[i]);
+ }
+ *s = sum;
+}
+
 inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
 #ifndef GGML_USE_ACCELERATE
  float max = -INFINITY;
@@ -9298,7 +9306,7 @@ static void ggml_compute_forward_sum_f32(
  for (int64_t i03 = 0; i03 < ne03; i03++) {
  for (int64_t i02 = 0; i02 < ne02; i02++) {
  for (int64_t i01 = 0; i01 < ne01; i01++) {
- ggml_vec_sum_ggf(ne00,
+ ggml_vec_sum_f32_ggf(ne00,
  &row_sum,
  (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
  sum += row_sum;
@@ -9308,6 +9316,38 @@ static void ggml_compute_forward_sum_f32(
  ((float *) dst->data)[0] = sum;
 }
 
+static void ggml_compute_forward_sum_f16(
+ const struct ggml_compute_params * params,
+ const struct ggml_tensor * src0,
+ struct ggml_tensor * dst) {
+ assert(params->ith == 0);
+ assert(ggml_is_scalar(dst));
+
+ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
+ return;
+ }
+
+ assert(src0->nb[0] == sizeof(ggml_fp16_t));
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne);
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb);
+
+ float sum = 0;
+ float row_sum = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_f16_ggf(ne00,
+ &row_sum,
+ (ggml_fp16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ ((ggml_fp16_t *) dst->data)[0] = GGML_FP32_TO_FP16(sum);
+}
+
 static void ggml_compute_forward_sum(
  const struct ggml_compute_params * params,
  const struct ggml_tensor * src0,
@@ -9317,6 +9357,10 @@ static void ggml_compute_forward_sum(
  {
  ggml_compute_forward_sum_f32(params, src0, dst);
  } break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_sum_f16(params, src0, dst);
+ } break;
  default:
  {
  GGML_ASSERT(false);
@@ -15159,7 +15203,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
  case GGML_OP_RELU:
  {
  if (src0->grad) {
- src0->grad = ggml_sub_impl(ctx,
+ src0->grad = ggml_add_impl(ctx,
  src0->grad,
  ggml_mul(ctx,
  ggml_step(ctx, src0),

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -26,68 +26,68 @@ endif()
 
 if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
  message(STATUS "ARM detected")
- #set(GGML_C_FLAGS "${GGML_C_FLAGS} -mcpu=apple-m1")
+ #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=apple-m1")
 elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
  message(STATUS "PPC64 detected")
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mpower9-vector")
 else()
  message(STATUS "x86 detected")
- #set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx -mavx2 -mfma -mf16c")
+ #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx -mavx2 -mfma -mf16c")
  if (UNAME_S MATCHES "Darwin")
  execute_process(COMMAND sysctl machdep.cpu.features OUTPUT_VARIABLE AVX1_M)
  if (AVX1_M MATCHES "AVX1.0")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
  endif()
  execute_process(COMMAND sysctl machdep.cpu.leaf7_features OUTPUT_VARIABLE AVX2_M)
  if (AVX2_M MATCHES "AVX2")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx2")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
  endif()
  if (AVX1_M MATCHES "FMA")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mfma")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
  endif()
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mf16c")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
  elseif (UNAME_S MATCHES "Linux")
  message(STATUS "Linux detected")
  execute_process(COMMAND grep "avx " /proc/cpuinfo OUTPUT_VARIABLE AVX1_M)
  if (AVX1_M MATCHES "avx")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
  endif()
  execute_process(COMMAND grep "avx2 " /proc/cpuinfo OUTPUT_VARIABLE AVX2_M)
  if (AVX2_M MATCHES "avx2")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx2")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
  endif()
  execute_process(COMMAND grep "fma " /proc/cpuinfo OUTPUT_VARIABLE FMA_M)
  if (FMA_M MATCHES "fma")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mfma")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
  endif()
  execute_process(COMMAND grep "f16c " /proc/cpuinfo OUTPUT_VARIABLE F16C_M)
  if (F16C_M MATCHES "f16c")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mf16c")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
  endif()
  execute_process(COMMAND grep "sse3 " /proc/cpuinfo OUTPUT_VARIABLE SSE3_M)
  if (SSE3_M MATCHES "sse3")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -msse3")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3")
  endif()
  elseif (UNAME_S MATCHES "Haiku")
  message(STATUS "Haiku detected")
  execute_process(COMMAND sysinfo -cpu COMMAND grep "AVX " OUTPUT_VARIABLE AVX1_M)
  if (AVX1_M MATCHES "avx")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
  endif()
  execute_process(COMMAND sysinfo -cpu COMMAND grep "AVX2 " OUTPUT_VARIABLE AVX2_M)
  if (AVX2_M MATCHES "avx2")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mavx2")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
  endif()
  execute_process(COMMAND sysinfo -cpu COMMAND grep "FMA " OUTPUT_VARIABLE FMA_M)
  if (FMA_M MATCHES "fma")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mfma")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
  endif()
  execute_process(COMMAND sysinfo -cpu COMMAND grep "F16C " OUTPUT_VARIABLE F16C_M)
  if (F16C_M MATCHES "f16c")
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mf16c")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
  endif()
  else()
- set(GGML_C_FLAGS "${GGML_C_FLAGS} -mfma -mf16c -mavx -mavx2")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma -mf16c -mavx -mavx2")
  endif()
 endif()
 
@@ -142,8 +142,6 @@ if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86")
  set(TEST_TARGET test-vec1)
  add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
  target_link_libraries(${TEST_TARGET} PRIVATE ggml)
- #set_target_properties(${TEST_TARGET} PROPERTIES COMPILE_FLAGS "-mavx -mavx2 -mfma -mf16c")
- set_target_properties(${TEST_TARGET} PROPERTIES COMPILE_FLAGS ${GGML_C_FLAGS})
 endif()
 
 #
@@ -161,6 +159,7 @@ set(TEST_TARGET test-grad0)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-opt
@@ -169,6 +168,7 @@ set(TEST_TARGET test-opt)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-quantize-fns
@@ -177,6 +177,7 @@ set(TEST_TARGET test-quantize-fns)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-quantize-perf
@@ -185,6 +186,7 @@ set(TEST_TARGET test-quantize-perf)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-mul-mat0
@@ -194,6 +196,7 @@ add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml ${GGML_EXTRA_LIBS})
 target_compile_options(${TEST_TARGET} PRIVATE ${GGML_EXTRA_FLAGS})
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-mul-mat1 (arm)
@@ -214,6 +217,7 @@ if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" AND NOT GGML_NO_ACCELERATE)
  target_link_libraries(${TEST_TARGET} PRIVATE ggml ${GGML_EXTRA_LIBS})
  target_compile_options(${TEST_TARGET} PRIVATE ${GGML_EXTRA_FLAGS})
  add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}> 128 128 128)
+ set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 endif()
 
 #
@@ -223,6 +227,7 @@ set(TEST_TARGET test-mul-mat2)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test0
@@ -231,6 +236,7 @@ set(TEST_TARGET test0)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test1
@@ -242,6 +248,7 @@ if (MSVC)
  target_link_options(${TEST_TARGET} PRIVATE "/STACK: 8388608") # 8MB
 endif()
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test2
@@ -250,6 +257,7 @@ set(TEST_TARGET test2)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test3
@@ -258,6 +266,7 @@ set(TEST_TARGET test3)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-pool
@@ -266,6 +275,7 @@ set(TEST_TARGET test-pool)
 add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
 target_link_libraries(${TEST_TARGET} PRIVATE ggml)
 add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
+set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
 
 #
 # test-svd0 (arm/x86)