MNT Stricter tests (PABannier#75)

.gitignore

@@ -21,9 +21,8 @@ quantize
 !ggml_weights/ggml_vocab.bin
 !ggml_weights/vocab.txt
 
-tests/test-text-encoder
-tests/test-coarse-encoder
-tests/test-fine-encoder
+tests/test-gpt-eval
+tests/test-fine-gpt-eval
 
 tests/test-forward-coarse
 tests/test-forward-semantic

.vscode/launch.json

@@ -11,13 +11,13 @@
  "preLaunchTask": "build test fine encoder",
  },
  {
- "name": "Debug qkv",
+ "name": "Debug gpt-eval",
  "type": "lldb",
  "request": "launch",
- "program": "./tests/test-qkv",
+ "program": "./tests/test-gpt-eval",
  "args": [],
  "cwd": "${fileDirname}",
- "preLaunchTask": "build test qkv",
+ "preLaunchTask": "build test gpt eval",
  }
  ]
 }

.vscode/settings.json

@@ -68,6 +68,9 @@
  "typeinfo": "cpp",
  "unordered_map": "cpp",
  "variant": "cpp",
- "algorithm": "cpp"
+ "algorithm": "cpp",
+ "bit": "cpp",
+ "cinttypes": "cpp",
+ "codecvt": "cpp"
  }
 }

.vscode/tasks.json

@@ -7,13 +7,13 @@
  "tasks": [
  {
  "type": "shell",
- "label": "build test qkv",
+ "label": "build test gpt eval",
  "options": {
  "env": {
  "BARK_DEBUG": "1",
  }
  },
- "command": "make tests/test-qkv",
+ "command": "make tests/test-gpt-eval",
  "problemMatcher": [],
  },
  {

Makefile

@@ -2,7 +2,7 @@
 BUILD_TARGETS = main quantize
 
 # Binaries only useful for tests
-TEST_TARGETS = tests/test-tokenizer tests/test-text-encoder tests/test-coarse-encoder tests/test-fine-encoder
+TEST_TARGETS = tests/test-tokenizer tests/test-gpt-eval tests/test-fine-gpt-eval
 TEST_TARGETS += tests/test-forward-semantic tests/test-forward-coarse tests/test-forward-fine
 TEST_TARGETS += tests/test-forward-encodec
 
@@ -324,13 +324,10 @@ common.o: tests/common.cpp
 tests/test-tokenizer: tests/test-tokenizer.cpp ggml.o bark.o encodec.o $(OBJS)
  $(CXX) $(CXXFLAGS) $(filter-out %.txt,$^) -o $@ $(LDFLAGS)
 
-tests/test-text-encoder: tests/test-text-encoder.cpp ggml.o bark.o encodec.o common.o $(OBJS)
+tests/test-gpt-eval: tests/test-gpt-eval.cpp ggml.o bark.o encodec.o common.o $(OBJS)
  $(CXX) $(CXXFLAGS) $(filter-out %.txt,$^) -o $@ $(LDFLAGS)
 
-tests/test-coarse-encoder: tests/test-coarse-encoder.cpp ggml.o bark.o encodec.o common.o $(OBJS)
- $(CXX) $(CXXFLAGS) $(filter-out %.txt,$^) -o $@ $(LDFLAGS)
-
-tests/test-fine-encoder: tests/test-fine-encoder.cpp ggml.o bark.o encodec.o common.o $(OBJS)
+tests/test-fine-gpt-eval: tests/test-fine-gpt-eval.cpp ggml.o bark.o encodec.o common.o $(OBJS)
  $(CXX) $(CXXFLAGS) $(filter-out %.txt,$^) -o $@ $(LDFLAGS)
 
 tests/test-forward-semantic: tests/test-forward-semantic.cpp ggml.o bark.o encodec.o common.o $(OBJS)

bark.cpp

@@ -21,6 +21,8 @@ Author: Pierre-Antoine Bannier <[email protected]>
 #include <regex>
 #include <string>
 
+#define BARK_DEBUG 0
+
 bool allequal(struct ggml_tensor * a, struct ggml_tensor * b, std::string test_name) {
  assert(a->ne[0] == b->ne[0]);
  assert(a->ne[1] == b->ne[1]);
@@ -1177,7 +1179,7 @@ bool gpt_eval(
  cur = ggml_add(ctx0,
  ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
  cur);
-
+ 
  // GELU activation
  // [3072, N]
  cur = ggml_gelu(ctx0, cur);

tests/common.cpp

@@ -5,22 +5,21 @@
 #include "bark-util.h"
 #include "common.h"
 
-template <typename T, typename U>
-inline bool all_equal(std::vector<T> s1, std::vector<U> s2, int * n_violations) {
- if (s1.size() != s2.size()) { return false; }
- for (int i = 0; i < (int) s1.size(); i++) {
- if (s1[i] != s2[i])
- *n_violations += 1;
- }
- return *n_violations == 0;
+int64_t bytes_left(std::ifstream & f) {
+ // utils to check all bytes are read from stream
+ int64_t curr_pos = f.tellg();
+ f.seekg(0, std::ios::end);
+ int64_t file_size = f.tellg();
+ int64_t bytes_left_to_read = file_size - curr_pos;
+ return bytes_left_to_read;
 }
 
-template bool all_equal(std::vector<int> s1, std::vector<int> s2, int * n_violations);
-template bool all_equal(std::vector<float> s1, std::vector<float> s2, int * n_violations);
-
 template <typename T, typename U>
 inline bool all_close(
- std::vector<T> s1, std::vector<U> s2, float * max_violation, int * n_violations) {
+ std::vector<T> s1,
+ std::vector<U> s2,
+ float * max_violation,
+ int * n_violations) {
  if (s1.size() != s2.size()) { return false; }
  for (int i = 0; i < (int) s1.size(); i++) {
  float violation = fabs(s1[i] - s2[i]);
@@ -31,12 +30,23 @@ inline bool all_close(
  return *n_violations == 0;
 }
 
-template bool all_close(std::vector<int> s1, std::vector<int> s2, float * max_violation, int * n_violations);
-template bool all_close(std::vector<float> s1, std::vector<float> s2, float * max_violation, int * n_violations);
+template bool all_close(
+ std::vector<int> s1,
+ std::vector<int> s2,
+ float * max_violation,
+ int * n_violations);
 
-inline bool all_close_nested(
- std::vector<std::vector<float>> s1, std::vector<std::vector<float>> s2,
- float * max_violation, int * n_violations) {
+template bool all_close(
+ std::vector<float> s1,
+ std::vector<float> s2,
+ float * max_violation,
+ int * n_violations);
+
+inline bool all_close(
+ std::vector<std::vector<float>> s1,
+ std::vector<std::vector<float>> s2,
+ float * max_violation,
+ int * n_violations) {
  if (s1.size() != s2.size()) { return false; }
  for (int i = 0; i < (int) s1.size(); i++) {
  if (s1[i].size() != s2[i].size()) { return false; }
@@ -51,9 +61,23 @@ inline bool all_close_nested(
 }
 
 template <typename T, typename U>
-inline bool all_equal_nested(
- std::vector<std::vector<T>> s1, std::vector<std::vector<U>> s2,
- int * n_violations) {
+inline bool all_equal(std::vector<T> s1, std::vector<U> s2, int * n_violations) {
+ if (s1.size() != s2.size()) { return false; }
+ for (int i = 0; i < (int) s1.size(); i++) {
+ if (s1[i] != s2[i])
+ *n_violations += 1;
+ }
+ return *n_violations == 0;
+}
+
+template bool all_equal(std::vector<int> s1, std::vector<int> s2, int * n_violations);
+template bool all_equal(std::vector<float> s1, std::vector<float> s2, int * n_violations);
+
+template <typename T, typename U>
+inline bool all_equal(
+ std::vector<std::vector<T>> s1,
+ std::vector<std::vector<U>> s2,
+ int * n_violations) {
  if (s1.size() != s2.size()) { return false; }
  for (int i = 0; i < (int) s1.size(); i++) {
  if (s1[i].size() != s2[i].size()) { return false; }
@@ -65,25 +89,27 @@ inline bool all_equal_nested(
  return *n_violations == 0;
 }
 
-template bool all_equal_nested(std::vector<std::vector<int>>, std::vector<std::vector<int>>, int * n_violations);
+template bool all_equal(
+ std::vector<std::vector<int>>,
+ std::vector<std::vector<int>>,
+ int * n_violations);
 
-bool run_test_on_sequence(std::vector<float> truth, std::vector<float> result) {
+bool run_test(std::vector<float> truth, std::vector<float> result) {
  float max_violation = 0.0f;
  int n_violations = 0;
  if (!all_close(result, truth, &max_violation, &n_violations)) {
  if (n_violations == 0) {
  fprintf(stderr, "%s : wrong shape (%zu != %zu).\n", __func__, truth.size(), result.size());
  } else {
- fprintf(stderr, "\n");
- fprintf(stderr, " abs_tol=%.4f, abs max viol=%.4f, viol=%.1f%%", ABS_TOL, max_violation, (float)n_violations/truth.size()*100);
+ fprintf(stderr, "%s: abs_tol=%.4f, abs max viol=%.4f, viol=%.1f%%", __func__, ABS_TOL, max_violation, (float)n_violations/truth.size()*100);
  fprintf(stderr, "\n");
  }
  return false;
  }
  return true;
 }
 
-bool run_test_on_sequence(std::vector<int> truth, std::vector<int> result) {
+bool run_test(std::vector<int> truth, std::vector<int> result) {
  int n_violations = 0;
  if (!all_equal(result, truth, &n_violations)) {
  if (n_violations == 0) {
@@ -98,10 +124,10 @@ bool run_test_on_sequence(std::vector<int> truth, std::vector<int> result) {
  return true;
 }
 
-bool run_test_on_codes(logit_matrix truth, logit_matrix result) {
+bool run_test(logit_matrix truth, logit_matrix result) {
  float max_violation = 0.0f;
  int n_violations = 0;
- if (!all_close_nested(result, truth, &max_violation, &n_violations)) {
+ if (!all_close(result, truth, &max_violation, &n_violations)) {
  fprintf(stderr, "\n");
  fprintf(stderr, "%s : failed test\n", __func__);
  if (n_violations == 0) {
@@ -115,9 +141,9 @@ bool run_test_on_codes(logit_matrix truth, logit_matrix result) {
  return true;
 }
 
-bool run_test_on_codes(bark_codes truth, bark_codes result) {
+bool run_test(bark_codes truth, bark_codes result) {
  int n_violations = 0;
- if (!all_equal_nested(result, truth, &n_violations)) {
+ if (!all_equal(result, truth, &n_violations)) {
  fprintf(stderr, "\n");
  fprintf(stderr, "%s : failed test\n", __func__);
  if (n_violations == 0) {
@@ -171,14 +197,26 @@ void load_test_data(std::string fname, std::vector<T>& input, std::vector<U>& ou
  fin.read(reinterpret_cast<char *>(output.data()), nelements*sizeof(U));
  }
 
- assert(fin.eof());
+ if (bytes_left(fin) > 0) {
+ throw std::runtime_error("EOF not reached");
+ }
 }
 
-template void load_test_data(std::string fname, std::vector<int>& input, std::vector<float>& output);
-template void load_test_data(std::string fname, std::vector<int32_t>& input, std::vector<int32_t>& output);
+template void load_test_data(
+ std::string fname,
+ std::vector<int32_t> & input,
+ std::vector<float> & output);
+
+template void load_test_data(
+ std::string fname,
+ std::vector<int32_t> & input,
+ std::vector<int32_t> & output);
 
 template <typename T, typename U>
-void load_test_data(std::string fname, std::vector<T>& input, std::vector<std::vector<U>>& output) {
+void load_test_data(
+ std::string fname,
+ std::vector<T> & input,
+ std::vector<std::vector<U>> & output) {
  auto fin = std::ifstream(fname, std::ios::binary);
  if (!fin) {
  fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
@@ -216,7 +254,9 @@ void load_test_data(std::string fname, std::vector<T>& input, std::vector<std::v
  }
  }
 
- assert(fin.eof());
+ if (bytes_left(fin) > 0) {
+ throw std::runtime_error("EOF not reached");
+ }
 }
 
 template void load_test_data(std::string fname, std::vector<int32_t>& input, std::vector<std::vector<int32_t>>& output);
@@ -259,11 +299,13 @@ void load_test_data(std::string fname, std::vector<std::vector<int32_t>>& input,
  fin.read(reinterpret_cast<char *>(output.data()), nelements*sizeof(float));
  }
 
- assert(fin.eof());
+ if (bytes_left(fin) > 0) {
+ throw std::runtime_error("EOF not reached");
+ }
 }
 
 template <typename T>
-void load_nested_test_data(
+void load_test_data(
  std::string fname,
  std::vector<std::vector<int32_t>> & input,
  std::vector<std::vector<T>> & output) {
@@ -303,15 +345,17 @@ void load_nested_test_data(
  }
  }
 
- // assert(fin.eof());
+ if (bytes_left(fin) > 0) {
+ throw std::runtime_error("EOF not reached");
+ }
 }
 
-template void load_nested_test_data(
+template void load_test_data(
  std::string fname,
  std::vector<std::vector<int32_t>> & input,
  std::vector<std::vector<float>> & output);
 
-template void load_nested_test_data(
+template void load_test_data(
  std::string fname,
  std::vector<std::vector<int32_t>> & input,
  std::vector<std::vector<int32_t>> & output);

tests/common.h

@@ -9,12 +9,31 @@
 typedef std::vector<float> logit_sequence;
 typedef std::vector<std::vector<float>> logit_matrix;
 
-bool run_test_on_sequence(std::vector<int> truth, std::vector<int> result);
-bool run_test_on_sequence(std::vector<float> truth, std::vector<float> result);
+/* Comparison utils */
+template <typename T, typename U>
+inline bool all_equal(std::vector<T> s1, std::vector<U> s2, int * n_violations);
+
+template <typename T, typename U>
+inline bool all_equal(
+ std::vector<std::vector<T>> s1,
+ std::vector<std::vector<U>> s2,
+ int * n_violations);
+
+template <typename T, typename U>
+inline bool all_close(
+ std::vector<T> s1,
+ std::vector<U> s2,
+ float * max_violation,
+ int * n_violations);
+
+/* Test utils */
+bool run_test(std::vector<int> truth, std::vector<int> result);
+bool run_test(std::vector<float> truth, std::vector<float> result);
 
-bool run_test_on_codes(logit_matrix truth, logit_matrix result);
-bool run_test_on_codes(bark_codes truth, bark_codes result);
+bool run_test(logit_matrix truth, logit_matrix result);
+bool run_test(bark_codes truth, bark_codes result);
 
+/* Load utils */
 template <typename T, typename U>
 void load_test_data(std::string fname, std::vector<T>& input, std::vector<U>& output);
 
@@ -24,7 +43,7 @@ void load_test_data(std::string fname, std::vector<T>& input, std::vector<std::v
 void load_test_data(std::string fname, std::vector<std::vector<int32_t>>& input, std::vector<float>& output);
 
 template <typename T>
-void load_nested_test_data(
+void load_test_data(
  std::string fname,
  std::vector<std::vector<int32_t>> & input,
  std::vector<std::vector<T>> & output);