COSMIT Move tokenization in its own function (#32)

bark.cpp

@@ -1227,13 +1227,11 @@ bark_vocab::id gpt_sample(
  return gpt_multinomial_sample(logits, rng, temp, eos_p);
 }
 
-bark_sequence bark_tokenize_input(
- const char * text,
- const bark_vocab & vocab,
- const int32_t block_size) {
+bark_sequence bark_tokenize_input(const char * text, const bark_vocab & vocab, int32_t block_size) {
+ // max bark length: 256
  int32_t max_ctx_size = std::min(block_size, 256);
-
  int32_t n_tokens;
+
  bark_sequence tokens(max_ctx_size);
 
  bert_tokenize(vocab, text, tokens.data(), &n_tokens, max_ctx_size);
@@ -1494,8 +1492,6 @@ bool bark_generate_audio(
  const bark_vocab& vocab,
  const char * text,
  const int n_threads) {
- bark_sequence tokens;
-
  // TODO move into params
  // const int top_k = 10;
  const int seed = 0;
@@ -1513,8 +1509,8 @@ bool bark_generate_audio(
 
  std::mt19937 rng(seed);
 
- // bert tokenizer
- const int32_t block_size = model.text_model.hparams.block_size;
+ // tokenize input (bert tokenizer)
+ int32_t block_size = model.text_model.hparams.block_size;
  bark_sequence tokens = bark_tokenize_input(text, vocab, block_size);
 
  printf("%s: prompt: '%s'\n", __func__, text);