eos is always used for termination.

This could solve issue #26.
foldl · Jun 21, 2024 · b52435a · b52435a
1 parent 8bdacf8
commit b52435a
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/chat.cpp b/chat.cpp
@@ -168,8 +168,7 @@ namespace chatllm
 
  bool BaseTokenizer::is_terminate_token_id(int id) const
  {
- if (terminate_ids.size() < 1)
- return id == eos_token_id;
+ if (id == eos_token_id) return true;
 
  return terminate_ids.find(id) != terminate_ids.end();
  }