aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--llama.cpp4
1 files changed, 3 insertions, 1 deletions
diff --git a/llama.cpp b/llama.cpp
index a448977..a35c690 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -1924,7 +1924,9 @@ struct llama_tokenizer {
if (token == vocab_.token_to_id.end()) {
// output any symbols that did not form tokens as bytes.
for (int j = 0; j < (int) symbol.n; ++j) {
- llama_vocab::id token_id = static_cast<uint8_t>(symbol.text[j]) + 3;
+ // NOTE: old version, before #2420 - not sure what are the implications of this
+ //llama_vocab::id token_id = static_cast<uint8_t>(symbol.text[j]) + 3;
+ llama_vocab::id token_id = vocab_.token_to_id.at(std::string(1, symbol.text[j]));
output.push_back(token_id);
}
} else {