diff options
Diffstat (limited to 'convert-unversioned-ggml-to-ggml.py')
| -rw-r--r-- | convert-unversioned-ggml-to-ggml.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/convert-unversioned-ggml-to-ggml.py b/convert-unversioned-ggml-to-ggml.py index 33b6243..5151d90 100644 --- a/convert-unversioned-ggml-to-ggml.py +++ b/convert-unversioned-ggml-to-ggml.py @@ -44,7 +44,7 @@ def write_header(f_out, header): def write_tokens(fout, tokenizer): for i in range(tokenizer.vocab_size()): if tokenizer.is_unknown(i): - text = " \u2047 ".encode("utf-8") + text = " \u2047 ".encode() elif tokenizer.is_control(i): text = b"" elif tokenizer.is_byte(i): @@ -55,7 +55,7 @@ def write_tokens(fout, tokenizer): byte_value = int(piece[3:-1], 16) text = struct.pack("B", byte_value) else: - text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") + text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode() fout.write(struct.pack("i", len(text))) fout.write(text) fout.write(struct.pack("f", tokenizer.get_score(i))) |