diff options
author | Georgi Gerganov <ggerganov@gmail.com> | 2023-03-25 20:26:40 +0200 |
---|---|---|
committer | Georgi Gerganov <ggerganov@gmail.com> | 2023-03-25 20:26:40 +0200 |
commit | a316a425d04027453dc0fd45f003b647c12f66f9 (patch) | |
tree | b33d7c55741f10f1cc84f489df05e1fad96f0417 /tests | |
parent | ecbe466a364876927994e2f1ec14f4d82301d201 (diff) |
Overhaul the examples structure
- main -> examples
- utils -> examples (renamed to "common")
- quantize -> examples
- separate tools for "perplexity" and "embedding"
Hope I didn't break something !
Diffstat (limited to 'tests')
-rw-r--r-- | tests/CMakeLists.txt | 2 | ||||
-rw-r--r-- | tests/test-tokenizer-0.cpp | 6 |
2 files changed, 5 insertions, 3 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6a4170f..b44d7fe 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ function(llama_add_test source) get_filename_component(TEST_TARGET ${source} NAME_WE) add_executable(${TEST_TARGET} ${source}) - target_link_libraries(${TEST_TARGET} PRIVATE llama ggml utils) + target_link_libraries(${TEST_TARGET} PRIVATE llama) add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}> ${ARGN}) endfunction() diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 49bc232..3820553 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -1,9 +1,9 @@ -#include "utils.h" #include "llama.h" #include <cstdio> #include <string> #include <map> +#include <vector> static const std::map<std::string, std::vector<llama_token>> k_tests = { { "Hello World", { 1, 10994, 2787, }, }, @@ -48,7 +48,9 @@ int main(int argc, char **argv) { } for (const auto & test_kv : k_tests) { - const auto res = ::llama_tokenize(ctx, test_kv.first, true); + std::vector<llama_token> res(test_kv.first.size()); + const int n = llama_tokenize(ctx, test_kv.first.c_str(), res.data(), res.size(), true); + res.resize(n); bool correct = res.size() == test_kv.second.size(); |