From e216aa04633892b972d013719e38b59fd4917341 Mon Sep 17 00:00:00 2001 From: Evan Jones Date: Tue, 2 May 2023 22:26:13 -0400 Subject: llama : only copy used KV cache in get / set state (#1272) * llama : only copy used KV cache in get / set state * switch to ggml for copying k, v * avoid designated initializers --- llama.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'llama.h') diff --git a/llama.h b/llama.h index 81f4317..e993c46 100644 --- a/llama.h +++ b/llama.h @@ -23,7 +23,7 @@ #define LLAMA_FILE_MAGIC 'ggjt' #define LLAMA_FILE_MAGIC_UNVERSIONED 'ggml' #define LLAMA_SESSION_MAGIC 'ggsn' -#define LLAMA_SESSION_VERSION 0 +#define LLAMA_SESSION_VERSION 1 #ifdef __cplusplus extern "C" { @@ -127,7 +127,8 @@ extern "C" { // Sets the current rng seed. LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, int seed); - // Returns the size in bytes of the state (rng, logits, embedding and kv_cache) + // Returns the maximum size in bytes of the state (rng, logits, embedding + // and kv_cache) - will often be smaller after compacting tokens LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx); // Copies the state to the specified destination address. -- cgit v1.2.3