diff options
author | DannyDaemonic <DannyDaemonic@gmail.com> | 2023-05-29 05:13:40 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-29 05:13:40 -0700 |
commit | 248367605ead6fb7c36d2bfb1ebd8f00a23f7c71 (patch) | |
tree | 1844f52cc151369fe7dd8e188b81acecc5ce1205 /examples | |
parent | 0e730dd23b0fb5f93dba574e0a48d9a69dc5dbae (diff) |
Work around for recalculating logits in cached prompts (Fixes #1585) (#1609)
* Work around for recalculating logits in cached prompts
Diffstat (limited to 'examples')
-rw-r--r-- | examples/main/main.cpp | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/examples/main/main.cpp b/examples/main/main.cpp index c7c5915..6131f5b 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -360,6 +360,12 @@ int main(int argc, char ** argv) { } } if (i > 0) { + // check if we've used up all the prompt but not all cached tokens + if (embd.size() == i && n_session_consumed < (int) session_tokens.size()) { + // force revaluation of the last token to recalculate logits + i--; + n_past--; + } embd.erase(embd.begin(), embd.begin() + i); } } |