aboutsummaryrefslogtreecommitdiff
path: root/examples/main
diff options
context:
space:
mode:
authorDannyDaemonic <DannyDaemonic@gmail.com>2023-05-29 05:13:40 -0700
committerGitHub <noreply@github.com>2023-05-29 05:13:40 -0700
commit248367605ead6fb7c36d2bfb1ebd8f00a23f7c71 (patch)
tree1844f52cc151369fe7dd8e188b81acecc5ce1205 /examples/main
parent0e730dd23b0fb5f93dba574e0a48d9a69dc5dbae (diff)
Work around for recalculating logits in cached prompts (Fixes #1585) (#1609)
* Work around for recalculating logits in cached prompts
Diffstat (limited to 'examples/main')
-rw-r--r--examples/main/main.cpp6
1 files changed, 6 insertions, 0 deletions
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index c7c5915..6131f5b 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -360,6 +360,12 @@ int main(int argc, char ** argv) {
}
}
if (i > 0) {
+ // check if we've used up all the prompt but not all cached tokens
+ if (embd.size() == i && n_session_consumed < (int) session_tokens.size()) {
+ // force revaluation of the last token to recalculate logits
+ i--;
+ n_past--;
+ }
embd.erase(embd.begin(), embd.begin() + i);
}
}