aboutsummaryrefslogtreecommitdiff
path: root/examples/main
diff options
context:
space:
mode:
authorKerfuffle <44031344+KerfuffleV2@users.noreply.github.com>2023-06-11 08:19:17 -0600
committerGitHub <noreply@github.com>2023-06-11 08:19:17 -0600
commitfa84c4b3e80199a5683438f062009c031a06c4fa (patch)
treec0c088e617ec6137a1ea47dc0167fc164c05e2bb /examples/main
parent12b063f0ecf280e98028e444fc492ee6222cdcdc (diff)
Fix issue where interactive mode crashes when input exceeds ctx size (#1789)
* Fix issue where interactive mode in the main example crashes when input exceeds ctx size * Ensure the context size is at least 8 tokens in the main example. Closes #1768
Diffstat (limited to 'examples/main')
-rw-r--r--examples/main/main.cpp16
1 files changed, 16 insertions, 0 deletions
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index de63faa..66d5631 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -81,6 +81,9 @@ int main(int argc, char ** argv) {
if (params.n_ctx > 2048) {
fprintf(stderr, "%s: warning: model does not support context sizes greater than 2048 tokens (%d specified);"
"expect poor results\n", __func__, params.n_ctx);
+ } else if (params.n_ctx < 8) {
+ fprintf(stderr, "%s: warning: minimum context size is 8, using minimum size.\n", __func__);
+ params.n_ctx = 8;
}
fprintf(stderr, "%s: build = %d (%s)\n", __func__, BUILD_NUMBER, BUILD_COMMIT);
@@ -331,6 +334,19 @@ int main(int argc, char ** argv) {
while ((n_remain != 0 && !is_antiprompt) || params.interactive) {
// predict
if (embd.size() > 0) {
+ // Note: n_ctx - 4 here is to match the logic for commandline prompt handling via
+ // --prompt or --file which uses the same value.
+ auto max_embd_size = n_ctx - 4;
+ // Ensure the input doesn't exceed the context size by truncating embd if necessary.
+ if ((int)embd.size() > max_embd_size) {
+ auto skipped_tokens = embd.size() - max_embd_size;
+ console_set_color(con_st, CONSOLE_COLOR_ERROR);
+ printf("<<input too long: skipped %ld token%s>>", skipped_tokens, skipped_tokens != 1 ? "s" : "");
+ console_set_color(con_st, CONSOLE_COLOR_DEFAULT);
+ fflush(stdout);
+ embd.resize(max_embd_size);
+ }
+
// infinite text generation via context swapping
// if we run out of context:
// - take the n_keep first tokens from the original prompt (via n_past)