diff options
| -rw-r--r-- | examples/common.cpp | 7 | ||||
| -rw-r--r-- | examples/common.h | 1 | ||||
| -rw-r--r-- | examples/main/README.md | 8 | ||||
| -rw-r--r-- | examples/main/main.cpp | 9 |
4 files changed, 25 insertions, 0 deletions
diff --git a/examples/common.cpp b/examples/common.cpp index cd63000..97eded6 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -324,6 +324,12 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { break; } params.input_prefix = argv[i]; + } else if (arg == "--in-suffix") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.input_suffix = argv[i]; } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); gpt_print_usage(argc, argv, default_params); @@ -362,6 +368,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { fprintf(stderr, " --session FNAME file to cache model state in (may be large!) (default: none)\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); + fprintf(stderr, " --in-suffix STRING string to suffix after user inputs with (default: empty)\n"); fprintf(stderr, " -f FNAME, --file FNAME\n"); fprintf(stderr, " prompt file to start generation.\n"); fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict); diff --git a/examples/common.h b/examples/common.h index 138d0de..842e151 100644 --- a/examples/common.h +++ b/examples/common.h @@ -43,6 +43,7 @@ struct gpt_params { std::string prompt = ""; std::string path_session = ""; // path to file for saving/loading model eval state std::string input_prefix = ""; // string to prefix user inputs with + std::string input_suffix = ""; // string to suffix user inputs with std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted std::string lora_adapter = ""; // lora adapter path diff --git a/examples/main/README.md b/examples/main/README.md index 6b7facb..35f87bc 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -112,6 +112,14 @@ The `--in-prefix` flag is used to add a prefix to your input, primarily, this is ./main -r "User:" --in-prefix " " ``` +### In-Suffix + +The `--in-suffix` flag is used to add a suffix after your input. This is useful for adding an "Assistant:" prompt after the user's input. It's added after the new-line character (`\n`) that's automatically added to the end of the user's input. Here's an example of how to use the `--in-suffix` flag in conjunction with the `--reverse-prompt` flag: + +```sh +./main -r "User:" --in-prefix " " --in-suffix "Assistant:" +``` + ### Instruction Mode Instruction mode is particularly useful when working with Alpaca models, which are designed to follow user instructions for specific tasks: diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 17a5a90..43dca8e 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -260,6 +260,10 @@ int main(int argc, char ** argv) { if (!params.input_prefix.empty()) { fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str()); } + + if (!params.input_suffix.empty()) { + fprintf(stderr, "Input suffix: '%s'\n", params.input_suffix.c_str()); + } } fprintf(stderr, "sampling: repeat_last_n = %d, repeat_penalty = %f, presence_penalty = %f, frequency_penalty = %f, top_k = %d, tfs_z = %f, top_p = %f, typical_p = %f, temp = %f, mirostat = %d, mirostat_lr = %f, mirostat_ent = %f\n", params.repeat_last_n, params.repeat_penalty, params.presence_penalty, params.frequency_penalty, params.top_k, params.tfs_z, params.top_p, params.typical_p, params.temp, params.mirostat, params.mirostat_eta, params.mirostat_tau); @@ -567,6 +571,11 @@ int main(int argc, char ** argv) { // Add tokens to embd only if the input buffer is non-empty // Entering a empty line lets the user pass control back if (buffer.length() > 1) { + // append input suffix if any + if (!params.input_suffix.empty()) { + buffer += params.input_suffix; + printf("%s", params.input_suffix.c_str()); + } // instruct mode: insert instruction prefix if (params.instruct && !is_antiprompt) { |