aboutsummaryrefslogtreecommitdiff
path: root/examples
diff options
context:
space:
mode:
Diffstat (limited to 'examples')
-rw-r--r--examples/common.cpp262
-rw-r--r--examples/common.h30
-rw-r--r--examples/main/main.cpp172
3 files changed, 102 insertions, 362 deletions
diff --git a/examples/common.cpp b/examples/common.cpp
index eaa5ace..0772dbf 100644
--- a/examples/common.cpp
+++ b/examples/common.cpp
@@ -2,13 +2,10 @@
#include <cassert>
#include <cstring>
-#include <iostream>
#include <fstream>
-#include <sstream>
#include <string>
#include <iterator>
#include <algorithm>
-#include <regex>
#if defined (_WIN32)
#include <fcntl.h>
@@ -26,43 +23,6 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
#define CP_UTF8 65001
#endif
-void split_args(const std::string & args_string, std::vector<std::string> & output_args)
-{
- std::string current_arg = "";
- bool in_quotes = false;
- char quote_type;
-
- for (char c : args_string) {
- if (c == '"' || c == '\'') {
- if (!in_quotes) {
- in_quotes = true;
- quote_type = c;
- } else if (quote_type == c) {
- in_quotes = false;
- } else {
- current_arg += c;
- }
- } else if (in_quotes) {
- current_arg += c;
- } else if (std::isspace(c)) {
- if (current_arg != "") {
- output_args.push_back(current_arg);
- current_arg = "";
- }
- } else {
- current_arg += c;
- }
- }
-
- if (current_arg != "") {
- output_args.push_back(current_arg);
- }
-}
-
-std::string unescape(const std::string & str) {
- return std::regex_replace(str, std::regex("\\\\n"), "\n");
-}
-
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
// determine sensible default number of threads.
// std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
@@ -80,66 +40,35 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
std::string arg;
gpt_params default_params;
- // get additional arguments from config files
- std::vector<std::string> args;
for (int i = 1; i < argc; i++) {
arg = argv[i];
- if (arg == "--config") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::ifstream file(argv[i]);
- if (!file) {
- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
- invalid_param = true;
- break;
- }
- std::string args_string;
- std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(args_string));
- if (args_string.back() == '\n') {
- args_string.pop_back();
- }
- split_args(args_string, args);
- for (int j = 0; j < args.size(); j++) {
- args[j] = unescape(args[j]);
- }
- } else {
- args.emplace_back(argv[i]);
- }
- }
-
- // parse args
- int args_c = static_cast<int>(args.size());
- for (int i = 0; i < args_c && !invalid_param; i++) {
- arg = args[i];
if (arg == "-s" || arg == "--seed") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.seed = std::stoi(args[i]);
+ params.seed = std::stoi(argv[i]);
} else if (arg == "-t" || arg == "--threads") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_threads = std::stoi(args[i]);
+ params.n_threads = std::stoi(argv[i]);
} else if (arg == "-p" || arg == "--prompt") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.prompt = args[i];
+ params.prompt = argv[i];
} else if (arg == "-f" || arg == "--file") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- std::ifstream file(args[i]);
+ std::ifstream file(argv[i]);
if (!file) {
- fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str());
+ fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
invalid_param = true;
break;
}
@@ -148,100 +77,80 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
params.prompt.pop_back();
}
} else if (arg == "-n" || arg == "--n_predict") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_predict = std::stoi(args[i]);
+ params.n_predict = std::stoi(argv[i]);
} else if (arg == "--top_k") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.top_k = std::stoi(args[i]);
+ params.top_k = std::stoi(argv[i]);
} else if (arg == "-c" || arg == "--ctx_size") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_ctx = std::stoi(args[i]);
+ params.n_ctx = std::stoi(argv[i]);
} else if (arg == "--memory_f32") {
params.memory_f16 = false;
} else if (arg == "--top_p") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.top_p = std::stof(args[i]);
+ params.top_p = std::stof(argv[i]);
} else if (arg == "--temp") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.temp = std::stof(args[i]);
+ params.temp = std::stof(argv[i]);
} else if (arg == "--repeat_last_n") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.repeat_last_n = std::stoi(args[i]);
+ params.repeat_last_n = std::stoi(argv[i]);
} else if (arg == "--repeat_penalty") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.repeat_penalty = std::stof(args[i]);
+ params.repeat_penalty = std::stof(argv[i]);
} else if (arg == "-b" || arg == "--batch_size") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_batch = std::stoi(args[i]);
+ params.n_batch = std::stoi(argv[i]);
params.n_batch = std::min(512, params.n_batch);
} else if (arg == "--keep") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_keep = std::stoi(args[i]);
+ params.n_keep = std::stoi(argv[i]);
} else if (arg == "-m" || arg == "--model") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.model = args[i];
+ params.model = argv[i];
} else if (arg == "-i" || arg == "--interactive") {
params.interactive = true;
} else if (arg == "--embedding") {
params.embedding = true;
- } else if (arg == "--clean-interface") {
- params.clean_interface = true;
} else if (arg == "--interactive-start") {
params.interactive = true;
} else if (arg == "--interactive-first") {
params.interactive_start = true;
} else if (arg == "-ins" || arg == "--instruct") {
- fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n"
- "--clean-interface "
- "--interactive-first "
- "--keep -1 "
- "--ins-prefix-bos "
- "--ins-prefix \"\\n\\n### Instruction:\\n\\n\" "
- "--ins-suffix \"\\n\\n### Response:\\n\\n\" "
- "-r \"### Instruction:\\n\\n\" "
- "\n\n");
- // params.instruct = true;
- params.clean_interface = true;
- params.interactive_start = true;
- params.n_keep = -1;
- params.instruct_prefix_bos = true;
- params.instruct_prefix = "\n\n### Instruction:\n\n";
- params.instruct_suffix = "\n\n### Response:\n\n";
- params.antiprompt.push_back("### Instruction:\n\n");
+ params.instruct = true;
} else if (arg == "--color") {
params.use_color = true;
- } else if (arg == "--disable-multiline") {
- params.multiline_mode = false;
} else if (arg == "--mlock") {
params.use_mlock = true;
} else if (arg == "--no-mmap") {
@@ -251,94 +160,65 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
} else if (arg == "--verbose-prompt") {
params.verbose_prompt = true;
} else if (arg == "-r" || arg == "--reverse-prompt") {
- if (++i >= args_c) {
- invalid_param = true;
- break;
- }
- params.antiprompt.push_back(args[i]);
- } else if (arg == "--stop-prompt") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.stopprompt.push_back(args[i]);
- } else if (arg == "--rm-trailing-space-workaround") {
- params.rm_trailing_space_workaround = true;
+ params.antiprompt.push_back(argv[i]);
} else if (arg == "--perplexity") {
params.perplexity = true;
} else if (arg == "--ignore-eos") {
params.ignore_eos = true;
} else if (arg == "--n_parts") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.n_parts = std::stoi(args[i]);
+ params.n_parts = std::stoi(argv[i]);
} else if (arg == "-h" || arg == "--help") {
- gpt_print_usage(argv[0], default_params);
+ gpt_print_usage(argc, argv, default_params);
exit(0);
} else if (arg == "--random-prompt") {
params.random_prompt = true;
} else if (arg == "--in-prefix") {
- if (++i >= args_c) {
- invalid_param = true;
- break;
- }
- params.input_prefix = args[i];
- } else if (arg == "--ins-prefix-bos") {
- params.instruct_prefix_bos = true;
- } else if (arg == "--ins-prefix") {
- if (++i >= args_c) {
- invalid_param = true;
- break;
- }
- params.instruct_prefix = args[i];
- } else if (arg == "--ins-suffix-bos") {
- params.instruct_suffix_bos = true;
- } else if (arg == "--ins-suffix") {
- if (++i >= args_c) {
+ if (++i >= argc) {
invalid_param = true;
break;
}
- params.instruct_suffix = args[i];
+ params.input_prefix = argv[i];
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
- gpt_print_usage(argv[0], default_params);
+ gpt_print_usage(argc, argv, default_params);
exit(1);
}
}
if (invalid_param) {
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
- gpt_print_usage(argv[0], default_params);
+ gpt_print_usage(argc, argv, default_params);
exit(1);
}
return true;
}
-void gpt_print_usage(char * argv_0, const gpt_params & params) {
- fprintf(stderr, "usage: %s [options]\n", argv_0);
+void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
+ fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -i, --interactive run in interactive mode\n");
fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n");
- fprintf(stderr, " --clean-interface hides input prefix & suffix and displays '>' instead\n");
+ fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n");
fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n");
fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n");
fprintf(stderr, " specified more than once for multiple prompts).\n");
fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n");
- fprintf(stderr, " --disable-multiline disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
fprintf(stderr, " prompt to start generation with (default: empty)\n");
fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n");
- fprintf(stderr, " --ins-prefix STRING (instruct) prefix user inputs with tokenized string (default: empty)\n");
- fprintf(stderr, " --ins-prefix-bos (instruct) prepend bos token to instruct prefix.\n");
- fprintf(stderr, " --ins-suffix STRING (instruct) suffix user inputs with tokenized string (default: empty)\n");
- fprintf(stderr, " --ins-suffix-bos (instruct) prepend bos token to instruct suffix.\n");
fprintf(stderr, " -f FNAME, --file FNAME\n");
fprintf(stderr, " prompt file to start generation.\n");
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
@@ -448,61 +328,3 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
str = strTo;
}
#endif
-
-bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) {
- bool another_line = true;
- bool is_eof_multiline_toggled = false;
- do {
- std::string line;
-#if defined(_WIN32)
- auto & stdcin = std::wcin;
- std::wstring wline;
- if (!std::getline(stdcin, wline)) {
- // input stream is bad or EOF received
- if (stdcin.bad()) {
- fprintf(stderr, "%s: error: input stream bad\n", __func__);
- return 1;
- }
- }
- win32_utf8_encode(wline, line);
-#else
- auto & stdcin = std::cin;
- if (!std::getline(stdcin, line)) {
- // input stream is bad or EOF received
- if (stdcin.bad()) {
- fprintf(stderr, "%s: error: input stream bad\n", __func__);
- return 1;
- }
- }
-#endif
- if (stdcin.eof()) {
- stdcin.clear();
- stdcin.seekg(0, std::ios::beg);
- if (!eof_toggled_multiline_mode) {
- another_line = false;
- } else {
- is_eof_multiline_toggled = !is_eof_multiline_toggled;
- if (is_eof_multiline_toggled) {
- input_text += line;
- continue;
- }
- }
- }
- if (!eof_toggled_multiline_mode) {
- if (line.empty() || line.back() != '\\') {
- another_line = false;
- } else {
- line.pop_back(); // Remove the continue character
- }
- } else {
- if (!is_eof_multiline_toggled) {
- another_line = false;
- }
- }
- input_text += line;
- if (another_line) {
- input_text += '\n'; // Append the line to the result
- }
- } while (another_line);
- return true;
-}
diff --git a/examples/common.h b/examples/common.h
index df8e4c6..1ea6f74 100644
--- a/examples/common.h
+++ b/examples/common.h
@@ -14,14 +14,14 @@
//
struct gpt_params {
- int32_t seed = -1; // RNG seed
- int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default)
- int32_t n_predict = 128; // new tokens to predict
- int32_t repeat_last_n = 64; // last n tokens to penalize
- int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
- int32_t n_ctx = 512; // context size
- int32_t n_batch = 8; // batch size for prompt processing
- int32_t n_keep = 0; // number of tokens to keep from initial prompt (-1 for all)
+ int32_t seed = -1; // RNG seed
+ int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
+ int32_t n_predict = 128; // new tokens to predict
+ int32_t repeat_last_n = 64; // last n tokens to penalize
+ int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
+ int32_t n_ctx = 512; // context size
+ int32_t n_batch = 8; // batch size for prompt processing
+ int32_t n_keep = 0; // number of tokens to keep from initial prompt
// sampling parameters
int32_t top_k = 40;
@@ -33,15 +33,8 @@ struct gpt_params {
std::string prompt = "";
std::string input_prefix = ""; // string to prefix user inputs with
- std::string instruct_prefix = ""; // prefix user inputs with tokenized string
- bool instruct_prefix_bos = false; // prepend bos token to instruct prefix
- std::string instruct_suffix = ""; // suffix user inputs with tokenized string
- bool instruct_suffix_bos = false; // prepend bos token to instruct suffix
std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
- std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes)
-
- bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts
bool memory_f16 = true; // use f16 instead of f32 for memory kv
bool random_prompt = false; // do not randomize prompt if none provided
@@ -58,14 +51,11 @@ struct gpt_params {
bool use_mlock = false; // use mlock to keep model in memory
bool mem_test = false; // compute maximum memory usage
bool verbose_prompt = false; // print prompt tokens before generation
-
- bool clean_interface = false; // hides input prefix & suffix and displays '>'
- bool multiline_mode = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
};
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
-void gpt_print_usage(char * argv_0, const gpt_params & params);
+void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng);
@@ -105,5 +95,3 @@ void set_console_color(console_state & con_st, console_color_t color);
void win32_console_init(bool enable_color);
void win32_utf8_encode(const std::wstring & wstr, std::string & str);
#endif
-
-bool get_input_text(std::string & input_text, bool escape_newline_mode);
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
index 68b4b28..ba153cb 100644
--- a/examples/main/main.cpp
+++ b/examples/main/main.cpp
@@ -30,8 +30,7 @@ static bool is_interacting = false;
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
void sigint_handler(int signo) {
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
- fflush(stdout);
- fflush(stderr);
+ printf("\n"); // this also force flush stdout.
if (signo == SIGINT) {
if (!is_interacting) {
is_interacting=true;
@@ -90,8 +89,6 @@ int main(int argc, char ** argv) {
params.prompt = gpt_random_prompt(rng);
}
- bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty();
-
// params.prompt = R"(// this function checks if the number n is prime
//bool is_prime(int n) {)";
@@ -156,20 +153,22 @@ int main(int argc, char ** argv) {
}
// number of tokens to keep when resetting context
- if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) {
+ if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) {
params.n_keep = (int)embd_inp.size();
}
// prefix & suffix for instruct mode
- const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos);
- std::string instruct_suffix = params.instruct_suffix;
- if (params.rm_trailing_space_workaround) {
- if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); }
+ const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true);
+ const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false);
+
+ // in instruct mode, we inject a prefix and a suffix to each input by the user
+ if (params.instruct) {
+ params.interactive_start = true;
+ params.antiprompt.push_back("### Instruction:\n\n");
}
- const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos);
// enable interactive mode if reverse prompt or interactive start is specified
- if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) {
+ if (params.antiprompt.size() != 0 || params.interactive_start) {
params.interactive = true;
}
@@ -211,21 +210,10 @@ int main(int argc, char ** argv) {
fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str());
}
}
- if (params.stopprompt.size()) {
- for (auto stopprompt : params.stopprompt) {
- fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str());
- }
- }
if (!params.input_prefix.empty()) {
fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
}
- if (!params.instruct_prefix.empty()) {
- fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str());
- }
- if (!params.instruct_suffix.empty()) {
- fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str());
- }
}
fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n",
params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
@@ -241,29 +229,12 @@ int main(int argc, char ** argv) {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
" - Press Ctrl+C to interject at any time.\n"
#endif
- );
- if (params.multiline_mode) {
- fprintf(stderr, " - Press Return to return control to LLaMa.\n"
-#if defined (_WIN32)
- " - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n");
-#else
- " - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n");
-#endif
- }
- else {
- fprintf(stderr, " - Press Return to return control to LLaMa.\n"
- " - If you want to submit another line, end your input in '\\'.\n\n");
- }
+ " - Press Return to return control to LLaMa.\n"
+ " - If you want to submit another line, end your input in '\\'.\n\n");
is_interacting = params.interactive_start;
}
- struct Antiprompt {
- bool any = false;
- bool trailing_space = false;
- size_t len;
- bool is_stop_prompt = false;
- } antiprompt;
-
+ bool is_antiprompt = false;
bool input_noecho = false;
int n_past = 0;
@@ -333,7 +304,7 @@ int main(int argc, char ** argv) {
}
// replace end of text token with newline token when in interactive mode
- if (id == llama_token_eos() && params.interactive && !instruct_mode) {
+ if (id == llama_token_eos() && params.interactive && !params.instruct) {
id = llama_token_newline.front();
if (params.antiprompt.size() != 0) {
// tokenize and inject first reverse prompt
@@ -379,72 +350,27 @@ int main(int argc, char ** argv) {
// check if we should prompt the user for more
if (params.interactive && (int) embd_inp.size() <= n_consumed) {
- // check for reverse prompt or stop prompt
- if (params.antiprompt.size() || params.stopprompt.size()) {
+ // check for reverse prompt
+ if (params.antiprompt.size()) {
std::string last_output;
for (auto id : last_n_tokens) {
last_output += llama_token_to_str(ctx, id);
}
- antiprompt.any = false;
- antiprompt.is_stop_prompt = false;
+ is_antiprompt = false;
// Check if each of the reverse prompts appears at the end of the output.
- for (std::string & prompt : params.antiprompt) {
- if (params.rm_trailing_space_workaround) {
- antiprompt.trailing_space = prompt.back() == ' ';
- antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
- }
- if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
+ for (std::string & antiprompt : params.antiprompt) {
+ if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
is_interacting = true;
- antiprompt.any = true;
+ is_antiprompt = true;
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
fflush(stdout);
break;
}
}
- if (!antiprompt.any) {
- for (std::string & prompt : params.stopprompt) {
- if (params.rm_trailing_space_workaround) {
- antiprompt.trailing_space = prompt.back() == ' ';
- antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
- }
- if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
- is_interacting = true;
- antiprompt.any = true;
- antiprompt.is_stop_prompt = true;
- set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
- fflush(stdout);
- break;
- }
- }
- }
}
- if (n_past > 0 && is_interacting)
- {
- std::string buffer;
- if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) {
- // avoid printing again user's new line (TODO: try to revert enter press and print newline)
- int i = params.instruct_prefix.front() == '\n' ? 1 : 0;
- for (; i < inp_pfx.size(); i++) {
- printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
- }
- fflush(stdout);
- }
- if (params.rm_trailing_space_workaround) {
- // add only if not stopprompt (as stopprompt could be used to pause
- // assistant and then continue without input - adding back trailing
- // space may mess it up.)
- if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) {
- // add back removed trailing space to buffer(workaround)
- buffer += ' ';
- if (!params.clean_interface) {
- printf("%s", buffer.c_str());
- }
- fflush(stdout);
- }
- }
-
+ if (n_past > 0 && is_interacting) {
// potentially set color to indicate we are taking user input
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
@@ -453,45 +379,49 @@ int main(int argc, char ** argv) {
signal(SIGINT, sigint_handler);
#endif
- if (params.clean_interface) {
+ if (params.instruct) {
printf("\n> ");
}
+ std::string buffer;
if (!params.input_prefix.empty()) {
buffer += params.input_prefix;
printf("%s", buffer.c_str());
}
- if (!get_input_text(buffer, params.multiline_mode)) {
- // input stream is bad
- return 1;
- }
- if (!antiprompt.is_stop_prompt) {
- buffer += "\n";
- }
+ std::string line;
+ bool another_line = true;
+ do {
+#if defined(_WIN32)
+ std::wstring wline;
+ if (!std::getline(std::wcin, wline)) {
+ // input stream is bad or EOF received
+ return 0;
+ }
+ win32_utf8_encode(wline, line);
+#else
+ if (!std::getline(std::cin, line)) {
+ // input stream is bad or EOF received
+ return 0;
+ }
+#endif
+ if (line.empty() || line.back() != '\\') {
+ another_line = false;
+ } else {
+ line.pop_back(); // Remove the continue character
+ }
+ buffer += line + '\n'; // Append the line to the result
+ } while (another_line);
// done taking input, reset color
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
- if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
- // avoid printing again user's new line (TODO: try to revert enter press and print newline)
- int i = params.instruct_suffix.front() == '\n' ? 1 : 0;
- for (; i < inp_sfx.size(); i++) {
- printf("%s", llama_token_to_str(ctx, inp_sfx[i]));
- }
- // if (remove trailing space workaround) {
- // We won't add back removed trailing space here, because assistant continues here,
- // and it may mess up it's output (remove trailing space workaround).
- // }
- fflush(stdout);
- }
-
// Add tokens to embd only if the input buffer is non-empty
// Entering a empty line lets the user pass control back
if (buffer.length() > 1) {
- // insert input prefix
- if (!params.instruct_prefix.empty() && !antiprompt.any) {
+ // instruct mode: insert instruction prefix
+ if (params.instruct && !is_antiprompt) {
n_consumed = embd_inp.size();
embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
}
@@ -499,8 +429,8 @@ int main(int argc, char ** argv) {
auto line_inp = ::llama_tokenize(ctx, buffer, false);
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
- // insert response suffix
- if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
+ // instruct mode: insert response suffix
+ if (params.instruct) {
embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());
}
@@ -517,7 +447,7 @@ int main(int argc, char ** argv) {
// end of text token
if (!embd.empty() && embd.back() == llama_token_eos()) {
- if (instruct_mode) {
+ if (params.instruct) {
is_interacting = true;
} else {
fprintf(stderr, " [end of text]\n");
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-31 01:36:11 +0000