diff options
Diffstat (limited to 'examples/server/public')
-rw-r--r-- | examples/server/public/completion.js | 55 | ||||
-rw-r--r-- | examples/server/public/index.html | 133 |
2 files changed, 138 insertions, 50 deletions
diff --git a/examples/server/public/completion.js b/examples/server/public/completion.js index a43d5a7..0c9bd5f 100644 --- a/examples/server/public/completion.js +++ b/examples/server/public/completion.js @@ -43,6 +43,7 @@ export async function* llama(prompt, params = {}, config = {}) { const decoder = new TextDecoder(); let content = ""; + let leftover = ""; // Buffer for partially read lines try { let cont = true; @@ -53,29 +54,47 @@ export async function* llama(prompt, params = {}, config = {}) { break; } - // sse answers in the form multiple lines of: value\n with data always present as a key. in our case we - // mainly care about the data: key here, which we expect as json - const text = decoder.decode(result.value); + // Add any leftover data to the current chunk of data + const text = leftover + decoder.decode(result.value); - // parse all sse events and add them to result - const regex = /^(\S+):\s(.*)$/gm; - for (const match of text.matchAll(regex)) { - result[match[1]] = match[2] - } + // Check if the last character is a line break + const endsWithLineBreak = text.endsWith('\n'); - // since we know this is llama.cpp, let's just decode the json in data - result.data = JSON.parse(result.data); - content += result.data.content; + // Split the text into lines + let lines = text.split('\n'); - // yield - yield result; + // If the text doesn't end with a line break, then the last line is incomplete + // Store it in leftover to be added to the next chunk of data + if (!endsWithLineBreak) { + leftover = lines.pop(); + } else { + leftover = ""; // Reset leftover if we have a line break at the end + } - // if we got a stop token from server, we will break here - if (result.data.stop) { - if (result.data.generation_settings) { - generation_settings = result.data.generation_settings; + // Parse all sse events and add them to result + const regex = /^(\S+):\s(.*)$/gm; + for (const line of lines) { + const match = regex.exec(line); + if (match) { + result[match[1]] = match[2] + // since we know this is llama.cpp, let's just decode the json in data + if (result.data) { + result.data = JSON.parse(result.data); + content += result.data.content; + + // yield + yield result; + + // if we got a stop token from server, we will break here + if (result.data.stop) { + if (result.data.generation_settings) { + generation_settings = result.data.generation_settings; + } + cont = false; + break; + } + } } - break; } } } catch (e) { diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 8ace0b0..de41da1 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -3,12 +3,11 @@ <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" /> + <meta name="color-scheme" content="light dark"> <title>llama.cpp - chat</title> <style> body { - background-color: #fff; - color: #000; font-family: system-ui; font-size: 90%; } @@ -73,6 +72,37 @@ margin: 0; } + fieldset.two { + display: grid; + grid-template: "a a"; + gap: 1em; + } + + fieldset.three { + display: grid; + grid-template: "a a a"; + gap: 1em; + } + + details { + border: 1px solid #aaa; + border-radius: 4px; + padding: 0.5em 0.5em 0; + margin-top: 0.5em; + } + + summary { + font-weight: bold; + margin: -0.5em -0.5em 0; + padding: 0.5em; + cursor: pointer; + } + + details[open] { + padding: 0.5em; + } + + textarea { padding: 5px; flex-grow: 1; @@ -125,10 +155,17 @@ const params = signal({ n_predict: 400, temperature: 0.7, - repeat_last_n: 256, - repeat_penalty: 1.18, - top_k: 40, - top_p: 0.5, + repeat_last_n: 256, // 0 = disable penalty, -1 = context size + repeat_penalty: 1.18, // 1.0 = disabled + top_k: 40, // <= 0 to use vocab size + top_p: 0.5, // 1.0 = disabled + tfs_z: 1.0, // 1.0 = disabled + typical_p: 1.0, // 1.0 = disabled + presence_penalty: 0.0, // 0.0 = disabled + frequency_penalty: 0.0, // 0.0 = disabled + mirostat: 0, // 0/1/2 + mirostat_tau: 5, // target entropy + mirostat_eta: 0.1, // learning rate }) const llamaStats = signal(null) @@ -245,8 +282,9 @@ useEffect(() => { // scroll to bottom (if needed) - if (container.current && container.current.scrollHeight <= container.current.scrollTop + container.current.offsetHeight + 300) { - container.current.scrollTo(0, container.current.scrollHeight) + const parent = container.current.parentElement; + if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) { + parent.scrollTo(0, parent.scrollHeight) } }, [messages]) @@ -264,6 +302,27 @@ const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value } const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value } const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) } + const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) } + + const FloatField = ({label, max, min, name, step, value}) => { + return html` + <div> + <label for="${name}">${label}</label> + <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} /> + <span>${value}</span> + </div> + ` + }; + + const IntField = ({label, max, min, name, value}) => { + return html` + <div> + <label for="${name}">${label}</label> + <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} /> + <span>${value}</span> + </div> + ` + }; return html` <form> @@ -272,7 +331,9 @@ <label for="prompt">Prompt</label> <textarea type="text" name="prompt" value="${session.value.prompt}" rows=4 oninput=${updateSession}/> </div> + </fieldset> + <fieldset class="two"> <div> <label for="user">User name</label> <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} /> @@ -282,7 +343,9 @@ <label for="bot">Bot name</label> <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} /> </div> + </fieldset> + <fieldset> <div> <label for="template">Prompt template</label> <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/> @@ -292,38 +355,44 @@ <label for="template">Chat history template</label> <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/> </div> + </fieldset> - <div> - <label for="temperature">Temperature</label> - <input type="range" id="temperature" min="0.0" max="1.0" step="0.01" name="temperature" value="${params.value.temperature}" oninput=${updateParamsFloat} /> - <span>${params.value.temperature}</span> - </div> - - <div> - <label for="nPredict">Predictions</label> - <input type="range" id="nPredict" min="1" max="2048" step="1" name="n_predict" value="${params.value.n_predict}" oninput=${updateParamsFloat} /> - <span>${params.value.n_predict}</span> - </div> - - <div> - <label for="repeat_penalty">Penalize repeat sequence</label> - <input type="range" id="repeat_penalty" min="0.0" max="2.0" step="0.01" name="repeat_penalty" value="${params.value.repeat_penalty}" oninput=${updateParamsFloat} /> - <span>${params.value.repeat_penalty}</span> - </div> - - <div> - <label for="repeat_last_n">Consider N tokens for penalize</label> - <input type="range" id="repeat_last_n" min="0.0" max="2048" name="repeat_last_n" value="${params.value.repeat_last_n}" oninput=${updateParamsFloat} /> - <span>${params.value.repeat_last_n}</span> - </div> - + <fieldset class="two"> + ${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})} + ${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})} + ${FloatField({label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty})} + ${IntField({label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n})} + ${IntField({label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k})} + ${FloatField({label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p})} </fieldset> + <details> + <summary>More options</summary> + <fieldset class="two"> + ${FloatField({label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z})} + ${FloatField({label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p})} + ${FloatField({label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty})} + ${FloatField({label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty})} + </fieldset> + <hr /> + <fieldset class="three"> + <div> + <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label> + <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label> + <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label> + </div> + ${FloatField({label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau})} + ${FloatField({label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta})} + </fieldset> + </details> </form> ` } // poor mans markdown replacement const Markdownish = (params) => { const md = params.text + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') .replace(/^#{1,6} (.*)$/gim, '<h3>$1</h3>') .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>') .replace(/__(.*?)__/g, '<strong>$1</strong>') |