Increase max token

dist/index.js

@@ -10,4 +10,4 @@ basic_boolean ::= "true" | "false"
 basic_null ::= "null"
 basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]"
 basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}"
-ws ::= [\n\t]*`,grammarSelection.onchange=A=>{console.log("Grammar selection changed:",A.target.value),"json"===A.target.value?(ebnfContainer.classList.add("hidden"),schemaContainer.classList.remove("hidden"),useCustomGrammar=!1):(ebnfContainer.classList.remove("hidden"),schemaContainer.classList.add("hidden"),useCustomGrammar=!0)};const availableModels=_charliefruan_web_llm__WEBPACK_IMPORTED_MODULE_0__.xn.model_list.filter((A=>A.model_id.startsWith("Llama-3")||A.model_id.startsWith("Hermes-2")||A.model_id.startsWith("Hermes-3")||A.model_id.startsWith("Phi-3"))).map((A=>A.model_id));let selectedModel=availableModels[0];availableModels.forEach((A=>{const g=document.createElement("option");g.value=A,g.textContent=A,modelSelection.appendChild(g)})),modelSelection.value=selectedModel,modelSelection.onchange=A=>{selectedModel=A.target.value,engine=null};const editor=ace_builds__WEBPACK_IMPORTED_MODULE_1___default().edit("schema",{mode:"ace/mode/javascript",theme:"ace/theme/github",wrap:!0});editor.setTheme("ace/theme/github"),editor.setValue('Type.Object({\n "name": Type.String(),\n "house": Type.Enum({\n "Gryffindor": "Gryffindor",\n "Hufflepuff": "Hufflepuff",\n "Ravenclaw": "Ravenclaw",\n "Slytherin": "Slytherin",\n }),\n "blood_status": Type.Enum({\n "Pure-blood": "Pure-blood",\n "Half-blood": "Half-blood",\n "Muggle-born": "Muggle-born",\n }),\n "occupation": Type.Enum({\n "Student": "Student",\n "Professor": "Professor",\n "Ministry of Magic": "Ministry of Magic",\n "Other": "Other",\n }),\n "wand": Type.Object({\n "wood": Type.String(),\n "core": Type.String(),\n "length": Type.Number(),\n }),\n "alive": Type.Boolean(),\n "patronus": Type.String(),\n })'),promptTextarea.value="Hermione Granger is a character in Harry Potter. Please fill in the following information about this character in JSON format.\nName is a string of character name.\nHouse is one of Gryffindor, Hufflepuff, Ravenclaw, Slytherin.\nBlood status is one of Pure-blood, Half-blood, Muggle-born.\nOccupation is one of Student, Professor, Ministry of Magic, Other.\nWand is an object with wood, core, and length.\nAlive is a boolean.\nPatronus is a string.\n",document.getElementById("generate").onclick=async()=>{const schemaInput=editor.getValue();let T;try{T=eval(schemaInput)}catch(A){return void console.error("Invalid schema",A)}const schema=JSON.stringify(T);engine||(engine=await(0,_charliefruan_web_llm__WEBPACK_IMPORTED_MODULE_0__.L6)(selectedModel,{initProgressCallback:A=>{console.log(A),outputDiv.textContent=A.text}}));const request={stream:!0,stream_options:{include_usage:!0},messages:[{role:"user",content:promptTextarea.value}],max_tokens:128,response_format:useCustomGrammar?{type:"grammar",grammar:ebnfTextarea.value}:{type:"json_object",schema}};let curMessage="",usage=null;const generator=await engine.chatCompletion(request);for await(const A of generator){const g=A.choices[0]?.delta.content;g&&(curMessage+=g),A.usage&&(usage=A.usage),outputDiv.textContent=curMessage}const finalMessage=await engine.getMessage();if(outputDiv.innerHTML=highlight_js__WEBPACK_IMPORTED_MODULE_2__.A.highlight(finalMessage,{language:"json"}).value,usage){const A=[];usage.extra.prefill_tokens_per_s&&A.push(`Prefill: ${usage.extra.prefill_tokens_per_s.toFixed(1)} tok/s`),usage.extra.decode_tokens_per_s&&A.push(`Decode: ${usage.extra.decode_tokens_per_s.toFixed(1)} tok/s`),usage.extra.grammar_init_ms&&A.push(`Grammar Init Overhead: ${usage.extra.grammar_init_ms.toFixed(0)} ms`),usage.extra.grammar_per_token_ms&&A.push(`Grammar Per Token Overhead: ${usage.extra.grammar_per_token_ms.toFixed(2)} ms`),statsParagraph.textContent=A.join(", "),statsParagraph.classList.remove("hidden")}}}))})()})();
+ws ::= [\n\t]*`,grammarSelection.onchange=A=>{console.log("Grammar selection changed:",A.target.value),"json"===A.target.value?(ebnfContainer.classList.add("hidden"),schemaContainer.classList.remove("hidden"),useCustomGrammar=!1):(ebnfContainer.classList.remove("hidden"),schemaContainer.classList.add("hidden"),useCustomGrammar=!0)};const availableModels=_charliefruan_web_llm__WEBPACK_IMPORTED_MODULE_0__.xn.model_list.filter((A=>A.model_id.startsWith("Llama-3")||A.model_id.startsWith("Hermes-2")||A.model_id.startsWith("Hermes-3")||A.model_id.startsWith("Phi-3"))).map((A=>A.model_id));let selectedModel=availableModels[0];availableModels.forEach((A=>{const g=document.createElement("option");g.value=A,g.textContent=A,modelSelection.appendChild(g)})),modelSelection.value=selectedModel,modelSelection.onchange=A=>{selectedModel=A.target.value,engine=null};const editor=ace_builds__WEBPACK_IMPORTED_MODULE_1___default().edit("schema",{mode:"ace/mode/javascript",theme:"ace/theme/github",wrap:!0});editor.setTheme("ace/theme/github"),editor.setValue('Type.Object({\n "name": Type.String(),\n "house": Type.Enum({\n "Gryffindor": "Gryffindor",\n "Hufflepuff": "Hufflepuff",\n "Ravenclaw": "Ravenclaw",\n "Slytherin": "Slytherin",\n }),\n "blood_status": Type.Enum({\n "Pure-blood": "Pure-blood",\n "Half-blood": "Half-blood",\n "Muggle-born": "Muggle-born",\n }),\n "occupation": Type.Enum({\n "Student": "Student",\n "Professor": "Professor",\n "Ministry of Magic": "Ministry of Magic",\n "Other": "Other",\n }),\n "wand": Type.Object({\n "wood": Type.String(),\n "core": Type.String(),\n "length": Type.Number(),\n }),\n "alive": Type.Boolean(),\n "patronus": Type.String(),\n })'),promptTextarea.value="Hermione Granger is a character in Harry Potter. Please fill in the following information about this character in JSON format.\nName is a string of character name.\nHouse is one of Gryffindor, Hufflepuff, Ravenclaw, Slytherin.\nBlood status is one of Pure-blood, Half-blood, Muggle-born.\nOccupation is one of Student, Professor, Ministry of Magic, Other.\nWand is an object with wood, core, and length.\nAlive is a boolean.\nPatronus is a string.\n",document.getElementById("generate").onclick=async()=>{const schemaInput=editor.getValue();let T;try{T=eval(schemaInput)}catch(A){return void console.error("Invalid schema",A)}const schema=JSON.stringify(T);engine||(engine=await(0,_charliefruan_web_llm__WEBPACK_IMPORTED_MODULE_0__.L6)(selectedModel,{initProgressCallback:A=>{console.log(A),outputDiv.textContent=A.text}}));const request={stream:!0,stream_options:{include_usage:!0},messages:[{role:"user",content:promptTextarea.value}],max_tokens:512,response_format:useCustomGrammar?{type:"grammar",grammar:ebnfTextarea.value}:{type:"json_object",schema}};let curMessage="",usage=null;const generator=await engine.chatCompletion(request);for await(const A of generator){const g=A.choices[0]?.delta.content;g&&(curMessage+=g),A.usage&&(usage=A.usage),outputDiv.textContent=curMessage}const finalMessage=await engine.getMessage();if(outputDiv.innerHTML=highlight_js__WEBPACK_IMPORTED_MODULE_2__.A.highlight(finalMessage,{language:"json"}).value,usage){const A=[];console.log(usage),usage.extra.prefill_tokens_per_s&&A.push(`Prefill Speed: ${usage.extra.prefill_tokens_per_s.toFixed(1)} tok/s`),usage.extra.decode_tokens_per_s&&A.push(`Decode Speed: ${usage.extra.decode_tokens_per_s.toFixed(1)} tok/s`),usage.extra.grammar_init_ms&&A.push(`Grammar Init Overhead: ${usage.extra.grammar_init_ms.toFixed(0)} ms`),usage.extra.grammar_per_token_ms&&A.push(`Grammar Per-token Overhead: ${usage.extra.grammar_per_token_ms.toFixed(2)} ms`),statsParagraph.textContent=A.join(", "),statsParagraph.classList.remove("hidden")}}}))})()})();

src/index.js

@@ -150,7 +150,7 @@ Patronus is a string.
  stream: true,
  stream_options: { include_usage: true },
  messages: [{ role: "user", content: promptTextarea.value }],
- max_tokens: 128,
+ max_tokens: 512,
  response_format: useCustomGrammar
  ? { type: "grammar", grammar: ebnfTextarea.value }
  : { type: "json_object", schema: schema },
@@ -174,13 +174,14 @@ Patronus is a string.
 
  if (usage) {
  const statsTextParts = [];
+ console.log(usage);
  if (usage.extra.prefill_tokens_per_s) {
- statsTextParts.push(`Prefill: ${usage.extra.prefill_tokens_per_s.toFixed(
+ statsTextParts.push(`Prefill Speed: ${usage.extra.prefill_tokens_per_s.toFixed(
  1
  )} tok/s`);
  }
  if (usage.extra.decode_tokens_per_s) {
- statsTextParts.push(`Decode: ${usage.extra.decode_tokens_per_s.toFixed(
+ statsTextParts.push(`Decode Speed: ${usage.extra.decode_tokens_per_s.toFixed(
  1
  )} tok/s`);
  }
@@ -190,7 +191,7 @@ Patronus is a string.
  )} ms`);
  }
  if (usage.extra.grammar_per_token_ms) {
- statsTextParts.push(`Grammar Per Token Overhead: ${usage.extra.grammar_per_token_ms.toFixed(
+ statsTextParts.push(`Grammar Per-token Overhead: ${usage.extra.grammar_per_token_ms.toFixed(
  2
  )} ms`);
  }