Fix title on mobile + add model functionality

public/tokenizer.css

@@ -13,6 +13,11 @@ body {
     background-color: #3c3c43;
   }
 }
+@media (max-width: 640px) {
+  h1 img {
+    display: none;
+  }
+}
 h1 {
   display: flex;
   justify-content: space-around;

public/tokenizer.html

@@ -13,13 +13,19 @@ <h1><img src="favicons/token.svg" alt="Token"><img src="favicons/token.svg" alt=
         <p>
             A pure Javascript tokenizer running in your browser that can load <code>tokenizer.json</code> and <code>tokenizer_config.json</code> from any repository on Huggingface. You can use it to count tokens and compare how different large language model vocabularies work. It's also useful for debugging prompt templates.
         </p>
+        <!-- TODO: take text from URL params? -->
         <textarea id="textInput" name="textInput" autofocus placeholder="Enter the text you want to tokenize" style="height: 5em;">
 [INST] <<SYS>>
 You are a friendly Llama.
 <</SYS>>
 
 Do you spit at people?[/INST]</textarea>
         <ul id="models"></ul>
+        <div id="addModel">
+            <!-- TODO: add styling -->
+            <input placeholder="mistralai/Mixtral-8x7B-v0.1" />
+            <button>Add tokenizer from HuggingFace</button>
+        </div>
     </body>
     <script type="module" async src="tokenizer.js"></script>
 </html>

public/tokenizer.js

@@ -17,6 +17,7 @@ const COLOURS = [
 
 let models = []
 
+// TODO: take model list from URL params?
 function loadModels() {
   const storedModels = localStorage.getItem(KEY_MODELS)
   try {
@@ -30,10 +31,10 @@ function loadModels() {
       'mistralai/Mistral-7B-v0.1',
       'hf-internal-testing/llama-tokenizer',
       'deepseek-ai/deepseek-coder-6.7b-instruct',
-      'microsoft/phi-1_5',
-      '01-ai/Yi-34B',
-      'Xenova/bert-base-cased',
-      'Xenova/t5-small',
+      // 'microsoft/phi-1_5',
+      // '01-ai/Yi-34B',
+      // 'Xenova/bert-base-cased',
+      // 'Xenova/t5-small',
       // 'obvious/error',
       // 'meta-llama/Llama-2-7b-chat-hf',
     ]
@@ -45,6 +46,10 @@ function saveModels() {
   localStorage.setItem(KEY_MODELS, JSON.stringify(models))
 }
 
+function addModel(name) {
+  localStorage.setItem(KEY_MODELS, JSON.stringify([...models, name]))
+}
+
 loadModels()
 
 const loadedModels = {}
@@ -62,36 +67,32 @@ textInput.addEventListener('input', (event) => {
 })
 
 async function loadTokenizers() {
-  modelsList.innerHTML = ''
-
+  console.log('Loading models...')
   for (const model of models) {
     if (!(model in loadedModels)) {
       try {
+        console.log('Loading model: ' + model)
         loadedModels[model] = await AutoTokenizer.from_pretrained(model)
       } catch (error) {
         console.error('Model loading error:' + error)
         loadedModels[model] = { error }
       }
-      // some tokenizers strip spaces, let's prevent it so we can render text with the token numbers
+
+      console.log('Loaded model', loadedModels[model])
+      // some tokenizers strip spaces, let's prevent it so we can render them with the token numbers
       if (loadedModels[model]?.decoder?.decoders?.at(-1)?.config?.type === 'Strip') {
         loadedModels[model].decoder.decoders.pop()
       }
-    }
 
-    const newModelListItem = document.createElement('li')
-    newModelListItem.dataset.model = model
-    // TODO: add delete button
-    // TODO: make it possible to reorder them?
-    // TODO: add token count to each box
-    modelsList.appendChild(newModelListItem)
-
-    updateTokens()
+      const newModelListItem = document.createElement('li')
+      newModelListItem.dataset.model = model
+      // TODO: add delete button
+      // TODO: make it possible to reorder them?
+      modelsList.appendChild(newModelListItem)
+    }
   }
 
-  // const addModelListItem = document.createElement('li')
-  // addModelListItem.id = 'addModel'
-  // addModelListItem.innerHTML = `<input /><button class="addModel">Add</button>`
-  // modelsList.appendChild(addModelListItem)
+  updateTokens()
 }
 
 const renderTokenAndText = (acc, { token, text }, index) => {
@@ -132,3 +133,11 @@ function updateTokens() {
 }
 
 await loadTokenizers()
+
+const addModelBox = document.getElementById('addModel')
+addModelBox.querySelector('button').addEventListener('click', async () => {
+  addModel(addModelBox.querySelector('input').value)
+  loadModels()
+  await loadTokenizers()
+  window.scrollTo(0, document.body.scrollHeight)
+})