Show how to use local file

[ahuang11]

Something like this, maybe not use os.

"""
Demonstrates how to use the `ChatInterface` to create a chatbot using
[Mistral](https://docs.mistral.ai) through
[CTransformers](https://github.com/marella/ctransformers).
"""
import os

import panel as pn
from ctransformers import AutoConfig, AutoModelForCausalLM, Config

pn.extension()

llms = pn.state.cache["llms"] = pn.state.cache.get("llms", {})

MODEL_FILE = "mistral-7b-openorca.Q4_K_M.gguf"
MODEL_URL = f"https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF/resolve/main/{MODEL_FILE}"
CURL_CMD = f"curl -C - -o {MODEL_FILE} {MODEL_URL}"


async def callback(contents: str, user: str, instance: pn.chat.ChatInterface):
    if "mistral" not in llms:
        instance.placeholder_text = "Downloading model; please wait..."
        config = AutoConfig(
            config=Config(
                temperature=0.5, max_new_tokens=2048, context_length=2048, gpu_layers=1
            ),
        )
        if not os.path.exists(MODEL_FILE):
            return_code = os.system(CURL_CMD)
            if return_code != 0:
                raise RuntimeError(f"Could not download {MODEL_URL}")
        llms["mistral"] = AutoModelForCausalLM.from_pretrained(
            MODEL_FILE,
            config=config,
            local_files_only=True,
        )

    llm = llms["mistral"]
    response = llm(contents, stream=True)
    message = ""
    for token in response:
        message += token
        yield message


chat_interface = pn.chat.ChatInterface(
    callback=callback,
    callback_user="Mistral",
    reset_on_send=True,
)
chat_interface.send(
    "Send a message to get a reply from Mistral!", user="System", respond=False
)
chat_interface.servable()