Bielik_Streamlit_simple_app_tunnel_GGUF_Q4.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kcQyTiW0mK-b"
      },
      "source": [
        "<p align=\"center\">\n",
        "  <img src=\"https://huggingface.co/speakleash/Bielik-7B-Instruct-v0.1/raw/main/speakleash_cyfronet.png\">\n",
        "</p>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "auYFDEJyVcpI"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install -q streamlit accelerate transformers ctransformers[cuda]\n",
        "!npm install -g localtunnel"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sDfhIUWMXV0a"
      },
      "source": [
        "## Prosta aplikacja z wykorzystaniem Streamlit i Bielika w wersji skwantyzowanej GGUF (Q4)\n",
        "Metoda kwantyzacji: Group-wise Gradient Update Filtering (GGUF)\n",
        "Rozmiar: Q4_K_M\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wcT43Zm_X_Eh"
      },
      "outputs": [],
      "source": [
        "% % writefile\n",
        "app.py\n",
        "import streamlit as st\n",
        "from ctransformers import AutoModelForCausalLM\n",
        "from transformers import AutoTokenizer, TextIteratorStreamer\n",
        "from threading import Thread\n",
        "\n",
        "\n",
        "@st.cache_resource  # Dekorator, żeby model i inne rzeczy były trzymane w pamięci cache\n",
        "def prepare_model() -> tuple:\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"speakleash/Bielik-7B-Instruct-v0.1\")\n",
        "\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "            \"speakleash/Bielik-7B-Instruct-v0.1-GGUF\",\n",
        "            model_file=\"bielik-7b-instruct-v0.1.Q4_K_M.gguf\",\n",
        "            model_type=\"mistral\", gpu_layers=50, hf=True)\n",
        "\n",
        "    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)\n",
        "    return tokenizer, model, streamer\n",
        "\n",
        "\n",
        "def generate_text(tokenizer: AutoTokenizer, model: AutoModelForCausalLM,\n",
        "                  streamer: TextIteratorStreamer,\n",
        "                  prompt: str = \"\", temperature: float = 0.4,\n",
        "                  top_k: int = 200, top_p: float = 0.95, max_tokens: int = 512) -> str:\n",
        "    # Przygotowanie promptu przez tokenizer\n",
        "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n",
        "\n",
        "    # Przygotowanie zmiennych dla modelu\n",
        "    generation_kwargs = {\n",
        "        \"input_ids\": inputs[\"input_ids\"],\n",
        "        \"max_new_tokens\": max_tokens,\n",
        "        \"streamer\": streamer,\n",
        "        \"do_sample\": True if temperature else False,\n",
        "        'temperature': temperature,\n",
        "        'top_k': top_k,\n",
        "        'top_p': top_p\n",
        "    }\n",
        "\n",
        "    # Uruchamiamy generowanie modelu w osobnym wątku\n",
        "    thread = Thread(target=model.generate, kwargs=generation_kwargs)\n",
        "    thread.start()\n",
        "\n",
        "    for output in streamer:\n",
        "        yield output    # Zwracamy fragmenty odpowiedzi\n",
        "\n",
        "\n",
        "### ------------------------------------------------------------------------ ###\n",
        "# Aplikacja Streamlit\n",
        "\n",
        "# Historia czatu\n",
        "if \"messages\" not in st.session_state:\n",
        "    st.session_state.messages = []\n",
        "\n",
        "st.title(\"Bielik-v0.1-7B (Q4) LLM App\")\n",
        "\n",
        "# Ładowanie modelu\n",
        "with st.status(\"Pobieranie i przygotowanie modelu...\", expanded=True) as status:\n",
        "    tokenizer, model, streamer = prepare_model()\n",
        "    status.update(label=\"Model gotowy do rozmowy!\", state=\"complete\", expanded=False)\n",
        "\n",
        "# Ustawienia parametrów modelu\n",
        "col_temp, col_maxtokens, col_topk, col_topp = st.columns([1,1,1,1])\n",
        "with col_temp:\n",
        "    temperature = st.slider(\"Temperatura\", 0.01, 2.0, 0.4)\n",
        "with col_maxtokens:\n",
        "    max_tokens = st.slider(\"Max liczba tokenów:\", 1, 4096, 512)\n",
        "with col_topk:\n",
        "    top_k = st.slider(\"Top_k\", 1, 500, 200)\n",
        "with col_topp:\n",
        "    top_p = st.slider(\"Top_p\", 0.01, 1.0, 0.95)\n",
        "\n",
        "# Historia konwersacji\n",
        "for message in st.session_state.messages:\n",
        "    with st.chat_message(message[\"role\"]):\n",
        "        st.markdown(message[\"content\"])\n",
        "\n",
        "# Czat z modelem\n",
        "if human_prompt := st.chat_input(\"Witaj, o czym dzisiaj porozmawiamy?\"):\n",
        "    # Dodanie specjalnych tokenów, żeby model działał poprawnie\n",
        "    prompt = f\"<s>[INST]{human_prompt}[/INST]\"\n",
        "\n",
        "    # Zapisanie promptu do historii\n",
        "    st.session_state.messages.append({\"role\": \"user\", \"content\": human_prompt})\n",
        "\n",
        "    # Pokazanie wiadomości od użytkownika\n",
        "    with st.chat_message(\"user\"):\n",
        "        st.markdown(human_prompt)\n",
        "\n",
        "    # Odpowiedź modelu\n",
        "    with st.chat_message(\"assistant\"):\n",
        "        # Streaming - czyli wiadomość będzie uzupełniana jak tylko model zwróci kolejny token\n",
        "        # czyli będą się pojawiać kolejne tokeny i nie będziemy czekać na całą wiadomość\n",
        "        response_stream = st.write_stream(generate_text(tokenizer,\n",
        "                                                        model,\n",
        "                                                        streamer,\n",
        "                                                        prompt,\n",
        "                                                        temperature,\n",
        "                                                        top_k,\n",
        "                                                        top_p,\n",
        "                                                        max_tokens))\n",
        "\n",
        "    # Zapisanie odpowiedzi modelu do historii czatu\n",
        "    st.session_state.messages.append({\"role\": \"assistant\", \"content\": response_stream})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4hBLUhsIYnNm"
      },
      "outputs": [],
      "source": [
        "# Po uruchomieniu będzie adres IP - należy go skopiować i przejść na stronę, która będzie poniżej\n",
        "!streamlit run app.py &>/content/logs.txt &\n",
        "!npx localtunnel --port 8501 & curl ipv4.icanhazip.com"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}