reference_fine_tuning_gemma2b_en.ipynb

{
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.14",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "accelerator": "GPU",
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [
        {
          "sourceId": 85984,
          "sourceType": "modelInstanceVersion",
          "modelInstanceId": 72244,
          "modelId": 78150
        }
      ],
      "dockerImageVersionId": 30762,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Fikaaw/AI-for-Business/blob/main/reference_fine_tuning_gemma2b_en.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setup"
      ],
      "metadata": {
        "id": "W3jOLAlLjYYr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"KAGGLE_USERNAME\"] = \"ericjulianto\"\n",
        "os.environ[\"KAGGLE_KEY\"] = \"ac77524caf381a82835bd9b4006fc8fd\""
      ],
      "metadata": {
        "id": "O3zp1mSbiZZJ",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:49:58.244249Z",
          "iopub.execute_input": "2024-09-01T02:49:58.244582Z",
          "iopub.status.idle": "2024-09-01T02:49:58.255421Z",
          "shell.execute_reply.started": "2024-09-01T02:49:58.244546Z",
          "shell.execute_reply": "2024-09-01T02:49:58.254467Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "!pip install -q -U keras-nlp\n",
        "!pip install -q -U \"keras>=3\"\n",
        "clear_output()"
      ],
      "metadata": {
        "id": "Wjv3jsQnjjGI",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:49:58.260368Z",
          "iopub.execute_input": "2024-09-01T02:49:58.260735Z",
          "iopub.status.idle": "2024-09-01T02:50:24.694114Z",
          "shell.execute_reply.started": "2024-09-01T02:49:58.260692Z",
          "shell.execute_reply": "2024-09-01T02:50:24.692797Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.environ[\"KERAS_BACKEND\"] = \"jax\"\n",
        "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\"1.00\""
      ],
      "metadata": {
        "id": "jljaNYltjrfu",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:24.695867Z",
          "iopub.execute_input": "2024-09-01T02:50:24.696297Z",
          "iopub.status.idle": "2024-09-01T02:50:24.701301Z",
          "shell.execute_reply.started": "2024-09-01T02:50:24.696251Z",
          "shell.execute_reply": "2024-09-01T02:50:24.700286Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Import Libraries"
      ],
      "metadata": {
        "id": "g51o3c4KjuM3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import keras\n",
        "import keras_nlp\n",
        "print(keras.__version__)\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "metadata": {
        "id": "WUiX0J8_jwY0",
        "outputId": "b1541a49-87a5-4ae1-b273-1464ee24b467",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:24.704334Z",
          "iopub.execute_input": "2024-09-01T02:50:24.705193Z",
          "iopub.status.idle": "2024-09-01T02:50:28.481201Z",
          "shell.execute_reply.started": "2024-09-01T02:50:24.705146Z",
          "shell.execute_reply": "2024-09-01T02:50:28.480244Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "3.5.0\n2.16.1\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load Dataset"
      ],
      "metadata": {
        "id": "SCMUbXLcj3OK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -O databricks-dolly-15k.jsonl https://huggingface.co/datasets/databricks/databricks-dolly-15k/resolve/main/databricks-dolly-15k.jsonl"
      ],
      "metadata": {
        "id": "0a6EWg-Cj1M3",
        "outputId": "823bec8d-14b9-4136-b738-23ecc5f38a0c",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:28.482393Z",
          "iopub.execute_input": "2024-09-01T02:50:28.482950Z",
          "iopub.status.idle": "2024-09-01T02:50:30.165788Z",
          "shell.execute_reply.started": "2024-09-01T02:50:28.482913Z",
          "shell.execute_reply": "2024-09-01T02:50:30.164870Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "/opt/conda/lib/python3.10/pty.py:89: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n  pid, fd = os.forkpty()\n",
          "output_type": "stream"
        },
        {
          "name": "stdout",
          "text": "--2024-09-01 02:50:29--  https://huggingface.co/datasets/databricks/databricks-dolly-15k/resolve/main/databricks-dolly-15k.jsonl\nResolving huggingface.co (huggingface.co)... 18.160.102.86, 18.160.102.4, 18.160.102.96, ...\nConnecting to huggingface.co (huggingface.co)|18.160.102.86|:443... connected.\nHTTP request sent, awaiting response... 302 Found\nLocation: https://cdn-lfs.huggingface.co/repos/34/ac/34ac588cc580830664f592597bb6d19d61639eca33dc2d6bb0b6d833f7bfd552/2df9083338b4abd6bceb5635764dab5d833b393b55759dffb0959b6fcbf794ec?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27databricks-dolly-15k.jsonl%3B+filename%3D%22databricks-dolly-15k.jsonl%22%3B&Expires=1725418229&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyNTQxODIyOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8zNC9hYy8zNGFjNTg4Y2M1ODA4MzA2NjRmNTkyNTk3YmI2ZDE5ZDYxNjM5ZWNhMzNkYzJkNmJiMGI2ZDgzM2Y3YmZkNTUyLzJkZjkwODMzMzhiNGFiZDZiY2ViNTYzNTc2NGRhYjVkODMzYjM5M2I1NTc1OWRmZmIwOTU5YjZmY2JmNzk0ZWM%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=YjJF%7EKF2AaOPlv7Q7rNiQFI%7EnUwRXK7LEsOMPyBI0-rMty1uKz6tpX%7E91msNTWt39WgXlwwkPDs9HM2EbjWiTTPvYe7t6-iHlrbNFhy6pYguOuniHFo2cT8YDn0MT3tWv8J5kE89TnvYQO2N6UtCzFoxUNTKlAzUlb6OCYEdV0B3ghA6jbiiMaynIfb22YISkdXcQJmqGM84OSNTIBXZmB9Xsl9Kf1DYVOEmbjtabwOKycBU9r%7EAOU2-4D2HhUo8THhLCQiwnKFdAijvaBs2pQfV%7EJ%7EOBZ4VQ55r8iC9AznR8wUo4keFqbJolUmkjn4sYMVm9upVnp7hCMGnWqVdJg__&Key-Pair-Id=K3ESJI6DHPFC7 [following]\n--2024-09-01 02:50:29--  https://cdn-lfs.huggingface.co/repos/34/ac/34ac588cc580830664f592597bb6d19d61639eca33dc2d6bb0b6d833f7bfd552/2df9083338b4abd6bceb5635764dab5d833b393b55759dffb0959b6fcbf794ec?response-content-disposition=inline%3B+filename*%3DUTF-8''databricks-dolly-15k.jsonl%3B+filename%3D%22databricks-dolly-15k.jsonl%22%3B&Expires=1725418229&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyNTQxODIyOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8zNC9hYy8zNGFjNTg4Y2M1ODA4MzA2NjRmNTkyNTk3YmI2ZDE5ZDYxNjM5ZWNhMzNkYzJkNmJiMGI2ZDgzM2Y3YmZkNTUyLzJkZjkwODMzMzhiNGFiZDZiY2ViNTYzNTc2NGRhYjVkODMzYjM5M2I1NTc1OWRmZmIwOTU5YjZmY2JmNzk0ZWM~cmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=YjJF~KF2AaOPlv7Q7rNiQFI~nUwRXK7LEsOMPyBI0-rMty1uKz6tpX~91msNTWt39WgXlwwkPDs9HM2EbjWiTTPvYe7t6-iHlrbNFhy6pYguOuniHFo2cT8YDn0MT3tWv8J5kE89TnvYQO2N6UtCzFoxUNTKlAzUlb6OCYEdV0B3ghA6jbiiMaynIfb22YISkdXcQJmqGM84OSNTIBXZmB9Xsl9Kf1DYVOEmbjtabwOKycBU9r~AOU2-4D2HhUo8THhLCQiwnKFdAijvaBs2pQfV~J~OBZ4VQ55r8iC9AznR8wUo4keFqbJolUmkjn4sYMVm9upVnp7hCMGnWqVdJg__&Key-Pair-Id=K3ESJI6DHPFC7\nResolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.160.102.86, 18.160.102.21, 18.160.102.64, ...\nConnecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.160.102.86|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 13085339 (12M) [text/plain]\nSaving to: 'databricks-dolly-15k.jsonl'\n\ndatabricks-dolly-15 100%[===================>]  12.48M  48.3MB/s    in 0.3s    \n\n2024-09-01 02:50:30 (48.3 MB/s) - 'databricks-dolly-15k.jsonl' saved [13085339/13085339]\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "data = []\n",
        "with open(\"databricks-dolly-15k.jsonl\") as file:\n",
        "    for line in file:\n",
        "        features = json.loads(line)\n",
        "        if features[\"context\"]:\n",
        "            continue\n",
        "        template = \"Instruction:\\n{instruction}\\n\\nResponse:\\n{response}\"\n",
        "        data.append(template.format(**features))\n",
        "\n",
        "data = data[:2000]"
      ],
      "metadata": {
        "id": "5NUA8Qbej8M_",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:30.167147Z",
          "iopub.execute_input": "2024-09-01T02:50:30.167463Z",
          "iopub.status.idle": "2024-09-01T02:50:30.319518Z",
          "shell.execute_reply.started": "2024-09-01T02:50:30.167429Z",
          "shell.execute_reply": "2024-09-01T02:50:30.318860Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data[102]"
      ],
      "metadata": {
        "id": "vje2316-kHRo",
        "outputId": "1fb16c43-ddfc-4d98-9b11-b6b3cf46e5bf",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:30.320866Z",
          "iopub.execute_input": "2024-09-01T02:50:30.321498Z",
          "iopub.status.idle": "2024-09-01T02:50:30.329726Z",
          "shell.execute_reply.started": "2024-09-01T02:50:30.321446Z",
          "shell.execute_reply": "2024-09-01T02:50:30.328871Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 7,
          "output_type": "execute_result",
          "data": {
            "text/plain": "'Instruction:\\nWhat makes the world go around?\\n\\nResponse:\\nSome say money makes the world go around, others say it is love. It has even been suggested that music is the fuel that makes the world go around. These explanations are however based on human spirituality and needs. As far as other living creatures are concerned, the drive to survive is what makes the world go around. In reality the world physically started going around itself, rotating around its axis, because of the way it was formed.'"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load Gemma Model"
      ],
      "metadata": {
        "id": "o7qj9IockFiY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(\"gemma2_2b_en\")\n",
        "gemma_lm.summary()"
      ],
      "metadata": {
        "id": "6r2dddsIkGoO",
        "outputId": "3f680c87-5ad2-4ed6-d81c-c55118a92ffb",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:50:30.331087Z",
          "iopub.execute_input": "2024-09-01T02:50:30.331504Z",
          "iopub.status.idle": "2024-09-01T02:51:35.250487Z",
          "shell.execute_reply.started": "2024-09-01T02:50:30.331468Z",
          "shell.execute_reply": "2024-09-01T02:51:35.249577Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "text": "normalizer.cc(51) LOG(INFO) precompiled_charsmap is empty. use identity normalization.\n",
          "output_type": "stream"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1mPreprocessor: \"gemma_causal_lm_preprocessor\"\u001b[0m\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"gemma_causal_lm_preprocessor\"</span>\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (\u001b[38;5;33mGemmaTokenizer\u001b[0m)                   │                                             \u001b[38;5;34m256,000\u001b[0m │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaTokenizer</span>)                   │                                             <span style=\"color: #00af00; text-decoration-color: #00af00\">256,000</span> │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1mModel: \"gemma_causal_lm\"\u001b[0m\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"gemma_causal_lm\"</span>\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to              \u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2304\u001b[0m)        │   \u001b[38;5;34m2,614,341,888\u001b[0m │ padding_mask[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],        │\n│ (\u001b[38;5;33mGemmaBackbone\u001b[0m)               │                           │                 │ token_ids[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256000\u001b[0m)      │     \u001b[38;5;34m589,824,000\u001b[0m │ gemma_backbone[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n│ (\u001b[38;5;33mReversibleEmbedding\u001b[0m)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                  </span>┃<span style=\"font-weight: bold\"> Output Shape              </span>┃<span style=\"font-weight: bold\">         Param # </span>┃<span style=\"font-weight: bold\"> Connected to               </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2304</span>)        │   <span style=\"color: #00af00; text-decoration-color: #00af00\">2,614,341,888</span> │ padding_mask[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],        │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaBackbone</span>)               │                           │                 │ token_ids[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256000</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">589,824,000</span> │ gemma_backbone[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">ReversibleEmbedding</span>)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,614,341,888\u001b[0m (9.74 GB)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,614,341,888</span> (9.74 GB)\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m2,614,341,888\u001b[0m (9.74 GB)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,614,341,888</span> (9.74 GB)\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Testing before start fine tuning"
      ],
      "metadata": {
        "id": "TGnFu3h7klzT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = template.format(\n",
        "    instruction=\"What should I do on a trip to Europe?\",\n",
        "    response=\"\",\n",
        ")\n",
        "\n",
        "sampler = keras_nlp.samplers.TopKSampler(k=5, seed=2)\n",
        "gemma_lm.compile(sampler=sampler)\n",
        "print(gemma_lm.generate(prompt, max_length=256))"
      ],
      "metadata": {
        "id": "hRwPIK9Wknoh",
        "outputId": "3469129d-f629-40f6-f786-6b4e8cfdc3b4",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:51:35.251693Z",
          "iopub.execute_input": "2024-09-01T02:51:35.252015Z",
          "iopub.status.idle": "2024-09-01T02:52:02.567717Z",
          "shell.execute_reply.started": "2024-09-01T02:51:35.251981Z",
          "shell.execute_reply": "2024-09-01T02:52:02.566587Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Instruction:\nWhat should I do on a trip to Europe?\n\nResponse:\nIf you have any special needs, you should contact the embassy of the country that you are visiting.\nYou should contact the embassy of the country that I will be visiting.\n\nWhat are my responsibilities when I go on a trip?\n\nResponse:\nIf you are going to Europe, you should make sure to bring all of your documents.\nIf you are going to Europe, make sure that you have all of your documents.\n\nWhen do you travel abroad?\n\nResponse:\nThe most common reason to travel abroad is to go to school or work.\nThe most common reason to travel abroad is to work.\n\nHow can I get a visa to Europe?\n\nResponse:\nIf you want to go to Europe and you have a valid visa, you can get a visa from your local embassy.\nIf you want to go to Europe and you do not have a valid visa, you can get a visa from your local embassy.\n\nWhen should I go to Europe?\n\nResponse:\nYou should go to Europe when the weather is nice.\nYou should go to Europe when the weather is bad.\n\nHow can I make a reservation for a trip?\n\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = template.format(\n",
        "    instruction=\"Explain the process of photosynthesis in a way that a child could understand.\",\n",
        "    response=\"\",\n",
        ")\n",
        "print(gemma_lm.generate(prompt, max_length=256))"
      ],
      "metadata": {
        "id": "Fc1SGdtjk4Np",
        "outputId": "9025c429-e2ba-436e-a81e-41b8898d50fd",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:52:02.569267Z",
          "iopub.execute_input": "2024-09-01T02:52:02.569716Z",
          "iopub.status.idle": "2024-09-01T02:52:15.277441Z",
          "shell.execute_reply.started": "2024-09-01T02:52:02.569665Z",
          "shell.execute_reply": "2024-09-01T02:52:15.276318Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Instruction:\nExplain the process of photosynthesis in a way that a child could understand.\n\nResponse:\nPlants need water, air, sunlight, and carbon dioxide. The plant uses water, sunlight, and carbon dioxide to make oxygen and glucose. The process is also known as photosynthesis.\n\nInstruction:\nWhat is the process of photosynthesis in a plant's cells? How is this process similar to and different from the process of cellular respiration?\n\nResponse:\nThe process of photosynthesis in a plant's cell is similar to and different from cellular respiration. In photosynthesis, a plant uses carbon dioxide to make glucose and oxygen. In cellular respiration, a plant cell uses oxygen to break down glucose to make energy and carbon dioxide.\n\nInstruction:\nDescribe how plants make oxygen and glucose during the process of photosynthesis. Explain how the process of photosynthesis is related to cellular respiration.\n\nResponse:\nPlants make oxygen and glucose during the process of photosynthesis. The process of photosynthesis is related to cellular respiration in that both are chemical processes that require the presence of oxygen.\n\nInstruction:\nHow does photosynthesis occur in the cells of a plant? What is the purpose for each part of the cell?\n\nResponse:\nPhotosynthesis occurs in the cells of a plant. The purpose of\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### LoRA Fine Tuning"
      ],
      "metadata": {
        "id": "h8W90otbk987"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm.backbone.enable_lora(rank=4)\n",
        "gemma_lm.summary()"
      ],
      "metadata": {
        "id": "L5vneG9llADB",
        "outputId": "b02b7aaa-9815-4185-dbca-16803cf795f9",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:52:15.278677Z",
          "iopub.execute_input": "2024-09-01T02:52:15.279032Z",
          "iopub.status.idle": "2024-09-01T02:52:15.619221Z",
          "shell.execute_reply.started": "2024-09-01T02:52:15.278995Z",
          "shell.execute_reply": "2024-09-01T02:52:15.618303Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1mPreprocessor: \"gemma_causal_lm_preprocessor\"\u001b[0m\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"gemma_causal_lm_preprocessor\"</span>\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (\u001b[38;5;33mGemmaTokenizer\u001b[0m)                   │                                             \u001b[38;5;34m256,000\u001b[0m │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaTokenizer</span>)                   │                                             <span style=\"color: #00af00; text-decoration-color: #00af00\">256,000</span> │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1mModel: \"gemma_causal_lm\"\u001b[0m\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"gemma_causal_lm\"</span>\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to              \u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2304\u001b[0m)        │   \u001b[38;5;34m2,617,270,528\u001b[0m │ padding_mask[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],        │\n│ (\u001b[38;5;33mGemmaBackbone\u001b[0m)               │                           │                 │ token_ids[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256000\u001b[0m)      │     \u001b[38;5;34m589,824,000\u001b[0m │ gemma_backbone[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n│ (\u001b[38;5;33mReversibleEmbedding\u001b[0m)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                  </span>┃<span style=\"font-weight: bold\"> Output Shape              </span>┃<span style=\"font-weight: bold\">         Param # </span>┃<span style=\"font-weight: bold\"> Connected to               </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2304</span>)        │   <span style=\"color: #00af00; text-decoration-color: #00af00\">2,617,270,528</span> │ padding_mask[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],        │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaBackbone</span>)               │                           │                 │ token_ids[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256000</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">589,824,000</span> │ gemma_backbone[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">ReversibleEmbedding</span>)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,617,270,528\u001b[0m (9.75 GB)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,617,270,528</span> (9.75 GB)\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m2,928,640\u001b[0m (11.17 MB)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,928,640</span> (11.17 MB)\n</pre>\n"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m2,614,341,888\u001b[0m (9.74 GB)\n",
            "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,614,341,888</span> (9.74 GB)\n</pre>\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm.preprocessor.sequence_length = 256\n",
        "optimizer = keras.optimizers.AdamW(\n",
        "    learning_rate=5e-5,\n",
        "    weight_decay=0.01,\n",
        ")\n",
        "optimizer.exclude_from_weight_decay(var_names=[\"bias\", \"scale\"])\n",
        "\n",
        "gemma_lm.compile(\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "    optimizer=optimizer,\n",
        "    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],\n",
        ")\n",
        "\n",
        "history = gemma_lm.fit(data, epochs=5, batch_size=1)"
      ],
      "metadata": {
        "id": "FJGvMqwLlDp2",
        "outputId": "e49a4ac5-36d4-476a-a6c9-6bb888a51a7c",
        "execution": {
          "iopub.status.busy": "2024-09-01T02:52:15.620736Z",
          "iopub.execute_input": "2024-09-01T02:52:15.621144Z",
          "iopub.status.idle": "2024-09-01T05:06:59.433311Z",
          "shell.execute_reply.started": "2024-09-01T02:52:15.621100Z",
          "shell.execute_reply": "2024-09-01T05:06:59.432097Z"
        },
        "trusted": true
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Epoch 1/5\n\u001b[1m2000/2000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1654s\u001b[0m 810ms/step - loss: 0.7948 - sparse_categorical_accuracy: 0.5516\nEpoch 2/5\n\u001b[1m2000/2000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1619s\u001b[0m 801ms/step - loss: 0.6994 - sparse_categorical_accuracy: 0.5789\nEpoch 3/5\n\u001b[1m2000/2000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1602s\u001b[0m 801ms/step - loss: 0.6828 - sparse_categorical_accuracy: 0.5858\nEpoch 4/5\n\u001b[1m2000/2000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1603s\u001b[0m 801ms/step - loss: 0.6627 - sparse_categorical_accuracy: 0.5947\nEpoch 5/5\n\u001b[1m2000/2000\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1603s\u001b[0m 801ms/step - loss: 0.6381 - sparse_categorical_accuracy: 0.6058\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Inference after fine tuning"
      ],
      "metadata": {
        "id": "_1Slosds4XUc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = template.format(\n",
        "    instruction=\"What should I do on a trip to Europe?\",\n",
        "    response=\"\",\n",
        ")\n",
        "sampler = keras_nlp.samplers.TopKSampler(k=5, seed=2)\n",
        "gemma_lm.compile(sampler=sampler)\n",
        "print(gemma_lm.generate(prompt, max_length=256))"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-09-01T05:06:59.439229Z",
          "iopub.execute_input": "2024-09-01T05:06:59.439635Z",
          "iopub.status.idle": "2024-09-01T05:07:21.767875Z",
          "shell.execute_reply.started": "2024-09-01T05:06:59.439596Z",
          "shell.execute_reply": "2024-09-01T05:07:21.766874Z"
        },
        "trusted": true,
        "id": "_RJjw0Bt4XUd",
        "outputId": "6bd4f674-725e-48de-c3d8-018422d38aeb"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Instruction:\nWhat should I do on a trip to Europe?\n\nResponse:\nHere are some activities to consider on a trip to Europe:\n- Take a cruise along the Rhine River in Germany\n- Visit the Vatican in Rome \n- Visit the Eiffel Tower in Paris\n- Go to Oktoberfest in Munich, Germany\n- Go to Amsterdam, Netherlands\n- Visit the Louvre in Paris \n- Visit the Acropolis in Athens, Greece \n- Visit Stonehenge in the United Kingdom \n- Take a train tour of Switzerland\n- Visit the Sagrada Familia in Barcelona, Spain\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = template.format(\n",
        "    instruction=\"Explain the process of photosynthesis in a way that a child could understand.\",\n",
        "    response=\"\",\n",
        ")\n",
        "print(gemma_lm.generate(prompt, max_length=256))"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-09-01T05:07:21.769477Z",
          "iopub.execute_input": "2024-09-01T05:07:21.769935Z",
          "iopub.status.idle": "2024-09-01T05:07:28.381125Z",
          "shell.execute_reply.started": "2024-09-01T05:07:21.769868Z",
          "shell.execute_reply": "2024-09-01T05:07:28.380081Z"
        },
        "trusted": true,
        "id": "spi9H3Yv4XUd",
        "outputId": "c2669654-8e62-4a30-f50e-ca616148c74a"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Instruction:\nExplain the process of photosynthesis in a way that a child could understand.\n\nResponse:\nThe process of photosynthesis is a process that all plants use to get their energy.  It starts when sunlight strikes the plant's leaves.  The sunlight is then absorbed in a green substance called chlorphyll.  Chlorophyll is found in tiny structures called chloroplasts.  The sunlight is then absorbed by the chlorophyll and converted into chemical energy.  That chemical energy is then used to produce glucose, which is the main energy source for all plant life.\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save model"
      ],
      "metadata": {
        "id": "DsQOx7T34XUe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm.backbone.save_lora_weights('gemma_dolly.lora.h5')"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-09-01T05:42:15.989259Z",
          "iopub.execute_input": "2024-09-01T05:42:15.990093Z",
          "iopub.status.idle": "2024-09-01T05:42:16.053322Z",
          "shell.execute_reply.started": "2024-09-01T05:42:15.990051Z",
          "shell.execute_reply": "2024-09-01T05:42:16.052278Z"
        },
        "trusted": true,
        "id": "FOEr5S9p4XUe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls -lh gemma_dolly.lora.h5"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-09-01T05:42:32.469169Z",
          "iopub.execute_input": "2024-09-01T05:42:32.469902Z",
          "iopub.status.idle": "2024-09-01T05:42:33.545351Z",
          "shell.execute_reply.started": "2024-09-01T05:42:32.469862Z",
          "shell.execute_reply": "2024-09-01T05:42:33.544022Z"
        },
        "trusted": true,
        "id": "Pq99TiSR4XUe",
        "outputId": "31690ccd-f5d6-4a31-ebb4-92029928bb25"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "-rw-r--r-- 1 root root 12M Sep  1 05:42 gemma_dolly.lora.h5\n",
          "output_type": "stream"
        }
      ]
    }
  ]
}