Ollama_MMLU_Pro.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sam-paech/Ollama-MMLU-Pro-IRT/blob/main/Ollama_MMLU_Pro.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "88WyLLrDBe-o",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "325ab622-8059-4cd5-dde6-8a6264821733"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sun Jun 23 23:22:02 2024       \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
            "|-----------------------------------------+----------------------+----------------------+\n",
            "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                                         |                      |               MIG M. |\n",
            "|=========================================+======================+======================|\n",
            "|   0  Tesla T4                       Off | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   47C    P8               9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                                         |                      |                  N/A |\n",
            "+-----------------------------------------+----------------------+----------------------+\n",
            "                                                                                         \n",
            "+---------------------------------------------------------------------------------------+\n",
            "| Processes:                                                                            |\n",
            "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
            "|        ID   ID                                                             Usage      |\n",
            "|=======================================================================================|\n",
            "|  No running processes found                                                           |\n",
            "+---------------------------------------------------------------------------------------+\n",
            "Your runtime has 13.6 gigabytes of available RAM\n",
            "\n",
            "Not using a high-RAM runtime\n"
          ]
        }
      ],
      "source": [
        "#@title #Runtime Info\n",
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)\n",
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "emySi6TEC-M8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1efbf7e8-6e5b-4641-99bd-e8595f6ec339"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  pci.ids usb.ids\n",
            "The following NEW packages will be installed:\n",
            "  lshw pci.ids usb.ids\n",
            "0 upgraded, 3 newly installed, 0 to remove and 45 not upgraded.\n",
            "Need to get 790 kB of archives.\n",
            "After this operation, 2,988 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 lshw amd64 02.19.git.2021.06.19.996aaad9c7-2build1 [321 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 pci.ids all 0.0~2022.01.22-1 [251 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 usb.ids all 2022.04.02-1 [219 kB]\n",
            "Fetched 790 kB in 0s (6,545 kB/s)\n",
            "Selecting previously unselected package lshw.\n",
            "(Reading database ... 121925 files and directories currently installed.)\n",
            "Preparing to unpack .../lshw_02.19.git.2021.06.19.996aaad9c7-2build1_amd64.deb ...\n",
            "Unpacking lshw (02.19.git.2021.06.19.996aaad9c7-2build1) ...\n",
            "Selecting previously unselected package pci.ids.\n",
            "Preparing to unpack .../pci.ids_0.0~2022.01.22-1_all.deb ...\n",
            "Unpacking pci.ids (0.0~2022.01.22-1) ...\n",
            "Selecting previously unselected package usb.ids.\n",
            "Preparing to unpack .../usb.ids_2022.04.02-1_all.deb ...\n",
            "Unpacking usb.ids (2022.04.02-1) ...\n",
            "Setting up pci.ids (0.0~2022.01.22-1) ...\n",
            "Setting up lshw (02.19.git.2021.06.19.996aaad9c7-2build1) ...\n",
            "Setting up usb.ids (2022.04.02-1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            ">>> Downloading ollama...\n",
            "############################################################################################# 100.0%\n",
            ">>> Installing ollama to /usr/local/bin...\n",
            ">>> Creating ollama user...\n",
            ">>> Adding ollama user to video group...\n",
            ">>> Adding current user to ollama group...\n",
            ">>> Creating ollama systemd service...\n",
            ">>> NVIDIA GPU installed.\n",
            ">>> The Ollama API is now available at 127.0.0.1:11434.\n",
            ">>> Install complete. Run \"ollama\" from the command line.\n",
            "Cloning into 'Ollama-MMLU-Pro'...\n",
            "remote: Enumerating objects: 8, done.\u001b[K\n",
            "remote: Counting objects: 100% (8/8), done.\u001b[K\n",
            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
            "remote: Total 8 (delta 0), reused 3 (delta 0), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (8/8), 8.98 KiB | 8.98 MiB/s, done.\n",
            "/content/Ollama-MMLU-Pro\n",
            "Collecting datasets (from -r requirements.txt (line 1))\n",
            "  Downloading datasets-2.20.0-py3-none-any.whl (547 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m547.8/547.8 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting openai (from -r requirements.txt (line 2))\n",
            "  Downloading openai-1.35.3-py3-none-any.whl (327 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m327.4/327.4 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (3.15.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (1.25.2)\n",
            "Collecting pyarrow>=15.0.0 (from datasets->-r requirements.txt (line 1))\n",
            "  Downloading pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl (40.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 MB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets->-r requirements.txt (line 1))\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (2.0.3)\n",
            "Collecting requests>=2.32.2 (from datasets->-r requirements.txt (line 1))\n",
            "  Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (4.66.4)\n",
            "Collecting xxhash (from datasets->-r requirements.txt (line 1))\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->-r requirements.txt (line 1))\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.5.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (3.9.5)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.2 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (0.23.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 1)) (6.0.1)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 2)) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai->-r requirements.txt (line 2)) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai->-r requirements.txt (line 2))\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 2)) (2.7.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 2)) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai->-r requirements.txt (line 2)) (4.12.2)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 2)) (3.7)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai->-r requirements.txt (line 2)) (1.2.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (4.0.3)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 2)) (2024.6.2)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai->-r requirements.txt (line 2))\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai->-r requirements.txt (line 2))\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 2)) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai->-r requirements.txt (line 2)) (2.18.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 1)) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 1)) (2.0.7)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 1)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 1)) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 1)) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets->-r requirements.txt (line 1)) (1.16.0)\n",
            "Installing collected packages: xxhash, requests, pyarrow, h11, dill, multiprocess, httpcore, httpx, openai, datasets\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.31.0\n",
            "    Uninstalling requests-2.31.0:\n",
            "      Successfully uninstalled requests-2.31.0\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 16.1.0 which is incompatible.\n",
            "google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.32.3 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 16.1.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed datasets-2.20.0 dill-0.3.8 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 multiprocess-0.70.16 openai-1.35.3 pyarrow-16.1.0 requests-2.32.3 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "#@title # Setup\n",
        "%cd /content/\n",
        "!apt install lshw\n",
        "!curl -fsSL https://ollama.com/install.sh | sh\n",
        "!git clone --depth 1 https://github.com/sam-paech/Ollama-MMLU-Pro-IRT.git\n",
        "%cd Ollama-MMLU-Pro-IRT\n",
        "!pip install -r requirements.txt\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title # Run server\n",
        "\n",
        "%env OLLAMA_NUM_PARALLEL=8\n",
        "%env OLLAMA_FLASH_ATTENTION=1\n",
        "\n",
        "import threading\n",
        "import subprocess\n",
        "server = threading.Thread(target=lambda: subprocess.run(['ollama', 'serve'], check=True), daemon=True)\n",
        "server.start()\n",
        "import  time\n",
        "\n",
        "time.sleep(2)"
      ],
      "metadata": {
        "id": "q9IV1xQao-We",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ffed8c6-2a60-4bd9-b1a0-46b0225afc99"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "env: OLLAMA_NUM_PARALLEL=8\n",
            "env: OLLAMA_FLASH_ATTENTION=1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Download\n",
        "print(\"Downloading phi3\")\n",
        "!ollama pull phi3 &>/dev/null\n"
      ],
      "metadata": {
        "id": "-8jD_-APlf51",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb73e680-57a2-4876-dbd8-58d30cef6598"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading phi3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Run Test\n",
        "\n",
        "!rm -rf eval_results\n",
        "!python run_openai.py \\\n",
        "    --url http://localhost:11434/v1 \\\n",
        "    --model phi3 \\\n",
        "    --verbosity 0 \\\n",
        "    --parallel 2\n"
      ],
      "metadata": {
        "id": "iNvYwyAJcNnz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ee3407a2-ba89-4f3f-fec3-05eba9a8f609"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading readme: 100% 9.56k/9.56k [00:00<00:00, 24.9MB/s]\n",
            "Downloading data: 100% 4.15M/4.15M [00:00<00:00, 10.5MB/s]\n",
            "Downloading data: 100% 45.3k/45.3k [00:00<00:00, 170kB/s]\n",
            "Generating test split: 100% 12032/12032 [00:00<00:00, 153306.88 examples/s]\n",
            "Generating validation split: 100% 70/70 [00:00<00:00, 19903.82 examples/s]\n",
            "assigned subjects ['computer science']\n",
            "Testing computer science...\n",
            "100% 410/410 [1:05:38<00:00,  9.61s/it]\n",
            "\n",
            "Correct: 135/410, Score: 32.93%\n",
            "Finished testing computer science.\n",
            "Finished the benchmark in 1 hours, 5 minutes, 43 seconds.\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}